[dpdk-dev] [PATCH 1/2] net/mlx5: move getter functions from net to common

[dpdk-dev] [PATCH 1/2] net/mlx5: move getter functions from net to common

[Ori Kam]

From: Ophir Munk <ophirmu@mellanox.com>

Getter functions such as: 'mlx5_os_get_ctx_device_name',
'mlx5_os_get_ctx_device_path', 'mlx5_os_get_dev_device_name',
'mlx5_os_get_umem_id' are implemented under net directory. To enable
additional devices (e.g. regex, vdpa) to access these getter functions
they are moved under common directory.

Signed-off-by: Ophir Munk <ophirmu@mellanox.com>
---
 drivers/common/mlx5/linux/mlx5_common_os.h | 98 ++++++++++++++++++++++++++++++
 drivers/net/mlx5/linux/mlx5_os.c           | 74 +---------------------
 drivers/net/mlx5/mlx5.c                    |  1 +
 drivers/net/mlx5/mlx5.h                    |  4 --
 drivers/net/mlx5/mlx5_flow_dv.c            |  1 +
 drivers/net/mlx5/mlx5_rxq.c                |  1 +
 6 files changed, 102 insertions(+), 77 deletions(-)
 create mode 100644 drivers/common/mlx5/linux/mlx5_common_os.h

diff --git a/drivers/common/mlx5/linux/mlx5_common_os.h b/drivers/common/mlx5/linux/mlx5_common_os.h
new file mode 100644
index 0000000..9a6872c
--- /dev/null
+++ b/drivers/common/mlx5/linux/mlx5_common_os.h
@@ -0,0 +1,98 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_COMMON_OS_H_
+#define RTE_PMD_MLX5_COMMON_OS_H_
+
+#include <stdio.h>
+
+#include <rte_pci.h>
+#include <rte_debug.h>
+#include <rte_atomic.h>
+#include <rte_log.h>
+#include <rte_kvargs.h>
+#include <rte_devargs.h>
+
+#include "mlx5_autoconf.h"
+#ifdef HAVE_INFINIBAND_VERBS_H
+#include <infiniband/verbs.h>
+#endif
+#ifdef HAVE_INFINIBAND_MLX5DV_H
+#include <infiniband/mlx5dv.h>
+#endif
+
+/**
+ * Get device name. Given an ibv_device pointer - return a
+ * pointer to the corresponding device name.
+ *
+ * @param[in] dev
+ *   Pointer to ibv device.
+ *
+ * @return
+ *   Pointer to device name if dev is valid, NULL otherwise.
+ */
+static inline const char *
+mlx5_os_get_dev_device_name(void *dev)
+{
+	if (!dev)
+		return NULL;
+	return ((struct ibv_device *)dev)->name;
+}
+
+/**
+ * Get ibv device name. Given an ibv_context pointer - return a
+ * pointer to the corresponding device name.
+ *
+ * @param[in] ctx
+ *   Pointer to ibv context.
+ *
+ * @return
+ *   Pointer to device name if ctx is valid, NULL otherwise.
+ */
+static inline const char *
+mlx5_os_get_ctx_device_name(void *ctx)
+{
+	if (!ctx)
+		return NULL;
+	return ((struct ibv_context *)ctx)->device->name;
+}
+
+/**
+ * Get ibv device path name. Given an ibv_context pointer - return a
+ * pointer to the corresponding device path name.
+ *
+ * @param[in] ctx
+ *   Pointer to ibv context.
+ *
+ * @return
+ *   Pointer to device path name if ctx is valid, NULL otherwise.
+ */
+
+static inline const char *
+mlx5_os_get_ctx_device_path(void *ctx)
+{
+	if (!ctx)
+		return NULL;
+
+	return ((struct ibv_context *)ctx)->device->ibdev_path;
+}
+
+/**
+ * Get umem id. Given a pointer to umem object of type
+ * 'struct mlx5dv_devx_umem *' - return its id.
+ *
+ * @param[in] umem
+ *    Pointer to umem object.
+ *
+ * @return
+ *    The umem id if umem is valid, 0 otherwise.
+ */
+static inline uint32_t
+mlx5_os_get_umem_id(void *umem)
+{
+	if (!umem)
+		return 0;
+	return ((struct mlx5dv_devx_umem *)umem)->umem_id;
+}
+#endif /* RTE_PMD_MLX5_COMMON_OS_H_ */
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 844c6c0..854409e 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -45,6 +45,7 @@
 
 #include "mlx5_defs.h"
 #include "mlx5.h"
+#include "mlx5_common_os.h"
 #include "mlx5_utils.h"
 #include "mlx5_rxtx.h"
 #include "mlx5_autoconf.h"
@@ -64,79 +65,6 @@
 #endif
 
 /**
- * Get device name. Given an ibv_device pointer - return a
- * pointer to the corresponding device name.
- *
- * @param[in] dev
- *   Pointer to ibv device.
- *
- * @return
- *   Pointer to device name if dev is valid, NULL otherwise.
- */
-const char *
-mlx5_os_get_dev_device_name(void *dev)
-{
-	if (!dev)
-		return NULL;
-	return ((struct ibv_device *)dev)->name;
-}
-
-/**
- * Get ibv device name. Given an ibv_context pointer - return a
- * pointer to the corresponding device name.
- *
- * @param[in] ctx
- *   Pointer to ibv context.
- *
- * @return
- *   Pointer to device name if ctx is valid, NULL otherwise.
- */
-const char *
-mlx5_os_get_ctx_device_name(void *ctx)
-{
-	if (!ctx)
-		return NULL;
-	return ((struct ibv_context *)ctx)->device->name;
-}
-
-/**
- * Get ibv device path name. Given an ibv_context pointer - return a
- * pointer to the corresponding device path name.
- *
- * @param[in] ctx
- *   Pointer to ibv context.
- *
- * @return
- *   Pointer to device path name if ctx is valid, NULL otherwise.
- */
-const char *
-mlx5_os_get_ctx_device_path(void *ctx)
-{
-	if (!ctx)
-		return NULL;
-
-	return ((struct ibv_context *)ctx)->device->ibdev_path;
-}
-
-/**
- * Get umem id. Given a pointer to umem object of type
- * 'struct mlx5dv_devx_umem *' - return its id.
- *
- * @param[in] umem
- *   Pointer to umem object.
- *
- * @return
- *   The umem id if umem is valid, 0 otherwise.
- */
-uint32_t
-mlx5_os_get_umem_id(void *umem)
-{
-	if (!umem)
-		return 0;
-	return ((struct mlx5dv_devx_umem *)umem)->umem_id;
-}
-
-/**
  * Get mlx5 device attributes. The glue function query_device_ex() is called
  * with out parameter of type 'struct ibv_device_attr_ex *'. Then fill in mlx5
  * device attributes from the glue out parameter.
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index b22acbb..d914b10 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -38,6 +38,7 @@
 #include <mlx5_glue.h>
 #include <mlx5_devx_cmds.h>
 #include <mlx5_common.h>
+#include <mlx5_common_os.h>
 #include <mlx5_common_mp.h>
 
 #include "mlx5_defs.h"
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 94a3667..96c1eb1 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -921,10 +921,6 @@ struct mlx5_flow_meter *mlx5_flow_meter_attach
 
 /* mlx5_os.c */
 struct rte_pci_driver;
-const char *mlx5_os_get_ctx_device_name(void *ctx);
-const char *mlx5_os_get_ctx_device_path(void *ctx);
-const char *mlx5_os_get_dev_device_name(void *dev);
-uint32_t mlx5_os_get_umem_id(void *umem);
 int mlx5_os_get_dev_attr(void *ctx, struct mlx5_dev_attr *dev_attr);
 void mlx5_os_free_shared_dr(struct mlx5_priv *priv);
 int mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 5bb252e..f0ce195 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -36,6 +36,7 @@
 
 #include "mlx5_defs.h"
 #include "mlx5.h"
+#include "mlx5_common_os.h"
 #include "mlx5_flow.h"
 #include "mlx5_rxtx.h"
 
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index dda0073..efd1e3f 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -34,6 +34,7 @@
 
 #include "mlx5_defs.h"
 #include "mlx5.h"
+#include "mlx5_common_os.h"
 #include "mlx5_rxtx.h"
 #include "mlx5_utils.h"
 #include "mlx5_autoconf.h"
-- 
1.8.3.1

[dpdk-dev] [PATCH 2/2] common/mlx5: move doorbell record to common

[Ori Kam]

The creation of DBR can be used by a number of different
Mellanox PMDs. for example RegEx / Net / VDPA.

This commits moves the DBR creation and relase functions to common
folder.

Signed-off-by: Ori Kam <orika@mellanox.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/common/mlx5/linux/mlx5_common_os.c      |   1 +
 drivers/common/mlx5/mlx5_common.c               | 122 ++++++++++++++++++++++++
 drivers/common/mlx5/mlx5_common.h               |  38 +++++++-
 drivers/common/mlx5/mlx5_devx_cmds.h            |  13 ---
 drivers/common/mlx5/rte_common_mlx5_version.map |   3 +
 drivers/net/mlx5/mlx5.c                         | 120 -----------------------
 drivers/net/mlx5/mlx5.h                         |  21 +---
 drivers/net/mlx5/mlx5_rxq.c                     |   6 +-
 8 files changed, 168 insertions(+), 156 deletions(-)

diff --git a/drivers/common/mlx5/linux/mlx5_common_os.c b/drivers/common/mlx5/linux/mlx5_common_os.c
index 1b71347..823a0b5 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.c
+++ b/drivers/common/mlx5/linux/mlx5_common_os.c
@@ -315,3 +315,4 @@
 		" libmlx5)");
 	mlx5_glue = NULL;
 }
+
diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c
index db94d4a..693e2c6 100644
--- a/drivers/common/mlx5/mlx5_common.c
+++ b/drivers/common/mlx5/mlx5_common.c
@@ -7,8 +7,11 @@
 #include <stdio.h>
 
 #include <rte_errno.h>
+#include <rte_mempool.h>
+#include <rte_malloc.h>
 
 #include "mlx5_common.h"
+#include "mlx5_common_os.h"
 #include "mlx5_common_utils.h"
 
 int mlx5_common_logtype;
@@ -150,3 +153,122 @@ static inline void mlx5_cpu_id(unsigned int level,
 #endif
 	haswell_broadwell_cpu = 0;
 }
+
+/**
+ * Allocate page of door-bells and register it using DevX API.
+ *
+ * @param [in] ctx
+ *   Pointer to the device context.
+ *
+ * @return
+ *   Pointer to new page on success, NULL otherwise.
+ */
+static struct mlx5_devx_dbr_page *
+mlx5_alloc_dbr_page(void *ctx)
+{
+	struct mlx5_devx_dbr_page *page;
+
+	/* Allocate space for door-bell page and management data. */
+	page = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_devx_dbr_page),
+				 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+	if (!page) {
+		DRV_LOG(ERR, "cannot allocate dbr page");
+		return NULL;
+	}
+	/* Register allocated memory. */
+	page->umem = mlx5_glue->devx_umem_reg(ctx, page->dbrs,
+					      MLX5_DBR_PAGE_SIZE, 0);
+	if (!page->umem) {
+		DRV_LOG(ERR, "cannot umem reg dbr page");
+		rte_free(page);
+		return NULL;
+	}
+	return page;
+}
+
+/**
+ * Find the next available door-bell, allocate new page if needed.
+ *
+ * @param [in] ctx
+ *   Pointer to device context.
+ * @param [in] head
+ *   Pointer to the head of dbr pages list.
+ * @param [out] dbr_page
+ *   Door-bell page containing the page data.
+ *
+ * @return
+ *   Door-bell address offset on success, a negative error value otherwise.
+ */
+int64_t
+mlx5_get_dbr(void *ctx,  struct mlx5_dbr_page_list *head,
+	     struct mlx5_devx_dbr_page **dbr_page)
+{
+	struct mlx5_devx_dbr_page *page = NULL;
+	uint32_t i, j;
+
+	LIST_FOREACH(page, head, next)
+		if (page->dbr_count < MLX5_DBR_PER_PAGE)
+			break;
+	if (!page) { /* No page with free door-bell exists. */
+		page = mlx5_alloc_dbr_page(ctx);
+		if (!page) /* Failed to allocate new page. */
+			return (-1);
+		LIST_INSERT_HEAD(head, page, next);
+	}
+	/* Loop to find bitmap part with clear bit. */
+	for (i = 0;
+	     i < MLX5_DBR_BITMAP_SIZE && page->dbr_bitmap[i] == UINT64_MAX;
+	     i++)
+		; /* Empty. */
+	/* Find the first clear bit. */
+	MLX5_ASSERT(i < MLX5_DBR_BITMAP_SIZE);
+	j = rte_bsf64(~page->dbr_bitmap[i]);
+	page->dbr_bitmap[i] |= (UINT64_C(1) << j);
+	page->dbr_count++;
+	*dbr_page = page;
+	return (((i * 64) + j) * sizeof(uint64_t));
+}
+
+/**
+ * Release a door-bell record.
+ *
+ * @param [in] head
+ *   Pointer to the head of dbr pages list.
+ * @param [in] umem_id
+ *   UMEM ID of page containing the door-bell record to release.
+ * @param [in] offset
+ *   Offset of door-bell record in page.
+ *
+ * @return
+ *   0 on success, a negative error value otherwise.
+ */
+int32_t
+mlx5_release_dbr(struct mlx5_dbr_page_list *head, uint32_t umem_id,
+		 uint64_t offset)
+{
+	struct mlx5_devx_dbr_page *page = NULL;
+	int ret = 0;
+
+	LIST_FOREACH(page, head, next)
+		/* Find the page this address belongs to. */
+		if (mlx5_os_get_umem_id(page->umem) == umem_id)
+			break;
+	if (!page)
+		return -EINVAL;
+	page->dbr_count--;
+	if (!page->dbr_count) {
+		/* Page not used, free it and remove from list. */
+		LIST_REMOVE(page, next);
+		if (page->umem)
+			ret = -mlx5_glue->devx_umem_dereg(page->umem);
+		rte_free(page);
+	} else {
+		/* Mark in bitmap that this door-bell is not in use. */
+		offset /= MLX5_DBR_SIZE;
+		int i = offset / 64;
+		int j = offset % 64;
+
+		page->dbr_bitmap[i] &= ~(UINT64_C(1) << j);
+	}
+	return ret;
+}
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 77f10e6..8b8f3a0 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -15,6 +15,7 @@
 #include <rte_devargs.h>
 
 #include "mlx5_prm.h"
+#include "mlx5_devx_cmds.h"
 
 
 /* Bit-field manipulation. */
@@ -207,6 +208,36 @@ enum mlx5_class {
 	MLX5_CLASS_INVALID,
 };
 
+#define MLX5_DBR_PAGE_SIZE 4096 /* Must be >= 512. */
+#define MLX5_DBR_SIZE 8
+#define MLX5_DBR_PER_PAGE (MLX5_DBR_PAGE_SIZE / MLX5_DBR_SIZE)
+#define MLX5_DBR_BITMAP_SIZE (MLX5_DBR_PER_PAGE / 64)
+
+struct mlx5_devx_dbr_page {
+	/* Door-bell records, must be first member in structure. */
+	uint8_t dbrs[MLX5_DBR_PAGE_SIZE];
+	LIST_ENTRY(mlx5_devx_dbr_page) next; /* Pointer to the next element. */
+	void *umem;
+	uint32_t dbr_count; /* Number of door-bell records in use. */
+	/* 1 bit marks matching door-bell is in use. */
+	uint64_t dbr_bitmap[MLX5_DBR_BITMAP_SIZE];
+};
+
+/* devX creation object */
+struct mlx5_devx_obj {
+	void *obj; /* The DV object. */
+	int id; /* The object ID. */
+};
+
+/* UMR memory buffer used to define 1 entry in indirect mkey. */
+struct mlx5_klm {
+	uint32_t byte_count;
+	uint32_t mkey;
+	uint64_t address;
+};
+
+LIST_HEAD(mlx5_dbr_page_list, mlx5_devx_dbr_page);
+
 __rte_internal
 enum mlx5_class mlx5_class_get(struct rte_devargs *devargs);
 __rte_internal
@@ -214,7 +245,12 @@ void mlx5_translate_port_name(const char *port_name_in,
 			      struct mlx5_switch_info *port_info_out);
 void mlx5_glue_constructor(void);
 size_t mlx5_os_get_page_size(void);
-
+__rte_internal
+int64_t mlx5_get_dbr(void *ctx,  struct mlx5_dbr_page_list *head,
+		     struct mlx5_devx_dbr_page **dbr_page);
+__rte_internal
+int32_t mlx5_release_dbr(struct mlx5_dbr_page_list *head, uint32_t umem_id,
+			 uint64_t offset);
 extern uint8_t haswell_broadwell_cpu;
 
 #endif /* RTE_PMD_MLX5_COMMON_H_ */
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 49b174a..a2b2ad4 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -9,19 +9,6 @@
 #include "mlx5_prm.h"
 
 
-/* devX creation object */
-struct mlx5_devx_obj {
-	void *obj; /* The DV object. */
-	int id; /* The object ID. */
-};
-
-/* UMR memory buffer used to define 1 entry in indirect mkey. */
-struct mlx5_klm {
-	uint32_t byte_count;
-	uint32_t mkey;
-	uint64_t address;
-};
-
 /* This is limitation of libibverbs: in length variable type is u16. */
 #define MLX5_DEVX_MAX_KLM_ENTRIES ((UINT16_MAX - \
 		MLX5_ST_SZ_DW(create_mkey_in) * 4) / (MLX5_ST_SZ_DW(klm) * 4))
diff --git a/drivers/common/mlx5/rte_common_mlx5_version.map b/drivers/common/mlx5/rte_common_mlx5_version.map
index 350e771..3674083 100644
--- a/drivers/common/mlx5/rte_common_mlx5_version.map
+++ b/drivers/common/mlx5/rte_common_mlx5_version.map
@@ -31,6 +31,9 @@ INTERNAL {
 	mlx5_devx_cmd_query_virtq;
 	mlx5_devx_get_out_command_status;
 
+	mlx5_get_dbr;
+	mlx5_release_dbr;
+
 	mlx5_mp_init_primary;
 	mlx5_mp_uninit_primary;
 	mlx5_mp_init_secondary;
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index d914b10..8b5c967 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1664,126 +1664,6 @@ struct mlx5_dev_ctx_shared *
 	DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask);
 }
 
-/**
- * Allocate page of door-bells and register it using DevX API.
- *
- * @param [in] dev
- *   Pointer to Ethernet device.
- *
- * @return
- *   Pointer to new page on success, NULL otherwise.
- */
-static struct mlx5_devx_dbr_page *
-mlx5_alloc_dbr_page(struct rte_eth_dev *dev)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_devx_dbr_page *page;
-
-	/* Allocate space for door-bell page and management data. */
-	page = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_devx_dbr_page),
-				 RTE_CACHE_LINE_SIZE, dev->device->numa_node);
-	if (!page) {
-		DRV_LOG(ERR, "port %u cannot allocate dbr page",
-			dev->data->port_id);
-		return NULL;
-	}
-	/* Register allocated memory. */
-	page->umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, page->dbrs,
-					      MLX5_DBR_PAGE_SIZE, 0);
-	if (!page->umem) {
-		DRV_LOG(ERR, "port %u cannot umem reg dbr page",
-			dev->data->port_id);
-		rte_free(page);
-		return NULL;
-	}
-	return page;
-}
-
-/**
- * Find the next available door-bell, allocate new page if needed.
- *
- * @param [in] dev
- *   Pointer to Ethernet device.
- * @param [out] dbr_page
- *   Door-bell page containing the page data.
- *
- * @return
- *   Door-bell address offset on success, a negative error value otherwise.
- */
-int64_t
-mlx5_get_dbr(struct rte_eth_dev *dev, struct mlx5_devx_dbr_page **dbr_page)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_devx_dbr_page *page = NULL;
-	uint32_t i, j;
-
-	LIST_FOREACH(page, &priv->dbrpgs, next)
-		if (page->dbr_count < MLX5_DBR_PER_PAGE)
-			break;
-	if (!page) { /* No page with free door-bell exists. */
-		page = mlx5_alloc_dbr_page(dev);
-		if (!page) /* Failed to allocate new page. */
-			return (-1);
-		LIST_INSERT_HEAD(&priv->dbrpgs, page, next);
-	}
-	/* Loop to find bitmap part with clear bit. */
-	for (i = 0;
-	     i < MLX5_DBR_BITMAP_SIZE && page->dbr_bitmap[i] == UINT64_MAX;
-	     i++)
-		; /* Empty. */
-	/* Find the first clear bit. */
-	MLX5_ASSERT(i < MLX5_DBR_BITMAP_SIZE);
-	j = rte_bsf64(~page->dbr_bitmap[i]);
-	page->dbr_bitmap[i] |= (UINT64_C(1) << j);
-	page->dbr_count++;
-	*dbr_page = page;
-	return (((i * 64) + j) * sizeof(uint64_t));
-}
-
-/**
- * Release a door-bell record.
- *
- * @param [in] dev
- *   Pointer to Ethernet device.
- * @param [in] umem_id
- *   UMEM ID of page containing the door-bell record to release.
- * @param [in] offset
- *   Offset of door-bell record in page.
- *
- * @return
- *   0 on success, a negative error value otherwise.
- */
-int32_t
-mlx5_release_dbr(struct rte_eth_dev *dev, uint32_t umem_id, uint64_t offset)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_devx_dbr_page *page = NULL;
-	int ret = 0;
-
-	LIST_FOREACH(page, &priv->dbrpgs, next)
-		/* Find the page this address belongs to. */
-		if (mlx5_os_get_umem_id(page->umem) == umem_id)
-			break;
-	if (!page)
-		return -EINVAL;
-	page->dbr_count--;
-	if (!page->dbr_count) {
-		/* Page not used, free it and remove from list. */
-		LIST_REMOVE(page, next);
-		if (page->umem)
-			ret = -mlx5_glue->devx_umem_dereg(page->umem);
-		rte_free(page);
-	} else {
-		/* Mark in bitmap that this door-bell is not in use. */
-		offset /= MLX5_DBR_SIZE;
-		int i = offset / 64;
-		int j = offset % 64;
-
-		page->dbr_bitmap[i] &= ~(UINT64_C(1) << j);
-	}
-	return ret;
-}
-
 int
 rte_pmd_mlx5_get_dyn_flag_names(char *names[], unsigned int n)
 {
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 96c1eb1..5e10272 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -500,21 +500,6 @@ struct mlx5_flow_tbl_resource {
 #define MLX5_MAX_TABLES_EXTERNAL (MLX5_MAX_TABLES - 3)
 #define MLX5_MAX_TABLES_FDB UINT16_MAX
 
-#define MLX5_DBR_PAGE_SIZE 4096 /* Must be >= 512. */
-#define MLX5_DBR_SIZE 8
-#define MLX5_DBR_PER_PAGE (MLX5_DBR_PAGE_SIZE / MLX5_DBR_SIZE)
-#define MLX5_DBR_BITMAP_SIZE (MLX5_DBR_PER_PAGE / 64)
-
-struct mlx5_devx_dbr_page {
-	/* Door-bell records, must be first member in structure. */
-	uint8_t dbrs[MLX5_DBR_PAGE_SIZE];
-	LIST_ENTRY(mlx5_devx_dbr_page) next; /* Pointer to the next element. */
-	void *umem;
-	uint32_t dbr_count; /* Number of door-bell records in use. */
-	/* 1 bit marks matching door-bell is in use. */
-	uint64_t dbr_bitmap[MLX5_DBR_BITMAP_SIZE];
-};
-
 /* ID generation structure. */
 struct mlx5_flow_id_pool {
 	uint32_t *free_arr; /**< Pointer to the a array of free values. */
@@ -655,7 +640,7 @@ struct mlx5_priv {
 	/* Context for Verbs allocator. */
 	int nl_socket_rdma; /* Netlink socket (NETLINK_RDMA). */
 	int nl_socket_route; /* Netlink socket (NETLINK_ROUTE). */
-	LIST_HEAD(dbrpage, mlx5_devx_dbr_page) dbrpgs; /* Door-bell pages. */
+	struct mlx5_dbr_page_list dbrpgs; /* Door-bell pages. */
 	struct mlx5_nl_vlan_vmwa_context *vmwa_context; /* VLAN WA context. */
 	struct mlx5_flow_id_pool *qrss_id_pool;
 	struct mlx5_hlist *mreg_cp_tbl;
@@ -682,10 +667,6 @@ struct mlx5_priv {
 
 int mlx5_getenv_int(const char *);
 int mlx5_proc_priv_init(struct rte_eth_dev *dev);
-int64_t mlx5_get_dbr(struct rte_eth_dev *dev,
-		     struct mlx5_devx_dbr_page **dbr_page);
-int32_t mlx5_release_dbr(struct rte_eth_dev *dev, uint32_t umem_id,
-			 uint64_t offset);
 int mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev,
 			      struct rte_eth_udp_tunnel *udp_tunnel);
 uint16_t mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index efd1e3f..bd0037b 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1414,7 +1414,8 @@ struct mlx5_rxq_obj *
 		struct mlx5_devx_dbr_page *dbr_page;
 		int64_t dbr_offset;
 
-		dbr_offset = mlx5_get_dbr(dev, &dbr_page);
+		dbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs,
+					  &dbr_page);
 		if (dbr_offset < 0)
 			goto error;
 		rxq_ctrl->dbr_offset = dbr_offset;
@@ -2102,7 +2103,8 @@ struct mlx5_rxq_ctrl *
 		rxq_ctrl->obj = NULL;
 	if (rte_atomic32_dec_and_test(&rxq_ctrl->refcnt)) {
 		if (rxq_ctrl->dbr_umem_id_valid)
-			claim_zero(mlx5_release_dbr(dev, rxq_ctrl->dbr_umem_id,
+			claim_zero(mlx5_release_dbr(&priv->dbrpgs,
+						    rxq_ctrl->dbr_umem_id,
 						    rxq_ctrl->dbr_offset));
 		if (rxq_ctrl->type == MLX5_RXQ_TYPE_STANDARD)
 			mlx5_mr_btree_free(&rxq_ctrl->rxq.mr_ctrl.cache_bh);
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH 1/2] net/mlx5: move getter functions from net to common

[Matan Azrad]

From: Ori Kam:
> From: Ophir Munk <ophirmu@mellanox.com>
> 
> Getter functions such as: 'mlx5_os_get_ctx_device_name',
> 'mlx5_os_get_ctx_device_path', 'mlx5_os_get_dev_device_name',
> 'mlx5_os_get_umem_id' are implemented under net directory. To enable
> additional devices (e.g. regex, vdpa) to access these getter functions they
> are moved under common directory.
> 
> Signed-off-by: Ophir Munk <ophirmu@mellanox.com>
Acked-by: Matan Azrad <matan@mellanox.com>

[dpdk-dev] [PATCH v2 1/2] net/mlx5: move getter functions from net to common

[Ori Kam]

From: Ophir Munk <ophirmu@mellanox.com>

Getter functions such as: 'mlx5_os_get_ctx_device_name',
'mlx5_os_get_ctx_device_path', 'mlx5_os_get_dev_device_name',
'mlx5_os_get_umem_id' are implemented under net directory. To enable
additional devices (e.g. regex, vdpa) to access these getter functions
they are moved under common directory.

As part of this commit string sizes DEV_SYSFS_NAME_MAX and
DEV_SYSFS_PATH_MAX are increased by 1 to make sure that the destination
string size in strncpy() function is bigger than the source string size.
This update will avoid GCC version 8 error -Werror=stringop-truncation.

Signed-off-by: Ophir Munk <ophirmu@mellanox.com>
Acked-by: Matan Azrad <matan@mellanox.com>
---
v2:
* avoid GCC version 8 error.
---
 drivers/common/mlx5/linux/mlx5_common_os.h | 98 ++++++++++++++++++++++++++++++
 drivers/net/mlx5/linux/mlx5_os.c           | 74 +---------------------
 drivers/net/mlx5/linux/mlx5_os.h           |  4 +-
 drivers/net/mlx5/mlx5.c                    |  1 +
 drivers/net/mlx5/mlx5.h                    |  4 --
 drivers/net/mlx5/mlx5_flow_dv.c            |  1 +
 drivers/net/mlx5/mlx5_rxq.c                |  1 +
 7 files changed, 104 insertions(+), 79 deletions(-)
 create mode 100644 drivers/common/mlx5/linux/mlx5_common_os.h

diff --git a/drivers/common/mlx5/linux/mlx5_common_os.h b/drivers/common/mlx5/linux/mlx5_common_os.h
new file mode 100644
index 0000000..9a6872c
--- /dev/null
+++ b/drivers/common/mlx5/linux/mlx5_common_os.h
@@ -0,0 +1,98 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_COMMON_OS_H_
+#define RTE_PMD_MLX5_COMMON_OS_H_
+
+#include <stdio.h>
+
+#include <rte_pci.h>
+#include <rte_debug.h>
+#include <rte_atomic.h>
+#include <rte_log.h>
+#include <rte_kvargs.h>
+#include <rte_devargs.h>
+
+#include "mlx5_autoconf.h"
+#ifdef HAVE_INFINIBAND_VERBS_H
+#include <infiniband/verbs.h>
+#endif
+#ifdef HAVE_INFINIBAND_MLX5DV_H
+#include <infiniband/mlx5dv.h>
+#endif
+
+/**
+ * Get device name. Given an ibv_device pointer - return a
+ * pointer to the corresponding device name.
+ *
+ * @param[in] dev
+ *   Pointer to ibv device.
+ *
+ * @return
+ *   Pointer to device name if dev is valid, NULL otherwise.
+ */
+static inline const char *
+mlx5_os_get_dev_device_name(void *dev)
+{
+	if (!dev)
+		return NULL;
+	return ((struct ibv_device *)dev)->name;
+}
+
+/**
+ * Get ibv device name. Given an ibv_context pointer - return a
+ * pointer to the corresponding device name.
+ *
+ * @param[in] ctx
+ *   Pointer to ibv context.
+ *
+ * @return
+ *   Pointer to device name if ctx is valid, NULL otherwise.
+ */
+static inline const char *
+mlx5_os_get_ctx_device_name(void *ctx)
+{
+	if (!ctx)
+		return NULL;
+	return ((struct ibv_context *)ctx)->device->name;
+}
+
+/**
+ * Get ibv device path name. Given an ibv_context pointer - return a
+ * pointer to the corresponding device path name.
+ *
+ * @param[in] ctx
+ *   Pointer to ibv context.
+ *
+ * @return
+ *   Pointer to device path name if ctx is valid, NULL otherwise.
+ */
+
+static inline const char *
+mlx5_os_get_ctx_device_path(void *ctx)
+{
+	if (!ctx)
+		return NULL;
+
+	return ((struct ibv_context *)ctx)->device->ibdev_path;
+}
+
+/**
+ * Get umem id. Given a pointer to umem object of type
+ * 'struct mlx5dv_devx_umem *' - return its id.
+ *
+ * @param[in] umem
+ *    Pointer to umem object.
+ *
+ * @return
+ *    The umem id if umem is valid, 0 otherwise.
+ */
+static inline uint32_t
+mlx5_os_get_umem_id(void *umem)
+{
+	if (!umem)
+		return 0;
+	return ((struct mlx5dv_devx_umem *)umem)->umem_id;
+}
+#endif /* RTE_PMD_MLX5_COMMON_OS_H_ */
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 3792371..be51193 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -46,6 +46,7 @@
 
 #include "mlx5_defs.h"
 #include "mlx5.h"
+#include "mlx5_common_os.h"
 #include "mlx5_utils.h"
 #include "mlx5_rxtx.h"
 #include "mlx5_autoconf.h"
@@ -66,79 +67,6 @@
 #endif
 
 /**
- * Get device name. Given an ibv_device pointer - return a
- * pointer to the corresponding device name.
- *
- * @param[in] dev
- *   Pointer to ibv device.
- *
- * @return
- *   Pointer to device name if dev is valid, NULL otherwise.
- */
-const char *
-mlx5_os_get_dev_device_name(void *dev)
-{
-	if (!dev)
-		return NULL;
-	return ((struct ibv_device *)dev)->name;
-}
-
-/**
- * Get ibv device name. Given an ibv_context pointer - return a
- * pointer to the corresponding device name.
- *
- * @param[in] ctx
- *   Pointer to ibv context.
- *
- * @return
- *   Pointer to device name if ctx is valid, NULL otherwise.
- */
-const char *
-mlx5_os_get_ctx_device_name(void *ctx)
-{
-	if (!ctx)
-		return NULL;
-	return ((struct ibv_context *)ctx)->device->name;
-}
-
-/**
- * Get ibv device path name. Given an ibv_context pointer - return a
- * pointer to the corresponding device path name.
- *
- * @param[in] ctx
- *   Pointer to ibv context.
- *
- * @return
- *   Pointer to device path name if ctx is valid, NULL otherwise.
- */
-const char *
-mlx5_os_get_ctx_device_path(void *ctx)
-{
-	if (!ctx)
-		return NULL;
-
-	return ((struct ibv_context *)ctx)->device->ibdev_path;
-}
-
-/**
- * Get umem id. Given a pointer to umem object of type
- * 'struct mlx5dv_devx_umem *' - return its id.
- *
- * @param[in] umem
- *   Pointer to umem object.
- *
- * @return
- *   The umem id if umem is valid, 0 otherwise.
- */
-uint32_t
-mlx5_os_get_umem_id(void *umem)
-{
-	if (!umem)
-		return 0;
-	return ((struct mlx5dv_devx_umem *)umem)->umem_id;
-}
-
-/**
  * Get mlx5 device attributes. The glue function query_device_ex() is called
  * with out parameter of type 'struct ibv_device_attr_ex *'. Then fill in mlx5
  * device attributes from the glue out parameter.
diff --git a/drivers/net/mlx5/linux/mlx5_os.h b/drivers/net/mlx5/linux/mlx5_os.h
index f310f17..31add39 100644
--- a/drivers/net/mlx5/linux/mlx5_os.h
+++ b/drivers/net/mlx5/linux/mlx5_os.h
@@ -8,8 +8,8 @@
 
 /* verb enumerations translations to local enums. */
 enum {
-	DEV_SYSFS_NAME_MAX = IBV_SYSFS_NAME_MAX,
-	DEV_SYSFS_PATH_MAX = IBV_SYSFS_PATH_MAX
+	DEV_SYSFS_NAME_MAX = IBV_SYSFS_NAME_MAX + 1,
+	DEV_SYSFS_PATH_MAX = IBV_SYSFS_PATH_MAX + 1
 };
 
 #define PCI_DRV_FLAGS  (RTE_PCI_DRV_INTR_LSC | \
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 5c86f6f..da38eb9 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -38,6 +38,7 @@
 #include <mlx5_glue.h>
 #include <mlx5_devx_cmds.h>
 #include <mlx5_common.h>
+#include <mlx5_common_os.h>
 #include <mlx5_common_mp.h>
 
 #include "mlx5_defs.h"
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 5bd5acd..a2a2c9d 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -921,10 +921,6 @@ struct mlx5_flow_meter *mlx5_flow_meter_attach
 
 /* mlx5_os.c */
 struct rte_pci_driver;
-const char *mlx5_os_get_ctx_device_name(void *ctx);
-const char *mlx5_os_get_ctx_device_path(void *ctx);
-const char *mlx5_os_get_dev_device_name(void *dev);
-uint32_t mlx5_os_get_umem_id(void *umem);
 int mlx5_os_get_dev_attr(void *ctx, struct mlx5_dev_attr *dev_attr);
 void mlx5_os_free_shared_dr(struct mlx5_priv *priv);
 int mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 5bb252e..f0ce195 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -36,6 +36,7 @@
 
 #include "mlx5_defs.h"
 #include "mlx5.h"
+#include "mlx5_common_os.h"
 #include "mlx5_flow.h"
 #include "mlx5_rxtx.h"
 
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index dda0073..efd1e3f 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -34,6 +34,7 @@
 
 #include "mlx5_defs.h"
 #include "mlx5.h"
+#include "mlx5_common_os.h"
 #include "mlx5_rxtx.h"
 #include "mlx5_utils.h"
 #include "mlx5_autoconf.h"
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 2/2] common/mlx5: move doorbell record to common

[Ori Kam]

The creation of DBR can be used by a number of different
Mellanox PMDs. for example RegEx / Net / VDPA.

This commits moves the DBR creation and relase functions to common
folder.

Signed-off-by: Ori Kam <orika@mellanox.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
v2:
* No changes.
---
 drivers/common/mlx5/linux/mlx5_common_os.c      |   1 +
 drivers/common/mlx5/mlx5_common.c               | 122 ++++++++++++++++++++++++
 drivers/common/mlx5/mlx5_common.h               |  38 +++++++-
 drivers/common/mlx5/mlx5_devx_cmds.h            |  13 ---
 drivers/common/mlx5/rte_common_mlx5_version.map |   3 +
 drivers/net/mlx5/mlx5.c                         | 120 -----------------------
 drivers/net/mlx5/mlx5.h                         |  21 +---
 drivers/net/mlx5/mlx5_rxq.c                     |   6 +-
 8 files changed, 168 insertions(+), 156 deletions(-)

diff --git a/drivers/common/mlx5/linux/mlx5_common_os.c b/drivers/common/mlx5/linux/mlx5_common_os.c
index 1b71347..823a0b5 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.c
+++ b/drivers/common/mlx5/linux/mlx5_common_os.c
@@ -315,3 +315,4 @@
 		" libmlx5)");
 	mlx5_glue = NULL;
 }
+
diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c
index db94d4a..693e2c6 100644
--- a/drivers/common/mlx5/mlx5_common.c
+++ b/drivers/common/mlx5/mlx5_common.c
@@ -7,8 +7,11 @@
 #include <stdio.h>
 
 #include <rte_errno.h>
+#include <rte_mempool.h>
+#include <rte_malloc.h>
 
 #include "mlx5_common.h"
+#include "mlx5_common_os.h"
 #include "mlx5_common_utils.h"
 
 int mlx5_common_logtype;
@@ -150,3 +153,122 @@ static inline void mlx5_cpu_id(unsigned int level,
 #endif
 	haswell_broadwell_cpu = 0;
 }
+
+/**
+ * Allocate page of door-bells and register it using DevX API.
+ *
+ * @param [in] ctx
+ *   Pointer to the device context.
+ *
+ * @return
+ *   Pointer to new page on success, NULL otherwise.
+ */
+static struct mlx5_devx_dbr_page *
+mlx5_alloc_dbr_page(void *ctx)
+{
+	struct mlx5_devx_dbr_page *page;
+
+	/* Allocate space for door-bell page and management data. */
+	page = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_devx_dbr_page),
+				 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+	if (!page) {
+		DRV_LOG(ERR, "cannot allocate dbr page");
+		return NULL;
+	}
+	/* Register allocated memory. */
+	page->umem = mlx5_glue->devx_umem_reg(ctx, page->dbrs,
+					      MLX5_DBR_PAGE_SIZE, 0);
+	if (!page->umem) {
+		DRV_LOG(ERR, "cannot umem reg dbr page");
+		rte_free(page);
+		return NULL;
+	}
+	return page;
+}
+
+/**
+ * Find the next available door-bell, allocate new page if needed.
+ *
+ * @param [in] ctx
+ *   Pointer to device context.
+ * @param [in] head
+ *   Pointer to the head of dbr pages list.
+ * @param [out] dbr_page
+ *   Door-bell page containing the page data.
+ *
+ * @return
+ *   Door-bell address offset on success, a negative error value otherwise.
+ */
+int64_t
+mlx5_get_dbr(void *ctx,  struct mlx5_dbr_page_list *head,
+	     struct mlx5_devx_dbr_page **dbr_page)
+{
+	struct mlx5_devx_dbr_page *page = NULL;
+	uint32_t i, j;
+
+	LIST_FOREACH(page, head, next)
+		if (page->dbr_count < MLX5_DBR_PER_PAGE)
+			break;
+	if (!page) { /* No page with free door-bell exists. */
+		page = mlx5_alloc_dbr_page(ctx);
+		if (!page) /* Failed to allocate new page. */
+			return (-1);
+		LIST_INSERT_HEAD(head, page, next);
+	}
+	/* Loop to find bitmap part with clear bit. */
+	for (i = 0;
+	     i < MLX5_DBR_BITMAP_SIZE && page->dbr_bitmap[i] == UINT64_MAX;
+	     i++)
+		; /* Empty. */
+	/* Find the first clear bit. */
+	MLX5_ASSERT(i < MLX5_DBR_BITMAP_SIZE);
+	j = rte_bsf64(~page->dbr_bitmap[i]);
+	page->dbr_bitmap[i] |= (UINT64_C(1) << j);
+	page->dbr_count++;
+	*dbr_page = page;
+	return (((i * 64) + j) * sizeof(uint64_t));
+}
+
+/**
+ * Release a door-bell record.
+ *
+ * @param [in] head
+ *   Pointer to the head of dbr pages list.
+ * @param [in] umem_id
+ *   UMEM ID of page containing the door-bell record to release.
+ * @param [in] offset
+ *   Offset of door-bell record in page.
+ *
+ * @return
+ *   0 on success, a negative error value otherwise.
+ */
+int32_t
+mlx5_release_dbr(struct mlx5_dbr_page_list *head, uint32_t umem_id,
+		 uint64_t offset)
+{
+	struct mlx5_devx_dbr_page *page = NULL;
+	int ret = 0;
+
+	LIST_FOREACH(page, head, next)
+		/* Find the page this address belongs to. */
+		if (mlx5_os_get_umem_id(page->umem) == umem_id)
+			break;
+	if (!page)
+		return -EINVAL;
+	page->dbr_count--;
+	if (!page->dbr_count) {
+		/* Page not used, free it and remove from list. */
+		LIST_REMOVE(page, next);
+		if (page->umem)
+			ret = -mlx5_glue->devx_umem_dereg(page->umem);
+		rte_free(page);
+	} else {
+		/* Mark in bitmap that this door-bell is not in use. */
+		offset /= MLX5_DBR_SIZE;
+		int i = offset / 64;
+		int j = offset % 64;
+
+		page->dbr_bitmap[i] &= ~(UINT64_C(1) << j);
+	}
+	return ret;
+}
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 77f10e6..8b8f3a0 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -15,6 +15,7 @@
 #include <rte_devargs.h>
 
 #include "mlx5_prm.h"
+#include "mlx5_devx_cmds.h"
 
 
 /* Bit-field manipulation. */
@@ -207,6 +208,36 @@ enum mlx5_class {
 	MLX5_CLASS_INVALID,
 };
 
+#define MLX5_DBR_PAGE_SIZE 4096 /* Must be >= 512. */
+#define MLX5_DBR_SIZE 8
+#define MLX5_DBR_PER_PAGE (MLX5_DBR_PAGE_SIZE / MLX5_DBR_SIZE)
+#define MLX5_DBR_BITMAP_SIZE (MLX5_DBR_PER_PAGE / 64)
+
+struct mlx5_devx_dbr_page {
+	/* Door-bell records, must be first member in structure. */
+	uint8_t dbrs[MLX5_DBR_PAGE_SIZE];
+	LIST_ENTRY(mlx5_devx_dbr_page) next; /* Pointer to the next element. */
+	void *umem;
+	uint32_t dbr_count; /* Number of door-bell records in use. */
+	/* 1 bit marks matching door-bell is in use. */
+	uint64_t dbr_bitmap[MLX5_DBR_BITMAP_SIZE];
+};
+
+/* devX creation object */
+struct mlx5_devx_obj {
+	void *obj; /* The DV object. */
+	int id; /* The object ID. */
+};
+
+/* UMR memory buffer used to define 1 entry in indirect mkey. */
+struct mlx5_klm {
+	uint32_t byte_count;
+	uint32_t mkey;
+	uint64_t address;
+};
+
+LIST_HEAD(mlx5_dbr_page_list, mlx5_devx_dbr_page);
+
 __rte_internal
 enum mlx5_class mlx5_class_get(struct rte_devargs *devargs);
 __rte_internal
@@ -214,7 +245,12 @@ void mlx5_translate_port_name(const char *port_name_in,
 			      struct mlx5_switch_info *port_info_out);
 void mlx5_glue_constructor(void);
 size_t mlx5_os_get_page_size(void);
-
+__rte_internal
+int64_t mlx5_get_dbr(void *ctx,  struct mlx5_dbr_page_list *head,
+		     struct mlx5_devx_dbr_page **dbr_page);
+__rte_internal
+int32_t mlx5_release_dbr(struct mlx5_dbr_page_list *head, uint32_t umem_id,
+			 uint64_t offset);
 extern uint8_t haswell_broadwell_cpu;
 
 #endif /* RTE_PMD_MLX5_COMMON_H_ */
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 49b174a..a2b2ad4 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -9,19 +9,6 @@
 #include "mlx5_prm.h"
 
 
-/* devX creation object */
-struct mlx5_devx_obj {
-	void *obj; /* The DV object. */
-	int id; /* The object ID. */
-};
-
-/* UMR memory buffer used to define 1 entry in indirect mkey. */
-struct mlx5_klm {
-	uint32_t byte_count;
-	uint32_t mkey;
-	uint64_t address;
-};
-
 /* This is limitation of libibverbs: in length variable type is u16. */
 #define MLX5_DEVX_MAX_KLM_ENTRIES ((UINT16_MAX - \
 		MLX5_ST_SZ_DW(create_mkey_in) * 4) / (MLX5_ST_SZ_DW(klm) * 4))
diff --git a/drivers/common/mlx5/rte_common_mlx5_version.map b/drivers/common/mlx5/rte_common_mlx5_version.map
index 68f1207..208f90d 100644
--- a/drivers/common/mlx5/rte_common_mlx5_version.map
+++ b/drivers/common/mlx5/rte_common_mlx5_version.map
@@ -34,6 +34,9 @@ INTERNAL {
 	mlx5_devx_cmd_query_virtq;
 	mlx5_devx_get_out_command_status;
 
+	mlx5_get_dbr;
+	mlx5_release_dbr;
+
 	mlx5_mp_init_primary;
 	mlx5_mp_uninit_primary;
 	mlx5_mp_init_secondary;
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index da38eb9..9e63bd5 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1666,126 +1666,6 @@ struct mlx5_dev_ctx_shared *
 	DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask);
 }
 
-/**
- * Allocate page of door-bells and register it using DevX API.
- *
- * @param [in] dev
- *   Pointer to Ethernet device.
- *
- * @return
- *   Pointer to new page on success, NULL otherwise.
- */
-static struct mlx5_devx_dbr_page *
-mlx5_alloc_dbr_page(struct rte_eth_dev *dev)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_devx_dbr_page *page;
-
-	/* Allocate space for door-bell page and management data. */
-	page = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_devx_dbr_page),
-				 RTE_CACHE_LINE_SIZE, dev->device->numa_node);
-	if (!page) {
-		DRV_LOG(ERR, "port %u cannot allocate dbr page",
-			dev->data->port_id);
-		return NULL;
-	}
-	/* Register allocated memory. */
-	page->umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, page->dbrs,
-					      MLX5_DBR_PAGE_SIZE, 0);
-	if (!page->umem) {
-		DRV_LOG(ERR, "port %u cannot umem reg dbr page",
-			dev->data->port_id);
-		rte_free(page);
-		return NULL;
-	}
-	return page;
-}
-
-/**
- * Find the next available door-bell, allocate new page if needed.
- *
- * @param [in] dev
- *   Pointer to Ethernet device.
- * @param [out] dbr_page
- *   Door-bell page containing the page data.
- *
- * @return
- *   Door-bell address offset on success, a negative error value otherwise.
- */
-int64_t
-mlx5_get_dbr(struct rte_eth_dev *dev, struct mlx5_devx_dbr_page **dbr_page)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_devx_dbr_page *page = NULL;
-	uint32_t i, j;
-
-	LIST_FOREACH(page, &priv->dbrpgs, next)
-		if (page->dbr_count < MLX5_DBR_PER_PAGE)
-			break;
-	if (!page) { /* No page with free door-bell exists. */
-		page = mlx5_alloc_dbr_page(dev);
-		if (!page) /* Failed to allocate new page. */
-			return (-1);
-		LIST_INSERT_HEAD(&priv->dbrpgs, page, next);
-	}
-	/* Loop to find bitmap part with clear bit. */
-	for (i = 0;
-	     i < MLX5_DBR_BITMAP_SIZE && page->dbr_bitmap[i] == UINT64_MAX;
-	     i++)
-		; /* Empty. */
-	/* Find the first clear bit. */
-	MLX5_ASSERT(i < MLX5_DBR_BITMAP_SIZE);
-	j = rte_bsf64(~page->dbr_bitmap[i]);
-	page->dbr_bitmap[i] |= (UINT64_C(1) << j);
-	page->dbr_count++;
-	*dbr_page = page;
-	return (((i * 64) + j) * sizeof(uint64_t));
-}
-
-/**
- * Release a door-bell record.
- *
- * @param [in] dev
- *   Pointer to Ethernet device.
- * @param [in] umem_id
- *   UMEM ID of page containing the door-bell record to release.
- * @param [in] offset
- *   Offset of door-bell record in page.
- *
- * @return
- *   0 on success, a negative error value otherwise.
- */
-int32_t
-mlx5_release_dbr(struct rte_eth_dev *dev, uint32_t umem_id, uint64_t offset)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_devx_dbr_page *page = NULL;
-	int ret = 0;
-
-	LIST_FOREACH(page, &priv->dbrpgs, next)
-		/* Find the page this address belongs to. */
-		if (mlx5_os_get_umem_id(page->umem) == umem_id)
-			break;
-	if (!page)
-		return -EINVAL;
-	page->dbr_count--;
-	if (!page->dbr_count) {
-		/* Page not used, free it and remove from list. */
-		LIST_REMOVE(page, next);
-		if (page->umem)
-			ret = -mlx5_glue->devx_umem_dereg(page->umem);
-		rte_free(page);
-	} else {
-		/* Mark in bitmap that this door-bell is not in use. */
-		offset /= MLX5_DBR_SIZE;
-		int i = offset / 64;
-		int j = offset % 64;
-
-		page->dbr_bitmap[i] &= ~(UINT64_C(1) << j);
-	}
-	return ret;
-}
-
 int
 rte_pmd_mlx5_get_dyn_flag_names(char *names[], unsigned int n)
 {
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index a2a2c9d..83bbf42 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -500,21 +500,6 @@ struct mlx5_flow_tbl_resource {
 #define MLX5_MAX_TABLES_EXTERNAL (MLX5_MAX_TABLES - 3)
 #define MLX5_MAX_TABLES_FDB UINT16_MAX
 
-#define MLX5_DBR_PAGE_SIZE 4096 /* Must be >= 512. */
-#define MLX5_DBR_SIZE 8
-#define MLX5_DBR_PER_PAGE (MLX5_DBR_PAGE_SIZE / MLX5_DBR_SIZE)
-#define MLX5_DBR_BITMAP_SIZE (MLX5_DBR_PER_PAGE / 64)
-
-struct mlx5_devx_dbr_page {
-	/* Door-bell records, must be first member in structure. */
-	uint8_t dbrs[MLX5_DBR_PAGE_SIZE];
-	LIST_ENTRY(mlx5_devx_dbr_page) next; /* Pointer to the next element. */
-	void *umem;
-	uint32_t dbr_count; /* Number of door-bell records in use. */
-	/* 1 bit marks matching door-bell is in use. */
-	uint64_t dbr_bitmap[MLX5_DBR_BITMAP_SIZE];
-};
-
 /* ID generation structure. */
 struct mlx5_flow_id_pool {
 	uint32_t *free_arr; /**< Pointer to the a array of free values. */
@@ -655,7 +640,7 @@ struct mlx5_priv {
 	/* Context for Verbs allocator. */
 	int nl_socket_rdma; /* Netlink socket (NETLINK_RDMA). */
 	int nl_socket_route; /* Netlink socket (NETLINK_ROUTE). */
-	LIST_HEAD(dbrpage, mlx5_devx_dbr_page) dbrpgs; /* Door-bell pages. */
+	struct mlx5_dbr_page_list dbrpgs; /* Door-bell pages. */
 	struct mlx5_nl_vlan_vmwa_context *vmwa_context; /* VLAN WA context. */
 	struct mlx5_flow_id_pool *qrss_id_pool;
 	struct mlx5_hlist *mreg_cp_tbl;
@@ -682,10 +667,6 @@ struct mlx5_priv {
 
 int mlx5_getenv_int(const char *);
 int mlx5_proc_priv_init(struct rte_eth_dev *dev);
-int64_t mlx5_get_dbr(struct rte_eth_dev *dev,
-		     struct mlx5_devx_dbr_page **dbr_page);
-int32_t mlx5_release_dbr(struct rte_eth_dev *dev, uint32_t umem_id,
-			 uint64_t offset);
 int mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev,
 			      struct rte_eth_udp_tunnel *udp_tunnel);
 uint16_t mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index efd1e3f..bd0037b 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1414,7 +1414,8 @@ struct mlx5_rxq_obj *
 		struct mlx5_devx_dbr_page *dbr_page;
 		int64_t dbr_offset;
 
-		dbr_offset = mlx5_get_dbr(dev, &dbr_page);
+		dbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs,
+					  &dbr_page);
 		if (dbr_offset < 0)
 			goto error;
 		rxq_ctrl->dbr_offset = dbr_offset;
@@ -2102,7 +2103,8 @@ struct mlx5_rxq_ctrl *
 		rxq_ctrl->obj = NULL;
 	if (rte_atomic32_dec_and_test(&rxq_ctrl->refcnt)) {
 		if (rxq_ctrl->dbr_umem_id_valid)
-			claim_zero(mlx5_release_dbr(dev, rxq_ctrl->dbr_umem_id,
+			claim_zero(mlx5_release_dbr(&priv->dbrpgs,
+						    rxq_ctrl->dbr_umem_id,
 						    rxq_ctrl->dbr_offset));
 		if (rxq_ctrl->type == MLX5_RXQ_TYPE_STANDARD)
 			mlx5_mr_btree_free(&rxq_ctrl->rxq.mr_ctrl.cache_bh);
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v2 1/2] net/mlx5: move getter functions from net to common

[Raslan Darawsheh]

Hi,

> -----Original Message-----
> From: Ori Kam <orika@mellanox.com>
> Sent: Friday, June 19, 2020 10:30 AM
> To: Matan Azrad <matan@mellanox.com>; Slava Ovsiienko
> <viacheslavo@mellanox.com>; Shahaf Shuler <shahafs@mellanox.com>
> Cc: dev@dpdk.org; Ori Kam <orika@mellanox.com>; Raslan Darawsheh
> <rasland@mellanox.com>; Ophir Munk <ophirmu@mellanox.com>
> Subject: [PATCH v2 1/2] net/mlx5: move getter functions from net to
> common
> 
> From: Ophir Munk <ophirmu@mellanox.com>
> 
> Getter functions such as: 'mlx5_os_get_ctx_device_name',
> 'mlx5_os_get_ctx_device_path', 'mlx5_os_get_dev_device_name',
> 'mlx5_os_get_umem_id' are implemented under net directory. To enable
> additional devices (e.g. regex, vdpa) to access these getter functions
> they are moved under common directory.
> 
> As part of this commit string sizes DEV_SYSFS_NAME_MAX and
> DEV_SYSFS_PATH_MAX are increased by 1 to make sure that the destination
> string size in strncpy() function is bigger than the source string size.
> This update will avoid GCC version 8 error -Werror=stringop-truncation.
> 
> Signed-off-by: Ophir Munk <ophirmu@mellanox.com>
> Acked-by: Matan Azrad <matan@mellanox.com>
> ---
> v2:
> * avoid GCC version 8 error.
> ---



Series rebased and applied to next-net-mlx,

Kindest regards,
Raslan Darawsheh