[dpdk-dev] [PATCH 2/3] net/mlx5: remove device register remap

[Yongseok Koh <yskoh@mellanox.com>]

UAR (User Access Region) registers will be stored in a process-local table
and a process accesses a register in a table entry with index. Alloc/free
of table entry is managed by a global bitmap.

When there's a need to store a UAR register such as Tx BlueFlame register
for doorbell, an index should be allocated by mlx5_uar_alloc_index() and
address of the allocated table entry must be acquired by
mlx5_uar_get_addr_ptr() so that the table can be expanded if overflowed.
The local UAR register table doesn't cover all the indexes in the bitmap.
This will be expanded if more indexes are allocated than the current size
of the table.

For example, the BlueFlame register for Tx doorbell has to be remapped on
each secondary process. On initialization, primary process allocates an
index for the UAR register table and stores the register address in the
indexed entry of its own table when configuring a Tx queue. The index is
stored in the shared memory(txq->bfreg_idx) and visiable to secondary
processes. As secondary processes know the index, each process stores
remapped register in the same indexed entry of its local UAR register
table.

On the datapath of each process, the register can be referenced simply by
MLX5_UAR_REG(idx) which accesses its local UAR register table by the index.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5.c         | 262 +++++++++++++++++++++++++++-------------
 drivers/net/mlx5/mlx5.h         |  19 ++-
 drivers/net/mlx5/mlx5_rxtx.h    |   7 +-
 drivers/net/mlx5/mlx5_trigger.c |   2 +-
 drivers/net/mlx5/mlx5_txq.c     |  91 ++++++--------
 5 files changed, 236 insertions(+), 145 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 93c0fc8c20..1860273194 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -26,6 +26,7 @@
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
 
+#include <rte_bitmap.h>
 #include <rte_malloc.h>
 #include <rte_ethdev_driver.h>
 #include <rte_ethdev_pci.h>
@@ -132,7 +133,7 @@ struct mlx5_shared_data *mlx5_shared_data;
 static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 
 /* Process local data for secondary processes. */
-static struct mlx5_local_data mlx5_local_data;
+struct mlx5_local_data mlx5_local_data;
 
 /** Driver-specific log messages type. */
 int mlx5_logtype;
@@ -647,130 +648,225 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 
 static struct rte_pci_driver mlx5_driver;
 
+
+/**
+ * Expand the local UAR register table.
+ *
+ * @param size
+ *   Size of the table to be expanded
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
 static int
-find_lower_va_bound(const struct rte_memseg_list *msl,
-		const struct rte_memseg *ms, void *arg)
+uar_expand_table(uint32_t size)
 {
-	void **addr = arg;
+	struct mlx5_local_data *ld = &mlx5_local_data;
+	void *mem;
+	size_t tbl_sz = ld->uar_table_sz;
 
-	if (msl->external)
+	if (size <= tbl_sz)
 		return 0;
-	if (*addr == NULL)
-		*addr = ms->addr;
-	else
-		*addr = RTE_MIN(*addr, ms->addr);
-
+	tbl_sz = RTE_ALIGN_CEIL(size, RTE_BITMAP_SLAB_BIT_SIZE);
+	mem = rte_realloc(ld->uar_table, tbl_sz * sizeof(void *),
+			  RTE_CACHE_LINE_SIZE);
+	if (!mem) {
+		rte_errno = ENOMEM;
+		DRV_LOG(ERR, "failed to expand uar table");
+		return -rte_errno;
+	}
+	DRV_LOG(DEBUG, "UAR reg. table is expanded to %zu", tbl_sz);
+	ld->uar_table = mem;
+	ld->uar_table_sz = tbl_sz;
 	return 0;
 }
 
 /**
- * Reserve UAR address space for primary process.
+ * Return the pointer of the indexed slot in the local UAR register table.
  *
- * Process local resource is used by both primary and secondary to avoid
- * duplicate reservation. The space has to be available on both primary and
- * secondary process, TXQ UAR maps to this area using fixed mmap w/o double
- * check.
+ * The indexed slot must be allocated by mlx5_uar_alloc_index() in advance. And
+ * the table will be expanded if overflowed.
+ *
+ * @param idx
+ *   Index of the table.
  *
  * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
+ *   Pointer of table entry on success, NULL otherwise and rte_errno is set.
  */
-static int
-mlx5_uar_init_primary(void)
+void **
+mlx5_uar_get_addr_ptr(uint32_t idx)
+{
+	struct mlx5_local_data *ld = &mlx5_local_data;
+	int ret;
+
+	assert(idx < MLX5_UAR_TABLE_SIZE_MAX);
+	if (idx >= ld->uar_table_sz) {
+		ret = uar_expand_table(idx + 1);
+		if (ret)
+			return NULL;
+	}
+	return &(*ld->uar_table)[idx];
+}
+
+/**
+ * Allocate a slot of UAR register table.
+ *
+ * Allocation is done by scanning the global bitmap. The global spinlock should
+ * be held.
+ *
+ * @return
+ *   Index of a free slot on success, a negative errno value otherwise and
+ *   rte_errno is set.
+ */
+uint32_t
+mlx5_uar_alloc_index(void)
 {
 	struct mlx5_shared_data *sd = mlx5_shared_data;
-	void *addr = (void *)0;
+	uint32_t idx = 0;
+	uint64_t slab = 0;
+	int ret;
 
-	if (sd->uar_base)
-		return 0;
-	/* find out lower bound of hugepage segments */
-	rte_memseg_walk(find_lower_va_bound, &addr);
-	/* keep distance to hugepages to minimize potential conflicts. */
-	addr = RTE_PTR_SUB(addr, (uintptr_t)(MLX5_UAR_OFFSET + MLX5_UAR_SIZE));
-	/* anonymous mmap, no real memory consumption. */
-	addr = mmap(addr, MLX5_UAR_SIZE,
-		    PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-	if (addr == MAP_FAILED) {
-		DRV_LOG(ERR,
-			"Failed to reserve UAR address space, please"
-			" adjust MLX5_UAR_SIZE or try --base-virtaddr");
-		rte_errno = ENOMEM;
+	assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
+	rte_spinlock_lock(&sd->lock);
+	__rte_bitmap_scan_init(sd->uar_bmp);
+	ret = rte_bitmap_scan(sd->uar_bmp, &idx, &slab);
+	if (unlikely(!ret)) {
+		/*
+		 * This cannot happen unless there are unreasonably large number
+		 * of queues and ports.
+		 */
+		rte_errno = ENOSPC;
+		rte_spinlock_unlock(&sd->lock);
 		return -rte_errno;
 	}
-	/* Accept either same addr or a new addr returned from mmap if target
-	 * range occupied.
-	 */
-	DRV_LOG(INFO, "Reserved UAR address space: %p", addr);
-	sd->uar_base = addr; /* for primary and secondary UAR re-mmap. */
-	return 0;
+	idx += __builtin_ctzll(slab);
+	/* Mark the slot is occupied. */
+	rte_bitmap_clear(sd->uar_bmp, idx);
+	rte_spinlock_unlock(&sd->lock);
+	DRV_LOG(DEBUG, "index %d is allocated in UAR reg. table", idx);
+	return idx;
 }
 
 /**
- * Unmap UAR address space reserved for primary process.
+ * Free a slot of UAR register table.
  */
-static void
-mlx5_uar_uninit_primary(void)
+void
+mlx5_uar_free_index(uint32_t idx)
 {
 	struct mlx5_shared_data *sd = mlx5_shared_data;
 
-	if (!sd->uar_base)
-		return;
-	munmap(sd->uar_base, MLX5_UAR_SIZE);
-	sd->uar_base = NULL;
+	assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
+	assert(idx < MLX5_UAR_TABLE_SIZE_MAX);
+	rte_spinlock_lock(&sd->lock);
+	/* Mark the slot is empty. */
+	rte_bitmap_set(sd->uar_bmp, idx);
+	rte_spinlock_unlock(&sd->lock);
+	DRV_LOG(DEBUG, "index %d is freed in UAR reg. table", idx);
 }
 
 /**
- * Reserve UAR address space for secondary process, align with primary process.
+ * Initialize UAR register table bitmap.
+ *
+ * UAR registers will be stored in a process-local table and the table is
+ * managed by a global bitmap. When there's a need to store a UAR register, an
+ * index should be allocated by mlx5_uar_alloc_index() and address of the
+ * allocated table entry must be acquired by mlx5_uar_get_addr_ptr() so that the
+ * table can be expanded if overflowed.
+ *
+ * The local UAR register table doesn't cover all the indexes in the bitmap.
+ * This will be expanded if more indexes are allocated than the current size of
+ * the table.
+ *
+ * Secondary process should have reference of the index and store remapped
+ * register at the same index in its local UAR register table.
+ *
+ * On the datapath of each process, the register can be referenced simply by
+ * MLX5_UAR_REG(idx).
  *
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx5_uar_init_secondary(void)
+uar_init_primary(void)
 {
 	struct mlx5_shared_data *sd = mlx5_shared_data;
-	struct mlx5_local_data *ld = &mlx5_local_data;
-	void *addr;
+	struct rte_bitmap *bmp;
+	void *bmp_mem;
+	uint32_t bmp_size;
+	unsigned int i;
 
-	if (ld->uar_base) { /* Already reserved. */
-		assert(sd->uar_base == ld->uar_base);
-		return 0;
-	}
-	assert(sd->uar_base);
-	/* anonymous mmap, no real memory consumption. */
-	addr = mmap(sd->uar_base, MLX5_UAR_SIZE,
-		    PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-	if (addr == MAP_FAILED) {
-		DRV_LOG(ERR, "UAR mmap failed: %p size: %llu",
-			sd->uar_base, MLX5_UAR_SIZE);
-		rte_errno = ENXIO;
+	bmp_size = rte_bitmap_get_memory_footprint(MLX5_UAR_TABLE_SIZE_MAX);
+	bmp_mem = rte_zmalloc("uar_table", bmp_size, RTE_CACHE_LINE_SIZE);
+	if (!bmp_mem) {
+		rte_errno = ENOMEM;
+		DRV_LOG(ERR, "failed to allocate memory for uar table");
 		return -rte_errno;
 	}
-	if (sd->uar_base != addr) {
-		DRV_LOG(ERR,
-			"UAR address %p size %llu occupied, please"
-			" adjust MLX5_UAR_OFFSET or try EAL parameter"
-			" --base-virtaddr",
-			sd->uar_base, MLX5_UAR_SIZE);
-		rte_errno = ENXIO;
-		return -rte_errno;
+	bmp = rte_bitmap_init(MLX5_UAR_TABLE_SIZE_MAX, bmp_mem, bmp_size);
+	/* Set the entire bitmap as 1 means vacant and 0 means empty. */
+	for (i = 0; i < bmp->array2_size; ++i)
+		rte_bitmap_set_slab(bmp, i * RTE_BITMAP_SLAB_BIT_SIZE, -1);
+	sd->uar_bmp = bmp;
+	return 0;
+}
+
+/**
+ * Un-initialize UAR register resources.
+ *
+ * The global bitmap and the register table of primary process are freed.
+ */
+static void
+uar_uninit_primary(void)
+{
+	struct mlx5_shared_data *sd = mlx5_shared_data;
+	struct mlx5_local_data *ld = &mlx5_local_data;
+
+	if (sd->uar_bmp) {
+		rte_bitmap_free(sd->uar_bmp);
+		rte_free(sd->uar_bmp);
+		sd->uar_bmp = NULL;
+	}
+	/* Free primary's table. */
+	if (ld->uar_table) {
+		rte_free(ld->uar_table);
+		ld->uar_table = NULL;
+		ld->uar_table_sz = 0;
 	}
-	ld->uar_base = addr;
-	DRV_LOG(INFO, "Reserved UAR address space: %p", addr);
+}
+
+/**
+ * Initialize UAR register resources for secondary process.
+ *
+ * Allocate the local UAR register table. Initially, the number of entries is
+ * same as the size of a bitmap slab.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+uar_init_secondary(void)
+{
+	/* Prepare at least a bitmap slab. */
+	uar_expand_table(RTE_BITMAP_SLAB_BIT_SIZE);
 	return 0;
 }
 
 /**
- * Unmap UAR address space reserved for secondary process.
+ * Un-initialize UAR register resources for secondary process.
+ *
+ * The local UAR register table is freed.
  */
 static void
-mlx5_uar_uninit_secondary(void)
+uar_uninit_secondary(void)
 {
 	struct mlx5_local_data *ld = &mlx5_local_data;
 
-	if (!ld->uar_base)
-		return;
-	munmap(ld->uar_base, MLX5_UAR_SIZE);
-	ld->uar_base = NULL;
+	/* Free process-local table. */
+	if (ld->uar_table) {
+		rte_free(ld->uar_table);
+		ld->uar_table = NULL;
+		ld->uar_table_sz = 0;
+	}
 }
 
 /**
@@ -804,7 +900,7 @@ mlx5_init_once(void)
 		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
 						mlx5_mr_mem_event_cb, NULL);
 		mlx5_mp_init_primary();
-		ret = mlx5_uar_init_primary();
+		ret = uar_init_primary();
 		if (ret)
 			goto error;
 		sd->init_done = true;
@@ -813,7 +909,7 @@ mlx5_init_once(void)
 		if (ld->init_done)
 			break;
 		mlx5_mp_init_secondary();
-		ret = mlx5_uar_init_secondary();
+		ret = uar_init_secondary();
 		if (ret)
 			goto error;
 		++sd->secondary_cnt;
@@ -827,12 +923,12 @@ mlx5_init_once(void)
 error:
 	switch (rte_eal_process_type()) {
 	case RTE_PROC_PRIMARY:
-		mlx5_uar_uninit_primary();
+		uar_uninit_primary();
 		mlx5_mp_uninit_primary();
 		rte_mem_event_callback_unregister("MLX5_MEM_EVENT_CB", NULL);
 		break;
 	case RTE_PROC_SECONDARY:
-		mlx5_uar_uninit_secondary();
+		uar_uninit_secondary();
 		mlx5_mp_uninit_secondary();
 		break;
 	default:
@@ -1058,7 +1154,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			goto error;
 		}
 		/* Remap UAR for Tx queues. */
-		err = mlx5_tx_uar_remap(eth_dev, err);
+		err = mlx5_txq_uar_init(eth_dev, err);
 		if (err) {
 			err = rte_errno;
 			goto error;
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index a82972d166..5fe8f6ed0c 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -97,8 +97,8 @@ struct mlx5_shared_data {
 	/* Global spinlock for primary and secondary processes. */
 	int init_done; /* Whether primary has done initialization. */
 	unsigned int secondary_cnt; /* Number of secondary processes init'd. */
-	void *uar_base;
-	/* Reserved UAR address space for TXQ UAR(hw doorbell) mapping. */
+	struct rte_bitmap *uar_bmp;
+	/* Bitmap to keep track of BlueFlame register table. */
 	struct mlx5_dev_list mem_event_cb_list;
 	rte_rwlock_t mem_event_rwlock;
 };
@@ -106,11 +106,19 @@ struct mlx5_shared_data {
 /* Per-process data structure, not visible to other processes. */
 struct mlx5_local_data {
 	int init_done; /* Whether a secondary has done initialization. */
-	void *uar_base;
-	/* Reserved UAR address space for TXQ UAR(hw doorbell) mapping. */
+	void *(*uar_table)[];
+	/* Table of BlueFlame registers for each process. */
+	size_t uar_table_sz;
+	/* Size of BlueFlame register table. */
 };
 
 extern struct mlx5_shared_data *mlx5_shared_data;
+extern struct mlx5_local_data mlx5_local_data;
+
+/* The maximum size of BlueFlame register table. */
+#define MLX5_UAR_TABLE_SIZE_MAX (RTE_MAX_ETHPORTS * RTE_MAX_QUEUES_PER_PORT)
+
+#define MLX5_UAR_REG(idx) ((*mlx5_local_data.uar_table)[(idx)])
 
 struct mlx5_counter_ctrl {
 	/* Name of the counter. */
@@ -301,6 +309,9 @@ struct mlx5_priv {
 /* mlx5.c */
 
 int mlx5_getenv_int(const char *);
+void **mlx5_uar_get_addr_ptr(uint32_t idx);
+uint32_t mlx5_uar_alloc_index(void);
+void mlx5_uar_free_index(uint32_t idx);
 
 /* mlx5_ethdev.c */
 
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 53115dde3d..1d28a9c5f2 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -202,7 +202,7 @@ struct mlx5_txq_data {
 	volatile void *wqes; /* Work queue (use volatile to write into). */
 	volatile uint32_t *qp_db; /* Work queue doorbell. */
 	volatile uint32_t *cq_db; /* Completion queue doorbell. */
-	volatile void *bf_reg; /* Blueflame register remapped. */
+	uint32_t bfreg_idx; /* Blueflame register index. */
 	struct rte_mbuf *(*elts)[]; /* TX elements. */
 	struct mlx5_txq_stats stats; /* TX queue counters. */
 #ifndef RTE_ARCH_64
@@ -231,7 +231,6 @@ struct mlx5_txq_ctrl {
 	struct mlx5_priv *priv; /* Back pointer to private data. */
 	struct mlx5_txq_data txq; /* Data path structure. */
 	off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */
-	volatile void *bf_reg_orig; /* Blueflame register from verbs. */
 	uint16_t idx; /* Queue index. */
 };
 
@@ -302,7 +301,7 @@ uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
 int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 			unsigned int socket, const struct rte_eth_txconf *conf);
 void mlx5_tx_queue_release(void *dpdk_txq);
-int mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd);
+int mlx5_txq_uar_init(struct rte_eth_dev *dev, int fd);
 struct mlx5_txq_ibv *mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
 struct mlx5_txq_ibv *mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
 int mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv);
@@ -701,7 +700,7 @@ static __rte_always_inline void
 mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe,
 		       int cond)
 {
-	uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg);
+	uint64_t *dst = MLX5_UAR_REG(txq->bfreg_idx);
 	volatile uint64_t *src = ((volatile uint64_t *)wqe);
 
 	rte_cio_wmb();
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 7c9ff921ab..d98ef87ef5 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -58,7 +58,7 @@ mlx5_txq_start(struct rte_eth_dev *dev)
 			goto error;
 		}
 	}
-	ret = mlx5_tx_uar_remap(dev, priv->ctx->cmd_fd);
+	ret = mlx5_txq_uar_init(dev, priv->ctx->cmd_fd);
 	if (ret) {
 		/* Adjust index for rollback. */
 		i = priv->txqs_n - 1;
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 5640fe1b91..0cea4ad45f 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -231,9 +231,13 @@ mlx5_tx_queue_release(void *dpdk_txq)
 
 
 /**
- * Mmap TX UAR(HW doorbell) pages into reserved UAR address space.
- * Both primary and secondary process do mmap to make UAR address
- * aligned.
+ * Initialize UAR register access for Tx.
+ *
+ * For both primary and secondary, initialize UAR locks for atomic access.
+ *
+ * For secondary, remap BlueFlame registers for secondary process. Remapped
+ * address is stored at the same indexed entry of the local UAR register table
+ * as primary process.
  *
  * @param[in] dev
  *   Pointer to Ethernet device.
@@ -244,75 +248,48 @@ mlx5_tx_queue_release(void *dpdk_txq)
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd)
+mlx5_txq_uar_init(struct rte_eth_dev *dev, int fd)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	unsigned int i, j;
-	uintptr_t pages[priv->txqs_n];
-	unsigned int pages_n = 0;
-	uintptr_t uar_va;
-	uintptr_t off;
-	void *addr;
-	void *ret;
 	struct mlx5_txq_data *txq;
 	struct mlx5_txq_ctrl *txq_ctrl;
-	int already_mapped;
+	void *addr;
+	void **addr_ptr;
 	size_t page_size = sysconf(_SC_PAGESIZE);
+	unsigned int i;
 #ifndef RTE_ARCH_64
 	unsigned int lock_idx;
 #endif
 
-	memset(pages, 0, priv->txqs_n * sizeof(uintptr_t));
-	/*
-	 * As rdma-core, UARs are mapped in size of OS page size.
-	 * Use aligned address to avoid duplicate mmap.
-	 * Ref to libmlx5 function: mlx5_init_context()
-	 */
 	for (i = 0; i != priv->txqs_n; ++i) {
 		if (!(*priv->txqs)[i])
 			continue;
 		txq = (*priv->txqs)[i];
 		txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq);
 		assert(txq_ctrl->idx == (uint16_t)i);
-		/* UAR addr form verbs used to find dup and offset in page. */
-		uar_va = (uintptr_t)txq_ctrl->bf_reg_orig;
-		off = uar_va & (page_size - 1); /* offset in page. */
-		uar_va = RTE_ALIGN_FLOOR(uar_va, page_size); /* page addr. */
-		already_mapped = 0;
-		for (j = 0; j != pages_n; ++j) {
-			if (pages[j] == uar_va) {
-				already_mapped = 1;
-				break;
-			}
-		}
-		/* new address in reserved UAR address space. */
-		addr = RTE_PTR_ADD(mlx5_shared_data->uar_base,
-				   uar_va & (uintptr_t)(MLX5_UAR_SIZE - 1));
-		if (!already_mapped) {
-			pages[pages_n++] = uar_va;
-			/* fixed mmap to specified address in reserved
-			 * address space.
+		if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
+			/*
+			 * As rdma-core, UARs are mapped in size of OS page
+			 * size. Ref to libmlx5 function: mlx5_init_context()
 			 */
-			ret = mmap(addr, page_size,
-				   PROT_WRITE, MAP_FIXED | MAP_SHARED, fd,
-				   txq_ctrl->uar_mmap_offset);
-			if (ret != addr) {
-				/* fixed mmap have to return same address */
+			addr = mmap(NULL, page_size, PROT_WRITE,
+				    MAP_FIXED | MAP_SHARED, fd,
+				    txq_ctrl->uar_mmap_offset);
+			if (addr == MAP_FAILED) {
 				DRV_LOG(ERR,
-					"port %u call to mmap failed on UAR"
-					" for txq %u",
+					"port %u mmap failed for BF reg."
+					" of txq %u",
 					dev->data->port_id, txq_ctrl->idx);
 				rte_errno = ENXIO;
 				return -rte_errno;
 			}
+			addr_ptr = mlx5_uar_get_addr_ptr(txq->bfreg_idx);
+			if (!addr_ptr)
+				return -rte_errno;
+			*addr_ptr = addr;
 		}
-		if (rte_eal_process_type() == RTE_PROC_PRIMARY) /* save once */
-			txq_ctrl->txq.bf_reg = RTE_PTR_ADD((void *)addr, off);
-		else
-			assert(txq_ctrl->txq.bf_reg ==
-			       RTE_PTR_ADD((void *)addr, off));
 #ifndef RTE_ARCH_64
-		/* Assign a UAR lock according to UAR page number */
+		/* Assign an UAR lock according to UAR page number */
 		lock_idx = (txq_ctrl->uar_mmap_offset / page_size) &
 			   MLX5_UAR_PAGE_NUM_MASK;
 		txq->uar_lock = &priv->uar_lock[lock_idx];
@@ -372,6 +349,7 @@ mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
 	struct mlx5dv_obj obj;
 	const int desc = 1 << txq_data->elts_n;
 	eth_tx_burst_t tx_pkt_burst = mlx5_select_tx_function(dev);
+	void **addr_ptr;
 	int ret = 0;
 
 	assert(txq_data);
@@ -507,7 +485,16 @@ mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
 	txq_data->wqes = qp.sq.buf;
 	txq_data->wqe_n = log2above(qp.sq.wqe_cnt);
 	txq_data->qp_db = &qp.dbrec[MLX5_SND_DBR];
-	txq_ctrl->bf_reg_orig = qp.bf.reg;
+	/* Allocate a new index in UAR table. */
+	ret = mlx5_uar_alloc_index();
+	if (ret < 0)
+		goto error;
+	txq_data->bfreg_idx = ret;
+	/* Store the BlueFlame register address in the local table. */
+	addr_ptr = mlx5_uar_get_addr_ptr(txq_data->bfreg_idx);
+	if (!addr_ptr)
+		goto error;
+	*addr_ptr = qp.bf.reg;
 	txq_data->cq_db = cq_info.dbrec;
 	txq_data->cqes =
 		(volatile struct mlx5_cqe (*)[])
@@ -589,6 +576,7 @@ mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv)
 {
 	assert(txq_ibv);
 	if (rte_atomic32_dec_and_test(&txq_ibv->refcnt)) {
+		mlx5_uar_free_index(txq_ibv->txq_ctrl->txq.bfreg_idx);
 		claim_zero(mlx5_glue->destroy_qp(txq_ibv->qp));
 		claim_zero(mlx5_glue->destroy_cq(txq_ibv->cq));
 		LIST_REMOVE(txq_ibv, next);
@@ -837,15 +825,12 @@ mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_txq_ctrl *txq;
-	size_t page_size = sysconf(_SC_PAGESIZE);
 
 	if (!(*priv->txqs)[idx])
 		return 0;
 	txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
 	if (txq->ibv && !mlx5_txq_ibv_release(txq->ibv))
 		txq->ibv = NULL;
-	munmap((void *)RTE_ALIGN_FLOOR((uintptr_t)txq->txq.bf_reg, page_size),
-	       page_size);
 	if (rte_atomic32_dec_and_test(&txq->refcnt)) {
 		txq_free_elts(txq);
 		mlx5_mr_btree_free(&txq->txq.mr_ctrl.cache_bh);
-- 
2.11.0