Re: [dpdk-dev] [PATCH v2] net/mlx5: share Memory Regions for multiport device

[Yongseok Koh <yskoh@mellanox.com>]

> On Apr 24, 2019, at 1:44 AM, Viacheslav Ovsiienko <viacheslavo@mellanox.com> wrote:
> 
> The multiport Infiniband device support was introduced [1].
> All active ports, belonging to the same Infiniband device use the signle
> shared Infiniband context of that device and share the resources:
>  - QPs are created within shared context
>  - Verbs flows are also created with specifying port index
>  - DV/DR resources
>  - Protection Domain
>  - Event Handlers
> 
> This patchset adds support for Memory Regions sharing between
> portes, created on the base of multiport Infiniban device.
> The datapath of mlx5 uses the layered cache subsystem for
> allocating/releasing Memory Regions, only the lowest layer L3
> is subject to share due to performance issues.
> 
> [1] http://patches.dpdk.org/cover/51800/
> 
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> ---
Acked-by: Yongseok Koh <yskoh@mellanox.com>

> v2:
>  - intendation issues
>  - comments cleanup
> 
> v1:
>  http://patches.dpdk.org/patch/52723/
> 
> drivers/net/mlx5/mlx5.c     |  40 +++++++----
> drivers/net/mlx5/mlx5.h     |  15 ++--
> drivers/net/mlx5/mlx5_mr.c  | 164 ++++++++++++++++++++++----------------------
> drivers/net/mlx5/mlx5_mr.h  |   5 +-
> drivers/net/mlx5/mlx5_txq.c |   2 +-
> 5 files changed, 121 insertions(+), 105 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
> index 1bb58b1..b563e0f 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -147,6 +147,7 @@ struct mlx5_dev_spawn_data {
> 	struct mlx5_switch_info info; /**< Switch information. */
> 	struct ibv_device *ibv_dev; /**< Associated IB device. */
> 	struct rte_eth_dev *eth_dev; /**< Associated Ethernet device. */
> +	struct rte_pci_device *pci_dev; /**< Backend PCI device. */
> };
> 
> static LIST_HEAD(, mlx5_ibv_shared) mlx5_ibv_list = LIST_HEAD_INITIALIZER();
> @@ -225,6 +226,7 @@ struct mlx5_dev_spawn_data {
> 		sizeof(sh->ibdev_name));
> 	strncpy(sh->ibdev_path, sh->ctx->device->ibdev_path,
> 		sizeof(sh->ibdev_path));
> +	sh->pci_dev = spawn->pci_dev;
> 	pthread_mutex_init(&sh->intr_mutex, NULL);
> 	/*
> 	 * Setting port_id to max unallowed value means
> @@ -239,6 +241,22 @@ struct mlx5_dev_spawn_data {
> 		err = ENOMEM;
> 		goto error;
> 	}
> +	/*
> +	 * Once the device is added to the list of memory event
> +	 * callback, its global MR cache table cannot be expanded
> +	 * on the fly because of deadlock. If it overflows, lookup
> +	 * should be done by searching MR list linearly, which is slow.
> +	 *
> +	 * At this point the device is not added to the memory
> +	 * event list yet, context is just being created.
> +	 */
> +	err = mlx5_mr_btree_init(&sh->mr.cache,
> +				 MLX5_MR_BTREE_CACHE_N * 2,
> +				 sh->pci_dev->device.numa_node);
> +	if (err) {
> +		err = rte_errno;
> +		goto error;
> +	}
> 	LIST_INSERT_HEAD(&mlx5_ibv_list, sh, next);
> exit:
> 	pthread_mutex_unlock(&mlx5_ibv_list_mutex);
> @@ -286,6 +304,8 @@ struct mlx5_dev_spawn_data {
> 	assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
> 	if (--sh->refcnt)
> 		goto exit;
> +	/* Release created Memory Regions. */
> +	mlx5_mr_release(sh);
> 	LIST_REMOVE(sh, next);
> 	/*
> 	 *  Ensure there is no async event handler installed.
> @@ -651,7 +671,10 @@ struct mlx5_dev_spawn_data {
> 	}
> 	mlx5_proc_priv_uninit(dev);
> 	mlx5_mprq_free_mp(dev);
> -	mlx5_mr_release(dev);
> +	/* Remove from memory callback device list. */
> +	rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
> +	LIST_REMOVE(priv, mem_event_cb);
> +	rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
> 	assert(priv->sh);
> 	mlx5_free_shared_dr(priv);
> 	if (priv->rss_conf.rss_key != NULL)
> @@ -1548,19 +1571,6 @@ struct mlx5_dev_spawn_data {
> 		goto error;
> 	}
> 	priv->config.flow_prio = err;
> -	/*
> -	 * Once the device is added to the list of memory event
> -	 * callback, its global MR cache table cannot be expanded
> -	 * on the fly because of deadlock. If it overflows, lookup
> -	 * should be done by searching MR list linearly, which is slow.
> -	 */
> -	err = mlx5_mr_btree_init(&priv->mr.cache,
> -				 MLX5_MR_BTREE_CACHE_N * 2,
> -				 eth_dev->device->numa_node);
> -	if (err) {
> -		err = rte_errno;
> -		goto error;
> -	}
> 	/* Add device to memory callback list. */
> 	rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
> 	LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list,
> @@ -1757,6 +1767,7 @@ struct mlx5_dev_spawn_data {
> 			list[ns].ibv_port = i;
> 			list[ns].ibv_dev = ibv_match[0];
> 			list[ns].eth_dev = NULL;
> +			list[ns].pci_dev = pci_dev;
> 			list[ns].ifindex = mlx5_nl_ifindex
> 					(nl_rdma, list[ns].ibv_dev->name, i);
> 			if (!list[ns].ifindex) {
> @@ -1823,6 +1834,7 @@ struct mlx5_dev_spawn_data {
> 			list[ns].ibv_port = 1;
> 			list[ns].ibv_dev = ibv_match[i];
> 			list[ns].eth_dev = NULL;
> +			list[ns].pci_dev = pci_dev;
> 			list[ns].ifindex = 0;
> 			if (nl_rdma >= 0)
> 				list[ns].ifindex = mlx5_nl_ifindex
> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
> index 0a6d7f1..2575732 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -275,6 +275,14 @@ struct mlx5_ibv_shared {
> 	char ibdev_name[IBV_SYSFS_NAME_MAX]; /* IB device name. */
> 	char ibdev_path[IBV_SYSFS_PATH_MAX]; /* IB device path for secondary */
> 	struct ibv_device_attr_ex device_attr; /* Device properties. */
> +	struct rte_pci_device *pci_dev; /* Backend PCI device. */
> +	struct {
> +		uint32_t dev_gen; /* Generation number to flush local caches. */
> +		rte_rwlock_t rwlock; /* MR Lock. */
> +		struct mlx5_mr_btree cache; /* Global MR cache table. */
> +		struct mlx5_mr_list mr_list; /* Registered MR list. */
> +		struct mlx5_mr_list mr_free_list; /* Freed MR list. */
> +	} mr;
> 	/* Shared DV/DR flow data section. */
> 	pthread_mutex_t dv_mutex; /* DV context mutex. */
> 	uint32_t dv_refcnt; /* DV/DR data reference counter. */
> @@ -347,13 +355,6 @@ struct mlx5_priv {
> 	struct mlx5_flows ctrl_flows; /* Control flow rules. */
> 	LIST_HEAD(counters, mlx5_flow_counter) flow_counters;
> 	/* Flow counters. */
> -	struct {
> -		uint32_t dev_gen; /* Generation number to flush local caches. */
> -		rte_rwlock_t rwlock; /* MR Lock. */
> -		struct mlx5_mr_btree cache; /* Global MR cache table. */
> -		struct mlx5_mr_list mr_list; /* Registered MR list. */
> -		struct mlx5_mr_list mr_free_list; /* Freed MR list. */
> -	} mr;
> 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
> 	LIST_HEAD(rxqibv, mlx5_rxq_ibv) rxqsibv; /* Verbs Rx queues. */
> 	LIST_HEAD(hrxq, mlx5_hrxq) hrxqs; /* Verbs Hash Rx queues. */
> diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
> index e4957a4..a7a63b1 100644
> --- a/drivers/net/mlx5/mlx5_mr.c
> +++ b/drivers/net/mlx5/mlx5_mr.c
> @@ -36,7 +36,7 @@ struct mr_update_mp_data {
> 
> /**
>  * Expand B-tree table to a given size. Can't be called with holding
> - * memory_hotplug_lock or priv->mr.rwlock due to rte_realloc().
> + * memory_hotplug_lock or sh->mr.rwlock due to rte_realloc().
>  *
>  * @param bt
>  *   Pointer to B-tree structure.
> @@ -350,7 +350,7 @@ struct mr_update_mp_data {
> 		n = mr_find_next_chunk(mr, &entry, n);
> 		if (!entry.end)
> 			break;
> -		if (mr_btree_insert(&priv->mr.cache, &entry) < 0) {
> +		if (mr_btree_insert(&priv->sh->mr.cache, &entry) < 0) {
> 			/*
> 			 * Overflowed, but the global table cannot be expanded
> 			 * because of deadlock.
> @@ -382,7 +382,7 @@ struct mr_update_mp_data {
> 	struct mlx5_mr *mr;
> 
> 	/* Iterate all the existing MRs. */
> -	LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
> +	LIST_FOREACH(mr, &priv->sh->mr.mr_list, mr) {
> 		unsigned int n;
> 
> 		if (mr->ms_n == 0)
> @@ -420,6 +420,7 @@ struct mr_update_mp_data {
> 	      uintptr_t addr)
> {
> 	struct mlx5_priv *priv = dev->data->dev_private;
> +	struct mlx5_ibv_shared *sh = priv->sh;
> 	uint16_t idx;
> 	uint32_t lkey = UINT32_MAX;
> 	struct mlx5_mr *mr;
> @@ -430,10 +431,10 @@ struct mr_update_mp_data {
> 	 * has to be searched by traversing the original MR list instead, which
> 	 * is very slow path. Otherwise, the global cache is all inclusive.
> 	 */
> -	if (!unlikely(priv->mr.cache.overflow)) {
> -		lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
> +	if (!unlikely(sh->mr.cache.overflow)) {
> +		lkey = mr_btree_lookup(&sh->mr.cache, &idx, addr);
> 		if (lkey != UINT32_MAX)
> -			*entry = (*priv->mr.cache.table)[idx];
> +			*entry = (*sh->mr.cache.table)[idx];
> 	} else {
> 		/* Falling back to the slowest path. */
> 		mr = mr_lookup_dev_list(dev, entry, addr);
> @@ -468,13 +469,12 @@ struct mr_update_mp_data {
> /**
>  * Release resources of detached MR having no online entry.
>  *
> - * @param dev
> - *   Pointer to Ethernet device.
> + * @param sh
> + *   Pointer to Ethernet device shared context.
>  */
> static void
> -mlx5_mr_garbage_collect(struct rte_eth_dev *dev)
> +mlx5_mr_garbage_collect(struct mlx5_ibv_shared *sh)
> {
> -	struct mlx5_priv *priv = dev->data->dev_private;
> 	struct mlx5_mr *mr_next;
> 	struct mlx5_mr_list free_list = LIST_HEAD_INITIALIZER(free_list);
> 
> @@ -484,11 +484,11 @@ struct mr_update_mp_data {
> 	 * MR can't be freed with holding the lock because rte_free() could call
> 	 * memory free callback function. This will be a deadlock situation.
> 	 */
> -	rte_rwlock_write_lock(&priv->mr.rwlock);
> +	rte_rwlock_write_lock(&sh->mr.rwlock);
> 	/* Detach the whole free list and release it after unlocking. */
> -	free_list = priv->mr.mr_free_list;
> -	LIST_INIT(&priv->mr.mr_free_list);
> -	rte_rwlock_write_unlock(&priv->mr.rwlock);
> +	free_list = sh->mr.mr_free_list;
> +	LIST_INIT(&sh->mr.mr_free_list);
> +	rte_rwlock_write_unlock(&sh->mr.rwlock);
> 	/* Release resources. */
> 	mr_next = LIST_FIRST(&free_list);
> 	while (mr_next != NULL) {
> @@ -548,12 +548,12 @@ struct mr_update_mp_data {
> 		      dev->data->port_id, (void *)addr);
> 		return UINT32_MAX;
> 	}
> -	rte_rwlock_read_lock(&priv->mr.rwlock);
> +	rte_rwlock_read_lock(&priv->sh->mr.rwlock);
> 	/* Fill in output data. */
> 	mr_lookup_dev(dev, entry, addr);
> 	/* Lookup can't fail. */
> 	assert(entry->lkey != UINT32_MAX);
> -	rte_rwlock_read_unlock(&priv->mr.rwlock);
> +	rte_rwlock_read_unlock(&priv->sh->mr.rwlock);
> 	DEBUG("port %u MR CREATED by primary process for %p:\n"
> 	      "  [0x%" PRIxPTR ", 0x%" PRIxPTR "), lkey=0x%x",
> 	      dev->data->port_id, (void *)addr,
> @@ -582,6 +582,7 @@ struct mr_update_mp_data {
> 		       uintptr_t addr)
> {
> 	struct mlx5_priv *priv = dev->data->dev_private;
> +	struct mlx5_ibv_shared *sh = priv->sh;
> 	struct mlx5_dev_config *config = &priv->config;
> 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
> 	const struct rte_memseg_list *msl;
> @@ -602,12 +603,12 @@ struct mr_update_mp_data {
> 		dev->data->port_id, (void *)addr);
> 	/*
> 	 * Release detached MRs if any. This can't be called with holding either
> -	 * memory_hotplug_lock or priv->mr.rwlock. MRs on the free list have
> +	 * memory_hotplug_lock or sh->mr.rwlock. MRs on the free list have
> 	 * been detached by the memory free event but it couldn't be released
> 	 * inside the callback due to deadlock. As a result, releasing resources
> 	 * is quite opportunistic.
> 	 */
> -	mlx5_mr_garbage_collect(dev);
> +	mlx5_mr_garbage_collect(sh);
> 	/*
> 	 * If enabled, find out a contiguous virtual address chunk in use, to
> 	 * which the given address belongs, in order to register maximum range.
> @@ -710,7 +711,7 @@ struct mr_update_mp_data {
> 		goto alloc_resources;
> 	}
> 	assert(data.msl == data_re.msl);
> -	rte_rwlock_write_lock(&priv->mr.rwlock);
> +	rte_rwlock_write_lock(&sh->mr.rwlock);
> 	/*
> 	 * Check the address is really missing. If other thread already created
> 	 * one or it is not found due to overflow, abort and return.
> @@ -721,10 +722,10 @@ struct mr_update_mp_data {
> 		 * low-on-memory. Then, this entry will have to be searched
> 		 * here again.
> 		 */
> -		mr_btree_insert(&priv->mr.cache, entry);
> +		mr_btree_insert(&sh->mr.cache, entry);
> 		DEBUG("port %u found MR for %p on final lookup, abort",
> 		      dev->data->port_id, (void *)addr);
> -		rte_rwlock_write_unlock(&priv->mr.rwlock);
> +		rte_rwlock_write_unlock(&sh->mr.rwlock);
> 		rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
> 		/*
> 		 * Must be unlocked before calling rte_free() because
> @@ -769,7 +770,7 @@ struct mr_update_mp_data {
> 	 * mlx5_alloc_buf_extern() which eventually calls rte_malloc_socket()
> 	 * through mlx5_alloc_verbs_buf().
> 	 */
> -	mr->ibv_mr = mlx5_glue->reg_mr(priv->sh->pd, (void *)data.start, len,
> +	mr->ibv_mr = mlx5_glue->reg_mr(sh->pd, (void *)data.start, len,
> 				       IBV_ACCESS_LOCAL_WRITE);
> 	if (mr->ibv_mr == NULL) {
> 		DEBUG("port %u fail to create a verbs MR for address (%p)",
> @@ -779,7 +780,7 @@ struct mr_update_mp_data {
> 	}
> 	assert((uintptr_t)mr->ibv_mr->addr == data.start);
> 	assert(mr->ibv_mr->length == len);
> -	LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
> +	LIST_INSERT_HEAD(&sh->mr.mr_list, mr, mr);
> 	DEBUG("port %u MR CREATED (%p) for %p:\n"
> 	      "  [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
> 	      " lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
> @@ -792,11 +793,11 @@ struct mr_update_mp_data {
> 	mr_lookup_dev(dev, entry, addr);
> 	/* Lookup can't fail. */
> 	assert(entry->lkey != UINT32_MAX);
> -	rte_rwlock_write_unlock(&priv->mr.rwlock);
> +	rte_rwlock_write_unlock(&sh->mr.rwlock);
> 	rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
> 	return entry->lkey;
> err_mrlock:
> -	rte_rwlock_write_unlock(&priv->mr.rwlock);
> +	rte_rwlock_write_unlock(&sh->mr.rwlock);
> err_memlock:
> 	rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
> err_nolock:
> @@ -854,14 +855,15 @@ struct mr_update_mp_data {
> mr_rebuild_dev_cache(struct rte_eth_dev *dev)
> {
> 	struct mlx5_priv *priv = dev->data->dev_private;
> +	struct mlx5_ibv_shared *sh = priv->sh;
> 	struct mlx5_mr *mr;
> 
> 	DRV_LOG(DEBUG, "port %u rebuild dev cache[]", dev->data->port_id);
> 	/* Flush cache to rebuild. */
> -	priv->mr.cache.len = 1;
> -	priv->mr.cache.overflow = 0;
> +	sh->mr.cache.len = 1;
> +	sh->mr.cache.overflow = 0;
> 	/* Iterate all the existing MRs. */
> -	LIST_FOREACH(mr, &priv->mr.mr_list, mr)
> +	LIST_FOREACH(mr, &sh->mr.mr_list, mr)
> 		if (mr_insert_dev_cache(dev, mr) < 0)
> 			return;
> }
> @@ -888,6 +890,7 @@ struct mr_update_mp_data {
> mlx5_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
> {
> 	struct mlx5_priv *priv = dev->data->dev_private;
> +	struct mlx5_ibv_shared *sh = priv->sh;
> 	const struct rte_memseg_list *msl;
> 	struct mlx5_mr *mr;
> 	int ms_n;
> @@ -901,7 +904,7 @@ struct mr_update_mp_data {
> 	assert((uintptr_t)addr == RTE_ALIGN((uintptr_t)addr, msl->page_sz));
> 	assert(len == RTE_ALIGN(len, msl->page_sz));
> 	ms_n = len / msl->page_sz;
> -	rte_rwlock_write_lock(&priv->mr.rwlock);
> +	rte_rwlock_write_lock(&sh->mr.rwlock);
> 	/* Clear bits of freed memsegs from MR. */
> 	for (i = 0; i < ms_n; ++i) {
> 		const struct rte_memseg *ms;
> @@ -928,7 +931,7 @@ struct mr_update_mp_data {
> 		rte_bitmap_clear(mr->ms_bmp, pos);
> 		if (--mr->ms_n == 0) {
> 			LIST_REMOVE(mr, mr);
> -			LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
> +			LIST_INSERT_HEAD(&sh->mr.mr_free_list, mr, mr);
> 			DEBUG("port %u remove MR(%p) from list",
> 			      dev->data->port_id, (void *)mr);
> 		}
> @@ -949,12 +952,12 @@ struct mr_update_mp_data {
> 		 * generation below) will be guaranteed to be seen by other core
> 		 * before the core sees the newly allocated memory.
> 		 */
> -		++priv->mr.dev_gen;
> +		++sh->mr.dev_gen;
> 		DEBUG("broadcasting local cache flush, gen=%d",
> -		      priv->mr.dev_gen);
> +		      sh->mr.dev_gen);
> 		rte_smp_wmb();
> 	}
> -	rte_rwlock_write_unlock(&priv->mr.rwlock);
> +	rte_rwlock_write_unlock(&sh->mr.rwlock);
> }
> 
> /**
> @@ -1013,6 +1016,7 @@ struct mr_update_mp_data {
> 		   struct mlx5_mr_cache *entry, uintptr_t addr)
> {
> 	struct mlx5_priv *priv = dev->data->dev_private;
> +	struct mlx5_ibv_shared *sh = priv->sh;
> 	struct mlx5_mr_btree *bt = &mr_ctrl->cache_bh;
> 	uint16_t idx;
> 	uint32_t lkey;
> @@ -1021,12 +1025,12 @@ struct mr_update_mp_data {
> 	if (unlikely(bt->len == bt->size))
> 		mr_btree_expand(bt, bt->size << 1);
> 	/* Look up in the global cache. */
> -	rte_rwlock_read_lock(&priv->mr.rwlock);
> -	lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
> +	rte_rwlock_read_lock(&sh->mr.rwlock);
> +	lkey = mr_btree_lookup(&sh->mr.cache, &idx, addr);
> 	if (lkey != UINT32_MAX) {
> 		/* Found. */
> -		*entry = (*priv->mr.cache.table)[idx];
> -		rte_rwlock_read_unlock(&priv->mr.rwlock);
> +		*entry = (*sh->mr.cache.table)[idx];
> +		rte_rwlock_read_unlock(&sh->mr.rwlock);
> 		/*
> 		 * Update local cache. Even if it fails, return the found entry
> 		 * to update top-half cache. Next time, this entry will be found
> @@ -1035,7 +1039,7 @@ struct mr_update_mp_data {
> 		mr_btree_insert(bt, entry);
> 		return lkey;
> 	}
> -	rte_rwlock_read_unlock(&priv->mr.rwlock);
> +	rte_rwlock_read_unlock(&sh->mr.rwlock);
> 	/* First time to see the address? Create a new MR. */
> 	lkey = mlx5_mr_create(dev, entry, addr);
> 	/*
> @@ -1261,6 +1265,7 @@ struct mr_update_mp_data {
> 	struct mr_update_mp_data *data = opaque;
> 	struct rte_eth_dev *dev = data->dev;
> 	struct mlx5_priv *priv = dev->data->dev_private;
> +	struct mlx5_ibv_shared *sh = priv->sh;
> 	struct mlx5_mr_ctrl *mr_ctrl = data->mr_ctrl;
> 	struct mlx5_mr *mr = NULL;
> 	uintptr_t addr = (uintptr_t)memhdr->addr;
> @@ -1270,9 +1275,9 @@ struct mr_update_mp_data {
> 
> 	assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
> 	/* If already registered, it should return. */
> -	rte_rwlock_read_lock(&priv->mr.rwlock);
> +	rte_rwlock_read_lock(&sh->mr.rwlock);
> 	lkey = mr_lookup_dev(dev, &entry, addr);
> -	rte_rwlock_read_unlock(&priv->mr.rwlock);
> +	rte_rwlock_read_unlock(&sh->mr.rwlock);
> 	if (lkey != UINT32_MAX)
> 		return;
> 	DRV_LOG(DEBUG, "port %u register MR for chunk #%d of mempool (%s)",
> @@ -1286,11 +1291,11 @@ struct mr_update_mp_data {
> 		data->ret = -1;
> 		return;
> 	}
> -	rte_rwlock_write_lock(&priv->mr.rwlock);
> -	LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
> +	rte_rwlock_write_lock(&sh->mr.rwlock);
> +	LIST_INSERT_HEAD(&sh->mr.mr_list, mr, mr);
> 	/* Insert to the global cache table. */
> 	mr_insert_dev_cache(dev, mr);
> -	rte_rwlock_write_unlock(&priv->mr.rwlock);
> +	rte_rwlock_write_unlock(&sh->mr.rwlock);
> 	/* Insert to the local cache table */
> 	mlx5_mr_addr2mr_bh(dev, mr_ctrl, addr);
> }
> @@ -1339,6 +1344,7 @@ struct mr_update_mp_data {
> 	struct rte_eth_dev *dev;
> 	struct mlx5_mr *mr;
> 	struct mlx5_priv *priv;
> +	struct mlx5_ibv_shared *sh;
> 
> 	dev = pci_dev_to_eth_dev(pdev);
> 	if (!dev) {
> @@ -1355,11 +1361,12 @@ struct mr_update_mp_data {
> 		rte_errno = EINVAL;
> 		return -1;
> 	}
> -	rte_rwlock_write_lock(&priv->mr.rwlock);
> -	LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
> +	sh = priv->sh;
> +	rte_rwlock_write_lock(&sh->mr.rwlock);
> +	LIST_INSERT_HEAD(&sh->mr.mr_list, mr, mr);
> 	/* Insert to the global cache table. */
> 	mr_insert_dev_cache(dev, mr);
> -	rte_rwlock_write_unlock(&priv->mr.rwlock);
> +	rte_rwlock_write_unlock(&sh->mr.rwlock);
> 	return 0;
> }
> 
> @@ -1384,6 +1391,7 @@ struct mr_update_mp_data {
> {
> 	struct rte_eth_dev *dev;
> 	struct mlx5_priv *priv;
> +	struct mlx5_ibv_shared *sh;
> 	struct mlx5_mr *mr;
> 	struct mlx5_mr_cache entry;
> 
> @@ -1395,10 +1403,11 @@ struct mr_update_mp_data {
> 		return -1;
> 	}
> 	priv = dev->data->dev_private;
> -	rte_rwlock_read_lock(&priv->mr.rwlock);
> +	sh = priv->sh;
> +	rte_rwlock_read_lock(&sh->mr.rwlock);
> 	mr = mr_lookup_dev_list(dev, &entry, (uintptr_t)addr);
> 	if (!mr) {
> -		rte_rwlock_read_unlock(&priv->mr.rwlock);
> +		rte_rwlock_read_unlock(&sh->mr.rwlock);
> 		DRV_LOG(WARNING, "address 0x%" PRIxPTR " wasn't registered "
> 				 "to PCI device %p", (uintptr_t)addr,
> 				 (void *)pdev);
> @@ -1406,7 +1415,7 @@ struct mr_update_mp_data {
> 		return -1;
> 	}
> 	LIST_REMOVE(mr, mr);
> -	LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
> +	LIST_INSERT_HEAD(&sh->mr.mr_free_list, mr, mr);
> 	DEBUG("port %u remove MR(%p) from list", dev->data->port_id,
> 	      (void *)mr);
> 	mr_rebuild_dev_cache(dev);
> @@ -1419,11 +1428,10 @@ struct mr_update_mp_data {
> 	 * generation below) will be guaranteed to be seen by other core
> 	 * before the core sees the newly allocated memory.
> 	 */
> -	++priv->mr.dev_gen;
> -	DEBUG("broadcasting local cache flush, gen=%d",
> -			priv->mr.dev_gen);
> +	++sh->mr.dev_gen;
> +	DEBUG("broadcasting local cache flush, gen=%d",	sh->mr.dev_gen);
> 	rte_smp_wmb();
> -	rte_rwlock_read_unlock(&priv->mr.rwlock);
> +	rte_rwlock_read_unlock(&sh->mr.rwlock);
> 	return 0;
> }
> 
> @@ -1544,25 +1552,24 @@ struct mr_update_mp_data {
> /**
>  * Dump all the created MRs and the global cache entries.
>  *
> - * @param dev
> - *   Pointer to Ethernet device.
> + * @param sh
> + *   Pointer to Ethernet device shared context.
>  */
> void
> -mlx5_mr_dump_dev(struct rte_eth_dev *dev __rte_unused)
> +mlx5_mr_dump_dev(struct mlx5_ibv_shared *sh __rte_unused)
> {
> #ifndef NDEBUG
> -	struct mlx5_priv *priv = dev->data->dev_private;
> 	struct mlx5_mr *mr;
> 	int mr_n = 0;
> 	int chunk_n = 0;
> 
> -	rte_rwlock_read_lock(&priv->mr.rwlock);
> +	rte_rwlock_read_lock(&sh->mr.rwlock);
> 	/* Iterate all the existing MRs. */
> -	LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
> +	LIST_FOREACH(mr, &sh->mr.mr_list, mr) {
> 		unsigned int n;
> 
> -		DEBUG("port %u MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u",
> -		      dev->data->port_id, mr_n++,
> +		DEBUG("device %s MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u",
> +		      sh->ibdev_name, mr_n++,
> 		      rte_cpu_to_be_32(mr->ibv_mr->lkey),
> 		      mr->ms_n, mr->ms_bmp_n);
> 		if (mr->ms_n == 0)
> @@ -1577,45 +1584,40 @@ struct mr_update_mp_data {
> 			      chunk_n++, ret.start, ret.end);
> 		}
> 	}
> -	DEBUG("port %u dumping global cache", dev->data->port_id);
> -	mlx5_mr_btree_dump(&priv->mr.cache);
> -	rte_rwlock_read_unlock(&priv->mr.rwlock);
> +	DEBUG("device %s dumping global cache", sh->ibdev_name);
> +	mlx5_mr_btree_dump(&sh->mr.cache);
> +	rte_rwlock_read_unlock(&sh->mr.rwlock);
> #endif
> }
> 
> /**
> - * Release all the created MRs and resources. Remove device from memory callback
> + * Release all the created MRs and resources for shared device context.
>  * list.
>  *
> - * @param dev
> - *   Pointer to Ethernet device.
> + * @param sh
> + *   Pointer to Ethernet device shared context.
>  */
> void
> -mlx5_mr_release(struct rte_eth_dev *dev)
> +mlx5_mr_release(struct mlx5_ibv_shared *sh)
> {
> -	struct mlx5_priv *priv = dev->data->dev_private;
> 	struct mlx5_mr *mr_next;
> 
> -	/* Remove from memory callback device list. */
> -	rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
> -	LIST_REMOVE(priv, mem_event_cb);
> -	rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
> 	if (rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
> -		mlx5_mr_dump_dev(dev);
> -	rte_rwlock_write_lock(&priv->mr.rwlock);
> +		mlx5_mr_dump_dev(sh);
> +	rte_rwlock_write_lock(&sh->mr.rwlock);
> 	/* Detach from MR list and move to free list. */
> -	mr_next = LIST_FIRST(&priv->mr.mr_list);
> +	mr_next = LIST_FIRST(&sh->mr.mr_list);
> 	while (mr_next != NULL) {
> 		struct mlx5_mr *mr = mr_next;
> 
> 		mr_next = LIST_NEXT(mr, mr);
> 		LIST_REMOVE(mr, mr);
> -		LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
> +		LIST_INSERT_HEAD(&sh->mr.mr_free_list, mr, mr);
> 	}
> -	LIST_INIT(&priv->mr.mr_list);
> +	LIST_INIT(&sh->mr.mr_list);
> 	/* Free global cache. */
> -	mlx5_mr_btree_free(&priv->mr.cache);
> -	rte_rwlock_write_unlock(&priv->mr.rwlock);
> +	mlx5_mr_btree_free(&sh->mr.cache);
> +	rte_rwlock_write_unlock(&sh->mr.rwlock);
> 	/* Free all remaining MRs. */
> -	mlx5_mr_garbage_collect(dev);
> +	mlx5_mr_garbage_collect(sh);
> }
> diff --git a/drivers/net/mlx5/mlx5_mr.h b/drivers/net/mlx5/mlx5_mr.h
> index 786f6a3..89e89b7 100644
> --- a/drivers/net/mlx5/mlx5_mr.h
> +++ b/drivers/net/mlx5/mlx5_mr.h
> @@ -62,6 +62,7 @@ struct mlx5_mr_ctrl {
> 	struct mlx5_mr_btree cache_bh; /* Cache for bottom-half. */
> } __rte_packed;
> 
> +struct mlx5_ibv_shared;
> extern struct mlx5_dev_list  mlx5_mem_event_cb_list;
> extern rte_rwlock_t mlx5_mem_event_rwlock;
> 
> @@ -76,11 +77,11 @@ void mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
> 			  size_t len, void *arg);
> int mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
> 		      struct rte_mempool *mp);
> -void mlx5_mr_release(struct rte_eth_dev *dev);
> +void mlx5_mr_release(struct mlx5_ibv_shared *sh);
> 
> /* Debug purpose functions. */
> void mlx5_mr_btree_dump(struct mlx5_mr_btree *bt);
> -void mlx5_mr_dump_dev(struct rte_eth_dev *dev);
> +void mlx5_mr_dump_dev(struct mlx5_ibv_shared *sh);
> 
> /**
>  * Look up LKey from given lookup table by linear search. Firstly look up the
> diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
> index dbe074f..4d55fd4 100644
> --- a/drivers/net/mlx5/mlx5_txq.c
> +++ b/drivers/net/mlx5/mlx5_txq.c
> @@ -814,7 +814,7 @@ struct mlx5_txq_ctrl *
> 		goto error;
> 	}
> 	/* Save pointer of global generation number to check memory event. */
> -	tmpl->txq.mr_ctrl.dev_gen_ptr = &priv->mr.dev_gen;
> +	tmpl->txq.mr_ctrl.dev_gen_ptr = &priv->sh->mr.dev_gen;
> 	assert(desc > MLX5_TX_COMP_THRESH);
> 	tmpl->txq.offloads = conf->offloads |
> 			     dev->data->dev_conf.txmode.offloads;
> -- 
> 1.8.3.1
>