DPDK patches and discussions
 help / color / mirror / Atom feed
From: Shahaf Shuler <shahafs@mellanox.com>
To: anatoly.burakov@intel.com, yskoh@mellanox.com,
	thomas@monjalon.net, ferruh.yigit@intel.com,
	nhorman@tuxdriver.com, gaetan.rivet@6wind.com
Cc: dev@dpdk.org
Subject: [dpdk-dev] [PATCH v3 4/6] net/mlx5: refactor external memory registration
Date: Tue,  5 Mar 2019 15:59:44 +0200	[thread overview]
Message-ID: <47b74a157f1ab6227c4992162fb2af90beed3f30.1551793527.git.shahafs@mellanox.com> (raw)
In-Reply-To: <cover.1551793527.git.shahafs@mellanox.com>
In-Reply-To: <cover.1551793527.git.shahafs@mellanox.com>

Move the memory region creation to a separate function to
prepare the ground for the reuse of it on the PCI driver map and unmap
functions.

Signed-off-by: Shahaf Shuler <shahafs@mellanox.com>
---
 drivers/net/mlx5/mlx5_mr.c | 86 +++++++++++++++++++++++++++--------------
 1 file changed, 57 insertions(+), 29 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 700d83d1bc..43ee9c961b 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -1109,6 +1109,58 @@ mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl)
 }
 
 /**
+ * Creates a memory region for external memory, that is memory which is not
+ * part of the DPDK memory segments.
+ *
+ * @param dev
+ *   Pointer to the ethernet device.
+ * @param addr
+ *   Starting virtual address of memory.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @param socked_id
+ *   Socket to allocate heap memory for the control structures.
+ *
+ * @return
+ *   Pointer to MR structure on success, NULL otherwise.
+ */
+static struct mlx5_mr *
+mlx5_create_mr_ext(struct rte_eth_dev *dev, uintptr_t addr, size_t len,
+		   int socket_id)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_mr *mr = NULL;
+
+	mr = rte_zmalloc_socket(NULL,
+				RTE_ALIGN_CEIL(sizeof(*mr),
+					       RTE_CACHE_LINE_SIZE),
+				RTE_CACHE_LINE_SIZE, socket_id);
+	if (mr == NULL)
+		return NULL;
+	mr->ibv_mr = mlx5_glue->reg_mr(priv->pd, (void *)addr, len,
+				       IBV_ACCESS_LOCAL_WRITE);
+	if (mr->ibv_mr == NULL) {
+		DRV_LOG(WARNING,
+			"port %u fail to create a verbs MR for address (%p)",
+			dev->data->port_id, (void *)addr);
+		rte_free(mr);
+		return NULL;
+	}
+	mr->msl = NULL; /* Mark it is external memory. */
+	mr->ms_bmp = NULL;
+	mr->ms_n = 1;
+	mr->ms_bmp_n = 1;
+	DRV_LOG(DEBUG,
+		"port %u MR CREATED (%p) for external memory %p:\n"
+		"  [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
+		" lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
+		dev->data->port_id, (void *)mr, (void *)addr,
+		addr, addr + len, rte_cpu_to_be_32(mr->ibv_mr->lkey),
+		mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
+	return mr;
+}
+
+/**
  * Called during rte_mempool_mem_iter() by mlx5_mr_update_ext_mp().
  *
  * Externally allocated chunk is registered and a MR is created for the chunk.
@@ -1142,43 +1194,19 @@ mlx5_mr_update_ext_mp_cb(struct rte_mempool *mp, void *opaque,
 	rte_rwlock_read_unlock(&priv->mr.rwlock);
 	if (lkey != UINT32_MAX)
 		return;
-	mr = rte_zmalloc_socket(NULL,
-				RTE_ALIGN_CEIL(sizeof(*mr),
-					       RTE_CACHE_LINE_SIZE),
-				RTE_CACHE_LINE_SIZE, mp->socket_id);
-	if (mr == NULL) {
-		DRV_LOG(WARNING,
-			"port %u unable to allocate memory for a new MR of"
-			" mempool (%s).",
-			dev->data->port_id, mp->name);
-		data->ret = -1;
-		return;
-	}
 	DRV_LOG(DEBUG, "port %u register MR for chunk #%d of mempool (%s)",
 		dev->data->port_id, mem_idx, mp->name);
-	mr->ibv_mr = mlx5_glue->reg_mr(priv->pd, (void *)addr, len,
-				       IBV_ACCESS_LOCAL_WRITE);
-	if (mr->ibv_mr == NULL) {
+	mr = mlx5_create_mr_ext(dev, addr, len, mp->socket_id);
+	if (!mr) {
 		DRV_LOG(WARNING,
-			"port %u fail to create a verbs MR for address (%p)",
-			dev->data->port_id, (void *)addr);
-		rte_free(mr);
+			"port %u unable to allocate a new MR of"
+			" mempool (%s).",
+			dev->data->port_id, mp->name);
 		data->ret = -1;
 		return;
 	}
-	mr->msl = NULL; /* Mark it is external memory. */
-	mr->ms_bmp = NULL;
-	mr->ms_n = 1;
-	mr->ms_bmp_n = 1;
 	rte_rwlock_write_lock(&priv->mr.rwlock);
 	LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
-	DRV_LOG(DEBUG,
-		"port %u MR CREATED (%p) for external memory %p:\n"
-		"  [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
-		" lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
-		dev->data->port_id, (void *)mr, (void *)addr,
-		addr, addr + len, rte_cpu_to_be_32(mr->ibv_mr->lkey),
-		mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
 	/* Insert to the global cache table. */
 	mr_insert_dev_cache(dev, mr);
 	rte_rwlock_write_unlock(&priv->mr.rwlock);
-- 
2.12.0

  parent reply	other threads:[~2019-03-05 14:00 UTC|newest]

Thread overview: 79+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2019-02-13  9:10 [dpdk-dev] [PATCH 0/6] introduce DMA memory mapping for external memory Shahaf Shuler
2019-02-13  9:10 ` [dpdk-dev] [PATCH 1/6] vfio: allow DMA map of memory for the default vfio fd Shahaf Shuler
2019-02-13  9:45   ` Gaëtan Rivet
2019-02-13 11:38     ` Gaëtan Rivet
2019-02-13 15:23     ` Shahaf Shuler
2019-02-13 14:41   ` Burakov, Anatoly
2019-02-13  9:10 ` [dpdk-dev] [PATCH 2/6] vfio: don't fail to DMA map if memory is already mapped Shahaf Shuler
2019-02-13  9:58   ` Gaëtan Rivet
2019-02-13 19:52     ` Shahaf Shuler
2019-02-13  9:10 ` [dpdk-dev] [PATCH 3/6] bus: introduce DMA memory mapping for external memory Shahaf Shuler
2019-02-13 11:17   ` Gaëtan Rivet
2019-02-13 19:07     ` Shahaf Shuler
2019-02-14 14:00       ` Gaëtan Rivet
2019-02-17  6:23         ` Shahaf Shuler
2019-02-13  9:10 ` [dpdk-dev] [PATCH 4/6] net/mlx5: refactor external memory registration Shahaf Shuler
2019-02-13  9:10 ` [dpdk-dev] [PATCH 5/6] net/mlx5: support PCI device DMA map and unmap Shahaf Shuler
2019-02-13 11:35   ` Gaëtan Rivet
2019-02-13 11:44     ` Gaëtan Rivet
2019-02-13 19:11       ` Shahaf Shuler
2019-02-14 10:21         ` Gaëtan Rivet
2019-02-21  9:21           ` Shahaf Shuler
2019-02-13  9:10 ` [dpdk-dev] [PATCH 6/6] doc: deprecate VFIO DMA map APIs Shahaf Shuler
2019-02-13 11:43 ` [dpdk-dev] [PATCH 0/6] introduce DMA memory mapping for external memory Alejandro Lucero
2019-02-13 19:24   ` Shahaf Shuler
2019-02-14 10:19     ` Burakov, Anatoly
2019-02-14 13:28       ` Shahaf Shuler
2019-02-14 16:19         ` Burakov, Anatoly
2019-02-17  6:18           ` Shahaf Shuler
2019-02-18 12:21             ` Burakov, Anatoly
2019-02-14 12:22     ` Alejandro Lucero
2019-02-14 12:27       ` Alejandro Lucero
2019-02-14 13:41       ` Shahaf Shuler
2019-02-14 16:43         ` Burakov, Anatoly
2019-02-21 14:50 ` [dpdk-dev] [PATCH v2 " Shahaf Shuler
2019-03-05 13:59   ` [dpdk-dev] [PATCH v3 " Shahaf Shuler
2019-03-10  8:27     ` [dpdk-dev] [PATCH v4 " Shahaf Shuler
2019-03-10  8:27       ` [dpdk-dev] [PATCH v4 1/6] vfio: allow DMA map of memory for the default vfio fd Shahaf Shuler
2019-03-30  0:23         ` Thomas Monjalon
2019-03-30  0:23           ` Thomas Monjalon
2019-03-30 14:29           ` Thomas Monjalon
2019-03-30 14:29             ` Thomas Monjalon
2019-03-10  8:27       ` [dpdk-dev] [PATCH v4 2/6] vfio: don't fail to DMA map if memory is already mapped Shahaf Shuler
2019-03-10  8:28       ` [dpdk-dev] [PATCH v4 3/6] bus: introduce device level DMA memory mapping Shahaf Shuler
2019-03-11 10:19         ` Burakov, Anatoly
2019-03-13  9:56         ` Thomas Monjalon
2019-03-13 11:12           ` Shahaf Shuler
2019-03-13 11:19             ` Thomas Monjalon
2019-03-13 11:47               ` Burakov, Anatoly
2019-03-30 14:36             ` Thomas Monjalon
2019-03-30 14:36               ` Thomas Monjalon
2019-03-10  8:28       ` [dpdk-dev] [PATCH v4 4/6] net/mlx5: refactor external memory registration Shahaf Shuler
2019-03-10  8:28       ` [dpdk-dev] [PATCH v4 5/6] net/mlx5: support PCI device DMA map and unmap Shahaf Shuler
2019-03-10  8:28       ` [dpdk-dev] [PATCH v4 6/6] doc: deprecation notice for VFIO DMA map APIs Shahaf Shuler
2019-03-11 10:20         ` Burakov, Anatoly
2019-03-11 17:35           ` Rami Rosen
2019-10-01 15:20         ` David Marchand
2019-10-02  4:53           ` Shahaf Shuler
2019-10-02  7:51             ` David Marchand
2019-03-11  9:27       ` [dpdk-dev] [PATCH v4 0/6] introduce DMA memory mapping for external memory Gaëtan Rivet
2019-03-30 14:40       ` Thomas Monjalon
2019-03-30 14:40         ` Thomas Monjalon
2019-03-05 13:59   ` [dpdk-dev] [PATCH v3 1/6] vfio: allow DMA map of memory for the default vfio fd Shahaf Shuler
2019-03-05 13:59   ` [dpdk-dev] [PATCH v3 2/6] vfio: don't fail to DMA map if memory is already mapped Shahaf Shuler
2019-03-05 13:59   ` [dpdk-dev] [PATCH v3 3/6] bus: introduce device level DMA memory mapping Shahaf Shuler
2019-03-05 16:35     ` Burakov, Anatoly
2019-03-05 13:59   ` Shahaf Shuler [this message]
2019-03-05 13:59   ` [dpdk-dev] [PATCH v3 5/6] net/mlx5: support PCI device DMA map and unmap Shahaf Shuler
2019-03-05 13:59   ` [dpdk-dev] [PATCH v3 6/6] doc: deprecation notice for VFIO DMA map APIs Shahaf Shuler
2019-02-21 14:50 ` [dpdk-dev] [PATCH v2 1/6] vfio: allow DMA map of memory for the default vfio fd Shahaf Shuler
2019-02-28 11:56   ` Burakov, Anatoly
2019-02-21 14:50 ` [dpdk-dev] [PATCH v2 2/6] vfio: don't fail to DMA map if memory is already mapped Shahaf Shuler
2019-02-28 11:58   ` Burakov, Anatoly
2019-02-21 14:50 ` [dpdk-dev] [PATCH v2 3/6] bus: introduce device level DMA memory mapping Shahaf Shuler
2019-02-28 12:14   ` Burakov, Anatoly
2019-02-28 14:41     ` Burakov, Anatoly
2019-02-21 14:50 ` [dpdk-dev] [PATCH v2 4/6] net/mlx5: refactor external memory registration Shahaf Shuler
2019-02-21 14:50 ` [dpdk-dev] [PATCH v2 5/6] net/mlx5: support PCI device DMA map and unmap Shahaf Shuler
2019-02-21 14:50 ` [dpdk-dev] [PATCH v2 6/6] doc: deprecate VFIO DMA map APIs Shahaf Shuler
2019-02-21 15:50   ` David Marchand

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=47b74a157f1ab6227c4992162fb2af90beed3f30.1551793527.git.shahafs@mellanox.com \
    --to=shahafs@mellanox.com \
    --cc=anatoly.burakov@intel.com \
    --cc=dev@dpdk.org \
    --cc=ferruh.yigit@intel.com \
    --cc=gaetan.rivet@6wind.com \
    --cc=nhorman@tuxdriver.com \
    --cc=thomas@monjalon.net \
    --cc=yskoh@mellanox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).