DPDK patches and discussions
 help / color / mirror / Atom feed
From: Yongseok Koh <yskoh@mellanox.com>
To: shahafs@mellanox.com
Cc: dev@dpdk.org
Subject: [dpdk-dev] [PATCH v2 2/3] net/mlx4: add external allocator for Verbs object
Date: Mon, 25 Mar 2019 12:18:00 -0700	[thread overview]
Message-ID: <20190325191801.20841-3-yskoh@mellanox.com> (raw)
In-Reply-To: <20190325191801.20841-1-yskoh@mellanox.com>

To support secondary process, the memory allocated by library such as
completion rings (CQ) and buffer rings (WQ) must be manageable by EAL, in
order to share it with secondary processes. With new changes in rdma-core
and kernel driver, it is possible to provide an external allocator to the
library layer for this purpose. All such resources will now be allocated
within DPDK framework.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx4/Makefile    |  5 ++++
 drivers/net/mlx4/meson.build | 10 +++++++
 drivers/net/mlx4/mlx4.c      | 67 ++++++++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx4/mlx4.h      | 20 +++++++++++++
 drivers/net/mlx4/mlx4_rxq.c  |  4 +++
 drivers/net/mlx4/mlx4_txq.c  |  6 ++--
 6 files changed, 110 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile
index 1f1b927484..b527efd625 100644
--- a/drivers/net/mlx4/Makefile
+++ b/drivers/net/mlx4/Makefile
@@ -88,6 +88,11 @@ mlx4_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 	$Q $(RM) -f -- '$@'
 	$Q : > '$@'
 	$Q sh -- '$<' '$@' \
+		HAVE_IBV_MLX4_BUF_ALLOCATORS \
+		infiniband/mlx4dv.h \
+		enum MLX4DV_SET_CTX_ATTR_BUF_ALLOCATORS \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
 		HAVE_IBV_MLX4_WQE_LSO_SEG \
 		infiniband/mlx4dv.h \
 		type 'struct mlx4_wqe_lso_seg' \
diff --git a/drivers/net/mlx4/meson.build b/drivers/net/mlx4/meson.build
index b4f9672e73..650e2c8fbc 100644
--- a/drivers/net/mlx4/meson.build
+++ b/drivers/net/mlx4/meson.build
@@ -70,7 +70,17 @@ if build
 		[ 'HAVE_IBV_MLX4_WQE_LSO_SEG', 'infiniband/mlx4dv.h',
 		'struct mlx4_wqe_lso_seg', 'mss_hdr_size' ],
 	]
+	# input array for meson symbol search:
+	# [ "MACRO to define if found", "header for the search",
+	#   "symbol to search" ]
+	has_sym_args = [
+		[ 'HAVE_IBV_MLX4_BUF_ALLOCATORS', 'infiniband/mlx4dv.h',
+		'MLX4DV_SET_CTX_ATTR_BUF_ALLOCATORS' ],
+	]
 	config = configuration_data()
+	foreach arg:has_sym_args
+		config.set(arg[0], cc.has_header_symbol(arg[1], arg[2]))
+	endforeach
 	foreach arg:has_member_args
 		file_prefix = '#include<' + arg[1] + '>'
 		config.set(arg[0], cc.has_member(arg[2], arg[3],
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index bb6ab8ec6e..0e0b035df0 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -69,6 +69,62 @@ const char *pmd_mlx4_init_params[] = {
 
 static void mlx4_dev_stop(struct rte_eth_dev *dev);
 
+#ifdef HAVE_IBV_MLX4_BUF_ALLOCATORS
+/**
+ * Verbs callback to allocate a memory. This function should allocate the space
+ * according to the size provided residing inside a huge page.
+ * Please note that all allocation must respect the alignment from libmlx4
+ * (i.e. currently sysconf(_SC_PAGESIZE)).
+ *
+ * @param[in] size
+ *   The size in bytes of the memory to allocate.
+ * @param[in] data
+ *   A pointer to the callback data.
+ *
+ * @return
+ *   Allocated buffer, NULL otherwise and rte_errno is set.
+ */
+static void *
+mlx4_alloc_verbs_buf(size_t size, void *data)
+{
+	struct mlx4_priv *priv = data;
+	void *ret;
+	size_t alignment = sysconf(_SC_PAGESIZE);
+	unsigned int socket = SOCKET_ID_ANY;
+
+	if (priv->verbs_alloc_ctx.type == MLX4_VERBS_ALLOC_TYPE_TX_QUEUE) {
+		const struct txq *txq = priv->verbs_alloc_ctx.obj;
+
+		socket = txq->socket;
+	} else if (priv->verbs_alloc_ctx.type ==
+		   MLX4_VERBS_ALLOC_TYPE_RX_QUEUE) {
+		const struct rxq *rxq = priv->verbs_alloc_ctx.obj;
+
+		socket = rxq->socket;
+	}
+	assert(data != NULL);
+	ret = rte_malloc_socket(__func__, size, alignment, socket);
+	if (!ret && size)
+		rte_errno = ENOMEM;
+	return ret;
+}
+
+/**
+ * Verbs callback to free a memory.
+ *
+ * @param[in] ptr
+ *   A pointer to the memory to free.
+ * @param[in] data
+ *   A pointer to the callback data.
+ */
+static void
+mlx4_free_verbs_buf(void *ptr, void *data __rte_unused)
+{
+	assert(data != NULL);
+	rte_free(ptr);
+}
+#endif
+
 /**
  * DPDK callback for Ethernet device configuration.
  *
@@ -755,6 +811,17 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 		eth_dev->intr_handle = &priv->intr_handle;
 		priv->dev_data = eth_dev->data;
 		eth_dev->dev_ops = &mlx4_dev_ops;
+#ifdef HAVE_IBV_MLX4_BUF_ALLOCATORS
+		/* Hint libmlx4 to use PMD allocator for data plane resources */
+		struct mlx4dv_ctx_allocators alctr = {
+			.alloc = &mlx4_alloc_verbs_buf,
+			.free = &mlx4_free_verbs_buf,
+			.data = priv,
+		};
+		mlx4_glue->dv_set_context_attr
+			(ctx, MLX4DV_SET_CTX_ATTR_BUF_ALLOCATORS,
+			 (void *)((uintptr_t)&alctr));
+#endif
 		/* Bring Ethernet device up. */
 		DEBUG("forcing Ethernet interface up");
 		mlx4_dev_set_link_up(eth_dev);
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 51566caf7f..d43e05ea74 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -72,6 +72,24 @@ struct rxq;
 struct txq;
 struct rte_flow;
 
+/**
+ * Type of objet being allocated.
+ */
+enum mlx4_verbs_alloc_type {
+	MLX4_VERBS_ALLOC_TYPE_NONE,
+	MLX4_VERBS_ALLOC_TYPE_TX_QUEUE,
+	MLX4_VERBS_ALLOC_TYPE_RX_QUEUE,
+};
+
+/**
+ * Verbs allocator needs a context to know in the callback which kind of
+ * resources it is allocating.
+ */
+struct mlx4_verbs_alloc_ctx {
+	enum mlx4_verbs_alloc_type type; /* Kind of object being allocated. */
+	const void *obj; /* Pointer to the DPDK object. */
+};
+
 LIST_HEAD(mlx4_dev_list, mlx4_priv);
 LIST_HEAD(mlx4_mr_list, mlx4_mr);
 
@@ -111,6 +129,8 @@ struct mlx4_priv {
 	LIST_HEAD(, rte_flow) flows; /**< Configured flow rule handles. */
 	struct ether_addr mac[MLX4_MAX_MAC_ADDRESSES];
 	/**< Configured MAC addresses. Unused entries are zeroed. */
+	struct mlx4_verbs_alloc_ctx verbs_alloc_ctx;
+	/**< Context for Verbs allocator. */
 };
 
 #define PORT_ID(priv) ((priv)->dev_data->port_id)
diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c
index 50f33eb0c5..f45c1ff85c 100644
--- a/drivers/net/mlx4/mlx4_rxq.c
+++ b/drivers/net/mlx4/mlx4_rxq.c
@@ -513,6 +513,8 @@ mlx4_rxq_attach(struct rxq *rxq)
 	int ret;
 
 	assert(rte_is_power_of_2(elts_n));
+	priv->verbs_alloc_ctx.type = MLX4_VERBS_ALLOC_TYPE_RX_QUEUE;
+	priv->verbs_alloc_ctx.obj = rxq;
 	cq = mlx4_glue->create_cq(priv->ctx, elts_n / sges_n, NULL,
 				  rxq->channel, 0);
 	if (!cq) {
@@ -620,6 +622,7 @@ mlx4_rxq_attach(struct rxq *rxq)
 	rxq->rq_ci = elts_n / sges_n;
 	rte_wmb();
 	*rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
+	priv->verbs_alloc_ctx.type = MLX4_VERBS_ALLOC_TYPE_NONE;
 	return 0;
 error:
 	if (wq)
@@ -630,6 +633,7 @@ mlx4_rxq_attach(struct rxq *rxq)
 	rte_errno = ret;
 	ERROR("error while attaching Rx queue %p: %s: %s",
 	      (void *)rxq, msg, strerror(ret));
+	priv->verbs_alloc_ctx.type = MLX4_VERBS_ALLOC_TYPE_NONE;
 	return -ret;
 }
 
diff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c
index 352700820d..2dc198e77f 100644
--- a/drivers/net/mlx4/mlx4_txq.c
+++ b/drivers/net/mlx4/mlx4_txq.c
@@ -177,10 +177,8 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	uint64_t offloads;
 
 	offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
-
 	DEBUG("%p: configuring queue %u for %u descriptors",
 	      (void *)dev, idx, desc);
-
 	if (idx >= dev->data->nb_tx_queues) {
 		rte_errno = EOVERFLOW;
 		ERROR("%p: queue index out of range (%u >= %u)",
@@ -241,6 +239,8 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		.lb = !!priv->vf,
 		.bounce_buf = bounce_buf,
 	};
+	priv->verbs_alloc_ctx.type = MLX4_VERBS_ALLOC_TYPE_TX_QUEUE;
+	priv->verbs_alloc_ctx.obj = txq;
 	txq->cq = mlx4_glue->create_cq(priv->ctx, desc, NULL, NULL, 0);
 	if (!txq->cq) {
 		rte_errno = ENOMEM;
@@ -331,6 +331,7 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	txq->mr_ctrl.dev_gen_ptr = &priv->mr.dev_gen;
 	DEBUG("%p: adding Tx queue %p to list", (void *)dev, (void *)txq);
 	dev->data->tx_queues[idx] = txq;
+	priv->verbs_alloc_ctx.type = MLX4_VERBS_ALLOC_TYPE_NONE;
 	return 0;
 error:
 	dev->data->tx_queues[idx] = NULL;
@@ -338,6 +339,7 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	mlx4_tx_queue_release(txq);
 	rte_errno = ret;
 	assert(rte_errno > 0);
+	priv->verbs_alloc_ctx.type = MLX4_VERBS_ALLOC_TYPE_NONE;
 	return -rte_errno;
 }
 
-- 
2.11.0

  parent reply	other threads:[~2019-03-25 19:18 UTC|newest]

Thread overview: 30+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2019-03-07  7:39 [dpdk-dev] [PATCH 0/3] net/mlx4: add secondary process support Yongseok Koh
2019-03-07  7:39 ` [dpdk-dev] [PATCH 1/3] net/mlx4: change device reference for secondary process Yongseok Koh
2019-03-07  7:39 ` [dpdk-dev] [PATCH 2/3] net/mlx4: add external allocator for Verbs object Yongseok Koh
2019-03-07  7:39 ` [dpdk-dev] [PATCH 3/3] net/mlx4: add secondary process support Yongseok Koh
2019-03-25 19:17 ` [dpdk-dev] [PATCH v2 0/3] " Yongseok Koh
2019-03-25 19:17   ` Yongseok Koh
2019-03-25 19:17   ` [dpdk-dev] [PATCH v2 1/3] net/mlx4: change device reference for secondary process Yongseok Koh
2019-03-25 19:17     ` Yongseok Koh
2019-03-26 19:16     ` Shahaf Shuler
2019-03-26 19:16       ` Shahaf Shuler
2019-03-25 19:18   ` Yongseok Koh [this message]
2019-03-25 19:18     ` [dpdk-dev] [PATCH v2 2/3] net/mlx4: add external allocator for Verbs object Yongseok Koh
2019-03-26 19:21     ` Shahaf Shuler
2019-03-26 19:21       ` Shahaf Shuler
2019-03-25 19:18   ` [dpdk-dev] [PATCH v2 3/3] net/mlx4: add secondary process support Yongseok Koh
2019-03-25 19:18     ` Yongseok Koh
2019-03-26 19:33     ` Shahaf Shuler
2019-03-26 19:33       ` Shahaf Shuler
2019-03-28 19:01       ` Yongseok Koh
2019-03-28 19:01         ` Yongseok Koh
2019-04-01 21:15 ` [dpdk-dev] [PATCH v3 0/3] " Yongseok Koh
2019-04-01 21:15   ` Yongseok Koh
2019-04-01 21:15   ` [dpdk-dev] [PATCH v3 1/3] net/mlx4: change device reference for secondary process Yongseok Koh
2019-04-01 21:15     ` Yongseok Koh
2019-04-01 21:15   ` [dpdk-dev] [PATCH v3 2/3] net/mlx4: add external allocator for Verbs object Yongseok Koh
2019-04-01 21:15     ` Yongseok Koh
2019-04-01 21:15   ` [dpdk-dev] [PATCH v3 3/3] net/mlx4: add secondary process support Yongseok Koh
2019-04-01 21:15     ` Yongseok Koh
2019-04-02  7:12   ` [dpdk-dev] [PATCH v3 0/3] " Shahaf Shuler
2019-04-02  7:12     ` Shahaf Shuler

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20190325191801.20841-3-yskoh@mellanox.com \
    --to=yskoh@mellanox.com \
    --cc=dev@dpdk.org \
    --cc=shahafs@mellanox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).