DPDK patches and discussions
 help / color / mirror / Atom feed
From: Michael Baum <michaelba@nvidia.com>
To: dev@dpdk.org
Cc: Matan Azrad <matan@nvidia.com>,
	Raslan Darawsheh <rasland@nvidia.com>,
	Viacheslav Ovsiienko <viacheslavo@nvidia.com>
Subject: [dpdk-dev] [PATCH v2 02/17] common/mlx5: share DevX CQ creation
Date: Tue, 29 Dec 2020 08:52:09 +0000
Message-ID: <1609231944-29274-3-git-send-email-michaelba@nvidia.com> (raw)
In-Reply-To: <1609231944-29274-1-git-send-email-michaelba@nvidia.com>

The CQ object in DevX is created in several places and in several
different drivers.
In all places almost all the details are the same, and in particular the
allocations of the required resources.

Add a structure that contains all the resources, and provide creation
and release functions for it.

Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/meson.build        |   1 +
 drivers/common/mlx5/mlx5_common_devx.c | 157 +++++++++++++++++++++++++++++++++
 drivers/common/mlx5/mlx5_common_devx.h |  28 ++++++
 3 files changed, 186 insertions(+)
 create mode 100644 drivers/common/mlx5/mlx5_common_devx.c
 create mode 100644 drivers/common/mlx5/mlx5_common_devx.h

diff --git a/drivers/common/mlx5/meson.build b/drivers/common/mlx5/meson.build
index 3dacc6f..26cee06 100644
--- a/drivers/common/mlx5/meson.build
+++ b/drivers/common/mlx5/meson.build
@@ -16,6 +16,7 @@ sources += files(
 	'mlx5_common_mr.c',
 	'mlx5_malloc.c',
 	'mlx5_common_pci.c',
+	'mlx5_common_devx.c',
 )
 
 cflags_options = [
diff --git a/drivers/common/mlx5/mlx5_common_devx.c b/drivers/common/mlx5/mlx5_common_devx.c
new file mode 100644
index 0000000..324c6ea
--- /dev/null
+++ b/drivers/common/mlx5/mlx5_common_devx.c
@@ -0,0 +1,157 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+#include <stdint.h>
+
+#include <rte_errno.h>
+#include <rte_common.h>
+#include <rte_eal_paging.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common_os.h>
+
+#include "mlx5_prm.h"
+#include "mlx5_devx_cmds.h"
+#include "mlx5_common_utils.h"
+#include "mlx5_malloc.h"
+#include "mlx5_common.h"
+#include "mlx5_common_devx.h"
+
+/**
+ * Destroy DevX Completion Queue.
+ *
+ * @param[in] cq
+ *   DevX CQ to destroy.
+ */
+void
+mlx5_devx_cq_destroy(struct mlx5_devx_cq *cq)
+{
+	if (cq->cq)
+		claim_zero(mlx5_devx_cmd_destroy(cq->cq));
+	if (cq->umem_obj)
+		claim_zero(mlx5_glue->devx_umem_dereg(cq->umem_obj));
+	if (cq->umem_buf)
+		mlx5_free((void *)(uintptr_t)cq->umem_buf);
+}
+
+/* Mark all CQEs initially as invalid. */
+static void
+mlx5_cq_init(struct mlx5_devx_cq *cq_obj, uint16_t cq_size)
+{
+	volatile struct mlx5_cqe *cqe = cq_obj->cqes;
+	uint16_t i;
+
+	for (i = 0; i < cq_size; i++, cqe++)
+		cqe->op_own = (MLX5_CQE_INVALID << 4) | MLX5_CQE_OWNER_MASK;
+}
+
+/**
+ * Create Completion Queue using DevX API.
+ *
+ * Get a pointer to partially initialized attributes structure, and updates the
+ * following fields:
+ *   q_umem_valid
+ *   q_umem_id
+ *   q_umem_offset
+ *   db_umem_valid
+ *   db_umem_id
+ *   db_umem_offset
+ *   eqn
+ *   log_cq_size
+ *   log_page_size
+ * All other fields are updated by caller.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param[in/out] cq_obj
+ *   Pointer to CQ to create.
+ * @param[in] log_desc_n
+ *   Log of number of descriptors in queue.
+ * @param[in] attr
+ *   Pointer to CQ attributes structure.
+ * @param[in] socket
+ *   Socket to use for allocation.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_devx_cq_create(void *ctx, struct mlx5_devx_cq *cq_obj, uint16_t log_desc_n,
+		    struct mlx5_devx_cq_attr *attr, int socket)
+{
+	struct mlx5_devx_obj *cq = NULL;
+	struct mlx5dv_devx_umem *umem_obj = NULL;
+	void *umem_buf = NULL;
+	size_t page_size = rte_mem_page_size();
+	size_t alignment = MLX5_CQE_BUF_ALIGNMENT;
+	uint32_t umem_size, umem_dbrec;
+	uint32_t eqn;
+	uint16_t cq_size = 1 << log_desc_n;
+	int ret;
+
+	if (page_size == (size_t)-1 || alignment == (size_t)-1) {
+		DRV_LOG(ERR, "Failed to get page_size.");
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	/* Query first EQN. */
+	ret = mlx5_glue->devx_query_eqn(ctx, 0, &eqn);
+	if (ret) {
+		rte_errno = errno;
+		DRV_LOG(ERR, "Failed to query event queue number.");
+		goto error;
+	}
+	/* Allocate memory buffer for CQEs and doorbell record. */
+	umem_size = sizeof(struct mlx5_cqe) * cq_size;
+	umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);
+	umem_size += MLX5_DBR_SIZE;
+	umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size,
+			       alignment, socket);
+	if (!umem_buf) {
+		DRV_LOG(ERR, "Failed to allocate memory for CQ.");
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	/* Register allocated buffer in user space with DevX. */
+	umem_obj = mlx5_glue->devx_umem_reg(ctx, (void *)(uintptr_t)umem_buf,
+					    umem_size, IBV_ACCESS_LOCAL_WRITE);
+	if (!umem_obj) {
+		DRV_LOG(ERR, "Failed to register umem for CQ.");
+		rte_errno = errno;
+		goto error;
+	}
+	/* Fill attributes for CQ object creation. */
+	attr->q_umem_valid = 1;
+	attr->q_umem_id = mlx5_os_get_umem_id(umem_obj);
+	attr->q_umem_offset = 0;
+	attr->db_umem_valid = 1;
+	attr->db_umem_id = attr->q_umem_id;
+	attr->db_umem_offset = umem_dbrec;
+	attr->eqn = eqn;
+	attr->log_cq_size = log_desc_n;
+	attr->log_page_size = rte_log2_u32(page_size);
+	/* Create completion queue object with DevX. */
+	cq = mlx5_devx_cmd_create_cq(ctx, attr);
+	if (!cq) {
+		DRV_LOG(ERR, "Can't create DevX CQ object.");
+		rte_errno  = ENOMEM;
+		goto error;
+	}
+	cq_obj->umem_buf = umem_buf;
+	cq_obj->umem_obj = umem_obj;
+	cq_obj->cq = cq;
+	cq_obj->db_rec = RTE_PTR_ADD(cq_obj->umem_buf, umem_dbrec);
+	/* Mark all CQEs initially as invalid. */
+	mlx5_cq_init(cq_obj, cq_size);
+	return 0;
+error:
+	ret = rte_errno;
+	if (cq)
+		claim_zero(mlx5_devx_cmd_destroy(cq));
+	if (umem_obj)
+		claim_zero(mlx5_glue->devx_umem_dereg(umem_obj));
+	if (umem_buf)
+		mlx5_free((void *)(uintptr_t)umem_buf);
+	rte_errno = ret;
+	return -rte_errno;
+}
diff --git a/drivers/common/mlx5/mlx5_common_devx.h b/drivers/common/mlx5/mlx5_common_devx.h
new file mode 100644
index 0000000..31cb804
--- /dev/null
+++ b/drivers/common/mlx5/mlx5_common_devx.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_COMMON_DEVX_H_
+#define RTE_PMD_MLX5_COMMON_DEVX_H_
+
+#include "mlx5_devx_cmds.h"
+
+/* DevX Completion Queue structure. */
+struct mlx5_devx_cq {
+	struct mlx5_devx_obj *cq; /* The CQ DevX object. */
+	struct mlx5dv_devx_umem *umem_obj; /* The CQ umem object. */
+	union {
+		volatile void *umem_buf;
+		volatile struct mlx5_cqe *cqes; /* The CQ ring buffer. */
+	};
+	volatile uint32_t *db_rec; /* The CQ doorbell record. */
+};
+
+/* mlx5_common_devx.c */
+
+void mlx5_devx_cq_destroy(struct mlx5_devx_cq *cq);
+int mlx5_devx_cq_create(void *ctx, struct mlx5_devx_cq *cq_obj,
+			uint16_t log_desc_n, struct mlx5_devx_cq_attr *attr,
+			int socket);
+
+#endif /* RTE_PMD_MLX5_COMMON_DEVX_H_ */
-- 
1.8.3.1


  parent reply	other threads:[~2020-12-29  8:53 UTC|newest]

Thread overview: 57+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-12-17 11:44 [dpdk-dev] [PATCH 00/17] common/mlx5: share DevX resources creations Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 01/17] net/mlx5: fix ASO SQ creation error flow Michael Baum
2020-12-29  8:52   ` [dpdk-dev] [PATCH v2 00/17] common/mlx5: share DevX resources creations Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 01/17] net/mlx5: fix ASO SQ creation error flow Michael Baum
2021-01-06  8:19       ` [dpdk-dev] [PATCH v3 00/19] common/mlx5: share DevX resources creations Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 01/19] common/mlx5: fix completion queue entry size configuration Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 02/19] net/mlx5: remove CQE padding device argument Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 03/19] net/mlx5: fix ASO SQ creation error flow Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 04/19] common/mlx5: share DevX CQ creation Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 05/19] regex/mlx5: move DevX CQ creation to common Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 06/19] vdpa/mlx5: " Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 07/19] net/mlx5: move rearm and clock queue " Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 08/19] net/mlx5: move ASO " Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 09/19] net/mlx5: move Tx " Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 10/19] net/mlx5: move Rx " Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 11/19] common/mlx5: enhance page size configuration Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 12/19] common/mlx5: share DevX SQ creation Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 13/19] regex/mlx5: move DevX SQ creation to common Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 14/19] net/mlx5: move rearm and clock queue " Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 15/19] net/mlx5: move Tx " Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 16/19] net/mlx5: move ASO " Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 17/19] common/mlx5: share DevX RQ creation Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 18/19] net/mlx5: move Rx RQ creation to common Michael Baum
2021-01-06  8:19         ` [dpdk-dev] [PATCH v3 19/19] common/mlx5: remove doorbell allocation API Michael Baum
2021-01-12 21:39         ` [dpdk-dev] [PATCH v3 00/19] common/mlx5: share DevX resources creations Thomas Monjalon
2020-12-29  8:52     ` Michael Baum [this message]
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 03/17] regex/mlx5: move DevX CQ creation to common Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 04/17] vdpa/mlx5: " Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 05/17] net/mlx5: move rearm and clock queue " Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 06/17] net/mlx5: move ASO " Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 07/17] net/mlx5: move Tx " Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 08/17] net/mlx5: move Rx " Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 09/17] common/mlx5: enhance page size configuration Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 10/17] common/mlx5: share DevX SQ creation Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 11/17] regex/mlx5: move DevX SQ creation to common Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 12/17] net/mlx5: move rearm and clock queue " Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 13/17] net/mlx5: move Tx " Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 14/17] net/mlx5: move ASO " Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 15/17] common/mlx5: share DevX RQ creation Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 16/17] net/mlx5: move Rx RQ creation to common Michael Baum
2020-12-29  8:52     ` [dpdk-dev] [PATCH v2 17/17] common/mlx5: remove doorbell allocation API Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 02/17] common/mlx5: share DevX CQ creation Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 03/17] regex/mlx5: move DevX CQ creation to common Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 04/17] vdpa/mlx5: " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 05/17] net/mlx5: move rearm and clock queue " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 06/17] net/mlx5: move ASO " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 07/17] net/mlx5: move Tx " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 08/17] net/mlx5: move Rx " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 09/17] common/mlx5: enhance page size configuration Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 10/17] common/mlx5: share DevX SQ creation Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 11/17] regex/mlx5: move DevX SQ creation to common Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 12/17] net/mlx5: move rearm and clock queue " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 13/17] net/mlx5: move Tx " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 14/17] net/mlx5: move ASO " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 15/17] common/mlx5: share DevX RQ creation Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 16/17] net/mlx5: move Rx RQ creation to common Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 17/17] common/mlx5: remove doorbell allocation API Michael Baum

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1609231944-29274-3-git-send-email-michaelba@nvidia.com \
    --to=michaelba@nvidia.com \
    --cc=dev@dpdk.org \
    --cc=matan@nvidia.com \
    --cc=rasland@nvidia.com \
    --cc=viacheslavo@nvidia.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

DPDK patches and discussions

This inbox may be cloned and mirrored by anyone:

	git clone --mirror http://inbox.dpdk.org/dev/0 dev/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 dev dev/ http://inbox.dpdk.org/dev \
		dev@dpdk.org
	public-inbox-index dev

Example config snippet for mirrors.
Newsgroup available over NNTP:
	nntp://inbox.dpdk.org/inbox.dpdk.dev


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git