From: Michael Baum <michaelba@nvidia.com>
To: dev@dpdk.org
Cc: Matan Azrad <matan@nvidia.com>,
Raslan Darawsheh <rasland@nvidia.com>,
Viacheslav Ovsiienko <viacheslavo@nvidia.com>
Subject: [dpdk-dev] [PATCH v3 15/19] net/mlx5: move Tx SQ creation to common
Date: Wed, 6 Jan 2021 08:19:37 +0000 [thread overview]
Message-ID: <1609921181-5019-16-git-send-email-michaelba@nvidia.com> (raw)
In-Reply-To: <1609921181-5019-1-git-send-email-michaelba@nvidia.com>
Using common function for Tx SQ creation.
Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
drivers/net/mlx5/mlx5.h | 8 +--
drivers/net/mlx5/mlx5_devx.c | 162 ++++++++++---------------------------------
2 files changed, 40 insertions(+), 130 deletions(-)
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 0a0a943..62d7c89 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -841,11 +841,9 @@ struct mlx5_txq_obj {
struct {
struct rte_eth_dev *dev;
struct mlx5_devx_cq cq_obj;
- struct mlx5_devx_obj *sq_devx;
- void *sq_umem;
- void *sq_buf;
- int64_t sq_dbrec_offset;
- struct mlx5_devx_dbr_page *sq_dbrec_page;
+ /* DevX CQ object and its resources. */
+ struct mlx5_devx_sq sq_obj;
+ /* DevX SQ object and its resources. */
};
};
};
diff --git a/drivers/net/mlx5/mlx5_devx.c b/drivers/net/mlx5/mlx5_devx.c
index 913f169..96c44d4 100644
--- a/drivers/net/mlx5/mlx5_devx.c
+++ b/drivers/net/mlx5/mlx5_devx.c
@@ -115,7 +115,7 @@
else
msq_attr.sq_state = MLX5_SQC_STATE_RDY;
msq_attr.state = MLX5_SQC_STATE_RST;
- ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr);
+ ret = mlx5_devx_cmd_modify_sq(obj->sq_obj.sq, &msq_attr);
if (ret) {
DRV_LOG(ERR, "Cannot change the Tx SQ state to RESET"
" %s", strerror(errno));
@@ -127,7 +127,7 @@
/* Change queue state to ready. */
msq_attr.sq_state = MLX5_SQC_STATE_RST;
msq_attr.state = MLX5_SQC_STATE_RDY;
- ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr);
+ ret = mlx5_devx_cmd_modify_sq(obj->sq_obj.sq, &msq_attr);
if (ret) {
DRV_LOG(ERR, "Cannot change the Tx SQ state to READY"
" %s", strerror(errno));
@@ -1052,36 +1052,6 @@
#if defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) || !defined(HAVE_INFINIBAND_VERBS_H)
/**
- * Release DevX SQ resources.
- *
- * @param txq_obj
- * DevX Tx queue object.
- */
-static void
-mlx5_txq_release_devx_sq_resources(struct mlx5_txq_obj *txq_obj)
-{
- if (txq_obj->sq_devx) {
- claim_zero(mlx5_devx_cmd_destroy(txq_obj->sq_devx));
- txq_obj->sq_devx = NULL;
- }
- if (txq_obj->sq_umem) {
- claim_zero(mlx5_os_umem_dereg(txq_obj->sq_umem));
- txq_obj->sq_umem = NULL;
- }
- if (txq_obj->sq_buf) {
- mlx5_free(txq_obj->sq_buf);
- txq_obj->sq_buf = NULL;
- }
- if (txq_obj->sq_dbrec_page) {
- claim_zero(mlx5_release_dbr(&txq_obj->txq_ctrl->priv->dbrpgs,
- mlx5_os_get_umem_id
- (txq_obj->sq_dbrec_page->umem),
- txq_obj->sq_dbrec_offset));
- txq_obj->sq_dbrec_page = NULL;
- }
-}
-
-/**
* Destroy the Tx queue DevX object.
*
* @param txq_obj
@@ -1090,7 +1060,8 @@
static void
mlx5_txq_release_devx_resources(struct mlx5_txq_obj *txq_obj)
{
- mlx5_txq_release_devx_sq_resources(txq_obj);
+ mlx5_devx_sq_destroy(&txq_obj->sq_obj);
+ memset(&txq_obj->sq_obj, 0, sizeof(txq_obj->sq_obj));
mlx5_devx_cq_destroy(&txq_obj->cq_obj);
memset(&txq_obj->cq_obj, 0, sizeof(txq_obj->cq_obj));
}
@@ -1102,100 +1073,39 @@
* Pointer to Ethernet device.
* @param idx
* Queue index in DPDK Tx queue array.
+ * @param[in] log_desc_n
+ * Log of number of descriptors in queue.
*
* @return
- * Number of WQEs in SQ, 0 otherwise and rte_errno is set.
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
*/
-static uint32_t
-mlx5_txq_create_devx_sq_resources(struct rte_eth_dev *dev, uint16_t idx)
+static int
+mlx5_txq_create_devx_sq_resources(struct rte_eth_dev *dev, uint16_t idx,
+ uint16_t log_desc_n)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_txq_data *txq_data = (*priv->txqs)[idx];
struct mlx5_txq_ctrl *txq_ctrl =
container_of(txq_data, struct mlx5_txq_ctrl, txq);
struct mlx5_txq_obj *txq_obj = txq_ctrl->obj;
- struct mlx5_devx_create_sq_attr sq_attr = { 0 };
- size_t page_size;
- uint32_t wqe_n;
- int ret;
+ struct mlx5_devx_create_sq_attr sq_attr = {
+ .flush_in_error_en = 1,
+ .allow_multi_pkt_send_wqe = !!priv->config.mps,
+ .min_wqe_inline_mode = priv->config.hca_attr.vport_inline_mode,
+ .allow_swp = !!priv->config.swp,
+ .cqn = txq_obj->cq_obj.cq->id,
+ .tis_lst_sz = 1,
+ .tis_num = priv->sh->tis->id,
+ .wq_attr = (struct mlx5_devx_wq_attr){
+ .pd = priv->sh->pdn,
+ .uar_page =
+ mlx5_os_get_devx_uar_page_id(priv->sh->tx_uar),
+ },
+ };
- MLX5_ASSERT(txq_data);
- MLX5_ASSERT(txq_obj);
- page_size = rte_mem_page_size();
- if (page_size == (size_t)-1) {
- DRV_LOG(ERR, "Failed to get mem page size.");
- rte_errno = ENOMEM;
- return 0;
- }
- wqe_n = RTE_MIN(1UL << txq_data->elts_n,
- (uint32_t)priv->sh->device_attr.max_qp_wr);
- txq_obj->sq_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO,
- wqe_n * sizeof(struct mlx5_wqe),
- page_size, priv->sh->numa_node);
- if (!txq_obj->sq_buf) {
- DRV_LOG(ERR,
- "Port %u Tx queue %u cannot allocate memory (SQ).",
- dev->data->port_id, txq_data->idx);
- rte_errno = ENOMEM;
- goto error;
- }
- /* Register allocated buffer in user space with DevX. */
- txq_obj->sq_umem = mlx5_os_umem_reg
- (priv->sh->ctx,
- (void *)txq_obj->sq_buf,
- wqe_n * sizeof(struct mlx5_wqe),
- IBV_ACCESS_LOCAL_WRITE);
- if (!txq_obj->sq_umem) {
- rte_errno = errno;
- DRV_LOG(ERR,
- "Port %u Tx queue %u cannot register memory (SQ).",
- dev->data->port_id, txq_data->idx);
- goto error;
- }
- /* Allocate doorbell record for send queue. */
- txq_obj->sq_dbrec_offset = mlx5_get_dbr(priv->sh->ctx,
- &priv->dbrpgs,
- &txq_obj->sq_dbrec_page);
- if (txq_obj->sq_dbrec_offset < 0) {
- rte_errno = errno;
- DRV_LOG(ERR, "Failed to allocate SQ door-bell.");
- goto error;
- }
- sq_attr.tis_lst_sz = 1;
- sq_attr.tis_num = priv->sh->tis->id;
- sq_attr.state = MLX5_SQC_STATE_RST;
- sq_attr.cqn = txq_obj->cq_obj.cq->id;
- sq_attr.flush_in_error_en = 1;
- sq_attr.allow_multi_pkt_send_wqe = !!priv->config.mps;
- sq_attr.allow_swp = !!priv->config.swp;
- sq_attr.min_wqe_inline_mode = priv->config.hca_attr.vport_inline_mode;
- sq_attr.wq_attr.uar_page =
- mlx5_os_get_devx_uar_page_id(priv->sh->tx_uar);
- sq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC;
- sq_attr.wq_attr.pd = priv->sh->pdn;
- sq_attr.wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE);
- sq_attr.wq_attr.log_wq_sz = log2above(wqe_n);
- sq_attr.wq_attr.dbr_umem_valid = 1;
- sq_attr.wq_attr.dbr_addr = txq_obj->sq_dbrec_offset;
- sq_attr.wq_attr.dbr_umem_id =
- mlx5_os_get_umem_id(txq_obj->sq_dbrec_page->umem);
- sq_attr.wq_attr.wq_umem_valid = 1;
- sq_attr.wq_attr.wq_umem_id = mlx5_os_get_umem_id(txq_obj->sq_umem);
- sq_attr.wq_attr.wq_umem_offset = (uintptr_t)txq_obj->sq_buf % page_size;
/* Create Send Queue object with DevX. */
- txq_obj->sq_devx = mlx5_devx_cmd_create_sq(priv->sh->ctx, &sq_attr);
- if (!txq_obj->sq_devx) {
- rte_errno = errno;
- DRV_LOG(ERR, "Port %u Tx queue %u SQ creation failure.",
- dev->data->port_id, idx);
- goto error;
- }
- return wqe_n;
-error:
- ret = rte_errno;
- mlx5_txq_release_devx_sq_resources(txq_obj);
- rte_errno = ret;
- return 0;
+ return mlx5_devx_sq_create(priv->sh->ctx, &txq_obj->sq_obj, log_desc_n,
+ &sq_attr, priv->sh->numa_node);
}
#endif
@@ -1267,27 +1177,29 @@
txq_data->cq_db = txq_obj->cq_obj.db_rec;
*txq_data->cq_db = 0;
/* Create Send Queue object with DevX. */
- wqe_n = mlx5_txq_create_devx_sq_resources(dev, idx);
- if (!wqe_n) {
+ wqe_n = RTE_MIN(1UL << txq_data->elts_n,
+ (uint32_t)priv->sh->device_attr.max_qp_wr);
+ log_desc_n = log2above(wqe_n);
+ ret = mlx5_txq_create_devx_sq_resources(dev, idx, log_desc_n);
+ if (ret) {
+ DRV_LOG(ERR, "Port %u Tx queue %u SQ creation failure.",
+ dev->data->port_id, idx);
rte_errno = errno;
goto error;
}
/* Create the Work Queue. */
- txq_data->wqe_n = log2above(wqe_n);
+ txq_data->wqe_n = log_desc_n;
txq_data->wqe_s = 1 << txq_data->wqe_n;
txq_data->wqe_m = txq_data->wqe_s - 1;
- txq_data->wqes = (struct mlx5_wqe *)txq_obj->sq_buf;
+ txq_data->wqes = (struct mlx5_wqe *)(uintptr_t)txq_obj->sq_obj.wqes;
txq_data->wqes_end = txq_data->wqes + txq_data->wqe_s;
txq_data->wqe_ci = 0;
txq_data->wqe_pi = 0;
txq_data->wqe_comp = 0;
txq_data->wqe_thres = txq_data->wqe_s / MLX5_TX_COMP_THRESH_INLINE_DIV;
- txq_data->qp_db = (volatile uint32_t *)
- (txq_obj->sq_dbrec_page->dbrs +
- txq_obj->sq_dbrec_offset +
- MLX5_SND_DBR * sizeof(uint32_t));
+ txq_data->qp_db = txq_obj->sq_obj.db_rec;
*txq_data->qp_db = 0;
- txq_data->qp_num_8s = txq_obj->sq_devx->id << 8;
+ txq_data->qp_num_8s = txq_obj->sq_obj.sq->id << 8;
/* Change Send Queue state to Ready-to-Send. */
ret = mlx5_devx_modify_sq(txq_obj, MLX5_TXQ_MOD_RST2RDY, 0);
if (ret) {
--
1.8.3.1
next prev parent reply other threads:[~2021-01-06 8:22 UTC|newest]
Thread overview: 57+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-12-17 11:44 [dpdk-dev] [PATCH 00/17] common/mlx5: share DevX resources creations Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 01/17] net/mlx5: fix ASO SQ creation error flow Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 00/17] common/mlx5: share DevX resources creations Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 01/17] net/mlx5: fix ASO SQ creation error flow Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 00/19] common/mlx5: share DevX resources creations Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 01/19] common/mlx5: fix completion queue entry size configuration Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 02/19] net/mlx5: remove CQE padding device argument Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 03/19] net/mlx5: fix ASO SQ creation error flow Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 04/19] common/mlx5: share DevX CQ creation Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 05/19] regex/mlx5: move DevX CQ creation to common Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 06/19] vdpa/mlx5: " Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 07/19] net/mlx5: move rearm and clock queue " Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 08/19] net/mlx5: move ASO " Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 09/19] net/mlx5: move Tx " Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 10/19] net/mlx5: move Rx " Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 11/19] common/mlx5: enhance page size configuration Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 12/19] common/mlx5: share DevX SQ creation Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 13/19] regex/mlx5: move DevX SQ creation to common Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 14/19] net/mlx5: move rearm and clock queue " Michael Baum
2021-01-06 8:19 ` Michael Baum [this message]
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 16/19] net/mlx5: move ASO " Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 17/19] common/mlx5: share DevX RQ creation Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 18/19] net/mlx5: move Rx RQ creation to common Michael Baum
2021-01-06 8:19 ` [dpdk-dev] [PATCH v3 19/19] common/mlx5: remove doorbell allocation API Michael Baum
2021-01-12 21:39 ` [dpdk-dev] [PATCH v3 00/19] common/mlx5: share DevX resources creations Thomas Monjalon
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 02/17] common/mlx5: share DevX CQ creation Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 03/17] regex/mlx5: move DevX CQ creation to common Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 04/17] vdpa/mlx5: " Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 05/17] net/mlx5: move rearm and clock queue " Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 06/17] net/mlx5: move ASO " Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 07/17] net/mlx5: move Tx " Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 08/17] net/mlx5: move Rx " Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 09/17] common/mlx5: enhance page size configuration Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 10/17] common/mlx5: share DevX SQ creation Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 11/17] regex/mlx5: move DevX SQ creation to common Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 12/17] net/mlx5: move rearm and clock queue " Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 13/17] net/mlx5: move Tx " Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 14/17] net/mlx5: move ASO " Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 15/17] common/mlx5: share DevX RQ creation Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 16/17] net/mlx5: move Rx RQ creation to common Michael Baum
2020-12-29 8:52 ` [dpdk-dev] [PATCH v2 17/17] common/mlx5: remove doorbell allocation API Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 02/17] common/mlx5: share DevX CQ creation Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 03/17] regex/mlx5: move DevX CQ creation to common Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 04/17] vdpa/mlx5: " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 05/17] net/mlx5: move rearm and clock queue " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 06/17] net/mlx5: move ASO " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 07/17] net/mlx5: move Tx " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 08/17] net/mlx5: move Rx " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 09/17] common/mlx5: enhance page size configuration Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 10/17] common/mlx5: share DevX SQ creation Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 11/17] regex/mlx5: move DevX SQ creation to common Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 12/17] net/mlx5: move rearm and clock queue " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 13/17] net/mlx5: move Tx " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 14/17] net/mlx5: move ASO " Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 15/17] common/mlx5: share DevX RQ creation Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 16/17] net/mlx5: move Rx RQ creation to common Michael Baum
2020-12-17 11:44 ` [dpdk-dev] [PATCH 17/17] common/mlx5: remove doorbell allocation API Michael Baum
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1609921181-5019-16-git-send-email-michaelba@nvidia.com \
--to=michaelba@nvidia.com \
--cc=dev@dpdk.org \
--cc=matan@nvidia.com \
--cc=rasland@nvidia.com \
--cc=viacheslavo@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).