DPDK patches and discussions
 help / color / mirror / Atom feed
From: Michael Baum <michaelba@nvidia.com>
To: dev@dpdk.org
Cc: Matan Azrad <matan@nvidia.com>,
	Raslan Darawsheh <rasland@nvidia.com>,
	Viacheslav Ovsiienko <viacheslavo@nvidia.com>
Subject: [dpdk-dev] [PATCH v1 11/15] net/mlx5: separate Tx queue object modification
Date: Thu,  1 Oct 2020 14:09:22 +0000
Message-ID: <1601561366-1821-12-git-send-email-michaelba@nvidia.com> (raw)
In-Reply-To: <1601561366-1821-1-git-send-email-michaelba@nvidia.com>

Separate Tx object modification to the Verbs and DevX modules.

Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c    |   7 +++
 drivers/net/mlx5/linux/mlx5_verbs.c |  65 ++++++++++++++++++++++
 drivers/net/mlx5/mlx5.h             |   9 ++++
 drivers/net/mlx5/mlx5_devx.c        |  57 ++++++++++++++++++++
 drivers/net/mlx5/mlx5_rxtx.c        |  78 ++-------------------------
 drivers/net/mlx5/mlx5_txq.c         | 104 ++++--------------------------------
 6 files changed, 152 insertions(+), 168 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 0db2b5a..487714f 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1375,6 +1375,13 @@
 						ibv_obj_ops.drop_action_create;
 		priv->obj_ops.drop_action_destroy =
 						ibv_obj_ops.drop_action_destroy;
+#ifndef HAVE_MLX5DV_DEVX_UAR_OFFSET
+		priv->obj_ops.txq_obj_modify = ibv_obj_ops.txq_obj_modify;
+#else
+		if (!config->dv_esw_en)
+			priv->obj_ops.txq_obj_modify =
+						ibv_obj_ops.txq_obj_modify;
+#endif
 	} else {
 		priv->obj_ops = ibv_obj_ops;
 	}
diff --git a/drivers/net/mlx5/linux/mlx5_verbs.c b/drivers/net/mlx5/linux/mlx5_verbs.c
index 0476d94..7d5ea37 100644
--- a/drivers/net/mlx5/linux/mlx5_verbs.c
+++ b/drivers/net/mlx5/linux/mlx5_verbs.c
@@ -113,6 +113,70 @@
 }
 
 /**
+ * Modify QP using Verbs API.
+ *
+ * @param txq_obj
+ *   Verbs Tx queue object.
+ * @param type
+ *   Type of change queue state.
+ * @param dev_port
+ *   IB device port number.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_ibv_modify_qp(struct mlx5_txq_obj *obj, enum mlx5_txq_modify_type type,
+		   uint8_t dev_port)
+{
+	struct ibv_qp_attr mod = {
+		.qp_state = IBV_QPS_RESET,
+		.port_num = dev_port,
+	};
+	int attr_mask = (IBV_QP_STATE | IBV_QP_PORT);
+	int ret;
+
+	if (type != MLX5_TXQ_MOD_RST2RDY) {
+		ret = mlx5_glue->modify_qp(obj->qp, &mod, IBV_QP_STATE);
+		if (ret) {
+			DRV_LOG(ERR, "Cannot change Tx QP state to RESET %s",
+				strerror(errno));
+			rte_errno = errno;
+			return ret;
+		}
+		if (type == MLX5_TXQ_MOD_RDY2RST)
+			return 0;
+	}
+	if (type == MLX5_TXQ_MOD_ERR2RDY)
+		attr_mask = IBV_QP_STATE;
+	mod.qp_state = IBV_QPS_INIT;
+	ret = mlx5_glue->modify_qp(obj->qp, &mod, attr_mask);
+	if (ret) {
+		DRV_LOG(ERR, "Cannot change Tx QP state to INIT %s",
+			strerror(errno));
+		rte_errno = errno;
+		return ret;
+	}
+	mod.qp_state = IBV_QPS_RTR;
+	ret = mlx5_glue->modify_qp(obj->qp, &mod, IBV_QP_STATE);
+	if (ret) {
+		DRV_LOG(ERR, "Cannot change Tx QP state to RTR %s",
+			strerror(errno));
+		rte_errno = errno;
+		return ret;
+	}
+	mod.qp_state = IBV_QPS_RTS;
+	ret = mlx5_glue->modify_qp(obj->qp, &mod, IBV_QP_STATE);
+	if (ret) {
+		DRV_LOG(ERR, "Cannot change Tx QP state to RTS %s",
+			strerror(errno));
+		rte_errno = errno;
+		return ret;
+	}
+	return 0;
+}
+
+/**
  * Create a CQ Verbs object.
  *
  * @param dev
@@ -1043,5 +1107,6 @@ struct mlx5_obj_ops ibv_obj_ops = {
 	.drop_action_create = mlx5_ibv_drop_action_create,
 	.drop_action_destroy = mlx5_ibv_drop_action_destroy,
 	.txq_obj_new = mlx5_txq_ibv_obj_new,
+	.txq_obj_modify = mlx5_ibv_modify_qp,
 	.txq_obj_release = mlx5_txq_ibv_obj_release,
 };
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 3093f6e..7cbb09b 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -774,6 +774,13 @@ struct mlx5_txq_obj {
 	};
 };
 
+enum mlx5_txq_modify_type {
+	MLX5_TXQ_MOD_RDY2RDY, /* modify state from ready to ready. */
+	MLX5_TXQ_MOD_RST2RDY, /* modify state from reset to ready. */
+	MLX5_TXQ_MOD_RDY2RST, /* modify state from ready to reset. */
+	MLX5_TXQ_MOD_ERR2RDY, /* modify state from error to ready. */
+};
+
 /* HW objects operations structure. */
 struct mlx5_obj_ops {
 	int (*rxq_obj_modify_vlan_strip)(struct mlx5_rxq_obj *rxq_obj, int on);
@@ -790,6 +797,8 @@ struct mlx5_obj_ops {
 	int (*drop_action_create)(struct rte_eth_dev *dev);
 	void (*drop_action_destroy)(struct rte_eth_dev *dev);
 	int (*txq_obj_new)(struct rte_eth_dev *dev, uint16_t idx);
+	int (*txq_obj_modify)(struct mlx5_txq_obj *obj,
+			      enum mlx5_txq_modify_type type, uint8_t dev_port);
 	void (*txq_obj_release)(struct mlx5_txq_obj *txq_obj);
 };
 
diff --git a/drivers/net/mlx5/mlx5_devx.c b/drivers/net/mlx5/mlx5_devx.c
index 55fe946..7404a15 100644
--- a/drivers/net/mlx5/mlx5_devx.c
+++ b/drivers/net/mlx5/mlx5_devx.c
@@ -73,6 +73,62 @@
 }
 
 /**
+ * Modify SQ using DevX API.
+ *
+ * @param txq_obj
+ *   DevX Tx queue object.
+ * @param type
+ *   Type of change queue state.
+ * @param dev_port
+ *   Unnecessary.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_devx_modify_sq(struct mlx5_txq_obj *obj, enum mlx5_txq_modify_type type,
+		    uint8_t dev_port)
+{
+	struct mlx5_devx_modify_sq_attr msq_attr = { 0 };
+	int ret;
+
+	if (type != MLX5_TXQ_MOD_RST2RDY) {
+		/* Change queue state to reset. */
+		if (type == MLX5_TXQ_MOD_ERR2RDY)
+			msq_attr.sq_state = MLX5_SQC_STATE_ERR;
+		else
+			msq_attr.sq_state = MLX5_SQC_STATE_RDY;
+		msq_attr.state = MLX5_SQC_STATE_RST;
+		ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr);
+		if (ret) {
+			DRV_LOG(ERR, "Cannot change the Tx SQ state to RESET"
+				" %s", strerror(errno));
+			rte_errno = errno;
+			return ret;
+		}
+	}
+	if (type != MLX5_TXQ_MOD_RDY2RST) {
+		/* Change queue state to ready. */
+		msq_attr.sq_state = MLX5_SQC_STATE_RST;
+		msq_attr.state = MLX5_SQC_STATE_RDY;
+		ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr);
+		if (ret) {
+			DRV_LOG(ERR, "Cannot change the Tx SQ state to READY"
+				" %s", strerror(errno));
+			rte_errno = errno;
+			return ret;
+		}
+	}
+	/*
+	 * The dev_port variable is relevant only in Verbs API, and there is a
+	 * pointer that points to this function and a parallel function in verbs
+	 * intermittently, so they should have the same parameters.
+	 */
+	(void)dev_port;
+	return 0;
+}
+
+/**
  * Release the resources allocated for an RQ DevX object.
  *
  * @param rxq_ctrl
@@ -1298,5 +1354,6 @@ struct mlx5_obj_ops devx_obj_ops = {
 	.drop_action_create = mlx5_devx_drop_action_create,
 	.drop_action_destroy = mlx5_devx_drop_action_destroy,
 	.txq_obj_new = mlx5_txq_devx_obj_new,
+	.txq_obj_modify = mlx5_devx_modify_sq,
 	.txq_obj_release = mlx5_txq_devx_obj_release,
 };
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 0b87be1..af4b4ba 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -936,79 +936,11 @@ enum mlx5_txcmp_code {
 		struct mlx5_txq_ctrl *txq_ctrl =
 			container_of(txq, struct mlx5_txq_ctrl, txq);
 
-		if (txq_ctrl->obj->type == MLX5_TXQ_OBJ_TYPE_DEVX_SQ) {
-			struct mlx5_devx_modify_sq_attr msq_attr = { 0 };
-
-			/* Change queue state to reset. */
-			msq_attr.sq_state = MLX5_SQC_STATE_ERR;
-			msq_attr.state = MLX5_SQC_STATE_RST;
-			ret = mlx5_devx_cmd_modify_sq(txq_ctrl->obj->sq_devx,
-						      &msq_attr);
-			if (ret) {
-				DRV_LOG(ERR, "Cannot change the "
-					"Tx QP state to RESET %s",
-					strerror(errno));
-				rte_errno = errno;
-				return ret;
-			}
-			/* Change queue state to ready. */
-			msq_attr.sq_state = MLX5_SQC_STATE_RST;
-			msq_attr.state = MLX5_SQC_STATE_RDY;
-			ret = mlx5_devx_cmd_modify_sq(txq_ctrl->obj->sq_devx,
-						      &msq_attr);
-			if (ret) {
-				DRV_LOG(ERR, "Cannot change the "
-					"Tx QP state to READY %s",
-					strerror(errno));
-				rte_errno = errno;
-				return ret;
-			}
-		} else {
-			struct ibv_qp_attr mod = {
-				.qp_state = IBV_QPS_RESET,
-				.port_num = (uint8_t)priv->dev_port,
-			};
-			struct ibv_qp *qp = txq_ctrl->obj->qp;
-
-			MLX5_ASSERT
-				(txq_ctrl->obj->type == MLX5_TXQ_OBJ_TYPE_IBV);
-
-			ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-			if (ret) {
-				DRV_LOG(ERR, "Cannot change the "
-					"Tx QP state to RESET %s",
-					strerror(errno));
-				rte_errno = errno;
-				return ret;
-			}
-			mod.qp_state = IBV_QPS_INIT;
-			ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-			if (ret) {
-				DRV_LOG(ERR, "Cannot change the "
-					"Tx QP state to INIT %s",
-					strerror(errno));
-				rte_errno = errno;
-				return ret;
-			}
-			mod.qp_state = IBV_QPS_RTR;
-			ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-			if (ret) {
-				DRV_LOG(ERR, "Cannot change the "
-					"Tx QP state to RTR %s",
-					strerror(errno));
-				rte_errno = errno;
-				return ret;
-			}
-			mod.qp_state = IBV_QPS_RTS;
-			ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-			if (ret) {
-				DRV_LOG(ERR, "Cannot change the "
-					"Tx QP state to RTS %s",
-					strerror(errno));
-				rte_errno = errno;
-				return ret;
-			}
-		}
+		ret = priv->obj_ops.txq_obj_modify(txq_ctrl->obj,
+						   MLX5_TXQ_MOD_ERR2RDY,
+						   (uint8_t)priv->dev_port);
+		if (ret)
+			return ret;
 	}
 	return 0;
 }
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 23213d9..c31e446 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -182,37 +182,10 @@
 
 	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
 	/* Move QP to RESET state. */
-	if (txq_ctrl->obj->type == MLX5_TXQ_OBJ_TYPE_DEVX_SQ) {
-		struct mlx5_devx_modify_sq_attr msq_attr = { 0 };
-
-		/* Change queue state to reset with DevX. */
-		msq_attr.sq_state = MLX5_SQC_STATE_RDY;
-		msq_attr.state = MLX5_SQC_STATE_RST;
-		ret = mlx5_devx_cmd_modify_sq(txq_ctrl->obj->sq_devx,
-					      &msq_attr);
-		if (ret) {
-			DRV_LOG(ERR, "Cannot change the "
-				"Tx QP state to RESET %s",
-				strerror(errno));
-			rte_errno = errno;
-			return ret;
-		}
-	} else {
-		struct ibv_qp_attr mod = {
-			.qp_state = IBV_QPS_RESET,
-			.port_num = (uint8_t)priv->dev_port,
-		};
-		struct ibv_qp *qp = txq_ctrl->obj->qp;
-
-		/* Change queue state to reset with Verbs. */
-		ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-		if (ret) {
-			DRV_LOG(ERR, "Cannot change the Tx QP state to RESET "
-				"%s", strerror(errno));
-			rte_errno = errno;
-			return ret;
-		}
-	}
+	ret = priv->obj_ops.txq_obj_modify(txq_ctrl->obj, MLX5_TXQ_MOD_RDY2RST,
+					   (uint8_t)priv->dev_port);
+	if (ret)
+		return ret;
 	/* Handle all send completions. */
 	txq_sync_cq(txq);
 	/* Free elts stored in the SQ. */
@@ -281,70 +254,11 @@
 	int ret;
 
 	MLX5_ASSERT(rte_eal_process_type() ==  RTE_PROC_PRIMARY);
-	if (txq_ctrl->obj->type == MLX5_TXQ_OBJ_TYPE_DEVX_SQ) {
-		struct mlx5_devx_modify_sq_attr msq_attr = { 0 };
-		struct mlx5_txq_obj *obj = txq_ctrl->obj;
-
-		msq_attr.sq_state = MLX5_SQC_STATE_RDY;
-		msq_attr.state = MLX5_SQC_STATE_RST;
-		ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr);
-		if (ret) {
-			rte_errno = errno;
-			DRV_LOG(ERR,
-				"Cannot change the Tx QP state to RESET "
-				"%s", strerror(errno));
-			return ret;
-		}
-		msq_attr.sq_state = MLX5_SQC_STATE_RST;
-		msq_attr.state = MLX5_SQC_STATE_RDY;
-		ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr);
-		if (ret) {
-			rte_errno = errno;
-			DRV_LOG(ERR,
-				"Cannot change the Tx QP state to READY "
-				"%s", strerror(errno));
-			return ret;
-		}
-	} else {
-		struct ibv_qp_attr mod = {
-			.qp_state = IBV_QPS_RESET,
-			.port_num = (uint8_t)priv->dev_port,
-		};
-		struct ibv_qp *qp = txq_ctrl->obj->qp;
-
-		ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-		if (ret) {
-			DRV_LOG(ERR, "Cannot change the Tx QP state to RESET "
-				"%s", strerror(errno));
-			rte_errno = errno;
-			return ret;
-		}
-		mod.qp_state = IBV_QPS_INIT;
-		ret = mlx5_glue->modify_qp(qp, &mod,
-					   (IBV_QP_STATE | IBV_QP_PORT));
-		if (ret) {
-			DRV_LOG(ERR, "Cannot change Tx QP state to INIT %s",
-				strerror(errno));
-			rte_errno = errno;
-			return ret;
-		}
-		mod.qp_state = IBV_QPS_RTR;
-		ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-		if (ret) {
-			DRV_LOG(ERR, "Cannot change Tx QP state to RTR %s",
-				strerror(errno));
-			rte_errno = errno;
-			return ret;
-		}
-		mod.qp_state = IBV_QPS_RTS;
-		ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-		if (ret) {
-			DRV_LOG(ERR, "Cannot change Tx QP state to RTS %s",
-				strerror(errno));
-			rte_errno = errno;
-			return ret;
-		}
-	}
+	ret = priv->obj_ops.txq_obj_modify(txq_ctrl->obj,
+					   MLX5_TXQ_MOD_RDY2RDY,
+					   (uint8_t)priv->dev_port);
+	if (ret)
+		return ret;
 	txq_ctrl->txq.wqe_ci = 0;
 	txq_ctrl->txq.wqe_pi = 0;
 	txq_ctrl->txq.elts_comp = 0;
-- 
1.8.3.1


  parent reply	other threads:[~2020-10-01 14:14 UTC|newest]

Thread overview: 17+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-10-01 14:09 [dpdk-dev] [PATCH v1 00/15] mlx5 Tx DevX/Verbs separation Michael Baum
2020-10-01 14:09 ` [dpdk-dev] [PATCH v1 01/15] net/mlx5: fix send queue doorbell typo Michael Baum
2020-10-01 14:09 ` [dpdk-dev] [PATCH v1 02/15] net/mlx5: fix unused variable in Txq creation Michael Baum
2020-10-01 14:09 ` [dpdk-dev] [PATCH v1 03/15] net/mlx5: mitigate Tx queue reference counters Michael Baum
2020-10-01 14:09 ` [dpdk-dev] [PATCH v1 04/15] net/mlx5: reorder Tx queue DevX object creation Michael Baum
2020-10-01 14:09 ` [dpdk-dev] [PATCH v1 05/15] net/mlx5: reorder Tx queue Verbs " Michael Baum
2020-10-01 14:09 ` [dpdk-dev] [PATCH v1 06/15] net/mlx5: reposition the event queue number field Michael Baum
2020-10-01 14:09 ` [dpdk-dev] [PATCH v1 07/15] net/mlx5: separate Tx queue object creations Michael Baum
2020-10-01 14:09 ` [dpdk-dev] [PATCH v1 08/15] net/mlx5: share Tx control code Michael Baum
2020-10-01 14:09 ` [dpdk-dev] [PATCH v1 09/15] net/mlx5: rearrange SQ and CQ creation in DevX module Michael Baum
2020-10-01 14:09 ` [dpdk-dev] [PATCH v1 10/15] net/mlx5: rearrange QP creation in Verbs module Michael Baum
2020-10-01 14:09 ` Michael Baum [this message]
2020-10-01 14:09 ` [dpdk-dev] [PATCH v1 12/15] net/mlx5: share Tx queue object modification Michael Baum
2020-10-01 14:09 ` [dpdk-dev] [PATCH v1 13/15] net/mlx5: remove Tx queue object type field Michael Baum
2020-10-01 14:09 ` [dpdk-dev] [PATCH v1 14/15] net/mlx5: separate Rx queue state modification Michael Baum
2020-10-01 14:09 ` [dpdk-dev] [PATCH v1 15/15] net/mlx5: remove Rx queue object type field Michael Baum
2020-10-06 15:25 ` [dpdk-dev] [PATCH v1 00/15] mlx5 Tx DevX/Verbs separation Raslan Darawsheh

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1601561366-1821-12-git-send-email-michaelba@nvidia.com \
    --to=michaelba@nvidia.com \
    --cc=dev@dpdk.org \
    --cc=matan@nvidia.com \
    --cc=rasland@nvidia.com \
    --cc=viacheslavo@nvidia.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

DPDK patches and discussions

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://inbox.dpdk.org/dev/0 dev/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 dev dev/ https://inbox.dpdk.org/dev \
		dev@dpdk.org
	public-inbox-index dev

Example config snippet for mirrors.
Newsgroup available over NNTP:
	nntp://inbox.dpdk.org/inbox.dpdk.dev


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git