[dpdk-dev] [PATCH v1 11/15] net/mlx5: separate Tx queue object modification

[Michael Baum <michaelba@nvidia.com>]

Separate Tx object modification to the Verbs and DevX modules.

Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c    |   7 +++
 drivers/net/mlx5/linux/mlx5_verbs.c |  65 ++++++++++++++++++++++
 drivers/net/mlx5/mlx5.h             |   9 ++++
 drivers/net/mlx5/mlx5_devx.c        |  57 ++++++++++++++++++++
 drivers/net/mlx5/mlx5_rxtx.c        |  78 ++-------------------------
 drivers/net/mlx5/mlx5_txq.c         | 104 ++++--------------------------------
 6 files changed, 152 insertions(+), 168 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 0db2b5a..487714f 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1375,6 +1375,13 @@
 						ibv_obj_ops.drop_action_create;
 		priv->obj_ops.drop_action_destroy =
 						ibv_obj_ops.drop_action_destroy;
+#ifndef HAVE_MLX5DV_DEVX_UAR_OFFSET
+		priv->obj_ops.txq_obj_modify = ibv_obj_ops.txq_obj_modify;
+#else
+		if (!config->dv_esw_en)
+			priv->obj_ops.txq_obj_modify =
+						ibv_obj_ops.txq_obj_modify;
+#endif
 	} else {
 		priv->obj_ops = ibv_obj_ops;
 	}
diff --git a/drivers/net/mlx5/linux/mlx5_verbs.c b/drivers/net/mlx5/linux/mlx5_verbs.c
index 0476d94..7d5ea37 100644
--- a/drivers/net/mlx5/linux/mlx5_verbs.c
+++ b/drivers/net/mlx5/linux/mlx5_verbs.c
@@ -113,6 +113,70 @@
 }
 
 /**
+ * Modify QP using Verbs API.
+ *
+ * @param txq_obj
+ *   Verbs Tx queue object.
+ * @param type
+ *   Type of change queue state.
+ * @param dev_port
+ *   IB device port number.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_ibv_modify_qp(struct mlx5_txq_obj *obj, enum mlx5_txq_modify_type type,
+		   uint8_t dev_port)
+{
+	struct ibv_qp_attr mod = {
+		.qp_state = IBV_QPS_RESET,
+		.port_num = dev_port,
+	};
+	int attr_mask = (IBV_QP_STATE | IBV_QP_PORT);
+	int ret;
+
+	if (type != MLX5_TXQ_MOD_RST2RDY) {
+		ret = mlx5_glue->modify_qp(obj->qp, &mod, IBV_QP_STATE);
+		if (ret) {
+			DRV_LOG(ERR, "Cannot change Tx QP state to RESET %s",
+				strerror(errno));
+			rte_errno = errno;
+			return ret;
+		}
+		if (type == MLX5_TXQ_MOD_RDY2RST)
+			return 0;
+	}
+	if (type == MLX5_TXQ_MOD_ERR2RDY)
+		attr_mask = IBV_QP_STATE;
+	mod.qp_state = IBV_QPS_INIT;
+	ret = mlx5_glue->modify_qp(obj->qp, &mod, attr_mask);
+	if (ret) {
+		DRV_LOG(ERR, "Cannot change Tx QP state to INIT %s",
+			strerror(errno));
+		rte_errno = errno;
+		return ret;
+	}
+	mod.qp_state = IBV_QPS_RTR;
+	ret = mlx5_glue->modify_qp(obj->qp, &mod, IBV_QP_STATE);
+	if (ret) {
+		DRV_LOG(ERR, "Cannot change Tx QP state to RTR %s",
+			strerror(errno));
+		rte_errno = errno;
+		return ret;
+	}
+	mod.qp_state = IBV_QPS_RTS;
+	ret = mlx5_glue->modify_qp(obj->qp, &mod, IBV_QP_STATE);
+	if (ret) {
+		DRV_LOG(ERR, "Cannot change Tx QP state to RTS %s",
+			strerror(errno));
+		rte_errno = errno;
+		return ret;
+	}
+	return 0;
+}
+
+/**
  * Create a CQ Verbs object.
  *
  * @param dev
@@ -1043,5 +1107,6 @@ struct mlx5_obj_ops ibv_obj_ops = {
 	.drop_action_create = mlx5_ibv_drop_action_create,
 	.drop_action_destroy = mlx5_ibv_drop_action_destroy,
 	.txq_obj_new = mlx5_txq_ibv_obj_new,
+	.txq_obj_modify = mlx5_ibv_modify_qp,
 	.txq_obj_release = mlx5_txq_ibv_obj_release,
 };
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 3093f6e..7cbb09b 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -774,6 +774,13 @@ struct mlx5_txq_obj {
 	};
 };
 
+enum mlx5_txq_modify_type {
+	MLX5_TXQ_MOD_RDY2RDY, /* modify state from ready to ready. */
+	MLX5_TXQ_MOD_RST2RDY, /* modify state from reset to ready. */
+	MLX5_TXQ_MOD_RDY2RST, /* modify state from ready to reset. */
+	MLX5_TXQ_MOD_ERR2RDY, /* modify state from error to ready. */
+};
+
 /* HW objects operations structure. */
 struct mlx5_obj_ops {
 	int (*rxq_obj_modify_vlan_strip)(struct mlx5_rxq_obj *rxq_obj, int on);
@@ -790,6 +797,8 @@ struct mlx5_obj_ops {
 	int (*drop_action_create)(struct rte_eth_dev *dev);
 	void (*drop_action_destroy)(struct rte_eth_dev *dev);
 	int (*txq_obj_new)(struct rte_eth_dev *dev, uint16_t idx);
+	int (*txq_obj_modify)(struct mlx5_txq_obj *obj,
+			      enum mlx5_txq_modify_type type, uint8_t dev_port);
 	void (*txq_obj_release)(struct mlx5_txq_obj *txq_obj);
 };
 
diff --git a/drivers/net/mlx5/mlx5_devx.c b/drivers/net/mlx5/mlx5_devx.c
index 55fe946..7404a15 100644
--- a/drivers/net/mlx5/mlx5_devx.c
+++ b/drivers/net/mlx5/mlx5_devx.c
@@ -73,6 +73,62 @@
 }
 
 /**
+ * Modify SQ using DevX API.
+ *
+ * @param txq_obj
+ *   DevX Tx queue object.
+ * @param type
+ *   Type of change queue state.
+ * @param dev_port
+ *   Unnecessary.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_devx_modify_sq(struct mlx5_txq_obj *obj, enum mlx5_txq_modify_type type,
+		    uint8_t dev_port)
+{
+	struct mlx5_devx_modify_sq_attr msq_attr = { 0 };
+	int ret;
+
+	if (type != MLX5_TXQ_MOD_RST2RDY) {
+		/* Change queue state to reset. */
+		if (type == MLX5_TXQ_MOD_ERR2RDY)
+			msq_attr.sq_state = MLX5_SQC_STATE_ERR;
+		else
+			msq_attr.sq_state = MLX5_SQC_STATE_RDY;
+		msq_attr.state = MLX5_SQC_STATE_RST;
+		ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr);
+		if (ret) {
+			DRV_LOG(ERR, "Cannot change the Tx SQ state to RESET"
+				" %s", strerror(errno));
+			rte_errno = errno;
+			return ret;
+		}
+	}
+	if (type != MLX5_TXQ_MOD_RDY2RST) {
+		/* Change queue state to ready. */
+		msq_attr.sq_state = MLX5_SQC_STATE_RST;
+		msq_attr.state = MLX5_SQC_STATE_RDY;
+		ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr);
+		if (ret) {
+			DRV_LOG(ERR, "Cannot change the Tx SQ state to READY"
+				" %s", strerror(errno));
+			rte_errno = errno;
+			return ret;
+		}
+	}
+	/*
+	 * The dev_port variable is relevant only in Verbs API, and there is a
+	 * pointer that points to this function and a parallel function in verbs
+	 * intermittently, so they should have the same parameters.
+	 */
+	(void)dev_port;
+	return 0;
+}
+
+/**
  * Release the resources allocated for an RQ DevX object.
  *
  * @param rxq_ctrl
@@ -1298,5 +1354,6 @@ struct mlx5_obj_ops devx_obj_ops = {
 	.drop_action_create = mlx5_devx_drop_action_create,
 	.drop_action_destroy = mlx5_devx_drop_action_destroy,
 	.txq_obj_new = mlx5_txq_devx_obj_new,
+	.txq_obj_modify = mlx5_devx_modify_sq,
 	.txq_obj_release = mlx5_txq_devx_obj_release,
 };
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 0b87be1..af4b4ba 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -936,79 +936,11 @@ enum mlx5_txcmp_code {
 		struct mlx5_txq_ctrl *txq_ctrl =
 			container_of(txq, struct mlx5_txq_ctrl, txq);
 
-		if (txq_ctrl->obj->type == MLX5_TXQ_OBJ_TYPE_DEVX_SQ) {
-			struct mlx5_devx_modify_sq_attr msq_attr = { 0 };
-
-			/* Change queue state to reset. */
-			msq_attr.sq_state = MLX5_SQC_STATE_ERR;
-			msq_attr.state = MLX5_SQC_STATE_RST;
-			ret = mlx5_devx_cmd_modify_sq(txq_ctrl->obj->sq_devx,
-						      &msq_attr);
-			if (ret) {
-				DRV_LOG(ERR, "Cannot change the "
-					"Tx QP state to RESET %s",
-					strerror(errno));
-				rte_errno = errno;
-				return ret;
-			}
-			/* Change queue state to ready. */
-			msq_attr.sq_state = MLX5_SQC_STATE_RST;
-			msq_attr.state = MLX5_SQC_STATE_RDY;
-			ret = mlx5_devx_cmd_modify_sq(txq_ctrl->obj->sq_devx,
-						      &msq_attr);
-			if (ret) {
-				DRV_LOG(ERR, "Cannot change the "
-					"Tx QP state to READY %s",
-					strerror(errno));
-				rte_errno = errno;
-				return ret;
-			}
-		} else {
-			struct ibv_qp_attr mod = {
-				.qp_state = IBV_QPS_RESET,
-				.port_num = (uint8_t)priv->dev_port,
-			};
-			struct ibv_qp *qp = txq_ctrl->obj->qp;
-
-			MLX5_ASSERT
-				(txq_ctrl->obj->type == MLX5_TXQ_OBJ_TYPE_IBV);
-
-			ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-			if (ret) {
-				DRV_LOG(ERR, "Cannot change the "
-					"Tx QP state to RESET %s",
-					strerror(errno));
-				rte_errno = errno;
-				return ret;
-			}
-			mod.qp_state = IBV_QPS_INIT;
-			ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-			if (ret) {
-				DRV_LOG(ERR, "Cannot change the "
-					"Tx QP state to INIT %s",
-					strerror(errno));
-				rte_errno = errno;
-				return ret;
-			}
-			mod.qp_state = IBV_QPS_RTR;
-			ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-			if (ret) {
-				DRV_LOG(ERR, "Cannot change the "
-					"Tx QP state to RTR %s",
-					strerror(errno));
-				rte_errno = errno;
-				return ret;
-			}
-			mod.qp_state = IBV_QPS_RTS;
-			ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-			if (ret) {
-				DRV_LOG(ERR, "Cannot change the "
-					"Tx QP state to RTS %s",
-					strerror(errno));
-				rte_errno = errno;
-				return ret;
-			}
-		}
+		ret = priv->obj_ops.txq_obj_modify(txq_ctrl->obj,
+						   MLX5_TXQ_MOD_ERR2RDY,
+						   (uint8_t)priv->dev_port);
+		if (ret)
+			return ret;
 	}
 	return 0;
 }
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 23213d9..c31e446 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -182,37 +182,10 @@
 
 	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
 	/* Move QP to RESET state. */
-	if (txq_ctrl->obj->type == MLX5_TXQ_OBJ_TYPE_DEVX_SQ) {
-		struct mlx5_devx_modify_sq_attr msq_attr = { 0 };
-
-		/* Change queue state to reset with DevX. */
-		msq_attr.sq_state = MLX5_SQC_STATE_RDY;
-		msq_attr.state = MLX5_SQC_STATE_RST;
-		ret = mlx5_devx_cmd_modify_sq(txq_ctrl->obj->sq_devx,
-					      &msq_attr);
-		if (ret) {
-			DRV_LOG(ERR, "Cannot change the "
-				"Tx QP state to RESET %s",
-				strerror(errno));
-			rte_errno = errno;
-			return ret;
-		}
-	} else {
-		struct ibv_qp_attr mod = {
-			.qp_state = IBV_QPS_RESET,
-			.port_num = (uint8_t)priv->dev_port,
-		};
-		struct ibv_qp *qp = txq_ctrl->obj->qp;
-
-		/* Change queue state to reset with Verbs. */
-		ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-		if (ret) {
-			DRV_LOG(ERR, "Cannot change the Tx QP state to RESET "
-				"%s", strerror(errno));
-			rte_errno = errno;
-			return ret;
-		}
-	}
+	ret = priv->obj_ops.txq_obj_modify(txq_ctrl->obj, MLX5_TXQ_MOD_RDY2RST,
+					   (uint8_t)priv->dev_port);
+	if (ret)
+		return ret;
 	/* Handle all send completions. */
 	txq_sync_cq(txq);
 	/* Free elts stored in the SQ. */
@@ -281,70 +254,11 @@
 	int ret;
 
 	MLX5_ASSERT(rte_eal_process_type() ==  RTE_PROC_PRIMARY);
-	if (txq_ctrl->obj->type == MLX5_TXQ_OBJ_TYPE_DEVX_SQ) {
-		struct mlx5_devx_modify_sq_attr msq_attr = { 0 };
-		struct mlx5_txq_obj *obj = txq_ctrl->obj;
-
-		msq_attr.sq_state = MLX5_SQC_STATE_RDY;
-		msq_attr.state = MLX5_SQC_STATE_RST;
-		ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr);
-		if (ret) {
-			rte_errno = errno;
-			DRV_LOG(ERR,
-				"Cannot change the Tx QP state to RESET "
-				"%s", strerror(errno));
-			return ret;
-		}
-		msq_attr.sq_state = MLX5_SQC_STATE_RST;
-		msq_attr.state = MLX5_SQC_STATE_RDY;
-		ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr);
-		if (ret) {
-			rte_errno = errno;
-			DRV_LOG(ERR,
-				"Cannot change the Tx QP state to READY "
-				"%s", strerror(errno));
-			return ret;
-		}
-	} else {
-		struct ibv_qp_attr mod = {
-			.qp_state = IBV_QPS_RESET,
-			.port_num = (uint8_t)priv->dev_port,
-		};
-		struct ibv_qp *qp = txq_ctrl->obj->qp;
-
-		ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-		if (ret) {
-			DRV_LOG(ERR, "Cannot change the Tx QP state to RESET "
-				"%s", strerror(errno));
-			rte_errno = errno;
-			return ret;
-		}
-		mod.qp_state = IBV_QPS_INIT;
-		ret = mlx5_glue->modify_qp(qp, &mod,
-					   (IBV_QP_STATE | IBV_QP_PORT));
-		if (ret) {
-			DRV_LOG(ERR, "Cannot change Tx QP state to INIT %s",
-				strerror(errno));
-			rte_errno = errno;
-			return ret;
-		}
-		mod.qp_state = IBV_QPS_RTR;
-		ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-		if (ret) {
-			DRV_LOG(ERR, "Cannot change Tx QP state to RTR %s",
-				strerror(errno));
-			rte_errno = errno;
-			return ret;
-		}
-		mod.qp_state = IBV_QPS_RTS;
-		ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
-		if (ret) {
-			DRV_LOG(ERR, "Cannot change Tx QP state to RTS %s",
-				strerror(errno));
-			rte_errno = errno;
-			return ret;
-		}
-	}
+	ret = priv->obj_ops.txq_obj_modify(txq_ctrl->obj,
+					   MLX5_TXQ_MOD_RDY2RDY,
+					   (uint8_t)priv->dev_port);
+	if (ret)
+		return ret;
 	txq_ctrl->txq.wqe_ci = 0;
 	txq_ctrl->txq.wqe_pi = 0;
 	txq_ctrl->txq.elts_comp = 0;
-- 
1.8.3.1