[dpdk-dev] [PATCH v2 22/51] net/mlx4: revert resource domain support

[Adrien Mazarguil <adrien.mazarguil@6wind.com>]

This reverts commit 3e49c148b715c3c0a12c1200295bb9b312f7028e.

Resource domains are not part of the standard Verbs interface. The
performance improvement they bring will be restored later through a
different data path implementation.

This commit makes the PMD not rely on the non-standard QP allocation
interface.

Signed-off-by: Moti Haimovsky <motih@mellanox.com>
Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
---
 drivers/net/mlx4/mlx4.c      | 88 ++++-----------------------------------
 drivers/net/mlx4/mlx4.h      |  2 -
 drivers/net/mlx4/mlx4_flow.c | 30 +++++--------
 3 files changed, 20 insertions(+), 100 deletions(-)

diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index 682307f..7dbed93 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -697,17 +697,6 @@ txq_cleanup(struct txq *txq)
 		claim_zero(ibv_destroy_qp(txq->qp));
 	if (txq->cq != NULL)
 		claim_zero(ibv_destroy_cq(txq->cq));
-	if (txq->rd != NULL) {
-		struct ibv_exp_destroy_res_domain_attr attr = {
-			.comp_mask = 0,
-		};
-
-		assert(txq->priv != NULL);
-		assert(txq->priv->ctx != NULL);
-		claim_zero(ibv_exp_destroy_res_domain(txq->priv->ctx,
-						      txq->rd,
-						      &attr));
-	}
 	for (i = 0; (i != elemof(txq->mp2mr)); ++i) {
 		if (txq->mp2mr[i].mp == NULL)
 			break;
@@ -1175,9 +1164,7 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
 	};
 	union {
 		struct ibv_exp_query_intf_params params;
-		struct ibv_exp_qp_init_attr init;
-		struct ibv_exp_res_domain_init_attr rd;
-		struct ibv_exp_cq_init_attr cq;
+		struct ibv_qp_init_attr init;
 		struct ibv_qp_attr mod;
 	} attr;
 	enum ibv_exp_query_intf_status status;
@@ -1191,24 +1178,7 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
 		return EINVAL;
 	}
 	/* MRs will be registered in mp2mr[] later. */
-	attr.rd = (struct ibv_exp_res_domain_init_attr){
-		.comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL |
-			      IBV_EXP_RES_DOMAIN_MSG_MODEL),
-		.thread_model = IBV_EXP_THREAD_SINGLE,
-		.msg_model = IBV_EXP_MSG_HIGH_BW,
-	};
-	tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd);
-	if (tmpl.rd == NULL) {
-		ret = ENOMEM;
-		ERROR("%p: RD creation failure: %s",
-		      (void *)dev, strerror(ret));
-		goto error;
-	}
-	attr.cq = (struct ibv_exp_cq_init_attr){
-		.comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN,
-		.res_domain = tmpl.rd,
-	};
-	tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq);
+	tmpl.cq = ibv_create_cq(priv->ctx, desc, NULL, NULL, 0);
 	if (tmpl.cq == NULL) {
 		ret = ENOMEM;
 		ERROR("%p: CQ creation failure: %s",
@@ -1219,7 +1189,7 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
 	      priv->device_attr.max_qp_wr);
 	DEBUG("priv->device_attr.max_sge is %d",
 	      priv->device_attr.max_sge);
-	attr.init = (struct ibv_exp_qp_init_attr){
+	attr.init = (struct ibv_qp_init_attr){
 		/* CQ to be associated with the send queue. */
 		.send_cq = tmpl.cq,
 		/* CQ to be associated with the receive queue. */
@@ -1237,12 +1207,8 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
 		/* Do *NOT* enable this, completions events are managed per
 		 * TX burst. */
 		.sq_sig_all = 0,
-		.pd = priv->pd,
-		.res_domain = tmpl.rd,
-		.comp_mask = (IBV_EXP_QP_INIT_ATTR_PD |
-			      IBV_EXP_QP_INIT_ATTR_RES_DOMAIN),
 	};
-	tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init);
+	tmpl.qp = ibv_create_qp(priv->pd, &attr.init);
 	if (tmpl.qp == NULL) {
 		ret = (errno ? errno : EINVAL);
 		ERROR("%p: QP creation failure: %s",
@@ -1710,17 +1676,6 @@ rxq_cleanup(struct rxq *rxq)
 		claim_zero(ibv_destroy_cq(rxq->cq));
 	if (rxq->channel != NULL)
 		claim_zero(ibv_destroy_comp_channel(rxq->channel));
-	if (rxq->rd != NULL) {
-		struct ibv_exp_destroy_res_domain_attr attr = {
-			.comp_mask = 0,
-		};
-
-		assert(rxq->priv != NULL);
-		assert(rxq->priv->ctx != NULL);
-		claim_zero(ibv_exp_destroy_res_domain(rxq->priv->ctx,
-						      rxq->rd,
-						      &attr));
-	}
 	if (rxq->mr != NULL)
 		claim_zero(ibv_dereg_mr(rxq->mr));
 	memset(rxq, 0, sizeof(*rxq));
@@ -1890,10 +1845,9 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
  *   QP pointer or NULL in case of error.
  */
 static struct ibv_qp *
-rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc,
-	     struct ibv_exp_res_domain *rd)
+rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc)
 {
-	struct ibv_exp_qp_init_attr attr = {
+	struct ibv_qp_init_attr attr = {
 		/* CQ to be associated with the send queue. */
 		.send_cq = cq,
 		/* CQ to be associated with the receive queue. */
@@ -1907,13 +1861,9 @@ rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc,
 			.max_recv_sge = 1,
 		},
 		.qp_type = IBV_QPT_RAW_PACKET,
-		.comp_mask = (IBV_EXP_QP_INIT_ATTR_PD |
-			      IBV_EXP_QP_INIT_ATTR_RES_DOMAIN),
-		.pd = priv->pd,
-		.res_domain = rd,
 	};
 
-	return ibv_exp_create_qp(priv->ctx, &attr);
+	return ibv_create_qp(priv->pd, &attr);
 }
 
 /**
@@ -1949,8 +1899,6 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 	struct ibv_qp_attr mod;
 	union {
 		struct ibv_exp_query_intf_params params;
-		struct ibv_exp_cq_init_attr cq;
-		struct ibv_exp_res_domain_init_attr rd;
 	} attr;
 	enum ibv_exp_query_intf_status status;
 	struct ibv_recv_wr *bad_wr;
@@ -1988,19 +1936,6 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 		      (void *)dev, strerror(ret));
 		goto error;
 	}
-	attr.rd = (struct ibv_exp_res_domain_init_attr){
-		.comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL |
-			      IBV_EXP_RES_DOMAIN_MSG_MODEL),
-		.thread_model = IBV_EXP_THREAD_SINGLE,
-		.msg_model = IBV_EXP_MSG_HIGH_BW,
-	};
-	tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd);
-	if (tmpl.rd == NULL) {
-		ret = ENOMEM;
-		ERROR("%p: RD creation failure: %s",
-		      (void *)dev, strerror(ret));
-		goto error;
-	}
 	if (dev->data->dev_conf.intr_conf.rxq) {
 		tmpl.channel = ibv_create_comp_channel(priv->ctx);
 		if (tmpl.channel == NULL) {
@@ -2011,12 +1946,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 			goto error;
 		}
 	}
-	attr.cq = (struct ibv_exp_cq_init_attr){
-		.comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN,
-		.res_domain = tmpl.rd,
-	};
-	tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, tmpl.channel, 0,
-				    &attr.cq);
+	tmpl.cq = ibv_create_cq(priv->ctx, desc, NULL, tmpl.channel, 0);
 	if (tmpl.cq == NULL) {
 		ret = ENOMEM;
 		ERROR("%p: CQ creation failure: %s",
@@ -2027,7 +1957,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 	      priv->device_attr.max_qp_wr);
 	DEBUG("priv->device_attr.max_sge is %d",
 	      priv->device_attr.max_sge);
-	tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc, tmpl.rd);
+	tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc);
 	if (tmpl.qp == NULL) {
 		ret = (errno ? errno : EINVAL);
 		ERROR("%p: QP creation failure: %s",
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 9cbde1d..edec40c 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -167,7 +167,6 @@ struct rxq {
 	struct rxq_elt (*elts)[]; /* Rx elements. */
 	struct mlx4_rxq_stats stats; /* RX queue counters. */
 	unsigned int socket; /* CPU socket ID for allocations. */
-	struct ibv_exp_res_domain *rd; /* Resource Domain. */
 };
 
 /* TX element. */
@@ -204,7 +203,6 @@ struct txq {
 	unsigned int elts_comp_cd_init; /* Initial value for countdown. */
 	struct mlx4_txq_stats stats; /* TX queue counters. */
 	unsigned int socket; /* CPU socket ID for allocations. */
-	struct ibv_exp_res_domain *rd; /* Resource Domain. */
 };
 
 struct rte_flow;
diff --git a/drivers/net/mlx4/mlx4_flow.c b/drivers/net/mlx4/mlx4_flow.c
index 2c5dc3c..58d4698 100644
--- a/drivers/net/mlx4/mlx4_flow.c
+++ b/drivers/net/mlx4/mlx4_flow.c
@@ -752,29 +752,21 @@ mlx4_flow_create_drop_queue(struct priv *priv)
 		ERROR("Cannot allocate memory for drop struct");
 		goto err;
 	}
-	cq = ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
-			      &(struct ibv_exp_cq_init_attr){
-					.comp_mask = 0,
-			      });
+	cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
 	if (!cq) {
 		ERROR("Cannot create drop CQ");
 		goto err_create_cq;
 	}
-	qp = ibv_exp_create_qp(priv->ctx,
-			      &(struct ibv_exp_qp_init_attr){
-					.send_cq = cq,
-					.recv_cq = cq,
-					.cap = {
-						.max_recv_wr = 1,
-						.max_recv_sge = 1,
-					},
-					.qp_type = IBV_QPT_RAW_PACKET,
-					.comp_mask =
-						IBV_EXP_QP_INIT_ATTR_PD |
-						IBV_EXP_QP_INIT_ATTR_PORT,
-					.pd = priv->pd,
-					.port_num = priv->port,
-			      });
+	qp = ibv_create_qp(priv->pd,
+			   &(struct ibv_qp_init_attr){
+				.send_cq = cq,
+				.recv_cq = cq,
+				.cap = {
+					.max_recv_wr = 1,
+					.max_recv_sge = 1,
+				},
+				.qp_type = IBV_QPT_RAW_PACKET,
+			   });
 	if (!qp) {
 		ERROR("Cannot create drop QP");
 		goto err_create_qp;
-- 
2.1.4