[dpdk-dev] [PATCH 07/24] mlx5: split RX queue structure

[Nelio Laranjeiro <nelio.laranjeiro@6wind.com>]

To keep the data path as efficient as possible, move fields only useful to
the control path into new structure rxq_ctrl.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
---
 drivers/net/mlx5/mlx5.c      |   6 +-
 drivers/net/mlx5/mlx5_fdir.c |   8 +-
 drivers/net/mlx5/mlx5_rxq.c  | 250 ++++++++++++++++++++++---------------------
 drivers/net/mlx5/mlx5_rxtx.c |   1 -
 drivers/net/mlx5/mlx5_rxtx.h |  13 ++-
 5 files changed, 148 insertions(+), 130 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cc30463..95279bd 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -122,12 +122,14 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 		usleep(1000);
 		for (i = 0; (i != priv->rxqs_n); ++i) {
 			struct rxq *rxq = (*priv->rxqs)[i];
+			struct rxq_ctrl *rxq_ctrl;
 
 			if (rxq == NULL)
 				continue;
+			rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq);
 			(*priv->rxqs)[i] = NULL;
-			rxq_cleanup(rxq);
-			rte_free(rxq);
+			rxq_cleanup(rxq_ctrl);
+			rte_free(rxq_ctrl);
 		}
 		priv->rxqs_n = 0;
 		priv->rxqs = NULL;
diff --git a/drivers/net/mlx5/mlx5_fdir.c b/drivers/net/mlx5/mlx5_fdir.c
index 63e43ad..e3b97ba 100644
--- a/drivers/net/mlx5/mlx5_fdir.c
+++ b/drivers/net/mlx5/mlx5_fdir.c
@@ -424,7 +424,9 @@ create_flow:
 static struct fdir_queue *
 priv_get_fdir_queue(struct priv *priv, uint16_t idx)
 {
-	struct fdir_queue *fdir_queue = &(*priv->rxqs)[idx]->fdir_queue;
+	struct rxq_ctrl *rxq_ctrl =
+		container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
+	struct fdir_queue *fdir_queue = &rxq_ctrl->fdir_queue;
 	struct ibv_exp_rwq_ind_table *ind_table = NULL;
 	struct ibv_qp *qp = NULL;
 	struct ibv_exp_rwq_ind_table_init_attr ind_init_attr;
@@ -629,8 +631,10 @@ priv_fdir_disable(struct priv *priv)
 	/* Run on every RX queue to destroy related flow director QP and
 	 * indirection table. */
 	for (i = 0; (i != priv->rxqs_n); i++) {
-		fdir_queue = &(*priv->rxqs)[i]->fdir_queue;
+		struct rxq_ctrl *rxq_ctrl =
+			container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq);
 
+		fdir_queue = &rxq_ctrl->fdir_queue;
 		if (fdir_queue->qp != NULL) {
 			claim_zero(ibv_destroy_qp(fdir_queue->qp));
 			fdir_queue->qp = NULL;
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 4000624..8d32e74 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -636,7 +636,7 @@ priv_rehash_flows(struct priv *priv)
 /**
  * Allocate RX queue elements.
  *
- * @param rxq
+ * @param rxq_ctrl
  *   Pointer to RX queue structure.
  * @param elts_n
  *   Number of elements to allocate.
@@ -648,16 +648,17 @@ priv_rehash_flows(struct priv *priv)
  *   0 on success, errno value on failure.
  */
 static int
-rxq_alloc_elts(struct rxq *rxq, unsigned int elts_n, struct rte_mbuf **pool)
+rxq_alloc_elts(struct rxq_ctrl *rxq_ctrl, unsigned int elts_n,
+	       struct rte_mbuf **pool)
 {
 	unsigned int i;
 	struct rxq_elt (*elts)[elts_n] =
 		rte_calloc_socket("RXQ elements", 1, sizeof(*elts), 0,
-				  rxq->socket);
+				  rxq_ctrl->socket);
 	int ret = 0;
 
 	if (elts == NULL) {
-		ERROR("%p: can't allocate packets array", (void *)rxq);
+		ERROR("%p: can't allocate packets array", (void *)rxq_ctrl);
 		ret = ENOMEM;
 		goto error;
 	}
@@ -672,10 +673,10 @@ rxq_alloc_elts(struct rxq *rxq, unsigned int elts_n, struct rte_mbuf **pool)
 			assert(buf != NULL);
 			rte_pktmbuf_reset(buf);
 		} else
-			buf = rte_pktmbuf_alloc(rxq->mp);
+			buf = rte_pktmbuf_alloc(rxq_ctrl->rxq.mp);
 		if (buf == NULL) {
 			assert(pool == NULL);
-			ERROR("%p: empty mbuf pool", (void *)rxq);
+			ERROR("%p: empty mbuf pool", (void *)rxq_ctrl);
 			ret = ENOMEM;
 			goto error;
 		}
@@ -691,15 +692,15 @@ rxq_alloc_elts(struct rxq *rxq, unsigned int elts_n, struct rte_mbuf **pool)
 		sge->addr = (uintptr_t)
 			((uint8_t *)buf->buf_addr + RTE_PKTMBUF_HEADROOM);
 		sge->length = (buf->buf_len - RTE_PKTMBUF_HEADROOM);
-		sge->lkey = rxq->mr->lkey;
+		sge->lkey = rxq_ctrl->mr->lkey;
 		/* Redundant check for tailroom. */
 		assert(sge->length == rte_pktmbuf_tailroom(buf));
 	}
 	DEBUG("%p: allocated and configured %u single-segment WRs",
-	      (void *)rxq, elts_n);
-	rxq->elts_n = elts_n;
-	rxq->elts_head = 0;
-	rxq->elts = elts;
+	      (void *)rxq_ctrl, elts_n);
+	rxq_ctrl->rxq.elts_n = elts_n;
+	rxq_ctrl->rxq.elts_head = 0;
+	rxq_ctrl->rxq.elts = elts;
 	assert(ret == 0);
 	return 0;
 error:
@@ -714,7 +715,7 @@ error:
 		}
 		rte_free(elts);
 	}
-	DEBUG("%p: failed, freed everything", (void *)rxq);
+	DEBUG("%p: failed, freed everything", (void *)rxq_ctrl);
 	assert(ret > 0);
 	return ret;
 }
@@ -722,19 +723,19 @@ error:
 /**
  * Free RX queue elements.
  *
- * @param rxq
+ * @param rxq_ctrl
  *   Pointer to RX queue structure.
  */
 static void
-rxq_free_elts(struct rxq *rxq)
+rxq_free_elts(struct rxq_ctrl *rxq_ctrl)
 {
 	unsigned int i;
-	unsigned int elts_n = rxq->elts_n;
-	struct rxq_elt (*elts)[elts_n] = rxq->elts;
+	unsigned int elts_n = rxq_ctrl->rxq.elts_n;
+	struct rxq_elt (*elts)[elts_n] = rxq_ctrl->rxq.elts;
 
-	DEBUG("%p: freeing WRs", (void *)rxq);
-	rxq->elts_n = 0;
-	rxq->elts = NULL;
+	DEBUG("%p: freeing WRs", (void *)rxq_ctrl);
+	rxq_ctrl->rxq.elts_n = 0;
+	rxq_ctrl->rxq.elts = NULL;
 	if (elts == NULL)
 		return;
 	for (i = 0; (i != RTE_DIM(*elts)); ++i) {
@@ -752,58 +753,58 @@ rxq_free_elts(struct rxq *rxq)
  *
  * Destroy objects, free allocated memory and reset the structure for reuse.
  *
- * @param rxq
+ * @param rxq_ctrl
  *   Pointer to RX queue structure.
  */
 void
-rxq_cleanup(struct rxq *rxq)
+rxq_cleanup(struct rxq_ctrl *rxq_ctrl)
 {
 	struct ibv_exp_release_intf_params params;
 
-	DEBUG("cleaning up %p", (void *)rxq);
-	rxq_free_elts(rxq);
-	rxq->poll = NULL;
-	rxq->recv = NULL;
-	if (rxq->if_wq != NULL) {
-		assert(rxq->priv != NULL);
-		assert(rxq->priv->ctx != NULL);
-		assert(rxq->wq != NULL);
+	DEBUG("cleaning up %p", (void *)rxq_ctrl);
+	rxq_free_elts(rxq_ctrl);
+	rxq_ctrl->rxq.poll = NULL;
+	rxq_ctrl->rxq.recv = NULL;
+	if (rxq_ctrl->if_wq != NULL) {
+		assert(rxq_ctrl->rxq.priv != NULL);
+		assert(rxq_ctrl->rxq.priv->ctx != NULL);
+		assert(rxq_ctrl->rxq.wq != NULL);
 		params = (struct ibv_exp_release_intf_params){
 			.comp_mask = 0,
 		};
-		claim_zero(ibv_exp_release_intf(rxq->priv->ctx,
-						rxq->if_wq,
+		claim_zero(ibv_exp_release_intf(rxq_ctrl->rxq.priv->ctx,
+						rxq_ctrl->if_wq,
 						&params));
 	}
-	if (rxq->if_cq != NULL) {
-		assert(rxq->priv != NULL);
-		assert(rxq->priv->ctx != NULL);
-		assert(rxq->cq != NULL);
+	if (rxq_ctrl->if_cq != NULL) {
+		assert(rxq_ctrl->rxq.priv != NULL);
+		assert(rxq_ctrl->rxq.priv->ctx != NULL);
+		assert(rxq_ctrl->rxq.cq != NULL);
 		params = (struct ibv_exp_release_intf_params){
 			.comp_mask = 0,
 		};
-		claim_zero(ibv_exp_release_intf(rxq->priv->ctx,
-						rxq->if_cq,
+		claim_zero(ibv_exp_release_intf(rxq_ctrl->rxq.priv->ctx,
+						rxq_ctrl->if_cq,
 						&params));
 	}
-	if (rxq->wq != NULL)
-		claim_zero(ibv_exp_destroy_wq(rxq->wq));
-	if (rxq->cq != NULL)
-		claim_zero(ibv_destroy_cq(rxq->cq));
-	if (rxq->rd != NULL) {
+	if (rxq_ctrl->rxq.wq != NULL)
+		claim_zero(ibv_exp_destroy_wq(rxq_ctrl->rxq.wq));
+	if (rxq_ctrl->rxq.cq != NULL)
+		claim_zero(ibv_destroy_cq(rxq_ctrl->rxq.cq));
+	if (rxq_ctrl->rd != NULL) {
 		struct ibv_exp_destroy_res_domain_attr attr = {
 			.comp_mask = 0,
 		};
 
-		assert(rxq->priv != NULL);
-		assert(rxq->priv->ctx != NULL);
-		claim_zero(ibv_exp_destroy_res_domain(rxq->priv->ctx,
-						      rxq->rd,
+		assert(rxq_ctrl->rxq.priv != NULL);
+		assert(rxq_ctrl->rxq.priv->ctx != NULL);
+		claim_zero(ibv_exp_destroy_res_domain(rxq_ctrl->rxq.priv->ctx,
+						      rxq_ctrl->rd,
 						      &attr));
 	}
-	if (rxq->mr != NULL)
-		claim_zero(ibv_dereg_mr(rxq->mr));
-	memset(rxq, 0, sizeof(*rxq));
+	if (rxq_ctrl->mr != NULL)
+		claim_zero(ibv_dereg_mr(rxq_ctrl->mr));
+	memset(rxq_ctrl, 0, sizeof(*rxq_ctrl));
 }
 
 /**
@@ -815,37 +816,37 @@ rxq_cleanup(struct rxq *rxq)
  *
  * @param dev
  *   Pointer to Ethernet device structure.
- * @param rxq
+ * @param rxq_ctrl
  *   RX queue pointer.
  *
  * @return
  *   0 on success, errno value on failure.
  */
 int
-rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
+rxq_rehash(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl)
 {
-	struct priv *priv = rxq->priv;
-	struct rxq tmpl = *rxq;
+	struct priv *priv = rxq_ctrl->rxq.priv;
+	struct rxq_ctrl tmpl = *rxq_ctrl;
 	unsigned int mbuf_n;
 	unsigned int desc_n;
 	struct rte_mbuf **pool;
 	unsigned int i, k;
 	struct ibv_exp_wq_attr mod;
-	struct rxq_elt (*elts)[tmpl.elts_n];
+	struct rxq_elt (*elts)[tmpl.rxq.elts_n];
 	int err;
 
-	DEBUG("%p: rehashing queue %p", (void *)dev, (void *)rxq);
+	DEBUG("%p: rehashing queue %p", (void *)dev, (void *)rxq_ctrl);
 	/* Number of descriptors and mbufs currently allocated. */
-	desc_n = tmpl.elts_n;
+	desc_n = tmpl.rxq.elts_n;
 	mbuf_n = desc_n;
 	/* Toggle RX checksum offload if hardware supports it. */
 	if (priv->hw_csum) {
-		tmpl.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
-		rxq->csum = tmpl.csum;
+		tmpl.rxq.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
+		rxq_ctrl->rxq.csum = tmpl.rxq.csum;
 	}
 	if (priv->hw_csum_l2tun) {
-		tmpl.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
-		rxq->csum_l2tun = tmpl.csum_l2tun;
+		tmpl.rxq.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
+		rxq_ctrl->rxq.csum_l2tun = tmpl.rxq.csum_l2tun;
 	}
 	/* From now on, any failure will render the queue unusable.
 	 * Reinitialize WQ. */
@@ -853,7 +854,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
 		.attr_mask = IBV_EXP_WQ_ATTR_STATE,
 		.wq_state = IBV_EXP_WQS_RESET,
 	};
-	err = ibv_exp_modify_wq(tmpl.wq, &mod);
+	err = ibv_exp_modify_wq(tmpl.rxq.wq, &mod);
 	if (err) {
 		ERROR("%p: cannot reset WQ: %s", (void *)dev, strerror(err));
 		assert(err > 0);
@@ -867,7 +868,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
 	}
 	/* Snatch mbufs from original queue. */
 	k = 0;
-	elts = rxq->elts;
+	elts = rxq_ctrl->rxq.elts;
 	for (i = 0; (i != RTE_DIM(*elts)); ++i) {
 		struct rxq_elt *elt = &(*elts)[i];
 		struct rte_mbuf *buf = elt->buf;
@@ -875,9 +876,9 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
 		pool[k++] = buf;
 	}
 	assert(k == mbuf_n);
-	tmpl.elts_n = 0;
-	tmpl.elts = NULL;
-	assert((void *)&tmpl.elts == NULL);
+	tmpl.rxq.elts_n = 0;
+	tmpl.rxq.elts = NULL;
+	assert((void *)&tmpl.rxq.elts == NULL);
 	err = rxq_alloc_elts(&tmpl, desc_n, pool);
 	if (err) {
 		ERROR("%p: cannot reallocate WRs, aborting", (void *)dev);
@@ -885,18 +886,18 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
 		assert(err > 0);
 		return err;
 	}
-	assert(tmpl.elts_n == desc_n);
+	assert(tmpl.rxq.elts_n == desc_n);
 	rte_free(pool);
 	/* Clean up original data. */
-	rxq->elts_n = 0;
-	rte_free(rxq->elts);
-	rxq->elts = NULL;
+	rxq_ctrl->rxq.elts_n = 0;
+	rte_free(rxq_ctrl->rxq.elts);
+	rxq_ctrl->rxq.elts = NULL;
 	/* Change queue state to ready. */
 	mod = (struct ibv_exp_wq_attr){
 		.attr_mask = IBV_EXP_WQ_ATTR_STATE,
 		.wq_state = IBV_EXP_WQS_RDY,
 	};
-	err = ibv_exp_modify_wq(tmpl.wq, &mod);
+	err = ibv_exp_modify_wq(tmpl.rxq.wq, &mod);
 	if (err) {
 		ERROR("%p: WQ state to IBV_EXP_WQS_RDY failed: %s",
 		      (void *)dev, strerror(err));
@@ -904,10 +905,10 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
 	}
 	/* Post SGEs. */
 	assert(tmpl.if_wq != NULL);
-	elts = tmpl.elts;
+	elts = tmpl.rxq.elts;
 	for (i = 0; (i != RTE_DIM(*elts)); ++i) {
 		err = tmpl.if_wq->recv_burst(
-			tmpl.wq,
+			tmpl.rxq.wq,
 			&(*elts)[i].sge,
 			1);
 		if (err)
@@ -920,9 +921,9 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
 		err = EIO;
 		goto error;
 	}
-	tmpl.recv = tmpl.if_wq->recv_burst;
+	tmpl.rxq.recv = tmpl.if_wq->recv_burst;
 error:
-	*rxq = tmpl;
+	*rxq_ctrl = tmpl;
 	assert(err >= 0);
 	return err;
 }
@@ -932,7 +933,7 @@ error:
  *
  * @param dev
  *   Pointer to Ethernet device structure.
- * @param rxq
+ * @param rxq_ctrl
  *   Pointer to RX queue structure.
  * @param desc
  *   Number of descriptors to configure in queue.
@@ -947,15 +948,17 @@ error:
  *   0 on success, errno value on failure.
  */
 int
-rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
+rxq_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, uint16_t desc,
 	  unsigned int socket, const struct rte_eth_rxconf *conf,
 	  struct rte_mempool *mp)
 {
 	struct priv *priv = dev->data->dev_private;
-	struct rxq tmpl = {
-		.priv = priv,
-		.mp = mp,
-		.socket = socket
+	struct rxq_ctrl tmpl = {
+		.socket = socket,
+		.rxq = {
+			.priv = priv,
+			.mp = mp,
+		},
 	};
 	struct ibv_exp_wq_attr mod;
 	union {
@@ -978,9 +981,9 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 	}
 	/* Toggle RX checksum offload if hardware supports it. */
 	if (priv->hw_csum)
-		tmpl.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
+		tmpl.rxq.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
 	if (priv->hw_csum_l2tun)
-		tmpl.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
+		tmpl.rxq.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
 	(void)mb_len; /* I'll be back! */
 	/* Use the entire RX mempool as the memory region. */
 	tmpl.mr = mlx5_mp2mr(priv->pd, mp);
@@ -1007,9 +1010,9 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 		.comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN,
 		.res_domain = tmpl.rd,
 	};
-	tmpl.cq = ibv_exp_create_cq(priv->ctx, cq_size, NULL, NULL, 0,
-				    &attr.cq);
-	if (tmpl.cq == NULL) {
+	tmpl.rxq.cq = ibv_exp_create_cq(priv->ctx, cq_size, NULL, NULL, 0,
+					&attr.cq);
+	if (tmpl.rxq.cq == NULL) {
 		ret = ENOMEM;
 		ERROR("%p: CQ creation failure: %s",
 		      (void *)dev, strerror(ret));
@@ -1020,8 +1023,8 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 	DEBUG("priv->device_attr.max_sge is %d",
 	      priv->device_attr.max_sge);
 	/* Configure VLAN stripping. */
-	tmpl.vlan_strip = (priv->hw_vlan_strip &&
-			   !!dev->data->dev_conf.rxmode.hw_vlan_strip);
+	tmpl.rxq.vlan_strip = (priv->hw_vlan_strip &&
+			       !!dev->data->dev_conf.rxmode.hw_vlan_strip);
 	attr.wq = (struct ibv_exp_wq_init_attr){
 		.wq_context = NULL, /* Could be useful in the future. */
 		.wq_type = IBV_EXP_WQT_RQ,
@@ -1032,7 +1035,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 		/* Max number of scatter/gather elements in a WR. */
 		.max_recv_sge = 1,
 		.pd = priv->pd,
-		.cq = tmpl.cq,
+		.cq = tmpl.rxq.cq,
 		.comp_mask =
 			IBV_EXP_CREATE_WQ_RES_DOMAIN |
 #ifdef HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS
@@ -1041,7 +1044,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 			0,
 		.res_domain = tmpl.rd,
 #ifdef HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS
-		.vlan_offloads = (tmpl.vlan_strip ?
+		.vlan_offloads = (tmpl.rxq.vlan_strip ?
 				  IBV_EXP_RECEIVE_WQ_CVLAN_STRIP :
 				  0),
 #endif /* HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS */
@@ -1050,24 +1053,24 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 #ifdef HAVE_VERBS_FCS
 	/* By default, FCS (CRC) is stripped by hardware. */
 	if (dev->data->dev_conf.rxmode.hw_strip_crc) {
-		tmpl.crc_present = 0;
+		tmpl.rxq.crc_present = 0;
 	} else if (priv->hw_fcs_strip) {
 		/* Ask HW/Verbs to leave CRC in place when supported. */
 		attr.wq.flags |= IBV_EXP_CREATE_WQ_FLAG_SCATTER_FCS;
 		attr.wq.comp_mask |= IBV_EXP_CREATE_WQ_FLAGS;
-		tmpl.crc_present = 1;
+		tmpl.rxq.crc_present = 1;
 	} else {
 		WARN("%p: CRC stripping has been disabled but will still"
 		     " be performed by hardware, make sure MLNX_OFED and"
 		     " firmware are up to date",
 		     (void *)dev);
-		tmpl.crc_present = 0;
+		tmpl.rxq.crc_present = 0;
 	}
 	DEBUG("%p: CRC stripping is %s, %u bytes will be subtracted from"
 	      " incoming frames to hide it",
 	      (void *)dev,
-	      tmpl.crc_present ? "disabled" : "enabled",
-	      tmpl.crc_present << 2);
+	      tmpl.rxq.crc_present ? "disabled" : "enabled",
+	      tmpl.rxq.crc_present << 2);
 #endif /* HAVE_VERBS_FCS */
 
 #ifdef HAVE_VERBS_RX_END_PADDING
@@ -1075,7 +1078,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 		; /* Nothing else to do. */
 	else if (priv->hw_padding) {
 		INFO("%p: enabling packet padding on queue %p",
-		     (void *)dev, (void *)rxq);
+		     (void *)dev, (void *)rxq_ctrl);
 		attr.wq.flags |= IBV_EXP_CREATE_WQ_FLAG_RX_END_PADDING;
 		attr.wq.comp_mask |= IBV_EXP_CREATE_WQ_FLAGS;
 	} else
@@ -1085,8 +1088,8 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 		     (void *)dev);
 #endif /* HAVE_VERBS_RX_END_PADDING */
 
-	tmpl.wq = ibv_exp_create_wq(priv->ctx, &attr.wq);
-	if (tmpl.wq == NULL) {
+	tmpl.rxq.wq = ibv_exp_create_wq(priv->ctx, &attr.wq);
+	if (tmpl.rxq.wq == NULL) {
 		ret = (errno ? errno : EINVAL);
 		ERROR("%p: WQ creation failure: %s",
 		      (void *)dev, strerror(ret));
@@ -1099,15 +1102,15 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 		goto error;
 	}
 	/* Save port ID. */
-	tmpl.port_id = dev->data->port_id;
-	DEBUG("%p: RTE port ID: %u", (void *)rxq, tmpl.port_id);
+	tmpl.rxq.port_id = dev->data->port_id;
+	DEBUG("%p: RTE port ID: %u", (void *)rxq_ctrl, tmpl.rxq.port_id);
 	attr.params = (struct ibv_exp_query_intf_params){
 		.intf_scope = IBV_EXP_INTF_GLOBAL,
 #ifdef HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS
 		.intf_version = 1,
 #endif /* HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS */
 		.intf = IBV_EXP_INTF_CQ,
-		.obj = tmpl.cq,
+		.obj = tmpl.rxq.cq,
 	};
 	tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status);
 	if (tmpl.if_cq == NULL) {
@@ -1118,7 +1121,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 	attr.params = (struct ibv_exp_query_intf_params){
 		.intf_scope = IBV_EXP_INTF_GLOBAL,
 		.intf = IBV_EXP_INTF_WQ,
-		.obj = tmpl.wq,
+		.obj = tmpl.rxq.wq,
 	};
 	tmpl.if_wq = ibv_exp_query_intf(priv->ctx, &attr.params, &status);
 	if (tmpl.if_wq == NULL) {
@@ -1131,17 +1134,17 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 		.attr_mask = IBV_EXP_WQ_ATTR_STATE,
 		.wq_state = IBV_EXP_WQS_RDY,
 	};
-	ret = ibv_exp_modify_wq(tmpl.wq, &mod);
+	ret = ibv_exp_modify_wq(tmpl.rxq.wq, &mod);
 	if (ret) {
 		ERROR("%p: WQ state to IBV_EXP_WQS_RDY failed: %s",
 		      (void *)dev, strerror(ret));
 		goto error;
 	}
 	/* Post SGEs. */
-	elts = tmpl.elts;
+	elts = tmpl.rxq.elts;
 	for (i = 0; (i != RTE_DIM(*elts)); ++i) {
 		ret = tmpl.if_wq->recv_burst(
-			tmpl.wq,
+			tmpl.rxq.wq,
 			&(*elts)[i].sge,
 			1);
 		if (ret)
@@ -1155,18 +1158,18 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 		goto error;
 	}
 	/* Clean up rxq in case we're reinitializing it. */
-	DEBUG("%p: cleaning-up old rxq just in case", (void *)rxq);
-	rxq_cleanup(rxq);
-	*rxq = tmpl;
-	DEBUG("%p: rxq updated with %p", (void *)rxq, (void *)&tmpl);
+	DEBUG("%p: cleaning-up old rxq just in case", (void *)rxq_ctrl);
+	rxq_cleanup(rxq_ctrl);
+	*rxq_ctrl = tmpl;
+	DEBUG("%p: rxq updated with %p", (void *)rxq_ctrl, (void *)&tmpl);
 	assert(ret == 0);
 	/* Assign function in queue. */
 #ifdef HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS
-	rxq->poll = rxq->if_cq->poll_length_flags_cvlan;
+	rxq_ctrl->rxq.poll = rxq_ctrl->if_cq->poll_length_flags_cvlan;
 #else /* HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS */
-	rxq->poll = rxq->if_cq->poll_length_flags;
+	rxq_ctrl->rxq.poll = rxq_ctrl->if_cq->poll_length_flags;
 #endif /* HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS */
-	rxq->recv = rxq->if_wq->recv_burst;
+	rxq_ctrl->rxq.recv = rxq_ctrl->if_wq->recv_burst;
 	return 0;
 error:
 	rxq_cleanup(&tmpl);
@@ -1200,12 +1203,14 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 {
 	struct priv *priv = dev->data->dev_private;
 	struct rxq *rxq = (*priv->rxqs)[idx];
+	struct rxq_ctrl *rxq_ctrl;
 	int ret;
 
 	if (mlx5_is_secondary())
 		return -E_RTE_SECONDARY;
 
 	priv_lock(priv);
+	rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq);
 	DEBUG("%p: configuring queue %u for %u descriptors",
 	      (void *)dev, idx, desc);
 	if (idx >= priv->rxqs_n) {
@@ -1222,24 +1227,25 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 			return -EEXIST;
 		}
 		(*priv->rxqs)[idx] = NULL;
-		rxq_cleanup(rxq);
+		rxq_cleanup(rxq_ctrl);
 	} else {
-		rxq = rte_calloc_socket("RXQ", 1, sizeof(*rxq), 0, socket);
-		if (rxq == NULL) {
+		rxq_ctrl = rte_calloc_socket("RXQ", 1, sizeof(*rxq_ctrl), 0,
+					     socket);
+		if (rxq_ctrl == NULL) {
 			ERROR("%p: unable to allocate queue index %u",
 			      (void *)dev, idx);
 			priv_unlock(priv);
 			return -ENOMEM;
 		}
 	}
-	ret = rxq_setup(dev, rxq, desc, socket, conf, mp);
+	ret = rxq_setup(dev, rxq_ctrl, desc, socket, conf, mp);
 	if (ret)
-		rte_free(rxq);
+		rte_free(rxq_ctrl);
 	else {
-		rxq->stats.idx = idx;
+		rxq_ctrl->rxq.stats.idx = idx;
 		DEBUG("%p: adding RX queue %p to list",
-		      (void *)dev, (void *)rxq);
-		(*priv->rxqs)[idx] = rxq;
+		      (void *)dev, (void *)rxq_ctrl);
+		(*priv->rxqs)[idx] = &rxq_ctrl->rxq;
 		/* Update receive callback. */
 		dev->rx_pkt_burst = mlx5_rx_burst;
 	}
@@ -1257,6 +1263,7 @@ void
 mlx5_rx_queue_release(void *dpdk_rxq)
 {
 	struct rxq *rxq = (struct rxq *)dpdk_rxq;
+	struct rxq_ctrl *rxq_ctrl;
 	struct priv *priv;
 	unsigned int i;
 
@@ -1265,6 +1272,7 @@ mlx5_rx_queue_release(void *dpdk_rxq)
 
 	if (rxq == NULL)
 		return;
+	rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq);
 	priv = rxq->priv;
 	priv_lock(priv);
 	for (i = 0; (i != priv->rxqs_n); ++i)
@@ -1274,8 +1282,8 @@ mlx5_rx_queue_release(void *dpdk_rxq)
 			(*priv->rxqs)[i] = NULL;
 			break;
 		}
-	rxq_cleanup(rxq);
-	rte_free(rxq);
+	rxq_cleanup(rxq_ctrl);
+	rte_free(rxq_ctrl);
 	priv_unlock(priv);
 }
 
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 4ba88ea..f0b42e9 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -574,7 +574,6 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 
 		/* Reconfigure sge to use rep instead of seg. */
 		elt->sge.addr = (uintptr_t)rep->buf_addr + RTE_PKTMBUF_HEADROOM;
-		assert(elt->sge.lkey == rxq->mr->lkey);
 		elt->buf = rep;
 
 		/* Add SGE to array for repost. */
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 5baefcb..2c5e447 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -111,8 +111,11 @@ struct rxq {
 	unsigned int vlan_strip:1; /* Enable VLAN stripping. */
 	unsigned int crc_present:1; /* CRC must be subtracted. */
 	struct rxq_elt (*elts)[]; /* RX elements. */
-	unsigned int socket; /* CPU socket ID for allocations. */
 	struct mlx5_rxq_stats stats; /* RX queue counters. */
+} __rte_cache_aligned;
+
+/* RX queue control descriptor. */
+struct rxq_ctrl {
 	struct ibv_exp_res_domain *rd; /* Resource Domain. */
 	struct fdir_queue fdir_queue; /* Flow director queue. */
 	struct ibv_mr *mr; /* Memory Region (for mp). */
@@ -122,6 +125,8 @@ struct rxq {
 #else /* HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS */
 	struct ibv_exp_cq_family *if_cq; /* CQ interface. */
 #endif /* HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS */
+	unsigned int socket; /* CPU socket ID for allocations. */
+	struct rxq rxq; /* Data path structure. */
 };
 
 /* Hash RX queue types. */
@@ -285,9 +290,9 @@ int priv_create_hash_rxqs(struct priv *);
 void priv_destroy_hash_rxqs(struct priv *);
 int priv_allow_flow_type(struct priv *, enum hash_rxq_flow_type);
 int priv_rehash_flows(struct priv *);
-void rxq_cleanup(struct rxq *);
-int rxq_rehash(struct rte_eth_dev *, struct rxq *);
-int rxq_setup(struct rte_eth_dev *, struct rxq *, uint16_t, unsigned int,
+void rxq_cleanup(struct rxq_ctrl *);
+int rxq_rehash(struct rte_eth_dev *, struct rxq_ctrl *);
+int rxq_setup(struct rte_eth_dev *, struct rxq_ctrl *, uint16_t, unsigned int,
 	      const struct rte_eth_rxconf *, struct rte_mempool *);
 int mlx5_rx_queue_setup(struct rte_eth_dev *, uint16_t, uint16_t, unsigned int,
 			const struct rte_eth_rxconf *, struct rte_mempool *);
-- 
2.1.4