[dpdk-dev] [PATCH 08/16] mlx4: avoid looking up WR ID to improve RX performance

[Adrien Mazarguil <adrien.mazarguil@6wind.com>]

From: Alex Rosenbaum <Alexr@mellanox.com>

This is done by storing the current index in the RX queue structure.

Signed-off-by: Alex Rosenbaum <Alexr@mellanox.com>
Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
---
 drivers/net/mlx4/mlx4.c | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index 061f5e6..080602e 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -208,6 +208,7 @@ struct rxq {
 	struct ibv_exp_flow *allmulti_flow; /* Multicast flow. */
 	unsigned int port_id; /* Port ID for incoming packets. */
 	unsigned int elts_n; /* (*elts)[] length. */
+	unsigned int elts_head; /* Current index in (*elts)[]. */
 	union {
 		struct rxq_elt_sp (*sp)[]; /* Scattered RX elements. */
 		struct rxq_elt (*no_sp)[]; /* RX elements. */
@@ -1651,6 +1652,7 @@ rxq_alloc_elts_sp(struct rxq *rxq, unsigned int elts_n,
 	DEBUG("%p: allocated and configured %u WRs (%zu segments)",
 	      (void *)rxq, elts_n, (elts_n * elemof((*elts)[0].sges)));
 	rxq->elts_n = elts_n;
+	rxq->elts_head = 0;
 	rxq->elts.sp = elts;
 	assert(ret == 0);
 	return 0;
@@ -1795,6 +1797,7 @@ rxq_alloc_elts(struct rxq *rxq, unsigned int elts_n, struct rte_mbuf **pool)
 	DEBUG("%p: allocated and configured %u single-segment WRs",
 	      (void *)rxq, elts_n);
 	rxq->elts_n = elts_n;
+	rxq->elts_head = 0;
 	rxq->elts.no_sp = elts;
 	assert(ret == 0);
 	return 0;
@@ -2376,6 +2379,8 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
 	struct rxq *rxq = (struct rxq *)dpdk_rxq;
 	struct rxq_elt_sp (*elts)[rxq->elts_n] = rxq->elts.sp;
+	const unsigned int elts_n = rxq->elts_n;
+	unsigned int elts_head = rxq->elts_head;
 	struct ibv_exp_wc wcs[pkts_n];
 	struct ibv_recv_wr head;
 	struct ibv_recv_wr **next = &head.next;
@@ -2402,7 +2407,7 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		struct ibv_exp_wc *wc = &wcs[i];
 		uint64_t wr_id = wc->wr_id;
 		uint32_t len = wc->byte_len;
-		struct rxq_elt_sp *elt = &(*elts)[wr_id];
+		struct rxq_elt_sp *elt = &(*elts)[elts_head];
 		struct ibv_recv_wr *wr = &elt->wr;
 		struct rte_mbuf *pkt_buf = NULL; /* Buffer returned in pkts. */
 		struct rte_mbuf **pkt_buf_next = &pkt_buf;
@@ -2410,10 +2415,15 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		unsigned int j = 0;
 
 		/* Sanity checks. */
+#ifdef NDEBUG
+		(void)wr_id;
+#endif
 		assert(wr_id < rxq->elts_n);
 		assert(wr_id == wr->wr_id);
 		assert(wr->sg_list == elt->sges);
 		assert(wr->num_sge == elemof(elt->sges));
+		assert(elts_head < rxq->elts_n);
+		assert(rxq->elts_head < rxq->elts_n);
 		/* Link completed WRs together for repost. */
 		*next = wr;
 		next = &wr->next;
@@ -2522,6 +2532,8 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		rxq->stats.ibytes += wc->byte_len;
 #endif
 repost:
+		if (++elts_head >= elts_n)
+			elts_head = 0;
 		continue;
 	}
 	*next = NULL;
@@ -2539,6 +2551,7 @@ repost:
 		      strerror(i));
 		abort();
 	}
+	rxq->elts_head = elts_head;
 #ifdef MLX4_PMD_SOFT_COUNTERS
 	/* Increase packets counter. */
 	rxq->stats.ipackets += ret;
@@ -2568,6 +2581,8 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
 	struct rxq *rxq = (struct rxq *)dpdk_rxq;
 	struct rxq_elt (*elts)[rxq->elts_n] = rxq->elts.no_sp;
+	const unsigned int elts_n = rxq->elts_n;
+	unsigned int elts_head = rxq->elts_head;
 	struct ibv_exp_wc wcs[pkts_n];
 	struct ibv_recv_wr head;
 	struct ibv_recv_wr **next = &head.next;
@@ -2592,7 +2607,7 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		struct ibv_exp_wc *wc = &wcs[i];
 		uint64_t wr_id = wc->wr_id;
 		uint32_t len = wc->byte_len;
-		struct rxq_elt *elt = &(*elts)[WR_ID(wr_id).id];
+		struct rxq_elt *elt = &(*elts)[elts_head];
 		struct ibv_recv_wr *wr = &elt->wr;
 		struct rte_mbuf *seg =
 			(void *)(elt->sge.addr - WR_ID(wr_id).offset);
@@ -2603,6 +2618,8 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		assert(wr_id == wr->wr_id);
 		assert(wr->sg_list == &elt->sge);
 		assert(wr->num_sge == 1);
+		assert(elts_head < rxq->elts_n);
+		assert(rxq->elts_head < rxq->elts_n);
 		/* Link completed WRs together for repost. */
 		*next = wr;
 		next = &wr->next;
@@ -2663,6 +2680,8 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		rxq->stats.ibytes += wc->byte_len;
 #endif
 repost:
+		if (++elts_head >= elts_n)
+			elts_head = 0;
 		continue;
 	}
 	*next = NULL;
@@ -2680,6 +2699,7 @@ repost:
 		      strerror(i));
 		abort();
 	}
+	rxq->elts_head = elts_head;
 #ifdef MLX4_PMD_SOFT_COUNTERS
 	/* Increase packets counter. */
 	rxq->stats.ipackets += ret;
-- 
2.1.0