[dpdk-dev] [PATCH 0/9] net/sfc: support more features in EF10 Rx

[dpdk-dev] [PATCH 0/9] net/sfc: support more features in EF10 Rx

[Andrew Rybchenko]

Andrew Rybchenko (8):
  net/sfc: receive prepared packets even in Rx exception case
  net/sfc: use mbuf raw free instead of mempool put directly
  net/sfc: check mbufs allocated using mempool API for Rx
  net/sfc: avoid dummy writes to Rx queue state structure
  net/sfc: decrease number of variables maintained on EF10 Rx
  net/sfc: avoid usage of prepared packets number in EF10 Rx
  net/sfc: rename variable to prepare for scatter support
  net/sfc: support Rx scatter in EF10 Rx datapath

Igor Romanov (1):
  net/sfc: support Rx descriptor status on EF10 datapath

 doc/guides/nics/sfc_efx.rst            |   2 +-
 doc/guides/rel_notes/release_18_11.rst |   7 +
 drivers/net/sfc/sfc_ef10_essb_rx.c     |  16 +-
 drivers/net/sfc/sfc_ef10_rx.c          | 227 +++++++++++++++++--------
 drivers/net/sfc/sfc_rx.c               |   9 +-
 5 files changed, 184 insertions(+), 77 deletions(-)

-- 
2.17.1

[dpdk-dev] [PATCH 1/9] net/sfc: receive prepared packets even in Rx exception case

[Andrew Rybchenko]

Make sure that number of prepared packets, completed and added
Rx ring pointers are reset to zeros on queue purge at stop.

Fixes: 638bddc99faa ("net/sfc: implement EF10 native Rx datapath")
Cc: stable@dpdk.org

Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
 drivers/net/sfc/sfc_ef10_rx.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
index 6a5052b93..5c0152d01 100644
--- a/drivers/net/sfc/sfc_ef10_rx.c
+++ b/drivers/net/sfc/sfc_ef10_rx.c
@@ -359,11 +359,11 @@ sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	uint16_t n_rx_pkts;
 	efx_qword_t rx_ev;
 
+	n_rx_pkts = sfc_ef10_rx_prepared(rxq, rx_pkts, nb_pkts);
+
 	if (unlikely(rxq->flags &
 		     (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION)))
-		return 0;
-
-	n_rx_pkts = sfc_ef10_rx_prepared(rxq, rx_pkts, nb_pkts);
+		goto done;
 
 	evq_old_read_ptr = rxq->evq_read_ptr;
 	while (n_rx_pkts != nb_pkts && sfc_ef10_rx_get_event(rxq, &rx_ev)) {
@@ -383,6 +383,7 @@ sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	/* It is not a problem if we refill in the case of exception */
 	sfc_ef10_rx_qrefill(rxq);
 
+done:
 	return n_rx_pkts;
 }
 
@@ -594,8 +595,9 @@ sfc_ef10_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr)
 {
 	struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 
-	rxq->prepared = 0;
-	rxq->completed = rxq->added = 0;
+	SFC_ASSERT(rxq->prepared == 0);
+	SFC_ASSERT(rxq->completed == 0);
+	SFC_ASSERT(rxq->added == 0);
 
 	sfc_ef10_rx_qrefill(rxq);
 
@@ -642,12 +644,16 @@ sfc_ef10_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
 	unsigned int i;
 	struct sfc_ef10_rx_sw_desc *rxd;
 
+	rxq->prepared = 0;
+
 	for (i = rxq->completed; i != rxq->added; ++i) {
 		rxd = &rxq->sw_ring[i & rxq->ptr_mask];
 		rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf);
 		rxd->mbuf = NULL;
 	}
 
+	rxq->completed = rxq->added = 0;
+
 	rxq->flags &= ~SFC_EF10_RXQ_STARTED;
 }
 
-- 
2.17.1

[dpdk-dev] [PATCH 2/9] net/sfc: use mbuf raw free instead of mempool put directly

[Andrew Rybchenko]

mbuf raw free adds debug build checks to ensure that mbuf is
really OK to be returned to mempool.

Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Reviewed-by: Ivan Malov <ivan.malov@oktetlabs.ru>
---
 drivers/net/sfc/sfc_ef10_essb_rx.c | 4 ++--
 drivers/net/sfc/sfc_ef10_rx.c      | 4 ++--
 drivers/net/sfc/sfc_rx.c           | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/drivers/net/sfc/sfc_ef10_essb_rx.c b/drivers/net/sfc/sfc_ef10_essb_rx.c
index 81c8f7fbd..e5fc8506b 100644
--- a/drivers/net/sfc/sfc_ef10_essb_rx.c
+++ b/drivers/net/sfc/sfc_ef10_essb_rx.c
@@ -324,7 +324,7 @@ sfc_ef10_essb_rx_get_pending(struct sfc_ef10_essb_rxq *rxq,
 
 			/* Buffers to be discarded have 0 in packet type */
 			if (unlikely(m->packet_type == 0)) {
-				rte_mempool_put(rxq->refill_mb_pool, m);
+				rte_mbuf_raw_free(m);
 				goto next_buf;
 			}
 
@@ -687,7 +687,7 @@ sfc_ef10_essb_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
 		m = sfc_ef10_essb_mbuf_by_index(rxq, rxd->first_mbuf,
 				rxq->block_size - rxq->left_in_completed);
 		while (rxq->left_in_completed > 0) {
-			rte_mempool_put(rxq->refill_mb_pool, m);
+			rte_mbuf_raw_free(m);
 			m = sfc_ef10_essb_next_mbuf(rxq, m);
 			rxq->left_in_completed--;
 		}
diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
index 5c0152d01..946d96cac 100644
--- a/drivers/net/sfc/sfc_ef10_rx.c
+++ b/drivers/net/sfc/sfc_ef10_rx.c
@@ -238,7 +238,7 @@ sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 		rxq->completed += ready;
 		while (ready-- > 0) {
 			rxd = &rxq->sw_ring[completed++ & ptr_mask];
-			rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf);
+			rte_mbuf_raw_free(rxd->mbuf);
 		}
 		return 0;
 	}
@@ -648,7 +648,7 @@ sfc_ef10_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
 
 	for (i = rxq->completed; i != rxq->added; ++i) {
 		rxd = &rxq->sw_ring[i & rxq->ptr_mask];
-		rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf);
+		rte_mbuf_raw_free(rxd->mbuf);
 		rxd->mbuf = NULL;
 	}
 
diff --git a/drivers/net/sfc/sfc_rx.c b/drivers/net/sfc/sfc_rx.c
index 58c57acfe..0385fca16 100644
--- a/drivers/net/sfc/sfc_rx.c
+++ b/drivers/net/sfc/sfc_rx.c
@@ -296,7 +296,7 @@ sfc_efx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 discard:
 		discard_next = ((desc_flags & EFX_PKT_CONT) != 0);
-		rte_mempool_put(rxq->refill_mb_pool, m);
+		rte_mbuf_raw_free(m);
 		rxd->mbuf = NULL;
 	}
 
@@ -498,7 +498,7 @@ sfc_efx_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
 
 	for (i = rxq->completed; i != rxq->added; ++i) {
 		rxd = &rxq->sw_desc[i & rxq->ptr_mask];
-		rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf);
+		rte_mbuf_raw_free(rxd->mbuf);
 		rxd->mbuf = NULL;
 		/* Packed stream relies on 0 in inactive SW desc.
 		 * Rx queue stop is not performance critical, so
-- 
2.17.1

[dpdk-dev] [PATCH 3/9] net/sfc: check mbufs allocated using mempool API for Rx

[Andrew Rybchenko]

Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Reviewed-by: Ivan Malov <ivan.malov@oktetlabs.ru>
---
 drivers/net/sfc/sfc_ef10_essb_rx.c | 12 ++++++++++--
 drivers/net/sfc/sfc_ef10_rx.c      |  2 ++
 drivers/net/sfc/sfc_rx.c           |  5 ++---
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/drivers/net/sfc/sfc_ef10_essb_rx.c b/drivers/net/sfc/sfc_ef10_essb_rx.c
index e5fc8506b..a24f54e7b 100644
--- a/drivers/net/sfc/sfc_ef10_essb_rx.c
+++ b/drivers/net/sfc/sfc_ef10_essb_rx.c
@@ -123,14 +123,22 @@ static struct rte_mbuf *
 sfc_ef10_essb_next_mbuf(const struct sfc_ef10_essb_rxq *rxq,
 			struct rte_mbuf *mbuf)
 {
-	return (struct rte_mbuf *)((uintptr_t)mbuf + rxq->buf_stride);
+	struct rte_mbuf *m;
+
+	m = (struct rte_mbuf *)((uintptr_t)mbuf + rxq->buf_stride);
+	MBUF_RAW_ALLOC_CHECK(m);
+	return m;
 }
 
 static struct rte_mbuf *
 sfc_ef10_essb_mbuf_by_index(const struct sfc_ef10_essb_rxq *rxq,
 			    struct rte_mbuf *mbuf, unsigned int idx)
 {
-	return (struct rte_mbuf *)((uintptr_t)mbuf + idx * rxq->buf_stride);
+	struct rte_mbuf *m;
+
+	m = (struct rte_mbuf *)((uintptr_t)mbuf + idx * rxq->buf_stride);
+	MBUF_RAW_ALLOC_CHECK(m);
+	return m;
 }
 
 static struct rte_mbuf *
diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
index 946d96cac..8d0e69c64 100644
--- a/drivers/net/sfc/sfc_ef10_rx.c
+++ b/drivers/net/sfc/sfc_ef10_rx.c
@@ -133,6 +133,8 @@ sfc_ef10_rx_qrefill(struct sfc_ef10_rxq *rxq)
 			struct sfc_ef10_rx_sw_desc *rxd;
 			rte_iova_t phys_addr;
 
+			MBUF_RAW_ALLOC_CHECK(m);
+
 			SFC_ASSERT((id & ~ptr_mask) == 0);
 			rxd = &rxq->sw_ring[id];
 			rxd->mbuf = m;
diff --git a/drivers/net/sfc/sfc_rx.c b/drivers/net/sfc/sfc_rx.c
index 0385fca16..000cd0b20 100644
--- a/drivers/net/sfc/sfc_rx.c
+++ b/drivers/net/sfc/sfc_rx.c
@@ -96,13 +96,12 @@ sfc_efx_rx_qrefill(struct sfc_efx_rxq *rxq)
 		     ++i, id = (id + 1) & rxq->ptr_mask) {
 			m = objs[i];
 
+			MBUF_RAW_ALLOC_CHECK(m);
+
 			rxd = &rxq->sw_desc[id];
 			rxd->mbuf = m;
 
-			SFC_ASSERT(rte_mbuf_refcnt_read(m) == 1);
 			m->data_off = RTE_PKTMBUF_HEADROOM;
-			SFC_ASSERT(m->next == NULL);
-			SFC_ASSERT(m->nb_segs == 1);
 			m->port = port_id;
 
 			addr[i] = rte_pktmbuf_iova(m);
-- 
2.17.1

[dpdk-dev] [PATCH 4/9] net/sfc: avoid dummy writes to Rx queue state structure

[Andrew Rybchenko]

If there is no packets to be processed, it does not make sense
to write the same values back to Rx queue structure.

Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Reviewed-by: Ivan Malov <ivan.malov@oktetlabs.ru>
---
 drivers/net/sfc/sfc_ef10_rx.c | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
index 8d0e69c64..098610210 100644
--- a/drivers/net/sfc/sfc_ef10_rx.c
+++ b/drivers/net/sfc/sfc_ef10_rx.c
@@ -191,14 +191,18 @@ sfc_ef10_rx_prepared(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts,
 		     uint16_t nb_pkts)
 {
 	uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->prepared);
-	unsigned int completed = rxq->completed;
-	unsigned int i;
 
-	rxq->prepared -= n_rx_pkts;
-	rxq->completed = completed + n_rx_pkts;
+	if (n_rx_pkts != 0) {
+		unsigned int completed = rxq->completed;
 
-	for (i = 0; i < n_rx_pkts; ++i, ++completed)
-		rx_pkts[i] = rxq->sw_ring[completed & rxq->ptr_mask].mbuf;
+		rxq->prepared -= n_rx_pkts;
+		rxq->completed = completed + n_rx_pkts;
+
+		do {
+			*rx_pkts++ =
+				rxq->sw_ring[completed++ & rxq->ptr_mask].mbuf;
+		} while (completed != rxq->completed);
+	}
 
 	return n_rx_pkts;
 }
-- 
2.17.1

[dpdk-dev] [PATCH 5/9] net/sfc: decrease number of variables maintained on EF10 Rx

[Andrew Rybchenko]

Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Reviewed-by: Ivan Malov <ivan.malov@oktetlabs.ru>
---
 drivers/net/sfc/sfc_ef10_rx.c | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
index 098610210..d4af07d44 100644
--- a/drivers/net/sfc/sfc_ef10_rx.c
+++ b/drivers/net/sfc/sfc_ef10_rx.c
@@ -186,7 +186,7 @@ sfc_ef10_rx_prefetch_next(struct sfc_ef10_rxq *rxq, unsigned int next_id)
 	}
 }
 
-static uint16_t
+static struct rte_mbuf **
 sfc_ef10_rx_prepared(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts,
 		     uint16_t nb_pkts)
 {
@@ -204,7 +204,7 @@ sfc_ef10_rx_prepared(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts,
 		} while (completed != rxq->completed);
 	}
 
-	return n_rx_pkts;
+	return rx_pkts;
 }
 
 static uint16_t
@@ -219,9 +219,10 @@ sfc_ef10_rx_pseudo_hdr_get_hash(const uint8_t *pseudo_hdr)
 	return rte_le_to_cpu_32(*(const uint32_t *)pseudo_hdr);
 }
 
-static uint16_t
+static struct rte_mbuf **
 sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
-			  struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+			  struct rte_mbuf **rx_pkts,
+			  struct rte_mbuf ** const rx_pkts_end)
 {
 	const unsigned int ptr_mask = rxq->ptr_mask;
 	unsigned int completed = rxq->completed;
@@ -246,10 +247,10 @@ sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 			rxd = &rxq->sw_ring[completed++ & ptr_mask];
 			rte_mbuf_raw_free(rxd->mbuf);
 		}
-		return 0;
+		return rx_pkts;
 	}
 
-	n_rx_pkts = RTE_MIN(ready, nb_pkts);
+	n_rx_pkts = RTE_MIN(ready, rx_pkts_end - rx_pkts);
 	rxq->prepared = ready - n_rx_pkts;
 	rxq->completed += n_rx_pkts;
 
@@ -329,7 +330,7 @@ sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 		SFC_ASSERT(m->next == NULL);
 	}
 
-	return n_rx_pkts;
+	return rx_pkts;
 }
 
 static bool
@@ -361,26 +362,25 @@ static uint16_t
 sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 {
 	struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(rx_queue);
+	struct rte_mbuf ** const rx_pkts_end = &rx_pkts[nb_pkts];
 	unsigned int evq_old_read_ptr;
-	uint16_t n_rx_pkts;
 	efx_qword_t rx_ev;
 
-	n_rx_pkts = sfc_ef10_rx_prepared(rxq, rx_pkts, nb_pkts);
+	rx_pkts = sfc_ef10_rx_prepared(rxq, rx_pkts, nb_pkts);
 
 	if (unlikely(rxq->flags &
 		     (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION)))
 		goto done;
 
 	evq_old_read_ptr = rxq->evq_read_ptr;
-	while (n_rx_pkts != nb_pkts && sfc_ef10_rx_get_event(rxq, &rx_ev)) {
+	while (rx_pkts != rx_pkts_end && sfc_ef10_rx_get_event(rxq, &rx_ev)) {
 		/*
 		 * DROP_EVENT is an internal to the NIC, software should
 		 * never see it and, therefore, may ignore it.
 		 */
 
-		n_rx_pkts += sfc_ef10_rx_process_event(rxq, rx_ev,
-						       rx_pkts + n_rx_pkts,
-						       nb_pkts - n_rx_pkts);
+		rx_pkts = sfc_ef10_rx_process_event(rxq, rx_ev,
+						    rx_pkts, rx_pkts_end);
 	}
 
 	sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->ptr_mask, evq_old_read_ptr,
@@ -390,7 +390,7 @@ sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	sfc_ef10_rx_qrefill(rxq);
 
 done:
-	return n_rx_pkts;
+	return nb_pkts - (rx_pkts_end - rx_pkts);
 }
 
 const uint32_t *
-- 
2.17.1

[dpdk-dev] [PATCH 6/9] net/sfc: avoid usage of prepared packets number in EF10 Rx

[Andrew Rybchenko]

Number of prepared packets is good when one Rx descriptor is one packet.
Introduce pending Rx descriptor pointer which points to the first not
processed Rx descriptors. Rx descriptors from completed to pending have
buffers ready to be passed to application.

Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Reviewed-by: Ivan Malov <ivan.malov@oktetlabs.ru>
---
 drivers/net/sfc/sfc_ef10_rx.c | 53 +++++++++++++++++------------------
 1 file changed, 25 insertions(+), 28 deletions(-)

diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
index d4af07d44..9c921c057 100644
--- a/drivers/net/sfc/sfc_ef10_rx.c
+++ b/drivers/net/sfc/sfc_ef10_rx.c
@@ -57,7 +57,7 @@ struct sfc_ef10_rxq {
 #define SFC_EF10_RXQ_EXCEPTION		0x4
 #define SFC_EF10_RXQ_RSS_HASH		0x8
 	unsigned int			ptr_mask;
-	unsigned int			prepared;
+	unsigned int			pending;
 	unsigned int			completed;
 	unsigned int			evq_read_ptr;
 	efx_qword_t			*evq_hw_ring;
@@ -187,15 +187,14 @@ sfc_ef10_rx_prefetch_next(struct sfc_ef10_rxq *rxq, unsigned int next_id)
 }
 
 static struct rte_mbuf **
-sfc_ef10_rx_prepared(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts,
-		     uint16_t nb_pkts)
+sfc_ef10_rx_pending(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts,
+		    uint16_t nb_pkts)
 {
-	uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->prepared);
+	uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->pending - rxq->completed);
 
 	if (n_rx_pkts != 0) {
 		unsigned int completed = rxq->completed;
 
-		rxq->prepared -= n_rx_pkts;
 		rxq->completed = completed + n_rx_pkts;
 
 		do {
@@ -225,42 +224,40 @@ sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 			  struct rte_mbuf ** const rx_pkts_end)
 {
 	const unsigned int ptr_mask = rxq->ptr_mask;
-	unsigned int completed = rxq->completed;
+	unsigned int pending = rxq->pending;
 	unsigned int ready;
 	struct sfc_ef10_rx_sw_desc *rxd;
 	struct rte_mbuf *m;
 	struct rte_mbuf *m0;
-	uint16_t n_rx_pkts;
 	const uint8_t *pseudo_hdr;
 	uint16_t pkt_len;
 
-	ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - completed) &
+	ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - pending) &
 		EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
 	SFC_ASSERT(ready > 0);
 
+	rxq->pending = pending + ready;
+
 	if (rx_ev.eq_u64[0] &
 	    rte_cpu_to_le_64((1ull << ESF_DZ_RX_ECC_ERR_LBN) |
 			     (1ull << ESF_DZ_RX_ECRC_ERR_LBN))) {
-		SFC_ASSERT(rxq->prepared == 0);
-		rxq->completed += ready;
-		while (ready-- > 0) {
-			rxd = &rxq->sw_ring[completed++ & ptr_mask];
+		SFC_ASSERT(rxq->completed == pending);
+		do {
+			rxd = &rxq->sw_ring[pending++ & ptr_mask];
 			rte_mbuf_raw_free(rxd->mbuf);
-		}
+		} while (pending != rxq->pending);
+		rxq->completed = pending;
 		return rx_pkts;
 	}
 
-	n_rx_pkts = RTE_MIN(ready, rx_pkts_end - rx_pkts);
-	rxq->prepared = ready - n_rx_pkts;
-	rxq->completed += n_rx_pkts;
+	rxd = &rxq->sw_ring[pending++ & ptr_mask];
 
-	rxd = &rxq->sw_ring[completed++ & ptr_mask];
-
-	sfc_ef10_rx_prefetch_next(rxq, completed & ptr_mask);
+	sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask);
 
 	m = rxd->mbuf;
 
 	*rx_pkts++ = m;
+	rxq->completed = pending;
 
 	RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != sizeof(rxq->rearm_data));
 	m->rearm_data[0] = rxq->rearm_data;
@@ -294,15 +291,17 @@ sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 
 	/* Remember mbuf to copy offload flags and packet type from */
 	m0 = m;
-	for (--ready; ready > 0; --ready) {
-		rxd = &rxq->sw_ring[completed++ & ptr_mask];
+	while (pending != rxq->pending) {
+		rxd = &rxq->sw_ring[pending++ & ptr_mask];
 
-		sfc_ef10_rx_prefetch_next(rxq, completed & ptr_mask);
+		sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask);
 
 		m = rxd->mbuf;
 
-		if (ready > rxq->prepared)
+		if (rx_pkts != rx_pkts_end) {
 			*rx_pkts++ = m;
+			rxq->completed = pending;
+		}
 
 		RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) !=
 				 sizeof(rxq->rearm_data));
@@ -366,7 +365,7 @@ sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	unsigned int evq_old_read_ptr;
 	efx_qword_t rx_ev;
 
-	rx_pkts = sfc_ef10_rx_prepared(rxq, rx_pkts, nb_pkts);
+	rx_pkts = sfc_ef10_rx_pending(rxq, rx_pkts, nb_pkts);
 
 	if (unlikely(rxq->flags &
 		     (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION)))
@@ -601,8 +600,8 @@ sfc_ef10_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr)
 {
 	struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 
-	SFC_ASSERT(rxq->prepared == 0);
 	SFC_ASSERT(rxq->completed == 0);
+	SFC_ASSERT(rxq->pending == 0);
 	SFC_ASSERT(rxq->added == 0);
 
 	sfc_ef10_rx_qrefill(rxq);
@@ -650,15 +649,13 @@ sfc_ef10_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
 	unsigned int i;
 	struct sfc_ef10_rx_sw_desc *rxd;
 
-	rxq->prepared = 0;
-
 	for (i = rxq->completed; i != rxq->added; ++i) {
 		rxd = &rxq->sw_ring[i & rxq->ptr_mask];
 		rte_mbuf_raw_free(rxd->mbuf);
 		rxd->mbuf = NULL;
 	}
 
-	rxq->completed = rxq->added = 0;
+	rxq->completed = rxq->pending = rxq->added = 0;
 
 	rxq->flags &= ~SFC_EF10_RXQ_STARTED;
 }
-- 
2.17.1

[dpdk-dev] [PATCH 7/9] net/sfc: rename variable to prepare for scatter support

[Andrew Rybchenko]

In the case of scattered packet one Rx buffer is just a segment
of the whole packet.

Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Reviewed-by: Ivan Malov <ivan.malov@oktetlabs.ru>
---
 drivers/net/sfc/sfc_ef10_rx.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
index 9c921c057..78602bc3a 100644
--- a/drivers/net/sfc/sfc_ef10_rx.c
+++ b/drivers/net/sfc/sfc_ef10_rx.c
@@ -230,7 +230,7 @@ sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 	struct rte_mbuf *m;
 	struct rte_mbuf *m0;
 	const uint8_t *pseudo_hdr;
-	uint16_t pkt_len;
+	uint16_t seg_len;
 
 	ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - pending) &
 		EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
@@ -279,13 +279,13 @@ sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 	m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr);
 
 	if (ready == 1)
-		pkt_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES) -
+		seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES) -
 			rxq->prefix_size;
 	else
-		pkt_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr);
-	SFC_ASSERT(pkt_len > 0);
-	rte_pktmbuf_data_len(m) = pkt_len;
-	rte_pktmbuf_pkt_len(m) = pkt_len;
+		seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr);
+	SFC_ASSERT(seg_len > 0);
+	rte_pktmbuf_data_len(m) = seg_len;
+	rte_pktmbuf_pkt_len(m) = seg_len;
 
 	SFC_ASSERT(m->next == NULL);
 
@@ -321,10 +321,10 @@ sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 		 */
 		m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr);
 
-		pkt_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr);
-		SFC_ASSERT(pkt_len > 0);
-		rte_pktmbuf_data_len(m) = pkt_len;
-		rte_pktmbuf_pkt_len(m) = pkt_len;
+		seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr);
+		SFC_ASSERT(seg_len > 0);
+		rte_pktmbuf_data_len(m) = seg_len;
+		rte_pktmbuf_pkt_len(m) = seg_len;
 
 		SFC_ASSERT(m->next == NULL);
 	}
-- 
2.17.1

[dpdk-dev] [PATCH 8/9] net/sfc: support Rx scatter in EF10 Rx datapath

[Andrew Rybchenko]

Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Reviewed-by: Ivan Malov <ivan.malov@oktetlabs.ru>
---
 doc/guides/nics/sfc_efx.rst            |  2 +-
 doc/guides/rel_notes/release_18_11.rst |  6 +++
 drivers/net/sfc/sfc_ef10_rx.c          | 62 +++++++++++++++++++++++---
 3 files changed, 64 insertions(+), 6 deletions(-)

diff --git a/doc/guides/nics/sfc_efx.rst b/doc/guides/nics/sfc_efx.rst
index 425e669ed..a241f00bc 100644
--- a/doc/guides/nics/sfc_efx.rst
+++ b/doc/guides/nics/sfc_efx.rst
@@ -322,7 +322,7 @@ boolean parameters value.
   **efx** chooses libefx-based datapath which supports Rx scatter.
   **ef10** chooses EF10 (SFN7xxx, SFN8xxx, X2xxx) native datapath which is
   more efficient than libefx-based and provides richer packet type
-  classification, but lacks Rx scatter support.
+  classification.
   **ef10_esps** chooses SFNX2xxx equal stride packed stream datapath
   which may be used on DPDK firmware variant only
   (see notes about its limitations above).
diff --git a/doc/guides/rel_notes/release_18_11.rst b/doc/guides/rel_notes/release_18_11.rst
index a8327ea77..9d604d144 100644
--- a/doc/guides/rel_notes/release_18_11.rst
+++ b/doc/guides/rel_notes/release_18_11.rst
@@ -81,6 +81,12 @@ New Features
   * Support for runtime Rx and Tx queues setup.
   * Support multicast MAC address set.
 
+* **Updated Solarflare network PMD.**
+
+  Updated the sfc_efx driver including the following changes:
+
+  * Added support for Rx scatter in EF10 datapath implementation.
+
 * **Added Event Ethernet Tx Adapter.**
 
   Added event ethernet Tx adapter library that  provides configuration and
diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
index 78602bc3a..ff52a024b 100644
--- a/drivers/net/sfc/sfc_ef10_rx.c
+++ b/drivers/net/sfc/sfc_ef10_rx.c
@@ -63,6 +63,7 @@ struct sfc_ef10_rxq {
 	efx_qword_t			*evq_hw_ring;
 	struct sfc_ef10_rx_sw_desc	*sw_ring;
 	uint64_t			rearm_data;
+	struct rte_mbuf			*scatter_pkt;
 	uint16_t			prefix_size;
 
 	/* Used on refill */
@@ -192,6 +193,8 @@ sfc_ef10_rx_pending(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts,
 {
 	uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->pending - rxq->completed);
 
+	SFC_ASSERT(rxq->pending == rxq->completed || rxq->scatter_pkt == NULL);
+
 	if (n_rx_pkts != 0) {
 		unsigned int completed = rxq->completed;
 
@@ -234,7 +237,13 @@ sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 
 	ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - pending) &
 		EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
-	SFC_ASSERT(ready > 0);
+
+	if (ready == 0) {
+		/* Rx abort - it was no enough descriptors for Rx packet */
+		rte_pktmbuf_free(rxq->scatter_pkt);
+		rxq->scatter_pkt = NULL;
+		return rx_pkts;
+	}
 
 	rxq->pending = pending + ready;
 
@@ -250,15 +259,43 @@ sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 		return rx_pkts;
 	}
 
+	/* If scattered packet is in progress */
+	if (rxq->scatter_pkt != NULL) {
+		/* Events for scattered packet frags are not merged */
+		SFC_ASSERT(ready == 1);
+		SFC_ASSERT(rxq->completed == pending);
+
+		/* There is no pseudo-header in scatter segments. */
+		seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES);
+
+		rxd = &rxq->sw_ring[pending++ & ptr_mask];
+		m = rxd->mbuf;
+
+		MBUF_RAW_ALLOC_CHECK(m);
+
+		m->data_off = RTE_PKTMBUF_HEADROOM;
+		rte_pktmbuf_data_len(m) = seg_len;
+		rte_pktmbuf_pkt_len(m) = seg_len;
+
+		rxq->scatter_pkt->nb_segs++;
+		rte_pktmbuf_pkt_len(rxq->scatter_pkt) += seg_len;
+		rte_pktmbuf_lastseg(rxq->scatter_pkt)->next = m;
+
+		if (~rx_ev.eq_u64[0] &
+		    rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) {
+			*rx_pkts++ = rxq->scatter_pkt;
+			rxq->scatter_pkt = NULL;
+		}
+		rxq->completed = pending;
+		return rx_pkts;
+	}
+
 	rxd = &rxq->sw_ring[pending++ & ptr_mask];
 
 	sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask);
 
 	m = rxd->mbuf;
 
-	*rx_pkts++ = m;
-	rxq->completed = pending;
-
 	RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != sizeof(rxq->rearm_data));
 	m->rearm_data[0] = rxq->rearm_data;
 
@@ -289,6 +326,17 @@ sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 
 	SFC_ASSERT(m->next == NULL);
 
+	if (~rx_ev.eq_u64[0] & rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) {
+		*rx_pkts++ = m;
+		rxq->completed = pending;
+	} else {
+		/* Events with CONT bit are not merged */
+		SFC_ASSERT(ready == 1);
+		rxq->scatter_pkt = m;
+		rxq->completed = pending;
+		return rx_pkts;
+	}
+
 	/* Remember mbuf to copy offload flags and packet type from */
 	m0 = m;
 	while (pending != rxq->pending) {
@@ -649,6 +697,9 @@ sfc_ef10_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
 	unsigned int i;
 	struct sfc_ef10_rx_sw_desc *rxd;
 
+	rte_pktmbuf_free(rxq->scatter_pkt);
+	rxq->scatter_pkt = NULL;
+
 	for (i = rxq->completed; i != rxq->added; ++i) {
 		rxd = &rxq->sw_ring[i & rxq->ptr_mask];
 		rte_mbuf_raw_free(rxd->mbuf);
@@ -666,7 +717,8 @@ struct sfc_dp_rx sfc_ef10_rx = {
 		.type		= SFC_DP_RX,
 		.hw_fw_caps	= SFC_DP_HW_FW_CAP_EF10,
 	},
-	.features		= SFC_DP_RX_FEAT_MULTI_PROCESS |
+	.features		= SFC_DP_RX_FEAT_SCATTER |
+				  SFC_DP_RX_FEAT_MULTI_PROCESS |
 				  SFC_DP_RX_FEAT_TUNNELS |
 				  SFC_DP_RX_FEAT_CHECKSUM,
 	.get_dev_info		= sfc_ef10_rx_get_dev_info,
-- 
2.17.1

[dpdk-dev] [PATCH 9/9] net/sfc: support Rx descriptor status on EF10 datapath

[Andrew Rybchenko]

From: Igor Romanov <Igor.Romanov@oktetlabs.ru>

Signed-off-by: Igor Romanov <Igor.Romanov@oktetlabs.ru>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Reviewed-by: Ivan Malov <ivan.malov@oktetlabs.ru>
---
 doc/guides/rel_notes/release_18_11.rst |  1 +
 drivers/net/sfc/sfc_ef10_rx.c          | 46 ++++++++++++++++++++++----
 2 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/doc/guides/rel_notes/release_18_11.rst b/doc/guides/rel_notes/release_18_11.rst
index 9d604d144..e7d8d1fc7 100644
--- a/doc/guides/rel_notes/release_18_11.rst
+++ b/doc/guides/rel_notes/release_18_11.rst
@@ -86,6 +86,7 @@ New Features
   Updated the sfc_efx driver including the following changes:
 
   * Added support for Rx scatter in EF10 datapath implementation.
+  * Added support for Rx descriptor status API in EF10 datapath implementation.
 
 * **Added Event Ethernet Tx Adapter.**
 
diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
index ff52a024b..77ca580b5 100644
--- a/drivers/net/sfc/sfc_ef10_rx.c
+++ b/drivers/net/sfc/sfc_ef10_rx.c
@@ -500,21 +500,53 @@ sfc_ef10_supported_ptypes_get(uint32_t tunnel_encaps)
 
 static sfc_dp_rx_qdesc_npending_t sfc_ef10_rx_qdesc_npending;
 static unsigned int
-sfc_ef10_rx_qdesc_npending(__rte_unused struct sfc_dp_rxq *dp_rxq)
+sfc_ef10_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq)
 {
+	struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
+	efx_qword_t rx_ev;
+	const unsigned int evq_old_read_ptr = rxq->evq_read_ptr;
+	unsigned int pending = rxq->pending;
+	unsigned int ready;
+
+	if (unlikely(rxq->flags &
+		     (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION)))
+		goto done;
+
+	while (sfc_ef10_rx_get_event(rxq, &rx_ev)) {
+		ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) -
+			 pending) &
+			EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
+		pending += ready;
+	}
+
 	/*
-	 * Correct implementation requires EvQ polling and events
-	 * processing (keeping all ready mbufs in prepared).
+	 * The function does not process events, so return event queue read
+	 * pointer to the original position to allow the events that were
+	 * read to be processed later
 	 */
-	return -ENOTSUP;
+	rxq->evq_read_ptr = evq_old_read_ptr;
+
+done:
+	return pending - rxq->completed;
 }
 
 static sfc_dp_rx_qdesc_status_t sfc_ef10_rx_qdesc_status;
 static int
-sfc_ef10_rx_qdesc_status(__rte_unused struct sfc_dp_rxq *dp_rxq,
-			 __rte_unused uint16_t offset)
+sfc_ef10_rx_qdesc_status(struct sfc_dp_rxq *dp_rxq, uint16_t offset)
 {
-	return -ENOTSUP;
+	struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
+	unsigned int npending = sfc_ef10_rx_qdesc_npending(dp_rxq);
+
+	if (unlikely(offset > rxq->ptr_mask))
+		return -EINVAL;
+
+	if (offset < npending)
+		return RTE_ETH_RX_DESC_DONE;
+
+	if (offset < (rxq->added - rxq->completed))
+		return RTE_ETH_RX_DESC_AVAIL;
+
+	return RTE_ETH_RX_DESC_UNAVAIL;
 }
 
 
-- 
2.17.1

Re: [dpdk-dev] [PATCH 0/9] net/sfc: support more features in EF10 Rx

[Ferruh Yigit]

On 10/3/2018 10:03 AM, Andrew Rybchenko wrote:
> Andrew Rybchenko (8):
>   net/sfc: receive prepared packets even in Rx exception case
>   net/sfc: use mbuf raw free instead of mempool put directly
>   net/sfc: check mbufs allocated using mempool API for Rx
>   net/sfc: avoid dummy writes to Rx queue state structure
>   net/sfc: decrease number of variables maintained on EF10 Rx
>   net/sfc: avoid usage of prepared packets number in EF10 Rx
>   net/sfc: rename variable to prepare for scatter support
>   net/sfc: support Rx scatter in EF10 Rx datapath
> 
> Igor Romanov (1):
>   net/sfc: support Rx descriptor status on EF10 datapath

Series applied to dpdk-next-net/master, thanks.

Re: [dpdk-dev] [PATCH 1/9] net/sfc: receive prepared packets even in Rx exception case

[Ivan Malov]

On 10/3/2018 12:03 PM, Andrew Rybchenko wrote:
> Make sure that number of prepared packets, completed and added
> Rx ring pointers are reset to zeros on queue purge at stop.
> 
> Fixes: 638bddc99faa ("net/sfc: implement EF10 native Rx datapath")
> Cc: stable at dpdk.org
> 
> Signed-off-by: Andrew Rybchenko <arybchenko at solarflare.com>

Reviewed-by: Ivan Malov <Ivan.Malov@oktetlabs.ru>