[dpdk-dev] [PATCH 50/56] net/sfc: implement device callback to Rx burst of packets

[Andrew Rybchenko <arybchenko@solarflare.com>]

Reviewed-by: Andy Moreton <amoreton@solarflare.com>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
 drivers/net/sfc/efx/sfc_ethdev.c |  2 ++
 drivers/net/sfc/efx/sfc_ev.c     | 47 ++++++++++++++++++++++++--
 drivers/net/sfc/efx/sfc_rx.c     | 72 +++++++++++++++++++++++++++++++++++++++-
 drivers/net/sfc/efx/sfc_rx.h     |  7 ++++
 4 files changed, 125 insertions(+), 3 deletions(-)

diff --git a/drivers/net/sfc/efx/sfc_ethdev.c b/drivers/net/sfc/efx/sfc_ethdev.c
index c889325..61ae1bd 100644
--- a/drivers/net/sfc/efx/sfc_ethdev.c
+++ b/drivers/net/sfc/efx/sfc_ethdev.c
@@ -310,6 +310,7 @@ sfc_eth_dev_init(struct rte_eth_dev *dev)
 	ether_addr_copy(from, &dev->data->mac_addrs[0]);
 
 	dev->dev_ops = &sfc_eth_dev_ops;
+	dev->rx_pkt_burst = &sfc_recv_pkts;
 
 	sfc_adapter_unlock(sa);
 
@@ -347,6 +348,7 @@ sfc_eth_dev_uninit(struct rte_eth_dev *dev)
 	dev->data->mac_addrs = NULL;
 
 	dev->dev_ops = NULL;
+	dev->rx_pkt_burst = NULL;
 
 	sfc_kvargs_cleanup(sa);
 
diff --git a/drivers/net/sfc/efx/sfc_ev.c b/drivers/net/sfc/efx/sfc_ev.c
index 6750bfb..ae966fc 100644
--- a/drivers/net/sfc/efx/sfc_ev.c
+++ b/drivers/net/sfc/efx/sfc_ev.c
@@ -69,9 +69,52 @@ static boolean_t
 sfc_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, uint16_t flags)
 {
 	struct sfc_evq *evq = arg;
+	struct sfc_rxq *rxq;
+	unsigned int stop;
+	unsigned int pending_id;
+	unsigned int delta;
+	unsigned int i;
+	struct sfc_rx_sw_desc *rxd;
 
-	sfc_err(evq->sa, "EVQ %u unexpected Rx event", evq->evq_index);
-	return B_TRUE;
+	if (unlikely(evq->exception))
+		goto done;
+
+	rxq = evq->rxq;
+
+	SFC_ASSERT(rxq != NULL);
+	SFC_ASSERT(rxq->evq == evq);
+	SFC_ASSERT(rxq->state & SFC_RXQ_STARTED);
+
+	stop = (id + 1) & rxq->ptr_mask;
+	pending_id = rxq->pending & rxq->ptr_mask;
+	delta = (stop >= pending_id) ? (stop - pending_id) :
+		(rxq->ptr_mask + 1 - pending_id + stop);
+
+	if (unlikely(delta > rxq->batch_max)) {
+		evq->exception = B_TRUE;
+
+		sfc_err(evq->sa,
+			"EVQ %u RxQ %u completion out of order "
+			"(id=%#x delta=%u flags=%#x); needs restart\n",
+			evq->evq_index, sfc_rxq_sw_index(rxq), id, delta,
+			flags);
+
+		goto done;
+	}
+
+	for (i = pending_id; i != stop; i = (i + 1) & rxq->ptr_mask) {
+		rxd = &rxq->sw_desc[i];
+
+		rxd->flags = flags;
+
+		SFC_ASSERT(size < (1 << 16));
+		rxd->size = (uint16_t)size;
+	}
+
+	rxq->pending += delta;
+
+done:
+	return B_FALSE;
 }
 
 static boolean_t
diff --git a/drivers/net/sfc/efx/sfc_rx.c b/drivers/net/sfc/efx/sfc_rx.c
index 8e82ee0..307734e 100644
--- a/drivers/net/sfc/efx/sfc_rx.c
+++ b/drivers/net/sfc/efx/sfc_rx.c
@@ -32,6 +32,7 @@
 #include "efx.h"
 
 #include "sfc.h"
+#include "sfc_debug.h"
 #include "sfc_log.h"
 #include "sfc_ev.h"
 #include "sfc_rx.h"
@@ -129,6 +130,69 @@ sfc_rx_qrefill(struct sfc_rxq *rxq)
 	}
 }
 
+uint16_t
+sfc_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+	struct sfc_rxq *rxq = rx_queue;
+	unsigned int completed;
+	unsigned int prefix_size = rxq->prefix_size;
+	unsigned int done_pkts = 0;
+
+	if (unlikely((rxq->state & SFC_RXQ_RUNNING) == 0))
+		return 0;
+
+	sfc_ev_qpoll(rxq->evq);
+
+	completed = rxq->completed;
+	while (completed != rxq->pending && done_pkts < nb_pkts) {
+		unsigned int id;
+		struct sfc_rx_sw_desc *rxd;
+		struct rte_mbuf *m;
+		unsigned int seg_len;
+		unsigned int desc_flags;
+
+		id = completed++ & rxq->ptr_mask;
+		rxd = &rxq->sw_desc[id];
+		m = rxd->mbuf;
+		desc_flags = rxd->flags;
+
+		if (desc_flags & (EFX_ADDR_MISMATCH | EFX_DISCARD))
+			goto discard;
+
+		if (desc_flags & EFX_PKT_PREFIX_LEN) {
+			uint16_t tmp_size;
+			int rc;
+
+			rc = efx_psuedo_hdr_pkt_length_get(rxq->common,
+				rte_pktmbuf_mtod(m, uint8_t *), &tmp_size);
+			SFC_ASSERT(rc == 0);
+			seg_len = tmp_size;
+		} else {
+			seg_len = rxd->size - prefix_size;
+		}
+
+		m->data_off += prefix_size;
+		rte_pktmbuf_data_len(m) = seg_len;
+		rte_pktmbuf_pkt_len(m) = seg_len;
+
+		m->packet_type = RTE_PTYPE_L2_ETHER;
+
+		*rx_pkts++ = m;
+		done_pkts++;
+		continue;
+
+discard:
+		rte_mempool_put(rxq->refill_mb_pool, m);
+		rxd->mbuf = NULL;
+	}
+
+	rxq->completed = completed;
+
+	sfc_rx_qrefill(rxq);
+
+	return done_pkts;
+}
+
 static void
 sfc_rx_qpurge(struct sfc_rxq *rxq)
 {
@@ -226,7 +290,7 @@ sfc_rx_qstart(struct sfc_adapter *sa, unsigned int sw_index)
 
 	rxq->pending = rxq->completed = rxq->added = rxq->pushed = 0;
 
-	rxq->state |= SFC_RXQ_STARTED;
+	rxq->state |= (SFC_RXQ_STARTED | SFC_RXQ_RUNNING);
 
 	sfc_rx_qrefill(rxq);
 
@@ -271,6 +335,8 @@ sfc_rx_qstop(struct sfc_adapter *sa, unsigned int sw_index)
 	sa->eth_dev->data->rx_queue_state[sw_index] =
 		RTE_ETH_QUEUE_STATE_STOPPED;
 
+	rxq->state &= ~SFC_RXQ_RUNNING;
+
 	if (sw_index == 0)
 		efx_mac_filter_default_rxq_clear(sa->nic);
 
@@ -494,6 +560,10 @@ sfc_rx_qinit(struct sfc_adapter *sa, unsigned int sw_index,
 	rxq->hw_index = sw_index;
 	rxq->port_id = sa->eth_dev->data->port_id;
 
+	/* Cache limits required on datapath in RxQ structure */
+	rxq->batch_max = encp->enc_rx_batch_max;
+	rxq->prefix_size = encp->enc_rx_prefix_size;
+
 	rxq->state = SFC_RXQ_INITIALIZED;
 
 	rxq_info->rxq = rxq;
diff --git a/drivers/net/sfc/efx/sfc_rx.h b/drivers/net/sfc/efx/sfc_rx.h
index 1004a84..028b8e0 100644
--- a/drivers/net/sfc/efx/sfc_rx.h
+++ b/drivers/net/sfc/efx/sfc_rx.h
@@ -59,6 +59,8 @@ enum sfc_rxq_state_bit {
 #define	SFC_RXQ_INITIALIZED	(1 << SFC_RXQ_INITIALIZED_BIT)
 	SFC_RXQ_STARTED_BIT,
 #define	SFC_RXQ_STARTED		(1 << SFC_RXQ_STARTED_BIT)
+	SFC_RXQ_RUNNING_BIT,
+#define	SFC_RXQ_RUNNING		(1 << SFC_RXQ_RUNNING_BIT)
 	SFC_RXQ_FLUSHING_BIT,
 #define	SFC_RXQ_FLUSHING	(1 << SFC_RXQ_FLUSHING_BIT)
 	SFC_RXQ_FLUSHED_BIT,
@@ -79,6 +81,8 @@ struct sfc_rxq {
 	unsigned int		ptr_mask;
 	unsigned int		pending;
 	unsigned int		completed;
+	uint16_t		batch_max;
+	uint16_t		prefix_size;
 
 	/* Used on refill */
 	unsigned int		added;
@@ -132,6 +136,9 @@ void sfc_rx_qstop(struct sfc_adapter *sa, unsigned int sw_index);
 void sfc_rx_qflush_done(struct sfc_rxq *rxq);
 void sfc_rx_qflush_failed(struct sfc_rxq *rxq);
 
+uint16_t sfc_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+		       uint16_t nb_pkts);
+
 #ifdef __cplusplus
 }
 #endif
-- 
2.5.5