[dpdk-dev] [PATCH 28/38] net/sfc/base: add support for the Rx event mode w/o continue

[Andrew Rybchenko <arybchenko@solarflare.com>]

From: Mark Spender <mspender@solarflare.com>

The recently added NO_CONT_EV mode is recommended for when looking for
maximum throughput on 100G links as it allows the firmware to operate
more efficiently. The biggest benefit is when using scatter and jumbo
frames, but it is also necessary to achieve line rate in other cases.

Support for NO_CONT_EV when scatter is disabled is simple - the main
datapth change is to always read the packet length from the prefix,
not the event. This requires storing a flag with the event queue so
the event handler knows NO_CONT_EV mode is in use.

Supporting NO_CONT_EV with scatter would require an API change.

(Now the ee_flags field in efx_evq_t is used on the datapath, move it
next to other fields which are also read on the datapath to try to avoid
reading an additional cache line.)

Signed-off-by: Mark Spender <mspender@solarflare.com>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
 drivers/net/sfc/base/ef10_ev.c  | 65 +++++++++++++++++++++++++++------
 drivers/net/sfc/base/ef10_nic.c |  8 ++++
 drivers/net/sfc/base/ef10_rx.c  | 23 +++++++++---
 drivers/net/sfc/base/efx.h      | 11 ++++++
 drivers/net/sfc/base/efx_impl.h |  3 +-
 5 files changed, 91 insertions(+), 19 deletions(-)

diff --git a/drivers/net/sfc/base/ef10_ev.c b/drivers/net/sfc/base/ef10_ev.c
index b80cb0a35..202930876 100644
--- a/drivers/net/sfc/base/ef10_ev.c
+++ b/drivers/net/sfc/base/ef10_ev.c
@@ -457,6 +457,23 @@ ef10_ev_qcreate(
 		goto fail2;
 	}
 
+	/*
+	 * NO_CONT_EV mode is only requested from the firmware when creating
+	 * receive queues, but here it needs to be specified at event queue
+	 * creation, as the event handler needs to know which format is in use.
+	 *
+	 * If EFX_EVQ_FLAGS_NO_CONT_EV is specified, all receive queues for this
+	 * event queue will be created in NO_CONT_EV mode.
+	 *
+	 * See SF-109306-TC 5.11 "Events for RXQs in NO_CONT_EV mode".
+	 */
+	if (flags & EFX_EVQ_FLAGS_NO_CONT_EV) {
+		if (enp->en_nic_cfg.enc_no_cont_ev_mode_supported == B_FALSE) {
+			rc = EINVAL;
+			goto fail3;
+		}
+	}
+
 	/* Set up the handler table */
 	eep->ee_rx	= ef10_ev_rx;
 	eep->ee_tx	= ef10_ev_tx;
@@ -494,7 +511,7 @@ ef10_ev_qcreate(
 		rc = efx_mcdi_init_evq_v2(enp, index, esmp, ndescs, irq, us,
 		    flags);
 		if (rc != 0)
-			goto fail3;
+			goto fail4;
 	} else {
 		/*
 		 * On Huntington we need to specify the settings to use.
@@ -511,11 +528,13 @@ ef10_ev_qcreate(
 		rc = efx_mcdi_init_evq(enp, index, esmp, ndescs, irq, us, flags,
 		    low_latency);
 		if (rc != 0)
-			goto fail4;
+			goto fail5;
 	}
 
 	return (0);
 
+fail5:
+	EFSYS_PROBE(fail5);
 fail4:
 	EFSYS_PROBE(fail4);
 fail3:
@@ -912,23 +931,47 @@ ef10_ev_rx(
 	if (mac_class == ESE_DZ_MAC_CLASS_UCAST)
 		flags |= EFX_PKT_UNICAST;
 
-	/* Increment the count of descriptors read */
+	/*
+	 * Increment the count of descriptors read.
+	 *
+	 * In NO_CONT_EV mode, RX_DSC_PTR_LBITS is actually a packet count, but
+	 * when scatter is disabled, there is only one descriptor per packet and
+	 * so it can be treated the same.
+	 *
+	 * TODO: Support scatter in NO_CONT_EV mode.
+	 */
 	desc_count = (next_read_lbits - eersp->eers_rx_read_ptr) &
 	    EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
 	eersp->eers_rx_read_ptr += desc_count;
 
-	/*
-	 * FIXME: add error checking to make sure this a batched event.
-	 * This could also be an aborted scatter, see Bug36629.
-	 */
-	if (desc_count > 1) {
+	/* Calculate the index of the last descriptor consumed */
+	last_used_id = (eersp->eers_rx_read_ptr - 1) & eersp->eers_rx_mask;
+
+	if (eep->ee_flags & EFX_EVQ_FLAGS_NO_CONT_EV) {
+		if (desc_count > 1)
+			EFX_EV_QSTAT_INCR(eep, EV_RX_BATCH);
+
+		/* Always read the length from the prefix in NO_CONT_EV mode. */
+		flags |= EFX_PKT_PREFIX_LEN;
+
+		/*
+		 * Check for an aborted scatter, signalled by the ABORT bit in
+		 * NO_CONT_EV mode. The ABORT bit was not used before NO_CONT_EV
+		 * mode was added as it was broken in Huntington silicon.
+		 */
+		if (EFX_QWORD_FIELD(*eqp, ESF_EZ_RX_ABORT) != 0) {
+			flags |= EFX_DISCARD;
+			goto deliver;
+		}
+	} else if (desc_count > 1) {
+		/*
+		 * FIXME: add error checking to make sure this a batched event.
+		 * This could also be an aborted scatter, see Bug36629.
+		 */
 		EFX_EV_QSTAT_INCR(eep, EV_RX_BATCH);
 		flags |= EFX_PKT_PREFIX_LEN;
 	}
 
-	/* Calculate the index of the last descriptor consumed */
-	last_used_id = (eersp->eers_rx_read_ptr - 1) & eersp->eers_rx_mask;
-
 	/* Check for errors that invalidate checksum and L3/L4 fields */
 	if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_TRUNC_ERR) != 0) {
 		/* RX frame truncated */
diff --git a/drivers/net/sfc/base/ef10_nic.c b/drivers/net/sfc/base/ef10_nic.c
index d68920638..39ca53f03 100644
--- a/drivers/net/sfc/base/ef10_nic.c
+++ b/drivers/net/sfc/base/ef10_nic.c
@@ -1197,6 +1197,14 @@ ef10_get_datapath_caps(
 	else
 		encp->enc_init_evq_v2_supported = B_FALSE;
 
+	/*
+	 * Check if the NO_CONT_EV mode for RX events is supported.
+	 */
+	if (CAP_FLAGS2(req, INIT_RXQ_NO_CONT_EV))
+		encp->enc_no_cont_ev_mode_supported = B_TRUE;
+	else
+		encp->enc_no_cont_ev_mode_supported = B_FALSE;
+
 	/*
 	 * Check if firmware-verified NVRAM updates must be used.
 	 *
diff --git a/drivers/net/sfc/base/ef10_rx.c b/drivers/net/sfc/base/ef10_rx.c
index 23b80d78f..3b296e488 100644
--- a/drivers/net/sfc/base/ef10_rx.c
+++ b/drivers/net/sfc/base/ef10_rx.c
@@ -15,7 +15,7 @@ static	__checkReturn	efx_rc_t
 efx_mcdi_init_rxq(
 	__in		efx_nic_t *enp,
 	__in		uint32_t ndescs,
-	__in		uint32_t target_evq,
+	__in		efx_evq_t *eep,
 	__in		uint32_t label,
 	__in		uint32_t instance,
 	__in		efsys_mem_t *esmp,
@@ -38,6 +38,7 @@ efx_mcdi_init_rxq(
 	efx_rc_t rc;
 	uint32_t dma_mode;
 	boolean_t want_outer_classes;
+	boolean_t no_cont_ev;
 
 	EFSYS_ASSERT3U(ndescs, <=, encp->enc_rxq_max_ndescs);
 
@@ -47,6 +48,13 @@ efx_mcdi_init_rxq(
 		goto fail1;
 	}
 
+	no_cont_ev = (eep->ee_flags & EFX_EVQ_FLAGS_NO_CONT_EV);
+	if ((no_cont_ev == B_TRUE) && (disable_scatter == B_FALSE)) {
+		/* TODO: Support scatter in NO_CONT_EV mode */
+		rc = EINVAL;
+		goto fail2;
+	}
+
 	if (ps_bufsize > 0)
 		dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
 	else if (es_bufs_per_desc > 0)
@@ -81,10 +89,10 @@ efx_mcdi_init_rxq(
 	req.emr_out_length = MC_CMD_INIT_RXQ_V3_OUT_LEN;
 
 	MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs);
-	MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
+	MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, eep->ee_index);
 	MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
 	MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
-	MCDI_IN_POPULATE_DWORD_9(req, INIT_RXQ_EXT_IN_FLAGS,
+	MCDI_IN_POPULATE_DWORD_10(req, INIT_RXQ_EXT_IN_FLAGS,
 	    INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
 	    INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
 	    INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
@@ -94,7 +102,8 @@ efx_mcdi_init_rxq(
 	    INIT_RXQ_EXT_IN_DMA_MODE,
 	    dma_mode,
 	    INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize,
-	    INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes);
+	    INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes,
+	    INIT_RXQ_EXT_IN_FLAG_NO_CONT_EV, no_cont_ev);
 	MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
 	MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
 
@@ -127,11 +136,13 @@ efx_mcdi_init_rxq(
 
 	if (req.emr_rc != 0) {
 		rc = req.emr_rc;
-		goto fail2;
+		goto fail3;
 	}
 
 	return (0);
 
+fail3:
+	EFSYS_PROBE(fail3);
 fail2:
 	EFSYS_PROBE(fail2);
 fail1:
@@ -1122,7 +1133,7 @@ ef10_rx_qcreate(
 	else
 		want_inner_classes = B_FALSE;
 
-	if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep->ee_index, label, index,
+	if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep, label, index,
 		    esmp, disable_scatter, want_inner_classes,
 		    ps_buf_size, es_bufs_per_desc, es_max_dma_len,
 		    es_buf_stride, hol_block_timeout)) != 0)
diff --git a/drivers/net/sfc/base/efx.h b/drivers/net/sfc/base/efx.h
index f7d0a4f67..f5ad095d4 100644
--- a/drivers/net/sfc/base/efx.h
+++ b/drivers/net/sfc/base/efx.h
@@ -1370,6 +1370,7 @@ typedef struct efx_nic_cfg_s {
 	boolean_t		enc_allow_set_mac_with_installed_filters;
 	boolean_t		enc_enhanced_set_mac_supported;
 	boolean_t		enc_init_evq_v2_supported;
+	boolean_t		enc_no_cont_ev_mode_supported;
 	boolean_t		enc_rx_packed_stream_supported;
 	boolean_t		enc_rx_var_packed_stream_supported;
 	boolean_t		enc_rx_es_super_buffer_supported;
@@ -2000,6 +2001,16 @@ efx_evq_nbufs(
 #define	EFX_EVQ_FLAGS_NOTIFY_INTERRUPT	(0x0)	/* Interrupting (default) */
 #define	EFX_EVQ_FLAGS_NOTIFY_DISABLED	(0x4)	/* Non-interrupting */
 
+/*
+ * Use the NO_CONT_EV RX event format, which allows the firmware to operate more
+ * efficiently at high data rates. See SF-109306-TC 5.11 "Events for RXQs in
+ * NO_CONT_EV mode".
+ *
+ * NO_CONT_EV requires EVQ_RX_MERGE and RXQ_FORCED_EV_MERGING to both be set,
+ * which is the case when an event queue is set to THROUGHPUT mode.
+ */
+#define	EFX_EVQ_FLAGS_NO_CONT_EV	(0x10)
+
 extern	__checkReturn	efx_rc_t
 efx_ev_qcreate(
 	__in		efx_nic_t *enp,
diff --git a/drivers/net/sfc/base/efx_impl.h b/drivers/net/sfc/base/efx_impl.h
index bad23f819..e2cdba692 100644
--- a/drivers/net/sfc/base/efx_impl.h
+++ b/drivers/net/sfc/base/efx_impl.h
@@ -760,6 +760,7 @@ typedef struct efx_evq_rxq_state_s {
 
 struct efx_evq_s {
 	uint32_t			ee_magic;
+	uint32_t			ee_flags;
 	efx_nic_t			*ee_enp;
 	unsigned int			ee_index;
 	unsigned int			ee_mask;
@@ -778,8 +779,6 @@ struct efx_evq_s {
 #endif	/* EFSYS_OPT_MCDI */
 
 	efx_evq_rxq_state_t		ee_rxq_state[EFX_EV_RX_NLABELS];
-
-	uint32_t			ee_flags;
 };
 
 #define	EFX_EVQ_MAGIC	0x08081997
-- 
2.17.1