[dpdk-dev] [PATCH 25/56] net/sfc: import libefx support for Rx packed stream mode

[Andrew Rybchenko <arybchenko@solarflare.com>]

In packed stream mode, large buffers are provided to the NIC
into which many packets can be delivered. This reduces the
number of queue refills needed compared to delivering every
packet into a separate buffer.

EFSYS_OPT_RX_PACKED_STREAM should be enabled to use it.

>From Solarflare Communications Inc.

Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
 drivers/net/sfc/efx/base/ef10_ev.c   | 124 +++++++++++++++++++++++++++++++++++
 drivers/net/sfc/efx/base/ef10_impl.h |  46 +++++++++++++
 drivers/net/sfc/efx/base/ef10_rx.c   | 114 ++++++++++++++++++++++++++++++++
 drivers/net/sfc/efx/base/efx.h       |  48 ++++++++++++++
 drivers/net/sfc/efx/base/efx_check.h |   7 ++
 drivers/net/sfc/efx/base/efx_impl.h  |  11 ++++
 drivers/net/sfc/efx/base/efx_rx.c    |  84 ++++++++++++++++++++++++
 7 files changed, 434 insertions(+)

diff --git a/drivers/net/sfc/efx/base/ef10_ev.c b/drivers/net/sfc/efx/base/ef10_ev.c
index e93b458..3522674 100644
--- a/drivers/net/sfc/efx/base/ef10_ev.c
+++ b/drivers/net/sfc/efx/base/ef10_ev.c
@@ -759,6 +759,84 @@ ef10_ev_qstats_update(
 }
 #endif /* EFSYS_OPT_QSTATS */
 
+#if EFSYS_OPT_RX_PACKED_STREAM
+
+static	__checkReturn	boolean_t
+ef10_ev_rx_packed_stream(
+	__in		efx_evq_t *eep,
+	__in		efx_qword_t *eqp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	uint32_t label;
+	uint32_t next_read_lbits;
+	uint16_t flags;
+	boolean_t should_abort;
+	efx_evq_rxq_state_t *eersp;
+	unsigned int pkt_count;
+	unsigned int current_id;
+	boolean_t new_buffer;
+
+	next_read_lbits = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_DSC_PTR_LBITS);
+	label = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_QLABEL);
+	new_buffer = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_EV_ROTATE);
+
+	flags = 0;
+
+	eersp = &eep->ee_rxq_state[label];
+	pkt_count = (EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS) + 1 +
+	    next_read_lbits - eersp->eers_rx_stream_npackets) &
+	    EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
+	eersp->eers_rx_stream_npackets += pkt_count;
+
+	if (new_buffer) {
+		flags |= EFX_PKT_PACKED_STREAM_NEW_BUFFER;
+		if (eersp->eers_rx_packed_stream_credits <
+		    EFX_RX_PACKED_STREAM_MAX_CREDITS)
+			eersp->eers_rx_packed_stream_credits++;
+		eersp->eers_rx_read_ptr++;
+	}
+	current_id = eersp->eers_rx_read_ptr & eersp->eers_rx_mask;
+
+	/* Check for errors that invalidate checksum and L3/L4 fields */
+	if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ECC_ERR) != 0) {
+		/* RX frame truncated (error flag is misnamed) */
+		EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
+		flags |= EFX_DISCARD;
+		goto deliver;
+	}
+	if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ECRC_ERR) != 0) {
+		/* Bad Ethernet frame CRC */
+		EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
+		flags |= EFX_DISCARD;
+		goto deliver;
+	}
+
+	if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_PARSE_INCOMPLETE)) {
+		flags |= EFX_PKT_PACKED_STREAM_PARSE_INCOMPLETE;
+		goto deliver;
+	}
+
+	if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_IPCKSUM_ERR))
+		EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
+
+	if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_TCPUDP_CKSUM_ERR))
+		EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
+
+deliver:
+	/* If we're not discarding the packet then it is ok */
+	if (~flags & EFX_DISCARD)
+		EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
+
+	EFSYS_ASSERT(eecp->eec_rx_ps != NULL);
+	should_abort = eecp->eec_rx_ps(arg, label, current_id, pkt_count,
+	    flags);
+
+	return (should_abort);
+}
+
+#endif /* EFSYS_OPT_RX_PACKED_STREAM */
+
 static	__checkReturn	boolean_t
 ef10_ev_rx(
 	__in		efx_evq_t *eep,
@@ -791,6 +869,15 @@ ef10_ev_rx(
 	label = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_QLABEL);
 	eersp = &eep->ee_rxq_state[label];
 
+#if EFSYS_OPT_RX_PACKED_STREAM
+	/*
+	 * Packed stream events are very different,
+	 * so handle them separately
+	 */
+	if (eersp->eers_rx_packed_stream)
+		return (ef10_ev_rx_packed_stream(eep, eqp, eecp, arg));
+#endif
+
 	size = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_BYTES);
 	next_read_lbits = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_DSC_PTR_LBITS);
 	eth_tag_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ETH_TAG_CLASS);
@@ -1253,9 +1340,41 @@ ef10_ev_rxlabel_init(
 
 	EFSYS_ASSERT3U(eersp->eers_rx_mask, ==, 0);
 
+#if EFSYS_OPT_RX_PACKED_STREAM
+	/*
+	 * For packed stream modes, the very first event will
+	 * have a new buffer flag set, so it will be incremented,
+	 * yielding the correct pointer. That results in a simpler
+	 * code than trying to detect start-of-the-world condition
+	 * in the event handler.
+	 */
+	eersp->eers_rx_read_ptr = packed_stream ? ~0 : 0;
+#else
 	eersp->eers_rx_read_ptr = 0;
+#endif
 	eersp->eers_rx_mask = erp->er_mask;
+#if EFSYS_OPT_RX_PACKED_STREAM
+	eersp->eers_rx_stream_npackets = 0;
+	eersp->eers_rx_packed_stream = packed_stream;
+	if (packed_stream) {
+		eersp->eers_rx_packed_stream_credits = (eep->ee_mask + 1) /
+		    (EFX_RX_PACKED_STREAM_MEM_PER_CREDIT /
+		    EFX_RX_PACKED_STREAM_MIN_PACKET_SPACE);
+		EFSYS_ASSERT3U(eersp->eers_rx_packed_stream_credits, !=, 0);
+		/*
+		 * A single credit is allocated to the queue when it is started.
+		 * It is immediately spent by the first packet which has NEW
+		 * BUFFER flag set, though, but still we shall take into
+		 * account, as to not wrap around the maximum number of credits
+		 * accidentally
+		 */
+		eersp->eers_rx_packed_stream_credits--;
+		EFSYS_ASSERT3U(eersp->eers_rx_packed_stream_credits, <=,
+		    EFX_RX_PACKED_STREAM_MAX_CREDITS);
+	}
+#else
 	EFSYS_ASSERT(!packed_stream);
+#endif
 }
 
 		void
@@ -1272,6 +1391,11 @@ ef10_ev_rxlabel_fini(
 
 	eersp->eers_rx_read_ptr = 0;
 	eersp->eers_rx_mask = 0;
+#if EFSYS_OPT_RX_PACKED_STREAM
+	eersp->eers_rx_stream_npackets = 0;
+	eersp->eers_rx_packed_stream = B_FALSE;
+	eersp->eers_rx_packed_stream_credits = 0;
+#endif
 }
 
 #endif	/* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
diff --git a/drivers/net/sfc/efx/base/ef10_impl.h b/drivers/net/sfc/efx/base/ef10_impl.h
index e468b24..8c527b1 100644
--- a/drivers/net/sfc/efx/base/ef10_impl.h
+++ b/drivers/net/sfc/efx/base/ef10_impl.h
@@ -462,6 +462,22 @@ ef10_tx_qpush(
 	__in		unsigned int added,
 	__in		unsigned int pushed);
 
+#if EFSYS_OPT_RX_PACKED_STREAM
+extern			void
+ef10_rx_qps_update_credits(
+	__in	efx_rxq_t *erp);
+
+extern	__checkReturn	uint8_t *
+ef10_rx_qps_packet_info(
+	__in		efx_rxq_t *erp,
+	__in		uint8_t *buffer,
+	__in		uint32_t buffer_length,
+	__in		uint32_t current_offset,
+	__out		uint16_t *lengthp,
+	__out		uint32_t *next_offsetp,
+	__out		uint32_t *timestamp);
+#endif
+
 extern	__checkReturn	efx_rc_t
 ef10_tx_qpace(
 	__in		efx_txq_t *etp,
@@ -844,6 +860,36 @@ ef10_external_port_mapping(
 	__in		uint32_t port,
 	__out		uint8_t *external_portp);
 
+#if EFSYS_OPT_RX_PACKED_STREAM
+
+/* Data space per credit in packed stream mode */
+#define	EFX_RX_PACKED_STREAM_MEM_PER_CREDIT (1 << 16)
+
+/*
+ * Received packets are always aligned at this boundary. Also there always
+ * exists a gap of this size between packets.
+ * (see SF-112241-TC, 4.5)
+ */
+#define	EFX_RX_PACKED_STREAM_ALIGNMENT 64
+
+/*
+ * Size of a pseudo-header prepended to received packets
+ * in packed stream mode
+ */
+#define	EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE 8
+
+/* Minimum space for packet in packed stream mode */
+#define	EFX_RX_PACKED_STREAM_MIN_PACKET_SPACE		     \
+	P2ROUNDUP(EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE +	     \
+		  EFX_MAC_PDU_MIN +			     \
+		  EFX_RX_PACKED_STREAM_ALIGNMENT,	     \
+		  EFX_RX_PACKED_STREAM_ALIGNMENT)
+
+/* Maximum number of credits */
+#define	EFX_RX_PACKED_STREAM_MAX_CREDITS 127
+
+#endif /* EFSYS_OPT_RX_PACKED_STREAM */
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/drivers/net/sfc/efx/base/ef10_rx.c b/drivers/net/sfc/efx/base/ef10_rx.c
index 09a6314..2bcd823 100644
--- a/drivers/net/sfc/efx/base/ef10_rx.c
+++ b/drivers/net/sfc/efx/base/ef10_rx.c
@@ -714,6 +714,81 @@ ef10_rx_qpush(
 			    erp->er_index, &dword, B_FALSE);
 }
 
+#if EFSYS_OPT_RX_PACKED_STREAM
+
+			void
+ef10_rx_qps_update_credits(
+	__in	efx_rxq_t *erp)
+{
+	efx_nic_t *enp = erp->er_enp;
+	efx_dword_t dword;
+	efx_evq_rxq_state_t *rxq_state =
+		&erp->er_eep->ee_rxq_state[erp->er_label];
+
+	EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
+
+	if (rxq_state->eers_rx_packed_stream_credits == 0)
+		return;
+
+	EFX_POPULATE_DWORD_3(dword,
+	    ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
+	    ERF_DZ_RX_DESC_MAGIC_CMD,
+	    ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
+	    ERF_DZ_RX_DESC_MAGIC_DATA,
+	    rxq_state->eers_rx_packed_stream_credits);
+	EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
+	    erp->er_index, &dword, B_FALSE);
+
+	rxq_state->eers_rx_packed_stream_credits = 0;
+}
+
+	__checkReturn	uint8_t *
+ef10_rx_qps_packet_info(
+	__in		efx_rxq_t *erp,
+	__in		uint8_t *buffer,
+	__in		uint32_t buffer_length,
+	__in		uint32_t current_offset,
+	__out		uint16_t *lengthp,
+	__out		uint32_t *next_offsetp,
+	__out		uint32_t *timestamp)
+{
+	uint16_t buf_len;
+	uint8_t *pkt_start;
+	efx_qword_t *qwordp;
+	efx_evq_rxq_state_t *rxq_state =
+		&erp->er_eep->ee_rxq_state[erp->er_label];
+
+	EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
+
+	buffer += current_offset;
+	pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
+
+	qwordp = (efx_qword_t *)buffer;
+	*timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
+	*lengthp   = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
+	buf_len    = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
+
+	buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
+			    EFX_RX_PACKED_STREAM_ALIGNMENT);
+	*next_offsetp =
+	    current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
+
+	EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
+	EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
+
+	if ((*next_offsetp ^ current_offset) &
+	    EFX_RX_PACKED_STREAM_MEM_PER_CREDIT) {
+		if (rxq_state->eers_rx_packed_stream_credits <
+		    EFX_RX_PACKED_STREAM_MAX_CREDITS)
+			rxq_state->eers_rx_packed_stream_credits++;
+	}
+
+	return (pkt_start);
+}
+
+
+#endif
+
 	__checkReturn	efx_rc_t
 ef10_rx_qflush(
 	__in	efx_rxq_t *erp)
@@ -781,12 +856,45 @@ ef10_rx_qcreate(
 	case EFX_RXQ_TYPE_SCATTER:
 		ps_buf_size = 0;
 		break;
+#if EFSYS_OPT_RX_PACKED_STREAM
+	case EFX_RXQ_TYPE_PACKED_STREAM_1M:
+		ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
+		break;
+	case EFX_RXQ_TYPE_PACKED_STREAM_512K:
+		ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
+		break;
+	case EFX_RXQ_TYPE_PACKED_STREAM_256K:
+		ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
+		break;
+	case EFX_RXQ_TYPE_PACKED_STREAM_128K:
+		ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
+		break;
+	case EFX_RXQ_TYPE_PACKED_STREAM_64K:
+		ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
+		break;
+#endif /* EFSYS_OPT_RX_PACKED_STREAM */
 	default:
 		rc = ENOTSUP;
 		goto fail3;
 	}
 
+#if EFSYS_OPT_RX_PACKED_STREAM
+	if (ps_buf_size != 0) {
+		/* Check if datapath firmware supports packed stream mode */
+		if (encp->enc_rx_packed_stream_supported == B_FALSE) {
+			rc = ENOTSUP;
+			goto fail4;
+		}
+		/* Check if packed stream allows configurable buffer sizes */
+		if ((type != EFX_RXQ_TYPE_PACKED_STREAM_1M) &&
+		    (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
+			rc = ENOTSUP;
+			goto fail5;
+		}
+	}
+#else /* EFSYS_OPT_RX_PACKED_STREAM */
 	EFSYS_ASSERT(ps_buf_size == 0);
+#endif /* EFSYS_OPT_RX_PACKED_STREAM */
 
 	/* Scatter can only be disabled if the firmware supports doing so */
 	if (type == EFX_RXQ_TYPE_SCATTER)
@@ -807,6 +915,12 @@ ef10_rx_qcreate(
 
 fail6:
 	EFSYS_PROBE(fail6);
+#if EFSYS_OPT_RX_PACKED_STREAM
+fail5:
+	EFSYS_PROBE(fail5);
+fail4:
+	EFSYS_PROBE(fail4);
+#endif /* EFSYS_OPT_RX_PACKED_STREAM */
 fail3:
 	EFSYS_PROBE(fail3);
 fail2:
diff --git a/drivers/net/sfc/efx/base/efx.h b/drivers/net/sfc/efx/base/efx.h
index 4fc0207..1be64f2 100644
--- a/drivers/net/sfc/efx/base/efx.h
+++ b/drivers/net/sfc/efx/base/efx.h
@@ -1419,6 +1419,28 @@ typedef	__checkReturn	boolean_t
 	__in		uint32_t size,
 	__in		uint16_t flags);
 
+#if EFSYS_OPT_RX_PACKED_STREAM
+
+/*
+ * Packed stream mode is documented in SF-112241-TC.
+ * The general idea is that, instead of putting each incoming
+ * packet into a separate buffer which is specified in a RX
+ * descriptor, a large buffer is provided to the hardware and
+ * packets are put there in a continuous stream.
+ * The main advantage of such an approach is that RX queue refilling
+ * happens much less frequently.
+ */
+
+typedef	__checkReturn	boolean_t
+(*efx_rx_ps_ev_t)(
+	__in_opt	void *arg,
+	__in		uint32_t label,
+	__in		uint32_t id,
+	__in		uint32_t pkt_count,
+	__in		uint16_t flags);
+
+#endif
+
 typedef	__checkReturn	boolean_t
 (*efx_tx_ev_t)(
 	__in_opt	void *arg,
@@ -1508,6 +1530,9 @@ typedef __checkReturn	boolean_t
 typedef struct efx_ev_callbacks_s {
 	efx_initialized_ev_t		eec_initialized;
 	efx_rx_ev_t			eec_rx;
+#if EFSYS_OPT_RX_PACKED_STREAM
+	efx_rx_ps_ev_t			eec_rx_ps;
+#endif
 	efx_tx_ev_t			eec_tx;
 	efx_exception_ev_t		eec_exception;
 	efx_rxq_flush_done_ev_t		eec_rxq_flush_done;
@@ -1731,6 +1756,29 @@ efx_rx_qpush(
 	__in	unsigned int added,
 	__inout	unsigned int *pushedp);
 
+#if EFSYS_OPT_RX_PACKED_STREAM
+
+/*
+ * Fake length for RXQ descriptors in packed stream mode
+ * to make hardware happy
+ */
+#define	EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32
+
+extern			void
+efx_rx_qps_update_credits(
+	__in		efx_rxq_t *erp);
+
+extern	__checkReturn	uint8_t *
+efx_rx_qps_packet_info(
+	__in		efx_rxq_t *erp,
+	__in		uint8_t *buffer,
+	__in		uint32_t buffer_length,
+	__in		uint32_t current_offset,
+	__out		uint16_t *lengthp,
+	__out		uint32_t *next_offsetp,
+	__out		uint32_t *timestamp);
+#endif
+
 extern	__checkReturn	efx_rc_t
 efx_rx_qflush(
 	__in	efx_rxq_t *erp);
diff --git a/drivers/net/sfc/efx/base/efx_check.h b/drivers/net/sfc/efx/base/efx_check.h
index 35615e6..5522838 100644
--- a/drivers/net/sfc/efx/base/efx_check.h
+++ b/drivers/net/sfc/efx/base/efx_check.h
@@ -305,4 +305,11 @@
 # endif
 #endif /* EFSYS_OPT_ALLOW_UNCONFIGURED_NIC */
 
+#if EFSYS_OPT_RX_PACKED_STREAM
+/* Support packed stream mode */
+# if !(EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD)
+#  error "PACKED_STREAM requires HUNTINGTON or MEDFORD"
+# endif
+#endif
+
 #endif /* _SYS_EFX_CHECK_H */
diff --git a/drivers/net/sfc/efx/base/efx_impl.h b/drivers/net/sfc/efx/base/efx_impl.h
index c9d2d11..f563eda 100644
--- a/drivers/net/sfc/efx/base/efx_impl.h
+++ b/drivers/net/sfc/efx/base/efx_impl.h
@@ -166,6 +166,12 @@ typedef struct efx_rx_ops_s {
 				      unsigned int, unsigned int,
 				      unsigned int);
 	void		(*erxo_qpush)(efx_rxq_t *, unsigned int, unsigned int *);
+#if EFSYS_OPT_RX_PACKED_STREAM
+	void		(*erxo_qps_update_credits)(efx_rxq_t *);
+	uint8_t *	(*erxo_qps_packet_info)(efx_rxq_t *, uint8_t *,
+						uint32_t, uint32_t,
+						uint16_t *, uint32_t *, uint32_t *);
+#endif
 	efx_rc_t	(*erxo_qflush)(efx_rxq_t *);
 	void		(*erxo_qenable)(efx_rxq_t *);
 	efx_rc_t	(*erxo_qcreate)(efx_nic_t *enp, unsigned int,
@@ -525,6 +531,11 @@ typedef	boolean_t (*efx_ev_handler_t)(efx_evq_t *, efx_qword_t *,
 typedef struct efx_evq_rxq_state_s {
 	unsigned int			eers_rx_read_ptr;
 	unsigned int			eers_rx_mask;
+#if EFSYS_OPT_RX_PACKED_STREAM
+	unsigned int			eers_rx_stream_npackets;
+	boolean_t			eers_rx_packed_stream;
+	unsigned int			eers_rx_packed_stream_credits;
+#endif
 } efx_evq_rxq_state_t;
 
 struct efx_evq_s {
diff --git a/drivers/net/sfc/efx/base/efx_rx.c b/drivers/net/sfc/efx/base/efx_rx.c
index a884d43..e3ba8a3 100644
--- a/drivers/net/sfc/efx/base/efx_rx.c
+++ b/drivers/net/sfc/efx/base/efx_rx.c
@@ -98,6 +98,22 @@ siena_rx_qpush(
 	__in		unsigned int added,
 	__inout		unsigned int *pushedp);
 
+#if EFSYS_OPT_RX_PACKED_STREAM
+static		void
+siena_rx_qps_update_credits(
+	__in		efx_rxq_t *erp);
+
+static	__checkReturn	uint8_t *
+siena_rx_qps_packet_info(
+	__in		efx_rxq_t *erp,
+	__in		uint8_t *buffer,
+	__in		uint32_t buffer_length,
+	__in		uint32_t current_offset,
+	__out		uint16_t *lengthp,
+	__out		uint32_t *next_offsetp,
+	__out		uint32_t *timestamp);
+#endif
+
 static	__checkReturn	efx_rc_t
 siena_rx_qflush(
 	__in		efx_rxq_t *erp);
@@ -141,6 +157,10 @@ static const efx_rx_ops_t __efx_rx_siena_ops = {
 	siena_rx_prefix_pktlen,			/* erxo_prefix_pktlen */
 	siena_rx_qpost,				/* erxo_qpost */
 	siena_rx_qpush,				/* erxo_qpush */
+#if EFSYS_OPT_RX_PACKED_STREAM
+	siena_rx_qps_update_credits,		/* erxo_qps_update_credits */
+	siena_rx_qps_packet_info,		/* erxo_qps_packet_info */
+#endif
 	siena_rx_qflush,			/* erxo_qflush */
 	siena_rx_qenable,			/* erxo_qenable */
 	siena_rx_qcreate,			/* erxo_qcreate */
@@ -164,6 +184,10 @@ static const efx_rx_ops_t __efx_rx_ef10_ops = {
 	ef10_rx_prefix_pktlen,			/* erxo_prefix_pktlen */
 	ef10_rx_qpost,				/* erxo_qpost */
 	ef10_rx_qpush,				/* erxo_qpush */
+#if EFSYS_OPT_RX_PACKED_STREAM
+	ef10_rx_qps_update_credits,		/* erxo_qps_update_credits */
+	ef10_rx_qps_packet_info,		/* erxo_qps_packet_info */
+#endif
 	ef10_rx_qflush,				/* erxo_qflush */
 	ef10_rx_qenable,			/* erxo_qenable */
 	ef10_rx_qcreate,			/* erxo_qcreate */
@@ -425,6 +449,40 @@ efx_rx_qpost(
 	erxop->erxo_qpost(erp, addrp, size, n, completed, added);
 }
 
+#if EFSYS_OPT_RX_PACKED_STREAM
+
+			void
+efx_rx_qps_update_credits(
+	__in		efx_rxq_t *erp)
+{
+	efx_nic_t *enp = erp->er_enp;
+	const efx_rx_ops_t *erxop = enp->en_erxop;
+
+	EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
+
+	erxop->erxo_qps_update_credits(erp);
+}
+
+	__checkReturn	uint8_t *
+efx_rx_qps_packet_info(
+	__in		efx_rxq_t *erp,
+	__in		uint8_t *buffer,
+	__in		uint32_t buffer_length,
+	__in		uint32_t current_offset,
+	__out		uint16_t *lengthp,
+	__out		uint32_t *next_offsetp,
+	__out		uint32_t *timestamp)
+{
+	efx_nic_t *enp = erp->er_enp;
+	const efx_rx_ops_t *erxop = enp->en_erxop;
+
+	return (erxop->erxo_qps_packet_info(erp, buffer,
+		buffer_length, current_offset, lengthp,
+		next_offsetp, timestamp));
+}
+
+#endif /* EFSYS_OPT_RX_PACKED_STREAM */
+
 			void
 efx_rx_qpush(
 	__in		efx_rxq_t *erp,
@@ -1071,6 +1129,32 @@ siena_rx_qpush(
 			    erp->er_index, &dword, B_FALSE);
 }
 
+#if EFSYS_OPT_RX_PACKED_STREAM
+static		void
+siena_rx_qps_update_credits(
+	__in		efx_rxq_t *erp)
+{
+	/* Not supported by Siena hardware */
+	EFSYS_ASSERT(0);
+}
+
+static		uint8_t *
+siena_rx_qps_packet_info(
+	__in		efx_rxq_t *erp,
+	__in		uint8_t *buffer,
+	__in		uint32_t buffer_length,
+	__in		uint32_t current_offset,
+	__out		uint16_t *lengthp,
+	__out		uint32_t *next_offsetp,
+	__out		uint32_t *timestamp)
+{
+	/* Not supported by Siena hardware */
+	EFSYS_ASSERT(0);
+
+	return (NULL);
+}
+#endif /* EFSYS_OPT_RX_PACKED_STREAM */
+
 static	__checkReturn	efx_rc_t
 siena_rx_qflush(
 	__in	efx_rxq_t *erp)
-- 
2.5.5