[PATCH v4 0/2] example/l3fwd: relax l3fwd rx RSS/Offload if needed

[PATCH v4 0/2] example/l3fwd: relax l3fwd rx RSS/Offload if needed

[Trevor Tao]

This series try to relax l3fwd rx RSS/Offload mode requirement if they
are not supported by underlying hw or virtual devices, there is an
option named relax_rx_mode added to enable this option.

Trevor Tao (2):
  examples/l3fwd: relax RSS requirement with option
  examples/l3fwd: relax RX Offload with option

 examples/l3fwd/l3fwd.h     | 12 ++++++++++--
 examples/l3fwd/l3fwd_em.h  |  2 +-
 examples/l3fwd/l3fwd_lpm.h |  2 +-
 examples/l3fwd/main.c      | 35 ++++++++++++++++++++++++++++++++++-
 4 files changed, 46 insertions(+), 5 deletions(-)

-- 
2.34.1

[PATCH v4 1/2] examples/l3fwd: relax RSS requirement with option

[Trevor Tao]

Now the port Rx mq_mode had been set to RTE_ETH_MQ_RX_RSS
by default, but some hw and/or virtual interface does not
support the RSS and offload mode presupposed, e.g., some
virtio interfaces in the cloud don't support
RSS and the error msg may like:

virtio_dev_configure(): RSS support requested but not supported by
the device
Port0 dev_configure = -95

So to enable the l3fwd running in that environment, the Rx mode requirement
can be relaxed to reflect the hardware feature reality here, and the l3fwd
can run smoothly then.

An option named "disable-rss" is added to disable the RX RSS explicitly,
and it's disabled by default.

Signed-off-by: Trevor Tao <taozj888@163.com>
---
 examples/l3fwd/main.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/examples/l3fwd/main.c b/examples/l3fwd/main.c
index 6063eb1399..b42365ef1b 100644
--- a/examples/l3fwd/main.c
+++ b/examples/l3fwd/main.c
@@ -73,6 +73,7 @@ static enum L3FWD_LOOKUP_MODE lookup_mode;
 static int numa_on = 1; /**< NUMA is enabled by default. */
 static int parse_ptype; /**< Parse packet type using rx callback, and */
 			/**< disabled by default */
+static int disable_rss; /**< Disable the RX RSS mode */
 static int per_port_pool; /**< Use separate buffer pools per port; disabled */
 			  /**< by default */
 
@@ -678,6 +679,7 @@ static const char short_options[] =
 #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len"
 #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num"
 #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype"
+#define CMD_LINE_OPT_DISABLE_RSS "disable-rss"
 #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool"
 #define CMD_LINE_OPT_MODE "mode"
 #define CMD_LINE_OPT_EVENTQ_SYNC "eventq-sched"
@@ -705,6 +707,7 @@ enum {
 	CMD_LINE_OPT_MAX_PKT_LEN_NUM,
 	CMD_LINE_OPT_HASH_ENTRY_NUM_NUM,
 	CMD_LINE_OPT_PARSE_PTYPE_NUM,
+	CMD_LINE_OPT_DISABLE_RSS_NUM,
 	CMD_LINE_OPT_RULE_IPV4_NUM,
 	CMD_LINE_OPT_RULE_IPV6_NUM,
 	CMD_LINE_OPT_ALG_NUM,
@@ -728,6 +731,7 @@ static const struct option lgopts[] = {
 	{CMD_LINE_OPT_MAX_PKT_LEN, 1, 0, CMD_LINE_OPT_MAX_PKT_LEN_NUM},
 	{CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, CMD_LINE_OPT_HASH_ENTRY_NUM_NUM},
 	{CMD_LINE_OPT_PARSE_PTYPE, 0, 0, CMD_LINE_OPT_PARSE_PTYPE_NUM},
+	{CMD_LINE_OPT_DISABLE_RSS, 0, 0, CMD_LINE_OPT_DISABLE_RSS_NUM},
 	{CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PARSE_PER_PORT_POOL},
 	{CMD_LINE_OPT_MODE, 1, 0, CMD_LINE_OPT_MODE_NUM},
 	{CMD_LINE_OPT_EVENTQ_SYNC, 1, 0, CMD_LINE_OPT_EVENTQ_SYNC_NUM},
@@ -853,6 +857,11 @@ parse_args(int argc, char **argv)
 			parse_ptype = 1;
 			break;
 
+		case CMD_LINE_OPT_DISABLE_RSS_NUM:
+			printf("Disable RX RSS\n");
+			disable_rss = 1;
+			break;
+
 		case CMD_LINE_OPT_PARSE_PER_PORT_POOL:
 			printf("per port buffer pool is enabled\n");
 			per_port_pool = 1;
@@ -1257,7 +1266,7 @@ l3fwd_poll_resource_setup(void)
 		local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
 			dev_info.flow_type_rss_offloads;
 
-		if (dev_info.max_rx_queues == 1)
+		if (disable_rss == 1 || dev_info.max_rx_queues == 1)
 			local_port_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
 
 		if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
-- 
2.34.1

[PATCH v4 2/2] examples/l3fwd: relax RX Offload with option

[Trevor Tao]

Now the port Rx offload mode is set to RTE_ETH_RX_OFFLOAD_CHECKSUM
by default, but some hw and/or virtual interface does not support
the offload mode presupposed, e.g., some virtio interfaces in
the cloud may only partly support RTE_ETH_RX_OFFLOAD_UDP_CKSUM/
RTE_ETH_RX_OFFLOAD_TCP_CKSUM,
but not RTE_ETH_RX_OFFLOAD_IPV4_CKSUM, and the error msg here:

Ethdev port_id=0 requested Rx offloads 0xe does not match Rx offloads
capabilities 0x201d in rte_eth_dev_configure()

So to enable the l3fwd running in that environment, the Rx mode requirement
can be relaxed to reflect the hardware feature reality here with an option
relax_rx_offload, and the l3fwd can run smoothly then.
A warning msg would be provided to user in case it happens here.

On the other side, enabling the software cksum check in case missing the
hw support.

The relax action for rx cksum offload is just enabled when relax_rx_offload is
true which is false by default.

Signed-off-by: Trevor Tao <taozj888@163.com>
---
 examples/l3fwd/l3fwd.h     | 12 ++++++++++--
 examples/l3fwd/l3fwd_em.h  |  2 +-
 examples/l3fwd/l3fwd_lpm.h |  2 +-
 examples/l3fwd/main.c      | 24 ++++++++++++++++++++++++
 4 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/examples/l3fwd/l3fwd.h b/examples/l3fwd/l3fwd.h
index b55855c932..e7ae0e5834 100644
--- a/examples/l3fwd/l3fwd.h
+++ b/examples/l3fwd/l3fwd.h
@@ -159,7 +159,7 @@ send_single_packet(struct lcore_conf *qconf,
 
 #ifdef DO_RFC_1812_CHECKS
 static inline int
-is_valid_ipv4_pkt(struct rte_ipv4_hdr *pkt, uint32_t link_len)
+is_valid_ipv4_pkt(struct rte_ipv4_hdr *pkt, uint32_t link_len, uint64_t ol_flags)
 {
 	/* From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2 */
 	/*
@@ -170,7 +170,15 @@ is_valid_ipv4_pkt(struct rte_ipv4_hdr *pkt, uint32_t link_len)
 		return -1;
 
 	/* 2. The IP checksum must be correct. */
-	/* this is checked in H/W */
+	/* if this is not checked in H/W, check it. */
+	if ((ol_flags & RTE_MBUF_F_RX_IP_CKSUM_MASK) == RTE_MBUF_F_RX_IP_CKSUM_NONE) {
+		uint16_t actual_cksum, expected_cksum;
+		actual_cksum = pkt->hdr_checksum;
+		pkt->hdr_checksum = 0;
+		expected_cksum = rte_ipv4_cksum(pkt);
+		if (actual_cksum != expected_cksum)
+			return -2;
+	}
 
 	/*
 	 * 3. The IP version number must be 4. If the version number is not 4
diff --git a/examples/l3fwd/l3fwd_em.h b/examples/l3fwd/l3fwd_em.h
index 7d051fc076..1fee2e2e6c 100644
--- a/examples/l3fwd/l3fwd_em.h
+++ b/examples/l3fwd/l3fwd_em.h
@@ -20,7 +20,7 @@ l3fwd_em_handle_ipv4(struct rte_mbuf *m, uint16_t portid,
 
 #ifdef DO_RFC_1812_CHECKS
 	/* Check to make sure the packet is valid (RFC1812) */
-	if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt_len) < 0) {
+	if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt_len, m->ol_flags) < 0) {
 		rte_pktmbuf_free(m);
 		return BAD_PORT;
 	}
diff --git a/examples/l3fwd/l3fwd_lpm.h b/examples/l3fwd/l3fwd_lpm.h
index c61b969584..4ee61e8d88 100644
--- a/examples/l3fwd/l3fwd_lpm.h
+++ b/examples/l3fwd/l3fwd_lpm.h
@@ -22,7 +22,7 @@ l3fwd_lpm_simple_forward(struct rte_mbuf *m, uint16_t portid,
 
 #ifdef DO_RFC_1812_CHECKS
 		/* Check to make sure the packet is valid (RFC1812) */
-		if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt_len) < 0) {
+		if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt_len, m->ol_flags) < 0) {
 			rte_pktmbuf_free(m);
 			return;
 		}
diff --git a/examples/l3fwd/main.c b/examples/l3fwd/main.c
index b42365ef1b..851e7e68f9 100644
--- a/examples/l3fwd/main.c
+++ b/examples/l3fwd/main.c
@@ -74,6 +74,8 @@ static int numa_on = 1; /**< NUMA is enabled by default. */
 static int parse_ptype; /**< Parse packet type using rx callback, and */
 			/**< disabled by default */
 static int disable_rss; /**< Disable the RX RSS mode */
+static int relax_rx_offload; /**< Relax RX offload mode, and */
+			     /**< disabled by default */
 static int per_port_pool; /**< Use separate buffer pools per port; disabled */
 			  /**< by default */
 
@@ -680,6 +682,7 @@ static const char short_options[] =
 #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num"
 #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype"
 #define CMD_LINE_OPT_DISABLE_RSS "disable-rss"
+#define CMD_LINE_OPT_RELAX_RX_OFFLOAD "relax-rx-offload"
 #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool"
 #define CMD_LINE_OPT_MODE "mode"
 #define CMD_LINE_OPT_EVENTQ_SYNC "eventq-sched"
@@ -708,6 +711,7 @@ enum {
 	CMD_LINE_OPT_HASH_ENTRY_NUM_NUM,
 	CMD_LINE_OPT_PARSE_PTYPE_NUM,
 	CMD_LINE_OPT_DISABLE_RSS_NUM,
+	CMD_LINE_OPT_RELAX_RX_OFFLOAD_NUM,
 	CMD_LINE_OPT_RULE_IPV4_NUM,
 	CMD_LINE_OPT_RULE_IPV6_NUM,
 	CMD_LINE_OPT_ALG_NUM,
@@ -731,6 +735,7 @@ static const struct option lgopts[] = {
 	{CMD_LINE_OPT_MAX_PKT_LEN, 1, 0, CMD_LINE_OPT_MAX_PKT_LEN_NUM},
 	{CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, CMD_LINE_OPT_HASH_ENTRY_NUM_NUM},
 	{CMD_LINE_OPT_PARSE_PTYPE, 0, 0, CMD_LINE_OPT_PARSE_PTYPE_NUM},
+	{CMD_LINE_OPT_RELAX_RX_OFFLOAD, 0, 0, CMD_LINE_OPT_RELAX_RX_OFFLOAD_NUM},
 	{CMD_LINE_OPT_DISABLE_RSS, 0, 0, CMD_LINE_OPT_DISABLE_RSS_NUM},
 	{CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PARSE_PER_PORT_POOL},
 	{CMD_LINE_OPT_MODE, 1, 0, CMD_LINE_OPT_MODE_NUM},
@@ -857,6 +862,11 @@ parse_args(int argc, char **argv)
 			parse_ptype = 1;
 			break;
 
+		case CMD_LINE_OPT_RELAX_RX_OFFLOAD_NUM:
+			printf("Relax rx offload mode is enabled\n");
+			relax_rx_offload = 1;
+			break;
+
 		case CMD_LINE_OPT_DISABLE_RSS_NUM:
 			printf("Disable RX RSS\n");
 			disable_rss = 1;
@@ -1278,6 +1288,20 @@ l3fwd_poll_resource_setup(void)
 				local_port_conf.rx_adv_conf.rss_conf.rss_hf);
 		}
 
+		/* relax the rx offload requirement */
+		if ((local_port_conf.rxmode.offloads & dev_info.rx_offload_capa) !=
+			local_port_conf.rxmode.offloads) {
+			printf("Port %u requested Rx offloads 0x%"PRIx64" does not"
+				" match Rx offloads capabilities 0x%"PRIx64"\n",
+				portid, local_port_conf.rxmode.offloads,
+				dev_info.rx_offload_capa);
+			if (relax_rx_offload) {
+				local_port_conf.rxmode.offloads &= dev_info.rx_offload_capa;
+				printf("warning: modified the rx offload to 0x%"PRIx64" based on device"
+				" capability\n", local_port_conf.rxmode.offloads);
+			}
+		}
+
 		ret = rte_eth_dev_configure(portid, nb_rx_queue,
 					(uint16_t)n_tx_queue, &local_port_conf);
 		if (ret < 0)
-- 
2.34.1

Re: [PATCH v4 1/2] examples/l3fwd: relax RSS requirement with option

[Stephen Hemminger]

On Mon, 13 Nov 2023 16:02:54 +0000
Trevor Tao <taozj888@163.com> wrote:

> Now the port Rx mq_mode had been set to RTE_ETH_MQ_RX_RSS
> by default, but some hw and/or virtual interface does not
> support the RSS and offload mode presupposed, e.g., some
> virtio interfaces in the cloud don't support
> RSS and the error msg may like:
> 
> virtio_dev_configure(): RSS support requested but not supported by
> the device
> Port0 dev_configure = -95
> 
> So to enable the l3fwd running in that environment, the Rx mode requirement
> can be relaxed to reflect the hardware feature reality here, and the l3fwd
> can run smoothly then.
> 
> An option named "disable-rss" is added to disable the RX RSS explicitly,
> and it's disabled by default.
> 
> Signed-off-by: Trevor Tao <taozj888@163.com>

Why is running with > 1 rx queue and RSS disabled useful?
What happens is all packets arrive on 1st queue and you burn a thread
polling an always empty queue.

I would prefer not adding yet another command line option and
just "do the right thing". If number of rx queues > 1, check that device
supports RSS before using it. If device does not support RSS give
an error and exit.

Re: [PATCH v4 1/2] examples/l3fwd: relax RSS requirement with option

[Stephen Hemminger]

On Mon, 13 Nov 2023 08:34:47 -0800
Stephen Hemminger <stephen@networkplumber.org> wrote:

> On Mon, 13 Nov 2023 16:02:54 +0000
> Trevor Tao <taozj888@163.com> wrote:
> 
> > Now the port Rx mq_mode had been set to RTE_ETH_MQ_RX_RSS
> > by default, but some hw and/or virtual interface does not
> > support the RSS and offload mode presupposed, e.g., some
> > virtio interfaces in the cloud don't support
> > RSS and the error msg may like:
> > 
> > virtio_dev_configure(): RSS support requested but not supported by
> > the device
> > Port0 dev_configure = -95
> > 
> > So to enable the l3fwd running in that environment, the Rx mode requirement
> > can be relaxed to reflect the hardware feature reality here, and the l3fwd
> > can run smoothly then.
> > 
> > An option named "disable-rss" is added to disable the RX RSS explicitly,
> > and it's disabled by default.
> > 
> > Signed-off-by: Trevor Tao <taozj888@163.com>  
> 
> Why is running with > 1 rx queue and RSS disabled useful?
> What happens is all packets arrive on 1st queue and you burn a thread
> polling an always empty queue.
> 
> I would prefer not adding yet another command line option and
> just "do the right thing". If number of rx queues > 1, check that device
> supports RSS before using it. If device does not support RSS give
> an error and exit.

Something like this maybe:
diff --git a/examples/l3fwd/main.c b/examples/l3fwd/main.c
index 6063eb139900..c747ef8aadfb 100644
--- a/examples/l3fwd/main.c
+++ b/examples/l3fwd/main.c
@@ -1257,8 +1257,15 @@ l3fwd_poll_resource_setup(void)
 		local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
 			dev_info.flow_type_rss_offloads;
 
-		if (dev_info.max_rx_queues == 1)
+		if (nb_rx_queue > dev_info.max_rx_queues)
+			rte_exit(EXIT_FAILURE, "Port %u only supports %u queues\n",
+				 portid, dev_info.max_rx_queues);
+
+		if (nb_rx_queue == 1)
 			local_port_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
+		else if (!(dev_info.rx_offload_capa & RTE_ETH_RX_OFFLOAD_RSS_HASH))
+			rte_exit(EXIT_FAILURE, "Port %u does not support RSS but %u queues requested\n",
+				 portid, nb_rx_queue);
 
 		if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
 				port_conf.rx_adv_conf.rss_conf.rss_hf) {

RE: [PATCH v4 1/2] examples/l3fwd: relax RSS requirement with option

[Konstantin Ananyev]

> > > Now the port Rx mq_mode had been set to RTE_ETH_MQ_RX_RSS
> > > by default, but some hw and/or virtual interface does not
> > > support the RSS and offload mode presupposed, e.g., some
> > > virtio interfaces in the cloud don't support
> > > RSS and the error msg may like:
> > >
> > > virtio_dev_configure(): RSS support requested but not supported by
> > > the device
> > > Port0 dev_configure = -95
> > >
> > > So to enable the l3fwd running in that environment, the Rx mode requirement
> > > can be relaxed to reflect the hardware feature reality here, and the l3fwd
> > > can run smoothly then.
> > >
> > > An option named "disable-rss" is added to disable the RX RSS explicitly,
> > > and it's disabled by default.
> > >
> > > Signed-off-by: Trevor Tao <taozj888@163.com>
> >
> > Why is running with > 1 rx queue and RSS disabled useful?
> > What happens is all packets arrive on 1st queue and you burn a thread
> > polling an always empty queue.
> >
> > I would prefer not adding yet another command line option and
> > just "do the right thing". If number of rx queues > 1, check that device
> > supports RSS before using it. If device does not support RSS give
> > an error and exit.

I think that's pretty much what we have right now in l3fwd.
As I understand, the rational was: some virtual NICs does not support RSS (vritio?),
but there is still a desire to run l3fwd app on such devices.
I presume for performance comparison purposes.
So the idea was to relax NIC requirements and allow to run l3fwd with disabled RSS.
I suppose the main question here: what then will be the mechanism to distribute
packets over different RX queues?
From my understanding with '--disable-rss' option enabled it is undefined and 
depends on particular NIC.
In case of virtio, my guess - it will be hypervisor counterpart (vhost) to determine packet distribution.

> 
> Something like this maybe:
> diff --git a/examples/l3fwd/main.c b/examples/l3fwd/main.c
> index 6063eb139900..c747ef8aadfb 100644
> --- a/examples/l3fwd/main.c
> +++ b/examples/l3fwd/main.c
> @@ -1257,8 +1257,15 @@ l3fwd_poll_resource_setup(void)
>  		local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
>  			dev_info.flow_type_rss_offloads;
> 
> -		if (dev_info.max_rx_queues == 1)
> +		if (nb_rx_queue > dev_info.max_rx_queues)
> +			rte_exit(EXIT_FAILURE, "Port %u only supports %u queues\n",
> +				 portid, dev_info.max_rx_queues);
> +
> +		if (nb_rx_queue == 1)
>  			local_port_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
> +		else if (!(dev_info.rx_offload_capa & RTE_ETH_RX_OFFLOAD_RSS_HASH))
> +			rte_exit(EXIT_FAILURE, "Port %u does not support RSS but %u queues requested\n",
> +				 portid, nb_rx_queue);
> 
>  		if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
>  				port_conf.rx_adv_conf.rss_conf.rss_hf) {