[PATCH v2 0/3] example/l3fwd: relax l3fwd rx RSS/Offload if needed

[PATCH v2 0/3] example/l3fwd: relax l3fwd rx RSS/Offload if needed

[Trevor Tao]

This series tries to relax l3fwd rx RSS/Offload mode requirement if they
are not supported by underlying hw or virtual devices, there is an
option named relax_rx_mode added to enable this option.

Trevor Tao (3):
  examples/l3fwd: relax RSS requirement with option
  examples/l3fwd: relax the Offload requirement
  doc: add a relax rx mode requirement option

 doc/guides/rel_notes/release_23_11.rst  |  2 ++
 doc/guides/sample_app_ug/l3_forward.rst |  4 ++-
 examples/l3fwd/l3fwd.h                  | 12 +++++++--
 examples/l3fwd/l3fwd_em.h               |  2 +-
 examples/l3fwd/l3fwd_lpm.h              |  2 +-
 examples/l3fwd/main.c                   | 33 ++++++++++++++++++++++++-
 6 files changed, 49 insertions(+), 6 deletions(-)

-- 
2.34.1

[PATCH v2 1/3] examples/l3fwd: relax RSS requirement with option

[Trevor Tao]

Now the port Rx mq_mode had been set to RTE_ETH_MQ_RX_RSS
by default, but some hw and/or virtual interface does not
support the RSS and offload mode presupposed, e.g., some
virtio interfaces in the cloud don't support
RSS and the error msg may like:

virtio_dev_configure(): RSS support requested but not supported by
the device
Port0 dev_configure = -95

So to enable the l3fwd running in that environment, the Rx mode requirement
can be relaxed to reflect the hardware feature reality here, and the l3fwd
can run smoothly then.

An option named "relax-rx-mode" is added to enable the relax action
here, and it's disabled by default.

Signed-off-by: Trevor Tao <taozj888@163.com>
---
 examples/l3fwd/main.c | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/examples/l3fwd/main.c b/examples/l3fwd/main.c
index 6063eb1399..36a2a77756 100644
--- a/examples/l3fwd/main.c
+++ b/examples/l3fwd/main.c
@@ -73,6 +73,7 @@ static enum L3FWD_LOOKUP_MODE lookup_mode;
 static int numa_on = 1; /**< NUMA is enabled by default. */
 static int parse_ptype; /**< Parse packet type using rx callback, and */
 			/**< disabled by default */
+static int relax_rx_mode; /**< Relax RX mode is disabled by default */
 static int per_port_pool; /**< Use separate buffer pools per port; disabled */
 			  /**< by default */
 
@@ -678,6 +679,7 @@ static const char short_options[] =
 #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len"
 #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num"
 #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype"
+#define CMD_LINE_OPT_RELAX_RX_MODE "relax-rx-mode"
 #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool"
 #define CMD_LINE_OPT_MODE "mode"
 #define CMD_LINE_OPT_EVENTQ_SYNC "eventq-sched"
@@ -705,6 +707,7 @@ enum {
 	CMD_LINE_OPT_MAX_PKT_LEN_NUM,
 	CMD_LINE_OPT_HASH_ENTRY_NUM_NUM,
 	CMD_LINE_OPT_PARSE_PTYPE_NUM,
+	CMD_LINE_OPT_RELAX_RX_MODE_NUM,
 	CMD_LINE_OPT_RULE_IPV4_NUM,
 	CMD_LINE_OPT_RULE_IPV6_NUM,
 	CMD_LINE_OPT_ALG_NUM,
@@ -728,6 +731,7 @@ static const struct option lgopts[] = {
 	{CMD_LINE_OPT_MAX_PKT_LEN, 1, 0, CMD_LINE_OPT_MAX_PKT_LEN_NUM},
 	{CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, CMD_LINE_OPT_HASH_ENTRY_NUM_NUM},
 	{CMD_LINE_OPT_PARSE_PTYPE, 0, 0, CMD_LINE_OPT_PARSE_PTYPE_NUM},
+	{CMD_LINE_OPT_RELAX_RX_MODE, 0, 0, CMD_LINE_OPT_RELAX_RX_MODE_NUM},
 	{CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PARSE_PER_PORT_POOL},
 	{CMD_LINE_OPT_MODE, 1, 0, CMD_LINE_OPT_MODE_NUM},
 	{CMD_LINE_OPT_EVENTQ_SYNC, 1, 0, CMD_LINE_OPT_EVENTQ_SYNC_NUM},
@@ -853,6 +857,11 @@ parse_args(int argc, char **argv)
 			parse_ptype = 1;
 			break;
 
+		case CMD_LINE_OPT_RELAX_RX_MODE_NUM:
+			printf("Relax rx mode is enabled\n");
+			relax_rx_mode = 1;
+			break;
+
 		case CMD_LINE_OPT_PARSE_PER_PORT_POOL:
 			printf("per port buffer pool is enabled\n");
 			per_port_pool = 1;
@@ -1257,8 +1266,16 @@ l3fwd_poll_resource_setup(void)
 		local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
 			dev_info.flow_type_rss_offloads;
 
-		if (dev_info.max_rx_queues == 1)
+		if (dev_info.max_rx_queues == 1) {
 			local_port_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
+		}
+
+		/* relax the rx rss requirement */
+		if (relax_rx_mode && !local_port_conf.rx_adv_conf.rss_conf.rss_hf) {
+			printf("warning: modified the rx mq_mode to RTE_ETH_MQ_RX_NONE base on"
+					" device capability\n");
+				local_port_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
+		}
 
 		if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
 				port_conf.rx_adv_conf.rss_conf.rss_hf) {
-- 
2.34.1

[PATCH v2 2/3] examples/l3fwd: relax the Offload requirement

[Trevor Tao]

Now the port Rx offload mode is set to RTE_ETH_RX_OFFLOAD_CHECKSUM
by default, but some hw and/or virtual interface does not support
the offload mode presupposed, e.g., some virtio interfaces in
the cloud may only partly support RTE_ETH_RX_OFFLOAD_UDP_CKSUM/
RTE_ETH_RX_OFFLOAD_TCP_CKSUM,
but not RTE_ETH_RX_OFFLOAD_IPV4_CKSUM, and the error msg here:

Ethdev port_id=0 requested Rx offloads 0xe does not match Rx offloads
capabilities 0x201d in rte_eth_dev_configure()

So to enable the l3fwd running in that environment, the Rx mode requirement
can be relaxed to reflect the hardware feature reality here, and the l3fwd
can run smoothly then.
A warning msg would be provided to user in case it happens here.

On the other side, enabling the software cksum check in case missing the
hw support.

The relax action for rx cksum offload is just enabled when relax_rx_mode is
true which is false by default.

Signed-off-by: Trevor Tao <taozj888@163.com>
---
 examples/l3fwd/l3fwd.h     | 12 ++++++++++--
 examples/l3fwd/l3fwd_em.h  |  2 +-
 examples/l3fwd/l3fwd_lpm.h |  2 +-
 examples/l3fwd/main.c      | 14 ++++++++++++++
 4 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/examples/l3fwd/l3fwd.h b/examples/l3fwd/l3fwd.h
index b55855c932..fd98ad3373 100644
--- a/examples/l3fwd/l3fwd.h
+++ b/examples/l3fwd/l3fwd.h
@@ -159,7 +159,7 @@ send_single_packet(struct lcore_conf *qconf,
 
 #ifdef DO_RFC_1812_CHECKS
 static inline int
-is_valid_ipv4_pkt(struct rte_ipv4_hdr *pkt, uint32_t link_len)
+is_valid_ipv4_pkt(struct rte_ipv4_hdr *pkt, uint32_t link_len, uint64_t ol_flags)
 {
 	/* From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2 */
 	/*
@@ -170,7 +170,15 @@ is_valid_ipv4_pkt(struct rte_ipv4_hdr *pkt, uint32_t link_len)
 		return -1;
 
 	/* 2. The IP checksum must be correct. */
-	/* this is checked in H/W */
+	/* if this is not checked in H/W, check it. */
+	if ((ol_flags & RTE_ETH_RX_OFFLOAD_IPV4_CKSUM) == 0) {
+		uint16_t actual_cksum, expected_cksum;
+		actual_cksum = pkt->hdr_checksum;
+		pkt->hdr_checksum = 0;
+		expected_cksum = rte_ipv4_cksum(pkt);
+		if (actual_cksum != expected_cksum)
+			return -2;
+	}
 
 	/*
 	 * 3. The IP version number must be 4. If the version number is not 4
diff --git a/examples/l3fwd/l3fwd_em.h b/examples/l3fwd/l3fwd_em.h
index 7d051fc076..1fee2e2e6c 100644
--- a/examples/l3fwd/l3fwd_em.h
+++ b/examples/l3fwd/l3fwd_em.h
@@ -20,7 +20,7 @@ l3fwd_em_handle_ipv4(struct rte_mbuf *m, uint16_t portid,
 
 #ifdef DO_RFC_1812_CHECKS
 	/* Check to make sure the packet is valid (RFC1812) */
-	if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt_len) < 0) {
+	if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt_len, m->ol_flags) < 0) {
 		rte_pktmbuf_free(m);
 		return BAD_PORT;
 	}
diff --git a/examples/l3fwd/l3fwd_lpm.h b/examples/l3fwd/l3fwd_lpm.h
index c61b969584..4ee61e8d88 100644
--- a/examples/l3fwd/l3fwd_lpm.h
+++ b/examples/l3fwd/l3fwd_lpm.h
@@ -22,7 +22,7 @@ l3fwd_lpm_simple_forward(struct rte_mbuf *m, uint16_t portid,
 
 #ifdef DO_RFC_1812_CHECKS
 		/* Check to make sure the packet is valid (RFC1812) */
-		if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt_len) < 0) {
+		if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt_len, m->ol_flags) < 0) {
 			rte_pktmbuf_free(m);
 			return;
 		}
diff --git a/examples/l3fwd/main.c b/examples/l3fwd/main.c
index 36a2a77756..5f378ed812 100644
--- a/examples/l3fwd/main.c
+++ b/examples/l3fwd/main.c
@@ -1286,6 +1286,20 @@ l3fwd_poll_resource_setup(void)
 				local_port_conf.rx_adv_conf.rss_conf.rss_hf);
 		}
 
+		/* relax the rx offload requirement */
+		if ((local_port_conf.rxmode.offloads & dev_info.rx_offload_capa) !=
+			local_port_conf.rxmode.offloads) {
+			printf("Port %u requested Rx offloads 0x%"PRIx64" does not"
+				" match Rx offloads capabilities 0x%"PRIx64"\n",
+				portid, local_port_conf.rxmode.offloads,
+				dev_info.rx_offload_capa);
+			if (relax_rx_mode) {
+				local_port_conf.rxmode.offloads &= dev_info.rx_offload_capa;
+				printf("warning: modified the rx offload to 0x%"PRIx64" based on device"
+				" capability\n", local_port_conf.rxmode.offloads);
+			}
+		}
+
 		ret = rte_eth_dev_configure(portid, nb_rx_queue,
 					(uint16_t)n_tx_queue, &local_port_conf);
 		if (ret < 0)
-- 
2.34.1

[PATCH v2 3/3] doc: add a relax rx mode requirement option

[Trevor Tao]

Add an option to enable the RX mode requirement relax
in release notes and l3fwd sample guide.

Signed-off-by: Trevor Tao <taozj888@163.com>
---
 doc/guides/rel_notes/release_23_11.rst  | 2 ++
 doc/guides/sample_app_ug/l3_forward.rst | 4 +++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/guides/rel_notes/release_23_11.rst b/doc/guides/rel_notes/release_23_11.rst
index f09ecd50fe..2f9e4a54c8 100644
--- a/doc/guides/rel_notes/release_23_11.rst
+++ b/doc/guides/rel_notes/release_23_11.rst
@@ -84,6 +84,8 @@ New Features
   default. The implementation using C11 standard atomic operations is enabled
   via the ``enable_stdatomic`` build option.
 
+* sample: Added a command option ``--relax-rx-mode`` in l3fwd example
+  to relax the rx RSS/Offload mode requirement if needed.
 
 Removed Items
 -------------
diff --git a/doc/guides/sample_app_ug/l3_forward.rst b/doc/guides/sample_app_ug/l3_forward.rst
index 1cc2c1dd1d..00283f070c 100644
--- a/doc/guides/sample_app_ug/l3_forward.rst
+++ b/doc/guides/sample_app_ug/l3_forward.rst
@@ -126,6 +126,8 @@ Where,
 
 * ``--parse-ptype:`` Optional, set to use software to analyze packet type. Without this option, hardware will check the packet type.
 
+* ``--relax-rx-mode:`` Optional, set to enable rx mode relax when RSS/offload is not fully supported by the hardware. When the IPv4 cksum offload is relaxed, it is calculated by the software instead. Without this option, the RSS and cksum offload will be forced.
+
 * ``--per-port-pool:`` Optional, set to use independent buffer pools per port. Without this option, single buffer pool is used for all ports.
 
 * ``--mode:`` Optional, Packet transfer mode for I/O, poll or eventdev.
@@ -140,7 +142,7 @@ Where,
 
 * ``--event-vector-tmo:`` Optional, Max timeout to form vector in nanoseconds if event vectorization is enabled.
 
-* ``--alg=<val>:`` optional, ACL classify method to use, one of:
+* ``--alg=<val>:`` Optional, ACL classify method to use, one of:
   ``scalar|sse|avx2|neon|altivec|avx512x16|avx512x32``
 
 * ``-E:`` Optional, enable exact match,
-- 
2.34.1

Re: [PATCH v2 3/3] doc: add a relax rx mode requirement option

[Stephen Hemminger]

On Fri, 13 Oct 2023 04:27:22 +0000
Trevor Tao <taozj888@163.com> wrote:

> From: Trevor Tao <taozj888@163.com>
> To: dev@dpdk.org
> Cc: Trevor Tao <taozj888@163.com>
> Subject: [PATCH v2 3/3] doc: add a relax rx mode requirement option
> Date: Fri, 13 Oct 2023 04:27:22 +0000
> X-Mailer: git-send-email 2.34.1
> 
> Add an option to enable the RX mode requirement relax
> in release notes and l3fwd sample guide.
> 
> Signed-off-by: Trevor Tao <taozj888@163.com>

The patch you set is missing a bunch of other unrelated changes.
Looks like you were using a baseline with some other patches applied.

Please resubmit with just the relax rx mode change.

[PATCH v2 3/3] doc: add a relax rx mode requirement option

[Trevor Tao]

Add an option to enable the RX mode requirement relax
in release notes and l3fwd sample guide.

Signed-off-by: Trevor Tao <taozj888@163.com>
---
 doc/guides/rel_notes/release_23_11.rst  | 251 +++++++++++++++++++++---
 doc/guides/sample_app_ug/l3_forward.rst |   4 +-
 2 files changed, 230 insertions(+), 25 deletions(-)

diff --git a/doc/guides/rel_notes/release_23_11.rst b/doc/guides/rel_notes/release_23_11.rst
index f09ecd50fe..3f1d5039c3 100644
--- a/doc/guides/rel_notes/release_23_11.rst
+++ b/doc/guides/rel_notes/release_23_11.rst
@@ -20,26 +20,14 @@ DPDK Release 23.11
       ninja -C build doc
       xdg-open build/doc/guides/html/rel_notes/release_23_11.html
 
-* Build Requirements: From DPDK 23.11 onwards,
-  building DPDK will require a C compiler which supports the C11 standard,
-  including support for C11 standard atomics.
-
-  More specifically, the requirements will be:
 
-  * Support for flag "-std=c11" (or similar)
-  * __STDC_NO_ATOMICS__ is *not defined* when using c11 flag
+New Features
+------------
 
-  Please note:
+   * **Added support for models with multiple I/O in mldev library.**
 
-  * C11, including standard atomics, is supported from GCC version 5 onwards,
-    and is the default language version in that release
-    (Ref: https://gcc.gnu.org/gcc-5/changes.html)
-  * C11 is the default compilation mode in Clang from version 3.6,
-    which also added support for standard atomics
-    (Ref: https://releases.llvm.org/3.6.0/tools/clang/docs/ReleaseNotes.html)
+     Added support in mldev library for models with multiple inputs and outputs.
 
-New Features
-------------
 
 .. This section should contain new features added in this release.
    Sample format:
@@ -72,18 +60,177 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
-* build: Enabling deprecated libraries is now done using the new
-  ``enable_deprecated_libraries`` build option.
+* **Build requirements increased for C11.**
+
+  From DPDK 23.11 onwards,
+  building DPDK will require a C compiler which supports the C11 standard,
+  including support for C11 standard atomics.
+
+  More specifically, the requirements will be:
 
-* build: Optional libraries can now be selected with the new ``enable_libs``
-  build option similarly to the existing ``enable_drivers`` build option.
+  * Support for flag "-std=c11" (or similar)
+  * __STDC_NO_ATOMICS__ is *not defined* when using c11 flag
 
-* eal: Introduced a new API for atomic operations. This new API serves as a
-  wrapper for transitioning to standard atomic operations as described in the
-  C11 standard. This API implementation points at the compiler intrinsics by
-  default. The implementation using C11 standard atomic operations is enabled
+  Please note:
+
+  * C11, including standard atomics, is supported from GCC version 5 onwards,
+    and is the default language version in that release
+    (Ref: https://gcc.gnu.org/gcc-5/changes.html)
+  * C11 is the default compilation mode in Clang from version 3.6,
+    which also added support for standard atomics
+    (Ref: https://releases.llvm.org/3.6.0/tools/clang/docs/ReleaseNotes.html)
+
+* **Added new build options.**
+
+  * Enabling deprecated libraries is now done using
+    the new ``enable_deprecated_libraries`` build option.
+  * Optional libraries can now be selected with the new ``enable_libs``
+    build option similarly to the existing ``enable_drivers`` build option.
+
+* **Introduced a new API for atomic operations.**
+
+  This new API serves as a wrapper for transitioning
+  to standard atomic operations as described in the C11 standard.
+  This API implementation points at the compiler intrinsics by default.
+  The implementation using C11 standard atomic operations is enabled
   via the ``enable_stdatomic`` build option.
 
+* **Added support for power intrinsics with AMD processors.**
+
+* **Added support for allow/block list in vmbus bus driver.***
+
+  The ``vmbus`` bus driver now supports -a and -b EAL options for selecting
+  devices.
+
+* **Added mbuf recycling support.**
+
+  Added ``rte_eth_recycle_rx_queue_info_get`` and ``rte_eth_recycle_mbufs``
+  functions which allow the user to copy used mbufs from the Tx mbuf ring
+  into the Rx mbuf ring. This feature supports the case that the Rx Ethernet
+  device is different from the Tx Ethernet device with respective driver
+  callback functions in ``rte_eth_recycle_mbufs``.
+
+* **Added amd-pstate driver support to the power management library.**
+
+  Added support for amd-pstate driver which works on AMD EPYC processors.
+
+* **Added a flow action type for P4-defined actions.**
+
+  For P4-programmable devices, hardware pipeline can be configured through
+  a new "PROG" action type and its associated custom arguments.
+  Such P4 pipeline, not using the standard blocks of the flow API,
+  can be managed with ``RTE_FLOW_ITEM_TYPE_FLEX`` and ``RTE_FLOW_ACTION_TYPE_PROG``.
+
+* **Added flow group set miss actions.**
+
+  Introduced ``rte_flow_group_set_miss_actions()`` API to explicitly set
+  a group's miss actions, which are the actions to be performed on packets
+  that didn't match any of the flow rules in the group.
+
+* **Updated Marvell cnxk net driver.**
+
+  * Added support for ``RTE_FLOW_ITEM_TYPE_IPV6_ROUTING_EXT`` flow item.
+  * Added support for ``RTE_FLOW_ACTION_TYPE_AGE`` flow action.
+
+* **Updated Solarflare net driver.**
+
+  * Added support for transfer flow action ``INDIRECT`` with subtype ``VXLAN_ENCAP``.
+  * Supported packet replay (multi-count / multi-delivery) in transfer flows.
+
+* **Updated Netronome/Corigine nfp driver.**
+
+  * Added inline IPsec offload based on the security framework.
+
+* **Updated Wangxun ngbe driver.**
+
+  * Added 100M and auto-neg support in YT PHY fiber mode.
+
+* **Added support for TLS and DTLS record processing.**
+
+  Added TLS and DTLS record transform for security session
+  and added enhancements to ``rte_crypto_op`` fields
+  to handle all datapath requirements of TLS and DTLS.
+  The support was added for TLS 1.2, TLS 1.3 and DTLS 1.2.
+
+* **Added out of place processing support for inline ingress security session.**
+
+  Similar to out of place processing support for lookaside security session,
+  added the same support for inline ingress security session.
+
+* **Added security Rx inject API.**
+
+  Added Rx inject API to allow applications to submit packets
+  for protocol offload and have them injected back to ethdev Rx
+  so that further ethdev Rx actions (IP reassembly, packet parsing and flow lookups)
+  can happen based on inner packet.
+
+  The API when implemented by an ethdev, application would be able to process
+  packets that are received without/failed inline offload processing
+  (such as fragmented ESP packets with inline IPsec offload).
+  The API when implemented by a cryptodev, can be used for injecting packets
+  to ethdev Rx after IPsec processing and take advantage of ethdev Rx actions
+  for the inner packet which cannot be accelerated in inline protocol offload mode.
+
+* **Updated cryptodev scheduler driver.**
+
+  * Added support for DOCSIS security protocol
+    through the ``rte_security`` API callbacks.
+
+* **Updated ipsec_mb crypto driver.**
+
+  * Added support for digest encrypted to AESNI_MB asynchronous crypto driver.
+
+* **Updated Intel QuickAssist Technology driver.**
+
+  * Enabled support for QAT 2.0c (4944) devices in QAT crypto driver.
+
+* **Updated Marvell cnxk crypto driver.**
+
+  * Added SM2 algorithm support in asymmetric crypto operations.
+
+* **Updated Intel vRAN Boost baseband driver.**
+
+  * Added support for the new Intel vRAN Boost v2 device variant (GNR-D)
+    within the unified driver.
+
+* **Added new eventdev Ethernet Rx adapter create API.**
+
+  Added new function ``rte_event_eth_rx_adapter_create_ext_with_params()``
+  for creating Rx adapter instance for the applications desire to
+  control both the event port allocation and event buffer size.
+
+* **Added event DMA adapter library.**
+
+  * Added the Event DMA Adapter Library. This library extends the event-based
+    model by introducing APIs that allow applications to enqueue/dequeue DMA
+    operations to/from dmadev as events scheduled by an event device.
+
+* **Added eventdev support to link queues to port with link profile.**
+
+  Introduced event link profiles that can be used to associated links between
+  event queues and an event port with a unique identifier termed as link profile.
+  The profile can be used to switch between the associated links in fast-path
+  without the additional overhead of linking/unlinking and waiting for unlinking.
+
+  * Added ``rte_event_port_profile_links_set``, ``rte_event_port_profile_unlink``
+    ``rte_event_port_profile_links_get`` and ``rte_event_port_profile_switch``
+    functions to enable this feature.
+
+* **Updated Marvell cnxk eventdev driver.**
+
+  * Added support for ``remaining_ticks_get`` timer adapter PMD callback
+    to get the remaining ticks to expire for a given event timer.
+  * Added link profiles support, up to two link profiles are supported.
+
+* **Added dispatcher library.**
+
+  Added dispatcher library which purpose is to help decouple different
+  parts (modules) of an eventdev-based application.
+
+* **Added a command option ``--relax-rx-mode`` in l3fwd example.**
+
+  Added a command option ``--relax-rx-mode`` in l3fwd example
+  to relax the rx RSS/Offload mode requirement if needed.
 
 Removed Items
 -------------
@@ -99,10 +246,23 @@ Removed Items
 
 * eal: Removed deprecated ``RTE_FUNC_PTR_OR_*`` macros.
 
+* ethdev: Removed deprecated macro ``RTE_ETH_DEV_BONDED_SLAVE``.
+
 * flow_classify: Removed flow classification library and examples.
 
 * kni: Removed the Kernel Network Interface (KNI) library and driver.
 
+* cryptodev: Removed the arrays of algorithm strings ``rte_crypto_cipher_algorithm_strings``,
+  ``rte_crypto_auth_algorithm_strings``, ``rte_crypto_aead_algorithm_strings`` and
+  ``rte_crypto_asym_xform_strings``.
+
+* cryptodev: Removed explicit SM2 xform parameter in asymmetric xform.
+
+* security: Removed deprecated field ``reserved_opts``
+  from struct ``rte_security_ipsec_sa_options``.
+
+* mldev: Removed functions ``rte_ml_io_input_size_get`` and ``rte_ml_io_output_size_get``.
+
 
 API Changes
 -----------
@@ -125,6 +285,38 @@ API Changes
   except ``rte_thread_setname()`` and ``rte_ctrl_thread_create()`` which are
   replaced with ``rte_thread_set_name()`` and ``rte_thread_create_control()``.
 
+* eal: Removed ``RTE_CPUFLAG_NUMFLAGS`` to avoid misusage and theoretical ABI
+  compatibility issue when adding new cpuflags.
+
+* bonding: Replaced master/slave to main/member. The data structure
+  ``struct rte_eth_bond_8023ad_slave_info`` was renamed to
+  ``struct rte_eth_bond_8023ad_member_info`` in DPDK 23.11.
+  The following functions were removed in DPDK 23.11.
+  The old functions:
+  ``rte_eth_bond_8023ad_slave_info``,
+  ``rte_eth_bond_active_slaves_get``,
+  ``rte_eth_bond_slave_add``,
+  ``rte_eth_bond_slave_remove``, and
+  ``rte_eth_bond_slaves_get``
+  will be replaced by:
+  ``rte_eth_bond_8023ad_member_info``,
+  ``rte_eth_bond_active_members_get``,
+  ``rte_eth_bond_member_add``,
+  ``rte_eth_bond_member_remove``, and
+  ``rte_eth_bond_members_get``.
+
+* cryptodev: The elliptic curve asymmetric private and public keys can be maintained
+  per session. These keys are moved from per packet ``rte_crypto_ecdsa_op_param`` and
+  ``rte_crypto_sm2_op_param`` to generic EC xform ``rte_crypto_ec_xform``.
+
+* security: Structures ``rte_security_ops`` and ``rte_security_ctx`` were moved to
+  internal library headers not visible to application.
+
+* mldev: Updated the structure ``rte_ml_model_info`` to support input and output
+  with arbitrary shapes.
+  Updated ``rte_ml_op``, ``rte_ml_io_quantize`` and ``rte_ml_io_dequantize``
+  to support an array of ``rte_ml_buff_seg``.
+
 
 ABI Changes
 -----------
@@ -141,6 +333,17 @@ ABI Changes
    Also, make sure to start the actual text at the margin.
    =======================================================
 
+* ethdev: Added ``recycle_tx_mbufs_reuse`` and ``recycle_rx_descriptors_refill``
+  fields to ``rte_eth_dev`` structure.
+
+* ethdev: Structure ``rte_eth_fp_ops`` was affected to add
+  ``recycle_tx_mbufs_reuse`` and ``recycle_rx_descriptors_refill``
+  fields, to move ``rxq`` and ``txq`` fields, to change the size of
+  ``reserved1`` and ``reserved2`` fields.
+
+* security: struct ``rte_security_ipsec_sa_options`` was updated
+  due to inline out-of-place feature addition.
+
 
 Known Issues
 ------------
diff --git a/doc/guides/sample_app_ug/l3_forward.rst b/doc/guides/sample_app_ug/l3_forward.rst
index 1cc2c1dd1d..00283f070c 100644
--- a/doc/guides/sample_app_ug/l3_forward.rst
+++ b/doc/guides/sample_app_ug/l3_forward.rst
@@ -126,6 +126,8 @@ Where,
 
 * ``--parse-ptype:`` Optional, set to use software to analyze packet type. Without this option, hardware will check the packet type.
 
+* ``--relax-rx-mode:`` Optional, set to enable rx mode relax when RSS/offload is not fully supported by the hardware. When the IPv4 cksum offload is relaxed, it is calculated by the software instead. Without this option, the RSS and cksum offload will be forced.
+
 * ``--per-port-pool:`` Optional, set to use independent buffer pools per port. Without this option, single buffer pool is used for all ports.
 
 * ``--mode:`` Optional, Packet transfer mode for I/O, poll or eventdev.
@@ -140,7 +142,7 @@ Where,
 
 * ``--event-vector-tmo:`` Optional, Max timeout to form vector in nanoseconds if event vectorization is enabled.
 
-* ``--alg=<val>:`` optional, ACL classify method to use, one of:
+* ``--alg=<val>:`` Optional, ACL classify method to use, one of:
   ``scalar|sse|avx2|neon|altivec|avx512x16|avx512x32``
 
 * ``-E:`` Optional, enable exact match,
-- 
2.34.1

[PATCH v2 3/3] doc: add a relax rx mode requirement option

[Trevor Tao]

Add an option to enable the RX mode requirement relax
in release notes and l3fwd sample guide.

Signed-off-by: Trevor Tao <taozj888@163.com>
---
 doc/guides/rel_notes/release_23_11.rst  | 2 ++
 doc/guides/sample_app_ug/l3_forward.rst | 4 +++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/guides/rel_notes/release_23_11.rst b/doc/guides/rel_notes/release_23_11.rst
index f09ecd50fe..2f9e4a54c8 100644
--- a/doc/guides/rel_notes/release_23_11.rst
+++ b/doc/guides/rel_notes/release_23_11.rst
@@ -84,6 +84,8 @@ New Features
   default. The implementation using C11 standard atomic operations is enabled
   via the ``enable_stdatomic`` build option.
 
+* sample: Added a command option ``--relax-rx-mode`` in l3fwd example
+  to relax the rx RSS/Offload mode requirement if needed.
 
 Removed Items
 -------------
diff --git a/doc/guides/sample_app_ug/l3_forward.rst b/doc/guides/sample_app_ug/l3_forward.rst
index 1cc2c1dd1d..00283f070c 100644
--- a/doc/guides/sample_app_ug/l3_forward.rst
+++ b/doc/guides/sample_app_ug/l3_forward.rst
@@ -126,6 +126,8 @@ Where,
 
 * ``--parse-ptype:`` Optional, set to use software to analyze packet type. Without this option, hardware will check the packet type.
 
+* ``--relax-rx-mode:`` Optional, set to enable rx mode relax when RSS/offload is not fully supported by the hardware. When the IPv4 cksum offload is relaxed, it is calculated by the software instead. Without this option, the RSS and cksum offload will be forced.
+
 * ``--per-port-pool:`` Optional, set to use independent buffer pools per port. Without this option, single buffer pool is used for all ports.
 
 * ``--mode:`` Optional, Packet transfer mode for I/O, poll or eventdev.
@@ -140,7 +142,7 @@ Where,
 
 * ``--event-vector-tmo:`` Optional, Max timeout to form vector in nanoseconds if event vectorization is enabled.
 
-* ``--alg=<val>:`` optional, ACL classify method to use, one of:
+* ``--alg=<val>:`` Optional, ACL classify method to use, one of:
   ``scalar|sse|avx2|neon|altivec|avx512x16|avx512x32``
 
 * ``-E:`` Optional, enable exact match,
-- 
2.34.1

[PATCH v2 3/3] doc: add a relax rx mode requirement option

[Trevor Tao]

Add an option to enable the RX mode requirement relax
in release notes and l3fwd sample guide.

Signed-off-by: Trevor Tao <taozj888@163.com>
---
 doc/guides/rel_notes/release_23_11.rst  | 2 ++
 doc/guides/sample_app_ug/l3_forward.rst | 4 +++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/guides/rel_notes/release_23_11.rst b/doc/guides/rel_notes/release_23_11.rst
index f09ecd50fe..2f9e4a54c8 100644
--- a/doc/guides/rel_notes/release_23_11.rst
+++ b/doc/guides/rel_notes/release_23_11.rst
@@ -84,6 +84,8 @@ New Features
   default. The implementation using C11 standard atomic operations is enabled
   via the ``enable_stdatomic`` build option.
 
+* sample: Added a command option ``--relax-rx-mode`` in l3fwd example
+  to relax the rx RSS/Offload mode requirement if needed.
 
 Removed Items
 -------------
diff --git a/doc/guides/sample_app_ug/l3_forward.rst b/doc/guides/sample_app_ug/l3_forward.rst
index 1cc2c1dd1d..00283f070c 100644
--- a/doc/guides/sample_app_ug/l3_forward.rst
+++ b/doc/guides/sample_app_ug/l3_forward.rst
@@ -126,6 +126,8 @@ Where,
 
 * ``--parse-ptype:`` Optional, set to use software to analyze packet type. Without this option, hardware will check the packet type.
 
+* ``--relax-rx-mode:`` Optional, set to enable rx mode relax when RSS/offload is not fully supported by the hardware. When the IPv4 cksum offload is relaxed, it is calculated by the software instead. Without this option, the RSS and cksum offload will be forced.
+
 * ``--per-port-pool:`` Optional, set to use independent buffer pools per port. Without this option, single buffer pool is used for all ports.
 
 * ``--mode:`` Optional, Packet transfer mode for I/O, poll or eventdev.
@@ -140,7 +142,7 @@ Where,
 
 * ``--event-vector-tmo:`` Optional, Max timeout to form vector in nanoseconds if event vectorization is enabled.
 
-* ``--alg=<val>:`` optional, ACL classify method to use, one of:
+* ``--alg=<val>:`` Optional, ACL classify method to use, one of:
   ``scalar|sse|avx2|neon|altivec|avx512x16|avx512x32``
 
 * ``-E:`` Optional, enable exact match,
-- 
2.34.1