[PATCH] app/testpmd: add command line argument 'rx-metadata'

[PATCH] app/testpmd: add command line argument 'rx-metadata'

[Hanumanth Pothula]

Performance drop is observed by sending per packet rx
metadata (offloads).

Hence, introducing command line argument, 'rx-metadata' to
control passing rx metadata to PMD. By default it’s disabled.

Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
---
 app/test-pmd/parameters.c | 4 ++++
 app/test-pmd/testpmd.c    | 6 +++++-
 app/test-pmd/testpmd.h    | 2 ++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c
index e3c9757f3f..daf1218977 100644
--- a/app/test-pmd/parameters.c
+++ b/app/test-pmd/parameters.c
@@ -213,6 +213,7 @@ usage(char* progname)
 	printf("  --hairpin-mode=0xXX: bitmask set the hairpin port mode.\n"
 	       "    0x10 - explicit Tx rule, 0x02 - hairpin ports paired\n"
 	       "    0x01 - hairpin ports loop, 0x00 - hairpin port self\n");
+	printf(" --rx-metadata: send rx  metadata to driver \n");
 }
 
 #ifdef RTE_LIB_CMDLINE
@@ -710,6 +711,7 @@ launch_args_parse(int argc, char** argv)
 		{ "record-burst-stats",         0, 0, 0 },
 		{ PARAM_NUM_PROCS,              1, 0, 0 },
 		{ PARAM_PROC_ID,                1, 0, 0 },
+		{ "rx-metadata",                0, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1510,6 +1512,8 @@ launch_args_parse(int argc, char** argv)
 				num_procs = atoi(optarg);
 			if (!strcmp(lgopts[opt_idx].name, PARAM_PROC_ID))
 				proc_id = atoi(optarg);
+			if (!strcmp(lgopts[opt_idx].name, "rx-metadata"))
+				rx_metadata_negotiate = 1;
 			break;
 		case 'h':
 			usage(argv[0]);
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index addcbcac85..ebbde5dfc9 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -411,6 +411,9 @@ uint8_t clear_ptypes = true;
 /* Hairpin ports configuration mode. */
 uint16_t hairpin_mode;
 
+/* send rx metadata */
+uint8_t rx_metadata_negotiate;
+
 /* Pretty printing of ethdev events */
 static const char * const eth_event_desc[] = {
 	[RTE_ETH_EVENT_UNKNOWN] = "unknown",
@@ -1628,7 +1631,8 @@ init_config_port_offloads(portid_t pid, uint32_t socket_id)
 	int ret;
 	int i;
 
-	eth_rx_metadata_negotiate_mp(pid);
+	if (rx_metadata_negotiate)
+		eth_rx_metadata_negotiate_mp(pid);
 
 	port->dev_conf.txmode = tx_mode;
 	port->dev_conf.rxmode = rx_mode;
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index fb2f5195d3..8a9168c51e 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -621,6 +621,8 @@ extern struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
 extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
 
+extern uint8_t rx_metadata_negotiate;
+
 #ifdef RTE_LIB_GRO
 #define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
 #define GRO_DEFAULT_FLOW_NUM (RTE_GRO_MAX_BURST_ITEM_NUM / \
-- 
2.25.1

[PATCH] app/testpmd: add command line argument 'rx-metadata'

[Hanumanth Pothula]

Presently, rx metadata is sent to PMD by default, leading
to a performance drop as processing for the same in rx path
takes extra cycles.

Hence, introducing command line argument, 'rx-metadata' to
control passing rx metadata to PMD. By default it’s disabled.

ci: skip_checkpatch

Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
Change-Id: If7b6bbc7489d3e9df89c63e000d71ea2f7fe9afd
---
 app/test-pmd/parameters.c | 4 ++++
 app/test-pmd/testpmd.c    | 6 +++++-
 app/test-pmd/testpmd.h    | 2 ++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c
index e3c9757f3f..daf1218977 100644
--- a/app/test-pmd/parameters.c
+++ b/app/test-pmd/parameters.c
@@ -213,6 +213,7 @@ usage(char* progname)
 	printf("  --hairpin-mode=0xXX: bitmask set the hairpin port mode.\n"
 	       "    0x10 - explicit Tx rule, 0x02 - hairpin ports paired\n"
 	       "    0x01 - hairpin ports loop, 0x00 - hairpin port self\n");
+	printf(" --rx-metadata: send rx  metadata to driver \n");
 }
 
 #ifdef RTE_LIB_CMDLINE
@@ -710,6 +711,7 @@ launch_args_parse(int argc, char** argv)
 		{ "record-burst-stats",         0, 0, 0 },
 		{ PARAM_NUM_PROCS,              1, 0, 0 },
 		{ PARAM_PROC_ID,                1, 0, 0 },
+		{ "rx-metadata",                0, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1510,6 +1512,8 @@ launch_args_parse(int argc, char** argv)
 				num_procs = atoi(optarg);
 			if (!strcmp(lgopts[opt_idx].name, PARAM_PROC_ID))
 				proc_id = atoi(optarg);
+			if (!strcmp(lgopts[opt_idx].name, "rx-metadata"))
+				rx_metadata_negotiate = 1;
 			break;
 		case 'h':
 			usage(argv[0]);
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index addcbcac85..ebbde5dfc9 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -411,6 +411,9 @@ uint8_t clear_ptypes = true;
 /* Hairpin ports configuration mode. */
 uint16_t hairpin_mode;
 
+/* send rx metadata */
+uint8_t rx_metadata_negotiate;
+
 /* Pretty printing of ethdev events */
 static const char * const eth_event_desc[] = {
 	[RTE_ETH_EVENT_UNKNOWN] = "unknown",
@@ -1628,7 +1631,8 @@ init_config_port_offloads(portid_t pid, uint32_t socket_id)
 	int ret;
 	int i;
 
-	eth_rx_metadata_negotiate_mp(pid);
+	if (rx_metadata_negotiate)
+		eth_rx_metadata_negotiate_mp(pid);
 
 	port->dev_conf.txmode = tx_mode;
 	port->dev_conf.rxmode = rx_mode;
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index fb2f5195d3..8a9168c51e 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -621,6 +621,8 @@ extern struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
 extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
 
+extern uint8_t rx_metadata_negotiate;
+
 #ifdef RTE_LIB_GRO
 #define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
 #define GRO_DEFAULT_FLOW_NUM (RTE_GRO_MAX_BURST_ITEM_NUM / \
-- 
2.25.1

[PATCH] app/testpmd: add command line argument 'rx-metadata'

[Hanumanth Pothula]

Presently, rx metadata is sent to PMD by default, leading
to a performance drop as processing for the same in rx path
takes extra cycles.

Hence, introducing command line argument, 'rx-metadata' to
control passing rx metadata to PMD. By default it’s disabled.

Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
---
 app/test-pmd/parameters.c | 4 ++++
 app/test-pmd/testpmd.c    | 6 +++++-
 app/test-pmd/testpmd.h    | 2 ++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c
index e3c9757f3f..daf1218977 100644
--- a/app/test-pmd/parameters.c
+++ b/app/test-pmd/parameters.c
@@ -213,6 +213,7 @@ usage(char* progname)
 	printf("  --hairpin-mode=0xXX: bitmask set the hairpin port mode.\n"
 	       "    0x10 - explicit Tx rule, 0x02 - hairpin ports paired\n"
 	       "    0x01 - hairpin ports loop, 0x00 - hairpin port self\n");
+	printf(" --rx-metadata: send rx  metadata to driver \n");
 }
 
 #ifdef RTE_LIB_CMDLINE
@@ -710,6 +711,7 @@ launch_args_parse(int argc, char** argv)
 		{ "record-burst-stats",         0, 0, 0 },
 		{ PARAM_NUM_PROCS,              1, 0, 0 },
 		{ PARAM_PROC_ID,                1, 0, 0 },
+		{ "rx-metadata",                0, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1510,6 +1512,8 @@ launch_args_parse(int argc, char** argv)
 				num_procs = atoi(optarg);
 			if (!strcmp(lgopts[opt_idx].name, PARAM_PROC_ID))
 				proc_id = atoi(optarg);
+			if (!strcmp(lgopts[opt_idx].name, "rx-metadata"))
+				rx_metadata_negotiate = 1;
 			break;
 		case 'h':
 			usage(argv[0]);
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index addcbcac85..ebbde5dfc9 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -411,6 +411,9 @@ uint8_t clear_ptypes = true;
 /* Hairpin ports configuration mode. */
 uint16_t hairpin_mode;
 
+/* send rx metadata */
+uint8_t rx_metadata_negotiate;
+
 /* Pretty printing of ethdev events */
 static const char * const eth_event_desc[] = {
 	[RTE_ETH_EVENT_UNKNOWN] = "unknown",
@@ -1628,7 +1631,8 @@ init_config_port_offloads(portid_t pid, uint32_t socket_id)
 	int ret;
 	int i;
 
-	eth_rx_metadata_negotiate_mp(pid);
+	if (rx_metadata_negotiate)
+		eth_rx_metadata_negotiate_mp(pid);
 
 	port->dev_conf.txmode = tx_mode;
 	port->dev_conf.rxmode = rx_mode;
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index fb2f5195d3..8a9168c51e 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -621,6 +621,8 @@ extern struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
 extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
 
+extern uint8_t rx_metadata_negotiate;
+
 #ifdef RTE_LIB_GRO
 #define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
 #define GRO_DEFAULT_FLOW_NUM (RTE_GRO_MAX_BURST_ITEM_NUM / \
-- 
2.25.1

Re: [PATCH] app/testpmd: add command line argument 'rx-metadata'

[Ivan Malov]

[-- Attachment #1: Type: text/plain, Size: 3688 bytes --]

Hi Hanumanth,

Thanks for expanding support for the NIC-to-PMD Rx metadata feature.
I do not object the idea of the patch, it looks aceeptable.
However, please find my comments below.

On Mon, 1 Aug 2022, Hanumanth Pothula wrote:

> Presently, rx metadata is sent to PMD by default, leading
> to a performance drop as processing for the same in rx path
> takes extra cycles.
>
> Hence, introducing command line argument, 'rx-metadata' to
> control passing rx metadata to PMD. By default it’s disabled.
>
> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> ---
> app/test-pmd/parameters.c | 4 ++++
> app/test-pmd/testpmd.c | 6 +++++-
> app/test-pmd/testpmd.h | 2 ++
> 3 files changed, 11 insertions(+), 1 deletion(-)
>
> diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c
> index e3c9757f3f..daf1218977 100644
> --- a/app/test-pmd/parameters.c
> +++ b/app/test-pmd/parameters.c
> @@ -213,6 +213,7 @@ usage(char* progname)
> printf(" --hairpin-mode=0xXX: bitmask set the hairpin port mode.\n"
> " 0x10 - explicit Tx rule, 0x02 - hairpin ports paired\n"
> " 0x01 - hairpin ports loop, 0x00 - hairpin port self\n");
> + printf(" --rx-metadata: send rx metadata to driver \n");

In accordance with the above "printf", one should have 2 space
characters between the opening double quote character and "--".
Also, it is quite common to capitalise "R" in "Rx".
The space character before "\n" seems redundant.

Regarding the option name, I would appreciate if it gets more
precise and clear. Consider: "--nic-to-pmd-rx-metadata".
I do not insist on this particular name.

Perhaps it pays to have a bit more verbose description as well.
Consider: "let the NIC deliver per-packet Rx metadata to PMD".

> }
> #ifdef RTE_LIB_CMDLINE
> @@ -710,6 +711,7 @@ launch_args_parse(int argc, char** argv)
> { "record-burst-stats", 0, 0, 0 },
> { PARAM_NUM_PROCS, 1, 0, 0 },
> { PARAM_PROC_ID, 1, 0, 0 },
> + { "rx-metadata", 0, 0, 0 },
> { 0, 0, 0, 0 },
> };
> @@ -1510,6 +1512,8 @@ launch_args_parse(int argc, char** argv)
> num_procs = atoi(optarg);
> if (!strcmp(lgopts[opt_idx].name, PARAM_PROC_ID))
> proc_id = atoi(optarg);
> + if (!strcmp(lgopts[opt_idx].name, "rx-metadata"))
> + rx_metadata_negotiate = 1;
> break;
> case 'h':
> usage(argv[0]);
> diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
> index addcbcac85..ebbde5dfc9 100644
> --- a/app/test-pmd/testpmd.c
> +++ b/app/test-pmd/testpmd.c
> @@ -411,6 +411,9 @@ uint8_t clear_ptypes = true;
> /* Hairpin ports configuration mode. */
> uint16_t hairpin_mode;
> +/* send rx metadata */

Consider: "/* Send Rx metadata */".

> +uint8_t rx_metadata_negotiate;
> +
> /* Pretty printing of ethdev events */
> static const char * const eth_event_desc[] = {
> [RTE_ETH_EVENT_UNKNOWN] = "unknown",
> @@ -1628,7 +1631,8 @@ init_config_port_offloads(portid_t pid, uint32_t socket_id)
> int ret;
> int i;
> - eth_rx_metadata_negotiate_mp(pid);
> + if (rx_metadata_negotiate)
> + eth_rx_metadata_negotiate_mp(pid);
> port->dev_conf.txmode = tx_mode;
> port->dev_conf.rxmode = rx_mode;
> diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
> index fb2f5195d3..8a9168c51e 100644
> --- a/app/test-pmd/testpmd.h
> +++ b/app/test-pmd/testpmd.h
> @@ -621,6 +621,8 @@ extern struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
> extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
> extern uint32_t burst_tx_retry_num; /**< Burst tx retry number for mac-retry. */
> +extern uint8_t rx_metadata_negotiate;
> +
> #ifdef RTE_LIB_GRO
> #define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
> #define GRO_DEFAULT_FLOW_NUM (RTE_GRO_MAX_BURST_ITEM_NUM / \
>
> -- 
> 2.25.1
>
>

--
Best regards,
Ivan M

[PATCH] app/testpmd: add command line argument 'nic-to-pmd-rx-metadata'

[Hanumanth Pothula]

Presently, rx metadata is sent to PMD by default, leading
to a performance drop as processing for the same in rx path
takes extra cycles.

Hence, introducing command line argument, 'nic-to-pmd-rx-metadata'
to control passing rx metadata to PMD. By default it’s disabled.

Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>

v2:
- taken care alignment issues
- renamed command line argument from rx-metadata to nic-to-pmd-rx-metadata
- renamed variable name from rx-metadata to nic_to_pmd_rx_metadata
---
 app/test-pmd/parameters.c | 4 ++++
 app/test-pmd/testpmd.c    | 6 +++++-
 app/test-pmd/testpmd.h    | 2 ++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c
index e3c9757f3f..a381945492 100644
--- a/app/test-pmd/parameters.c
+++ b/app/test-pmd/parameters.c
@@ -213,6 +213,7 @@ usage(char* progname)
 	printf("  --hairpin-mode=0xXX: bitmask set the hairpin port mode.\n"
 	       "    0x10 - explicit Tx rule, 0x02 - hairpin ports paired\n"
 	       "    0x01 - hairpin ports loop, 0x00 - hairpin port self\n");
+	printf("  --nic-to-pmd-rx-metadata: let the NIC deliver per-packet Rx metadata to PMD\n");
 }
 
 #ifdef RTE_LIB_CMDLINE
@@ -710,6 +711,7 @@ launch_args_parse(int argc, char** argv)
 		{ "record-burst-stats",         0, 0, 0 },
 		{ PARAM_NUM_PROCS,              1, 0, 0 },
 		{ PARAM_PROC_ID,                1, 0, 0 },
+		{ "nic-to-pmd-rx-metadata",     0, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1510,6 +1512,8 @@ launch_args_parse(int argc, char** argv)
 				num_procs = atoi(optarg);
 			if (!strcmp(lgopts[opt_idx].name, PARAM_PROC_ID))
 				proc_id = atoi(optarg);
+			if (!strcmp(lgopts[opt_idx].name, "nic-to-pmd-rx-metadata"))
+				nic_to_pmd_rx_metadata = 1;
 			break;
 		case 'h':
 			usage(argv[0]);
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index addcbcac85..2b17d4f757 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -411,6 +411,9 @@ uint8_t clear_ptypes = true;
 /* Hairpin ports configuration mode. */
 uint16_t hairpin_mode;
 
+/* Send Rx metadata */
+uint8_t nic_to_pmd_rx_metadata;
+
 /* Pretty printing of ethdev events */
 static const char * const eth_event_desc[] = {
 	[RTE_ETH_EVENT_UNKNOWN] = "unknown",
@@ -1628,7 +1631,8 @@ init_config_port_offloads(portid_t pid, uint32_t socket_id)
 	int ret;
 	int i;
 
-	eth_rx_metadata_negotiate_mp(pid);
+	if (nic_to_pmd_rx_metadata)
+		eth_rx_metadata_negotiate_mp(pid);
 
 	port->dev_conf.txmode = tx_mode;
 	port->dev_conf.rxmode = rx_mode;
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index fb2f5195d3..294a9c8cf4 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -621,6 +621,8 @@ extern struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
 extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
 
+extern uint8_t nic_to_pmd_rx_metadata;
+
 #ifdef RTE_LIB_GRO
 #define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
 #define GRO_DEFAULT_FLOW_NUM (RTE_GRO_MAX_BURST_ITEM_NUM / \
-- 
2.25.1

[PATCH v2 1/2] version: 22.11-rc0

[Hanumanth Pothula]

From: David Marchand <david.marchand@redhat.com>

Start a new release cycle with empty release notes.

The ABI version becomes 23.0.
The map files are updated to the new ABI major number (23).
The ABI exceptions are dropped and CI ABI checks are disabled because
compatibility is not preserved.
Special handling of removed drivers is also dropped in check-abi.sh and
a note has been added in libabigail.abignore as a reminder.

Signed-off-by: David Marchand <david.marchand@redhat.com>
Acked-by: Thomas Monjalon <thomas@monjalon.net>
---
 .github/workflows/build.yml                |   4 +-
 .travis.yml                                |  21 +---
 ABI_VERSION                                |   2 +-
 VERSION                                    |   2 +-
 devtools/check-abi.sh                      |   4 -
 devtools/libabigail.abignore               |  37 ------
 doc/guides/rel_notes/index.rst             |   1 +
 doc/guides/rel_notes/release_22_11.rst     | 136 +++++++++++++++++++++
 drivers/baseband/acc100/version.map        |   2 +-
 drivers/baseband/fpga_5gnr_fec/version.map |   2 +-
 drivers/baseband/fpga_lte_fec/version.map  |   2 +-
 drivers/baseband/la12xx/version.map        |   2 +-
 drivers/baseband/null/version.map          |   2 +-
 drivers/baseband/turbo_sw/version.map      |   2 +-
 drivers/bus/fslmc/version.map              |   2 +-
 drivers/bus/ifpga/version.map              |   2 +-
 drivers/bus/pci/version.map                |   2 +-
 drivers/bus/vdev/version.map               |   2 +-
 drivers/bus/vmbus/version.map              |   2 +-
 drivers/common/qat/version.map             |   2 +-
 drivers/compress/isal/version.map          |   2 +-
 drivers/compress/mlx5/version.map          |   2 +-
 drivers/compress/octeontx/version.map      |   2 +-
 drivers/compress/zlib/version.map          |   2 +-
 drivers/crypto/armv8/version.map           |   2 +-
 drivers/crypto/bcmfs/version.map           |   2 +-
 drivers/crypto/caam_jr/version.map         |   2 +-
 drivers/crypto/ccp/version.map             |   2 +-
 drivers/crypto/ipsec_mb/version.map        |   2 +-
 drivers/crypto/mlx5/version.map            |   2 +-
 drivers/crypto/mvsam/version.map           |   2 +-
 drivers/crypto/nitrox/version.map          |   2 +-
 drivers/crypto/null/version.map            |   2 +-
 drivers/crypto/octeontx/version.map        |   2 +-
 drivers/crypto/openssl/version.map         |   2 +-
 drivers/crypto/scheduler/version.map       |   2 +-
 drivers/crypto/virtio/version.map          |   2 +-
 drivers/dma/cnxk/version.map               |   2 +-
 drivers/dma/dpaa/version.map               |   2 +-
 drivers/dma/dpaa2/version.map              |   2 +-
 drivers/dma/hisilicon/version.map          |   2 +-
 drivers/dma/idxd/version.map               |   2 +-
 drivers/dma/ioat/version.map               |   2 +-
 drivers/dma/skeleton/version.map           |   2 +-
 drivers/event/dlb2/version.map             |   2 +-
 drivers/event/dpaa/version.map             |   2 +-
 drivers/event/dpaa2/version.map            |   2 +-
 drivers/event/dsw/version.map              |   2 +-
 drivers/event/octeontx/version.map         |   2 +-
 drivers/event/opdl/version.map             |   2 +-
 drivers/event/skeleton/version.map         |   2 +-
 drivers/event/sw/version.map               |   2 +-
 drivers/gpu/cuda/version.map               |   2 +-
 drivers/mempool/bucket/version.map         |   2 +-
 drivers/mempool/dpaa2/version.map          |   2 +-
 drivers/mempool/octeontx/version.map       |   2 +-
 drivers/mempool/ring/version.map           |   2 +-
 drivers/mempool/stack/version.map          |   2 +-
 drivers/net/af_packet/version.map          |   2 +-
 drivers/net/af_xdp/version.map             |   2 +-
 drivers/net/ark/version.map                |   2 +-
 drivers/net/atlantic/version.map           |   2 +-
 drivers/net/avp/version.map                |   2 +-
 drivers/net/axgbe/version.map              |   2 +-
 drivers/net/bnx2x/version.map              |   2 +-
 drivers/net/bnxt/version.map               |   2 +-
 drivers/net/bonding/version.map            |   2 +-
 drivers/net/cnxk/version.map               |  12 +-
 drivers/net/cxgbe/version.map              |   2 +-
 drivers/net/dpaa/version.map               |   2 +-
 drivers/net/dpaa2/version.map              |   2 +-
 drivers/net/e1000/version.map              |   2 +-
 drivers/net/ena/version.map                |   2 +-
 drivers/net/enetc/version.map              |   2 +-
 drivers/net/enetfec/version.map            |   2 +-
 drivers/net/enic/version.map               |   2 +-
 drivers/net/failsafe/version.map           |   2 +-
 drivers/net/fm10k/version.map              |   2 +-
 drivers/net/hinic/version.map              |   2 +-
 drivers/net/hns3/version.map               |   2 +-
 drivers/net/i40e/version.map               |   2 +-
 drivers/net/iavf/version.map               |   2 +-
 drivers/net/ice/version.map                |   2 +-
 drivers/net/igc/version.map                |   2 +-
 drivers/net/ionic/version.map              |   2 +-
 drivers/net/ipn3ke/version.map             |   2 +-
 drivers/net/ixgbe/version.map              |   2 +-
 drivers/net/kni/version.map                |   2 +-
 drivers/net/liquidio/version.map           |   2 +-
 drivers/net/memif/version.map              |   2 +-
 drivers/net/mlx4/version.map               |   2 +-
 drivers/net/mlx5/version.map               |   2 +-
 drivers/net/mvneta/version.map             |   2 +-
 drivers/net/mvpp2/version.map              |   2 +-
 drivers/net/netvsc/version.map             |   2 +-
 drivers/net/nfb/version.map                |   2 +-
 drivers/net/nfp/version.map                |   2 +-
 drivers/net/ngbe/version.map               |   2 +-
 drivers/net/null/version.map               |   2 +-
 drivers/net/octeontx/version.map           |   2 +-
 drivers/net/octeontx_ep/version.map        |   2 +-
 drivers/net/pcap/version.map               |   2 +-
 drivers/net/pfe/version.map                |   2 +-
 drivers/net/qede/version.map               |   2 +-
 drivers/net/ring/version.map               |   2 +-
 drivers/net/sfc/version.map                |   2 +-
 drivers/net/softnic/version.map            |   2 +-
 drivers/net/tap/version.map                |   2 +-
 drivers/net/thunderx/version.map           |   2 +-
 drivers/net/txgbe/version.map              |   2 +-
 drivers/net/vdev_netvsc/version.map        |   2 +-
 drivers/net/vhost/version.map              |   2 +-
 drivers/net/virtio/version.map             |   2 +-
 drivers/net/vmxnet3/version.map            |   2 +-
 drivers/raw/cnxk_bphy/version.map          |   2 +-
 drivers/raw/cnxk_gpio/version.map          |   2 +-
 drivers/raw/dpaa2_cmdif/version.map        |   2 +-
 drivers/raw/ifpga/version.map              |  18 +--
 drivers/raw/ioat/version.map               |   2 +-
 drivers/raw/ntb/version.map                |   2 +-
 drivers/raw/skeleton/version.map           |   2 +-
 drivers/regex/cn9k/version.map             |   2 +-
 drivers/regex/mlx5/version.map             |   2 +-
 drivers/vdpa/ifc/version.map               |   2 +-
 drivers/vdpa/mlx5/version.map              |   2 +-
 drivers/vdpa/sfc/version.map               |   2 +-
 lib/acl/version.map                        |   2 +-
 lib/bbdev/version.map                      |   2 +-
 lib/bitratestats/version.map               |   2 +-
 lib/bpf/version.map                        |   2 +-
 lib/cfgfile/version.map                    |   2 +-
 lib/cmdline/version.map                    |   2 +-
 lib/cryptodev/version.map                  |   2 +-
 lib/distributor/version.map                |   2 +-
 lib/eal/version.map                        |   2 +-
 lib/efd/version.map                        |   2 +-
 lib/ethdev/version.map                     |   2 +-
 lib/eventdev/version.map                   |   2 +-
 lib/fib/version.map                        |   2 +-
 lib/gro/version.map                        |   2 +-
 lib/gso/version.map                        |   2 +-
 lib/hash/version.map                       |   2 +-
 lib/ip_frag/version.map                    |   2 +-
 lib/ipsec/version.map                      |   2 +-
 lib/jobstats/version.map                   |   2 +-
 lib/kni/version.map                        |   2 +-
 lib/kvargs/version.map                     |   2 +-
 lib/latencystats/version.map               |   2 +-
 lib/lpm/version.map                        |   2 +-
 lib/mbuf/version.map                       |   2 +-
 lib/member/version.map                     |   2 +-
 lib/mempool/version.map                    |   2 +-
 lib/meter/version.map                      |   2 +-
 lib/metrics/version.map                    |   2 +-
 lib/net/version.map                        |   2 +-
 lib/pci/version.map                        |   2 +-
 lib/pdump/version.map                      |   2 +-
 lib/pipeline/version.map                   |   2 +-
 lib/port/version.map                       |   2 +-
 lib/power/version.map                      |   2 +-
 lib/rawdev/version.map                     |   2 +-
 lib/rcu/version.map                        |   2 +-
 lib/reorder/version.map                    |   2 +-
 lib/rib/version.map                        |   2 +-
 lib/ring/version.map                       |   2 +-
 lib/sched/version.map                      |   2 +-
 lib/security/version.map                   |   2 +-
 lib/stack/version.map                      |   2 +-
 lib/table/version.map                      |   2 +-
 lib/telemetry/version.map                  |   2 +-
 lib/timer/version.map                      |   2 +-
 lib/vhost/version.map                      |   2 +-
 172 files changed, 319 insertions(+), 242 deletions(-)
 create mode 100644 doc/guides/rel_notes/release_22_11.rst

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 1106256539..6f04e7071c 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -25,7 +25,7 @@ jobs:
       MINGW: ${{ matrix.config.cross == 'mingw' }}
       MINI: ${{ matrix.config.mini != '' }}
       PPC64LE: ${{ matrix.config.cross == 'ppc64le' }}
-      REF_GIT_TAG: v22.03
+      REF_GIT_TAG: none
       RISCV64: ${{ matrix.config.cross == 'riscv64' }}
       RUN_TESTS: ${{ contains(matrix.config.checks, 'tests') }}
 
@@ -43,7 +43,7 @@ jobs:
           - os: ubuntu-20.04
             compiler: gcc
             library: shared
-            checks: abi+doc+tests
+            checks: doc+tests
           - os: ubuntu-20.04
             compiler: clang
             library: static
diff --git a/.travis.yml b/.travis.yml
index 5f46dccb54..4bb5bf629e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,7 +42,7 @@ script: ./.ci/${TRAVIS_OS_NAME}-build.sh
 env:
   global:
     - LIBABIGAIL_VERSION=libabigail-1.8
-    - REF_GIT_TAG=v22.03
+    - REF_GIT_TAG=none
 
 jobs:
   include:
@@ -61,14 +61,6 @@ jobs:
         packages:
           - *required_packages
           - *doc_packages
-  - env: DEF_LIB="shared" ABI_CHECKS=true
-    arch: amd64
-    compiler: gcc
-    addons:
-      apt:
-        packages:
-          - *required_packages
-          - *libabigail_build_packages
   # x86_64 clang jobs
   - env: DEF_LIB="static"
     arch: amd64
@@ -145,17 +137,6 @@ jobs:
         packages:
           - *required_packages
           - *doc_packages
-  - env: DEF_LIB="shared" ABI_CHECKS=true
-    dist: focal
-    arch: arm64-graviton2
-    virt: vm
-    group: edge
-    compiler: gcc
-    addons:
-      apt:
-        packages:
-          - *required_packages
-          - *libabigail_build_packages
   # aarch64 clang jobs
   - env: DEF_LIB="static"
     dist: focal
diff --git a/ABI_VERSION b/ABI_VERSION
index 95af471221..919c868b57 100644
--- a/ABI_VERSION
+++ b/ABI_VERSION
@@ -1 +1 @@
-22.2
+23.0
diff --git a/VERSION b/VERSION
index 97371cc668..4bc005def2 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-22.07.0
+22.11.0-rc0
diff --git a/devtools/check-abi.sh b/devtools/check-abi.sh
index d465c8cee0..c583eae2fd 100755
--- a/devtools/check-abi.sh
+++ b/devtools/check-abi.sh
@@ -37,10 +37,6 @@ fi
 error=
 for dump in $(find $refdir -name "*.dump"); do
 	name=$(basename $dump)
-	if grep -qE '\<librte_*(.*_octeontx2|raw_dpaa2_qdma)' $dump; then
-		echo "Skipped removed driver $name."
-		continue
-	fi
 	dump2=$(find $newdir -name $name)
 	if [ -z "$dump2" ] || [ ! -e "$dump2" ]; then
 		echo "Error: cannot find $name in $newdir" >&2
diff --git a/devtools/libabigail.abignore b/devtools/libabigail.abignore
index 8c8c648895..7a93de3ba1 100644
--- a/devtools/libabigail.abignore
+++ b/devtools/libabigail.abignore
@@ -34,40 +34,3 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ; Temporary exceptions till next major ABI version ;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-; Ignore ethdev event enum update
-; because new event cannot be received if not registered
-[suppress_type]
-        type_kind = enum
-        changed_enumerators = RTE_ETH_EVENT_MAX
-
-; Ignore fields inserted in cacheline boundary of rte_eth_rxq_info
-; because the struct size is unchanged
-[suppress_type]
-        name = rte_eth_rxq_info
-        has_data_member_inserted_between = {offset_of(rx_buf_size), end}
-
-; Ignore fields inserted in place of reserved_opts of rte_security_ipsec_sa_options
-[suppress_type]
-        name = rte_security_ipsec_sa_options
-        has_data_member_inserted_between = {offset_of(reserved_opts), end}
-
-; Ignore section attribute fixes in experimental regexdev library
-[suppress_file]
-        soname_regexp = ^librte_regexdev\.
-
-; Ignore changes in common mlx5 driver, should be all internal
-[suppress_file]
-        soname_regexp = ^librte_common_mlx5\.
-
-; Ignore visibility fix of local functions in experimental auxiliary driver
-[suppress_file]
-        soname_regexp = ^librte_bus_auxiliary\.
-
-; Ignore visibility fix of local functions in experimental gpudev library
-[suppress_file]
-        soname_regexp = ^librte_gpudev\.
-
-; Ignore libabigail false-positive in clang builds, after moving code.
-[suppress_function]
-	name = rte_eal_remote_launch
diff --git a/doc/guides/rel_notes/index.rst b/doc/guides/rel_notes/index.rst
index 93a3f7e5da..f6782b91db 100644
--- a/doc/guides/rel_notes/index.rst
+++ b/doc/guides/rel_notes/index.rst
@@ -8,6 +8,7 @@ Release Notes
     :maxdepth: 1
     :numbered:
 
+    release_22_11
     release_22_07
     release_22_03
     release_21_11
diff --git a/doc/guides/rel_notes/release_22_11.rst b/doc/guides/rel_notes/release_22_11.rst
new file mode 100644
index 0000000000..8c021cf050
--- /dev/null
+++ b/doc/guides/rel_notes/release_22_11.rst
@@ -0,0 +1,136 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright 2022 The DPDK contributors
+
+.. include:: <isonum.txt>
+
+DPDK Release 22.11
+==================
+
+.. **Read this first.**
+
+   The text in the sections below explains how to update the release notes.
+
+   Use proper spelling, capitalization and punctuation in all sections.
+
+   Variable and config names should be quoted as fixed width text:
+   ``LIKE_THIS``.
+
+   Build the docs and view the output file to ensure the changes are correct::
+
+      ninja -C build doc
+      xdg-open build/doc/guides/html/rel_notes/release_22_11.html
+
+
+New Features
+------------
+
+.. This section should contain new features added in this release.
+   Sample format:
+
+   * **Add a title in the past tense with a full stop.**
+
+     Add a short 1-2 sentence description in the past tense.
+     The description should be enough to allow someone scanning
+     the release notes to understand the new feature.
+
+     If the feature adds a lot of sub-features you can use a bullet list
+     like this:
+
+     * Added feature foo to do something.
+     * Enhanced feature bar to do something else.
+
+     Refer to the previous release notes for examples.
+
+     Suggested order in release notes items:
+     * Core libs (EAL, mempool, ring, mbuf, buses)
+     * Device abstraction libs and PMDs (ordered alphabetically by vendor name)
+       - ethdev (lib, PMDs)
+       - cryptodev (lib, PMDs)
+       - eventdev (lib, PMDs)
+       - etc
+     * Other libs
+     * Apps, Examples, Tools (if significant)
+
+     This section is a comment. Do not overwrite or remove it.
+     Also, make sure to start the actual text at the margin.
+     =======================================================
+
+
+Removed Items
+-------------
+
+.. This section should contain removed items in this release. Sample format:
+
+   * Add a short 1-2 sentence description of the removed item
+     in the past tense.
+
+   This section is a comment. Do not overwrite or remove it.
+   Also, make sure to start the actual text at the margin.
+   =======================================================
+
+
+API Changes
+-----------
+
+.. This section should contain API changes. Sample format:
+
+   * sample: Add a short 1-2 sentence description of the API change
+     which was announced in the previous releases and made in this release.
+     Start with a scope label like "ethdev:".
+     Use fixed width quotes for ``function_names`` or ``struct_names``.
+     Use the past tense.
+
+   This section is a comment. Do not overwrite or remove it.
+   Also, make sure to start the actual text at the margin.
+   =======================================================
+
+
+ABI Changes
+-----------
+
+.. This section should contain ABI changes. Sample format:
+
+   * sample: Add a short 1-2 sentence description of the ABI change
+     which was announced in the previous releases and made in this release.
+     Start with a scope label like "ethdev:".
+     Use fixed width quotes for ``function_names`` or ``struct_names``.
+     Use the past tense.
+
+   This section is a comment. Do not overwrite or remove it.
+   Also, make sure to start the actual text at the margin.
+   =======================================================
+
+
+Known Issues
+------------
+
+.. This section should contain new known issues in this release. Sample format:
+
+   * **Add title in present tense with full stop.**
+
+     Add a short 1-2 sentence description of the known issue
+     in the present tense. Add information on any known workarounds.
+
+   This section is a comment. Do not overwrite or remove it.
+   Also, make sure to start the actual text at the margin.
+   =======================================================
+
+
+Tested Platforms
+----------------
+
+.. This section should contain a list of platforms that were tested
+   with this release.
+
+   The format is:
+
+   * <vendor> platform with <vendor> <type of devices> combinations
+
+     * List of CPU
+     * List of OS
+     * List of devices
+     * Other relevant details...
+
+   This section is a comment. Do not overwrite or remove it.
+   Also, make sure to start the actual text at the margin.
+   =======================================================
diff --git a/drivers/baseband/acc100/version.map b/drivers/baseband/acc100/version.map
index 13f0398d71..b4ff13e38f 100644
--- a/drivers/baseband/acc100/version.map
+++ b/drivers/baseband/acc100/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
 
diff --git a/drivers/baseband/fpga_5gnr_fec/version.map b/drivers/baseband/fpga_5gnr_fec/version.map
index de4e5025bf..6b191cf330 100644
--- a/drivers/baseband/fpga_5gnr_fec/version.map
+++ b/drivers/baseband/fpga_5gnr_fec/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
 
diff --git a/drivers/baseband/fpga_lte_fec/version.map b/drivers/baseband/fpga_lte_fec/version.map
index e3bfa6edb0..aab28a9976 100644
--- a/drivers/baseband/fpga_lte_fec/version.map
+++ b/drivers/baseband/fpga_lte_fec/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
 
diff --git a/drivers/baseband/la12xx/version.map b/drivers/baseband/la12xx/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/baseband/la12xx/version.map
+++ b/drivers/baseband/la12xx/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/baseband/null/version.map b/drivers/baseband/null/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/baseband/null/version.map
+++ b/drivers/baseband/null/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/baseband/turbo_sw/version.map b/drivers/baseband/turbo_sw/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/baseband/turbo_sw/version.map
+++ b/drivers/baseband/turbo_sw/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/bus/fslmc/version.map b/drivers/bus/fslmc/version.map
index 79b29746b6..a25a9e8ca0 100644
--- a/drivers/bus/fslmc/version.map
+++ b/drivers/bus/fslmc/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_fslmc_vfio_mem_dmamap;
diff --git a/drivers/bus/ifpga/version.map b/drivers/bus/ifpga/version.map
index 8ac3a4d258..c0a1eecae8 100644
--- a/drivers/bus/ifpga/version.map
+++ b/drivers/bus/ifpga/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_ifpga_driver_register;
diff --git a/drivers/bus/pci/version.map b/drivers/bus/pci/version.map
index aa56439c2b..3d5c2f3068 100644
--- a/drivers/bus/pci/version.map
+++ b/drivers/bus/pci/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_pci_dump;
diff --git a/drivers/bus/vdev/version.map b/drivers/bus/vdev/version.map
index 0d60b7e2bc..7cc3098a50 100644
--- a/drivers/bus/vdev/version.map
+++ b/drivers/bus/vdev/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_vdev_add_custom_scan;
diff --git a/drivers/bus/vmbus/version.map b/drivers/bus/vmbus/version.map
index 3cadec7fae..84db31b6e9 100644
--- a/drivers/bus/vmbus/version.map
+++ b/drivers/bus/vmbus/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_vmbus_chan_close;
diff --git a/drivers/common/qat/version.map b/drivers/common/qat/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/common/qat/version.map
+++ b/drivers/common/qat/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/compress/isal/version.map b/drivers/compress/isal/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/compress/isal/version.map
+++ b/drivers/compress/isal/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/compress/mlx5/version.map b/drivers/compress/mlx5/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/compress/mlx5/version.map
+++ b/drivers/compress/mlx5/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/compress/octeontx/version.map b/drivers/compress/octeontx/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/compress/octeontx/version.map
+++ b/drivers/compress/octeontx/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/compress/zlib/version.map b/drivers/compress/zlib/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/compress/zlib/version.map
+++ b/drivers/compress/zlib/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/crypto/armv8/version.map b/drivers/crypto/armv8/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/crypto/armv8/version.map
+++ b/drivers/crypto/armv8/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/crypto/bcmfs/version.map b/drivers/crypto/bcmfs/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/crypto/bcmfs/version.map
+++ b/drivers/crypto/bcmfs/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/crypto/caam_jr/version.map b/drivers/crypto/caam_jr/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/crypto/caam_jr/version.map
+++ b/drivers/crypto/caam_jr/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/crypto/ccp/version.map b/drivers/crypto/ccp/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/crypto/ccp/version.map
+++ b/drivers/crypto/ccp/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/crypto/ipsec_mb/version.map b/drivers/crypto/ipsec_mb/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/crypto/ipsec_mb/version.map
+++ b/drivers/crypto/ipsec_mb/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/crypto/mlx5/version.map b/drivers/crypto/mlx5/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/crypto/mlx5/version.map
+++ b/drivers/crypto/mlx5/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/crypto/mvsam/version.map b/drivers/crypto/mvsam/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/crypto/mvsam/version.map
+++ b/drivers/crypto/mvsam/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/crypto/nitrox/version.map b/drivers/crypto/nitrox/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/crypto/nitrox/version.map
+++ b/drivers/crypto/nitrox/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/crypto/null/version.map b/drivers/crypto/null/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/crypto/null/version.map
+++ b/drivers/crypto/null/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/crypto/octeontx/version.map b/drivers/crypto/octeontx/version.map
index 997a95ea33..cc4b6b0970 100644
--- a/drivers/crypto/octeontx/version.map
+++ b/drivers/crypto/octeontx/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
 
diff --git a/drivers/crypto/openssl/version.map b/drivers/crypto/openssl/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/crypto/openssl/version.map
+++ b/drivers/crypto/openssl/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/crypto/scheduler/version.map b/drivers/crypto/scheduler/version.map
index 47e4487b75..74491beabb 100644
--- a/drivers/crypto/scheduler/version.map
+++ b/drivers/crypto/scheduler/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_cryptodev_scheduler_load_user_scheduler;
diff --git a/drivers/crypto/virtio/version.map b/drivers/crypto/virtio/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/crypto/virtio/version.map
+++ b/drivers/crypto/virtio/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/dma/cnxk/version.map b/drivers/dma/cnxk/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/dma/cnxk/version.map
+++ b/drivers/dma/cnxk/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/dma/dpaa/version.map b/drivers/dma/dpaa/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/dma/dpaa/version.map
+++ b/drivers/dma/dpaa/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/dma/dpaa2/version.map b/drivers/dma/dpaa2/version.map
index 38d3d225db..0c020e5249 100644
--- a/drivers/dma/dpaa2/version.map
+++ b/drivers/dma/dpaa2/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
 
diff --git a/drivers/dma/hisilicon/version.map b/drivers/dma/hisilicon/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/dma/hisilicon/version.map
+++ b/drivers/dma/hisilicon/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/dma/idxd/version.map b/drivers/dma/idxd/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/dma/idxd/version.map
+++ b/drivers/dma/idxd/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/dma/ioat/version.map b/drivers/dma/ioat/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/dma/ioat/version.map
+++ b/drivers/dma/ioat/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/dma/skeleton/version.map b/drivers/dma/skeleton/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/dma/skeleton/version.map
+++ b/drivers/dma/skeleton/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/event/dlb2/version.map b/drivers/event/dlb2/version.map
index c727207d1a..1327e3e335 100644
--- a/drivers/event/dlb2/version.map
+++ b/drivers/event/dlb2/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
 
diff --git a/drivers/event/dpaa/version.map b/drivers/event/dpaa/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/event/dpaa/version.map
+++ b/drivers/event/dpaa/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/event/dpaa2/version.map b/drivers/event/dpaa2/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/event/dpaa2/version.map
+++ b/drivers/event/dpaa2/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/event/dsw/version.map b/drivers/event/dsw/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/event/dsw/version.map
+++ b/drivers/event/dsw/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/event/octeontx/version.map b/drivers/event/octeontx/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/event/octeontx/version.map
+++ b/drivers/event/octeontx/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/event/opdl/version.map b/drivers/event/opdl/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/event/opdl/version.map
+++ b/drivers/event/opdl/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/event/skeleton/version.map b/drivers/event/skeleton/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/event/skeleton/version.map
+++ b/drivers/event/skeleton/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/event/sw/version.map b/drivers/event/sw/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/event/sw/version.map
+++ b/drivers/event/sw/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/gpu/cuda/version.map b/drivers/gpu/cuda/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/gpu/cuda/version.map
+++ b/drivers/gpu/cuda/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/mempool/bucket/version.map b/drivers/mempool/bucket/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/mempool/bucket/version.map
+++ b/drivers/mempool/bucket/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/mempool/dpaa2/version.map b/drivers/mempool/dpaa2/version.map
index cfd4ae617a..0023765843 100644
--- a/drivers/mempool/dpaa2/version.map
+++ b/drivers/mempool/dpaa2/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_dpaa2_mbuf_from_buf_addr;
diff --git a/drivers/mempool/octeontx/version.map b/drivers/mempool/octeontx/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/mempool/octeontx/version.map
+++ b/drivers/mempool/octeontx/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/mempool/ring/version.map b/drivers/mempool/ring/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/mempool/ring/version.map
+++ b/drivers/mempool/ring/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/mempool/stack/version.map b/drivers/mempool/stack/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/mempool/stack/version.map
+++ b/drivers/mempool/stack/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/af_packet/version.map b/drivers/net/af_packet/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/af_packet/version.map
+++ b/drivers/net/af_packet/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/af_xdp/version.map b/drivers/net/af_xdp/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/af_xdp/version.map
+++ b/drivers/net/af_xdp/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/ark/version.map b/drivers/net/ark/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/ark/version.map
+++ b/drivers/net/ark/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/atlantic/version.map b/drivers/net/atlantic/version.map
index d36fc61a84..e301b105fe 100644
--- a/drivers/net/atlantic/version.map
+++ b/drivers/net/atlantic/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
 
diff --git a/drivers/net/avp/version.map b/drivers/net/avp/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/avp/version.map
+++ b/drivers/net/avp/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/axgbe/version.map b/drivers/net/axgbe/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/axgbe/version.map
+++ b/drivers/net/axgbe/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/bnx2x/version.map b/drivers/net/bnx2x/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/bnx2x/version.map
+++ b/drivers/net/bnx2x/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/bnxt/version.map b/drivers/net/bnxt/version.map
index 2ba5ec5f6e..075bb37a36 100644
--- a/drivers/net/bnxt/version.map
+++ b/drivers/net/bnxt/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_pmd_bnxt_get_vf_rx_status;
diff --git a/drivers/net/bonding/version.map b/drivers/net/bonding/version.map
index d7142c4f94..9333923b4e 100644
--- a/drivers/net/bonding/version.map
+++ b/drivers/net/bonding/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_eth_bond_8023ad_agg_selection_get;
diff --git a/drivers/net/cnxk/version.map b/drivers/net/cnxk/version.map
index 0bcc65fe6f..3ef3e76bb0 100644
--- a/drivers/net/cnxk/version.map
+++ b/drivers/net/cnxk/version.map
@@ -1,15 +1,15 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
 
-INTERNAL {
-	global:
-	cnxk_nix_inb_mode_set;
-};
-
 EXPERIMENTAL {
 	# added in 22.07
 	global:
 	rte_pmd_cnxk_hw_sa_read;
 	rte_pmd_cnxk_hw_sa_write;
 };
+
+INTERNAL {
+	global:
+	cnxk_nix_inb_mode_set;
+};
diff --git a/drivers/net/cxgbe/version.map b/drivers/net/cxgbe/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/cxgbe/version.map
+++ b/drivers/net/cxgbe/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/dpaa/version.map b/drivers/net/dpaa/version.map
index 338ea2d8b2..5268d39ef6 100644
--- a/drivers/net/dpaa/version.map
+++ b/drivers/net/dpaa/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_pmd_dpaa_set_tx_loopback;
diff --git a/drivers/net/dpaa2/version.map b/drivers/net/dpaa2/version.map
index cc82b8579d..d6535343b1 100644
--- a/drivers/net/dpaa2/version.map
+++ b/drivers/net/dpaa2/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_pmd_dpaa2_mux_flow_create;
diff --git a/drivers/net/e1000/version.map b/drivers/net/e1000/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/e1000/version.map
+++ b/drivers/net/e1000/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/ena/version.map b/drivers/net/ena/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/ena/version.map
+++ b/drivers/net/ena/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/enetc/version.map b/drivers/net/enetc/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/enetc/version.map
+++ b/drivers/net/enetc/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/enetfec/version.map b/drivers/net/enetfec/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/enetfec/version.map
+++ b/drivers/net/enetfec/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/enic/version.map b/drivers/net/enic/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/enic/version.map
+++ b/drivers/net/enic/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/failsafe/version.map b/drivers/net/failsafe/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/failsafe/version.map
+++ b/drivers/net/failsafe/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/fm10k/version.map b/drivers/net/fm10k/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/fm10k/version.map
+++ b/drivers/net/fm10k/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/hinic/version.map b/drivers/net/hinic/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/hinic/version.map
+++ b/drivers/net/hinic/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/hns3/version.map b/drivers/net/hns3/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/hns3/version.map
+++ b/drivers/net/hns3/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/i40e/version.map b/drivers/net/i40e/version.map
index 5dd68158d3..561db50eac 100644
--- a/drivers/net/i40e/version.map
+++ b/drivers/net/i40e/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_pmd_i40e_add_vf_mac_addr;
diff --git a/drivers/net/iavf/version.map b/drivers/net/iavf/version.map
index 97f0f87311..4796c2884f 100644
--- a/drivers/net/iavf/version.map
+++ b/drivers/net/iavf/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
 
diff --git a/drivers/net/ice/version.map b/drivers/net/ice/version.map
index 60a3f17393..e7e8e50557 100644
--- a/drivers/net/ice/version.map
+++ b/drivers/net/ice/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
 
diff --git a/drivers/net/igc/version.map b/drivers/net/igc/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/igc/version.map
+++ b/drivers/net/igc/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/ionic/version.map b/drivers/net/ionic/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/ionic/version.map
+++ b/drivers/net/ionic/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/ipn3ke/version.map b/drivers/net/ipn3ke/version.map
index 4fd4e7c09b..4c48499993 100644
--- a/drivers/net/ipn3ke/version.map
+++ b/drivers/net/ipn3ke/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
 
diff --git a/drivers/net/ixgbe/version.map b/drivers/net/ixgbe/version.map
index f0f29d8749..94693ccc1a 100644
--- a/drivers/net/ixgbe/version.map
+++ b/drivers/net/ixgbe/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_pmd_ixgbe_bypass_event_show;
diff --git a/drivers/net/kni/version.map b/drivers/net/kni/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/kni/version.map
+++ b/drivers/net/kni/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/liquidio/version.map b/drivers/net/liquidio/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/liquidio/version.map
+++ b/drivers/net/liquidio/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/memif/version.map b/drivers/net/memif/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/memif/version.map
+++ b/drivers/net/memif/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/mlx4/version.map b/drivers/net/mlx4/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/mlx4/version.map
+++ b/drivers/net/mlx4/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/mlx5/version.map b/drivers/net/mlx5/version.map
index c97dfe440a..9942de5079 100644
--- a/drivers/net/mlx5/version.map
+++ b/drivers/net/mlx5/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
 
diff --git a/drivers/net/mvneta/version.map b/drivers/net/mvneta/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/mvneta/version.map
+++ b/drivers/net/mvneta/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/mvpp2/version.map b/drivers/net/mvpp2/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/mvpp2/version.map
+++ b/drivers/net/mvpp2/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/netvsc/version.map b/drivers/net/netvsc/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/netvsc/version.map
+++ b/drivers/net/netvsc/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/nfb/version.map b/drivers/net/nfb/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/nfb/version.map
+++ b/drivers/net/nfb/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/nfp/version.map b/drivers/net/nfp/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/nfp/version.map
+++ b/drivers/net/nfp/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/ngbe/version.map b/drivers/net/ngbe/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/ngbe/version.map
+++ b/drivers/net/ngbe/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/null/version.map b/drivers/net/null/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/null/version.map
+++ b/drivers/net/null/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/octeontx/version.map b/drivers/net/octeontx/version.map
index d12156778e..ae37d32d04 100644
--- a/drivers/net/octeontx/version.map
+++ b/drivers/net/octeontx/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_octeontx_pchan_map;
diff --git a/drivers/net/octeontx_ep/version.map b/drivers/net/octeontx_ep/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/octeontx_ep/version.map
+++ b/drivers/net/octeontx_ep/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/pcap/version.map b/drivers/net/pcap/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/pcap/version.map
+++ b/drivers/net/pcap/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/pfe/version.map b/drivers/net/pfe/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/pfe/version.map
+++ b/drivers/net/pfe/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/qede/version.map b/drivers/net/qede/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/qede/version.map
+++ b/drivers/net/qede/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/ring/version.map b/drivers/net/ring/version.map
index e43f5ea908..84e52064e0 100644
--- a/drivers/net/ring/version.map
+++ b/drivers/net/ring/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_eth_from_ring;
diff --git a/drivers/net/sfc/version.map b/drivers/net/sfc/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/sfc/version.map
+++ b/drivers/net/sfc/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/softnic/version.map b/drivers/net/softnic/version.map
index 01e1514276..4dac46ecd5 100644
--- a/drivers/net/softnic/version.map
+++ b/drivers/net/softnic/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_pmd_softnic_manage;
diff --git a/drivers/net/tap/version.map b/drivers/net/tap/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/tap/version.map
+++ b/drivers/net/tap/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/thunderx/version.map b/drivers/net/thunderx/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/thunderx/version.map
+++ b/drivers/net/thunderx/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/txgbe/version.map b/drivers/net/txgbe/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/txgbe/version.map
+++ b/drivers/net/txgbe/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/vdev_netvsc/version.map b/drivers/net/vdev_netvsc/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/vdev_netvsc/version.map
+++ b/drivers/net/vdev_netvsc/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/vhost/version.map b/drivers/net/vhost/version.map
index 1aa8abef75..e42c89f1eb 100644
--- a/drivers/net/vhost/version.map
+++ b/drivers/net/vhost/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_eth_vhost_get_queue_event;
diff --git a/drivers/net/virtio/version.map b/drivers/net/virtio/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/virtio/version.map
+++ b/drivers/net/virtio/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/net/vmxnet3/version.map b/drivers/net/vmxnet3/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/net/vmxnet3/version.map
+++ b/drivers/net/vmxnet3/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/raw/cnxk_bphy/version.map b/drivers/raw/cnxk_bphy/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/raw/cnxk_bphy/version.map
+++ b/drivers/raw/cnxk_bphy/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/raw/cnxk_gpio/version.map b/drivers/raw/cnxk_gpio/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/raw/cnxk_gpio/version.map
+++ b/drivers/raw/cnxk_gpio/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/raw/dpaa2_cmdif/version.map b/drivers/raw/dpaa2_cmdif/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/raw/dpaa2_cmdif/version.map
+++ b/drivers/raw/dpaa2_cmdif/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/raw/ifpga/version.map b/drivers/raw/ifpga/version.map
index ff71a453e2..f5c3959b7f 100644
--- a/drivers/raw/ifpga/version.map
+++ b/drivers/raw/ifpga/version.map
@@ -1,18 +1,18 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
+	rte_pmd_ifpga_cleanup;
 	rte_pmd_ifpga_get_dev_id;
-	rte_pmd_ifpga_get_rsu_status;
-	rte_pmd_ifpga_set_rsu_status;
-	rte_pmd_ifpga_get_property;
+	rte_pmd_ifpga_get_pci_bus;
 	rte_pmd_ifpga_get_phy_info;
-	rte_pmd_ifpga_update_flash;
-	rte_pmd_ifpga_stop_update;
+	rte_pmd_ifpga_get_property;
+	rte_pmd_ifpga_get_rsu_status;
+	rte_pmd_ifpga_partial_reconfigure;
 	rte_pmd_ifpga_reboot_try;
 	rte_pmd_ifpga_reload;
-	rte_pmd_ifpga_get_pci_bus;
-	rte_pmd_ifpga_partial_reconfigure;
-	rte_pmd_ifpga_cleanup;
+	rte_pmd_ifpga_set_rsu_status;
+	rte_pmd_ifpga_stop_update;
+	rte_pmd_ifpga_update_flash;
 
 	local: *;
 };
diff --git a/drivers/raw/ioat/version.map b/drivers/raw/ioat/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/raw/ioat/version.map
+++ b/drivers/raw/ioat/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/raw/ntb/version.map b/drivers/raw/ntb/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/raw/ntb/version.map
+++ b/drivers/raw/ntb/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/raw/skeleton/version.map b/drivers/raw/skeleton/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/raw/skeleton/version.map
+++ b/drivers/raw/skeleton/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/regex/cn9k/version.map b/drivers/regex/cn9k/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/regex/cn9k/version.map
+++ b/drivers/regex/cn9k/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/regex/mlx5/version.map b/drivers/regex/mlx5/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/regex/mlx5/version.map
+++ b/drivers/regex/mlx5/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/vdpa/ifc/version.map b/drivers/vdpa/ifc/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/vdpa/ifc/version.map
+++ b/drivers/vdpa/ifc/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/vdpa/mlx5/version.map b/drivers/vdpa/mlx5/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/vdpa/mlx5/version.map
+++ b/drivers/vdpa/mlx5/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/drivers/vdpa/sfc/version.map b/drivers/vdpa/sfc/version.map
index c2e0723b4c..78c3585d7c 100644
--- a/drivers/vdpa/sfc/version.map
+++ b/drivers/vdpa/sfc/version.map
@@ -1,3 +1,3 @@
-DPDK_22 {
+DPDK_23 {
 	local: *;
 };
diff --git a/lib/acl/version.map b/lib/acl/version.map
index 2b18c21601..4c15dbbb36 100644
--- a/lib/acl/version.map
+++ b/lib/acl/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_acl_add_rules;
diff --git a/lib/bbdev/version.map b/lib/bbdev/version.map
index cce3f3c8f5..25a0a22bd4 100644
--- a/lib/bbdev/version.map
+++ b/lib/bbdev/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_bbdev_allocate;
diff --git a/lib/bitratestats/version.map b/lib/bitratestats/version.map
index 01c61670dd..dc110440e0 100644
--- a/lib/bitratestats/version.map
+++ b/lib/bitratestats/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_stats_bitrate_calc;
diff --git a/lib/bpf/version.map b/lib/bpf/version.map
index 3b953f2f45..04bd657a85 100644
--- a/lib/bpf/version.map
+++ b/lib/bpf/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_bpf_destroy;
diff --git a/lib/cfgfile/version.map b/lib/cfgfile/version.map
index 02cbccb8ab..fdb0f13040 100644
--- a/lib/cfgfile/version.map
+++ b/lib/cfgfile/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_cfgfile_add_entry;
diff --git a/lib/cmdline/version.map b/lib/cmdline/version.map
index fc7fdd6ea4..e3d59aaf8d 100644
--- a/lib/cmdline/version.map
+++ b/lib/cmdline/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	cirbuf_add_buf_head;
diff --git a/lib/cryptodev/version.map b/lib/cryptodev/version.map
index dbf1f62199..5aee87c6f7 100644
--- a/lib/cryptodev/version.map
+++ b/lib/cryptodev/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_crypto_aead_algorithm_strings;
diff --git a/lib/distributor/version.map b/lib/distributor/version.map
index 4d9ff07373..7a34dfa2f2 100644
--- a/lib/distributor/version.map
+++ b/lib/distributor/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_distributor_clear_returns;
diff --git a/lib/eal/version.map b/lib/eal/version.map
index c2a2cebf69..1f293e768b 100644
--- a/lib/eal/version.map
+++ b/lib/eal/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	__rte_panic;
diff --git a/lib/efd/version.map b/lib/efd/version.map
index 0226285245..67886414ab 100644
--- a/lib/efd/version.map
+++ b/lib/efd/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_efd_create;
diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map
index d46f31b63f..03f52fee91 100644
--- a/lib/ethdev/version.map
+++ b/lib/ethdev/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_eth_add_first_rx_callback;
diff --git a/lib/eventdev/version.map b/lib/eventdev/version.map
index 886e2ec875..d99f996245 100644
--- a/lib/eventdev/version.map
+++ b/lib/eventdev/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	__rte_eventdev_trace_crypto_adapter_enqueue;
diff --git a/lib/fib/version.map b/lib/fib/version.map
index b23fa42b9b..a867d2b7d8 100644
--- a/lib/fib/version.map
+++ b/lib/fib/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_fib6_add;
diff --git a/lib/gro/version.map b/lib/gro/version.map
index f8a32e221c..105aa64ca3 100644
--- a/lib/gro/version.map
+++ b/lib/gro/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_gro_ctx_create;
diff --git a/lib/gso/version.map b/lib/gso/version.map
index 73767623b9..f6b552de6d 100644
--- a/lib/gso/version.map
+++ b/lib/gso/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_gso_segment;
diff --git a/lib/hash/version.map b/lib/hash/version.map
index c2e261ed13..bdcebd19c2 100644
--- a/lib/hash/version.map
+++ b/lib/hash/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_fbk_hash_create;
diff --git a/lib/ip_frag/version.map b/lib/ip_frag/version.map
index e537224293..b9c1ccaa1b 100644
--- a/lib/ip_frag/version.map
+++ b/lib/ip_frag/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_ip_frag_free_death_row;
diff --git a/lib/ipsec/version.map b/lib/ipsec/version.map
index 0af27ffd60..f17a49dd26 100644
--- a/lib/ipsec/version.map
+++ b/lib/ipsec/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_ipsec_pkt_crypto_group;
diff --git a/lib/jobstats/version.map b/lib/jobstats/version.map
index 89faa02004..bca7480afb 100644
--- a/lib/jobstats/version.map
+++ b/lib/jobstats/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_jobstats_abort;
diff --git a/lib/kni/version.map b/lib/kni/version.map
index cc7790651a..83bbbe880f 100644
--- a/lib/kni/version.map
+++ b/lib/kni/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_kni_alloc;
diff --git a/lib/kvargs/version.map b/lib/kvargs/version.map
index 0d42cd58a8..781f71cf23 100644
--- a/lib/kvargs/version.map
+++ b/lib/kvargs/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_kvargs_count;
diff --git a/lib/latencystats/version.map b/lib/latencystats/version.map
index be5b014cd7..79b8395f12 100644
--- a/lib/latencystats/version.map
+++ b/lib/latencystats/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_latencystats_get;
diff --git a/lib/lpm/version.map b/lib/lpm/version.map
index 0cdd04822e..e1a7aaedbb 100644
--- a/lib/lpm/version.map
+++ b/lib/lpm/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_lpm6_add;
diff --git a/lib/mbuf/version.map b/lib/mbuf/version.map
index 2745c2b77c..ed486ed14e 100644
--- a/lib/mbuf/version.map
+++ b/lib/mbuf/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	__rte_pktmbuf_linearize;
diff --git a/lib/member/version.map b/lib/member/version.map
index f287aabc91..19469c6aba 100644
--- a/lib/member/version.map
+++ b/lib/member/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_member_add;
diff --git a/lib/mempool/version.map b/lib/mempool/version.map
index 1b7d7c5456..b67d7aace7 100644
--- a/lib/mempool/version.map
+++ b/lib/mempool/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_mempool_audit;
diff --git a/lib/meter/version.map b/lib/meter/version.map
index befa3b7e32..b10b544641 100644
--- a/lib/meter/version.map
+++ b/lib/meter/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_meter_srtcm_config;
diff --git a/lib/metrics/version.map b/lib/metrics/version.map
index b11a9647cd..89ffa9be80 100644
--- a/lib/metrics/version.map
+++ b/lib/metrics/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_metrics_deinit;
diff --git a/lib/net/version.map b/lib/net/version.map
index 4f4330d1c4..e8fe2b7635 100644
--- a/lib/net/version.map
+++ b/lib/net/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_eth_random_addr;
diff --git a/lib/pci/version.map b/lib/pci/version.map
index 3f38303749..e9282ff49c 100644
--- a/lib/pci/version.map
+++ b/lib/pci/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_pci_addr_cmp;
diff --git a/lib/pdump/version.map b/lib/pdump/version.map
index ce5502d9cd..25df5a82c2 100644
--- a/lib/pdump/version.map
+++ b/lib/pdump/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_pdump_disable;
diff --git a/lib/pipeline/version.map b/lib/pipeline/version.map
index 8312307a7a..1baa74a77a 100644
--- a/lib/pipeline/version.map
+++ b/lib/pipeline/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_pipeline_ah_packet_drop;
diff --git a/lib/port/version.map b/lib/port/version.map
index 73d0825d2e..af6cf696fd 100644
--- a/lib/port/version.map
+++ b/lib/port/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_port_ethdev_reader_ops;
diff --git a/lib/power/version.map b/lib/power/version.map
index a687754f4a..f9b2947adf 100644
--- a/lib/power/version.map
+++ b/lib/power/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_power_exit;
diff --git a/lib/rawdev/version.map b/lib/rawdev/version.map
index 4f56870761..8278aacdea 100644
--- a/lib/rawdev/version.map
+++ b/lib/rawdev/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_rawdev_close;
diff --git a/lib/rcu/version.map b/lib/rcu/version.map
index b63c74f856..cabed64fca 100644
--- a/lib/rcu/version.map
+++ b/lib/rcu/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_rcu_log_type;
diff --git a/lib/reorder/version.map b/lib/reorder/version.map
index 250e6664f5..e114d17730 100644
--- a/lib/reorder/version.map
+++ b/lib/reorder/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_reorder_create;
diff --git a/lib/rib/version.map b/lib/rib/version.map
index f356fe8849..ca2815e44b 100644
--- a/lib/rib/version.map
+++ b/lib/rib/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_rib6_create;
diff --git a/lib/ring/version.map b/lib/ring/version.map
index 3377293ee4..4d7c27a6d9 100644
--- a/lib/ring/version.map
+++ b/lib/ring/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_ring_create;
diff --git a/lib/sched/version.map b/lib/sched/version.map
index 13d7510584..2f64834c8f 100644
--- a/lib/sched/version.map
+++ b/lib/sched/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_approx;
diff --git a/lib/security/version.map b/lib/security/version.map
index edf4887e12..c770b2e8f8 100644
--- a/lib/security/version.map
+++ b/lib/security/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_security_capabilities_get;
diff --git a/lib/stack/version.map b/lib/stack/version.map
index e145e32451..c0250f5cdf 100644
--- a/lib/stack/version.map
+++ b/lib/stack/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_stack_create;
diff --git a/lib/table/version.map b/lib/table/version.map
index 623709f886..e32e15a5fc 100644
--- a/lib/table/version.map
+++ b/lib/table/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_table_acl_ops;
diff --git a/lib/telemetry/version.map b/lib/telemetry/version.map
index 77528bb1fe..9794f9ea20 100644
--- a/lib/telemetry/version.map
+++ b/lib/telemetry/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_tel_data_add_array_container;
diff --git a/lib/timer/version.map b/lib/timer/version.map
index 4b782456da..101f5c18b5 100644
--- a/lib/timer/version.map
+++ b/lib/timer/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_timer_alt_dump_stats;
diff --git a/lib/vhost/version.map b/lib/vhost/version.map
index 9329f88e79..18574346d5 100644
--- a/lib/vhost/version.map
+++ b/lib/vhost/version.map
@@ -1,4 +1,4 @@
-DPDK_22 {
+DPDK_23 {
 	global:
 
 	rte_vdpa_find_device_by_name;
-- 
2.25.1

[PATCH v2 2/2] app/testpmd: add command line argument 'nic-to-pmd-rx-metadata'

[Hanumanth Pothula]

Presently, rx metadata is sent to PMD by default, leading
to a performance drop as processing for the same in rx path
takes extra cycles.

Hence, introducing command line argument, 'nic-to-pmd-rx-metadata'
to control passing rx metadata to PMD. By default it’s disabled.

Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>

v2:
- taken cared alignment issues
- renamed command line argument from rx-metadata to nic-to-pmd-rx-metadata
- renamed variable name from rx-metadata to nic_to_pmd_rx_metadata
---
 app/test-pmd/parameters.c | 4 ++++
 app/test-pmd/testpmd.c    | 6 +++++-
 app/test-pmd/testpmd.h    | 2 ++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c
index e3c9757f3f..a381945492 100644
--- a/app/test-pmd/parameters.c
+++ b/app/test-pmd/parameters.c
@@ -213,6 +213,7 @@ usage(char* progname)
 	printf("  --hairpin-mode=0xXX: bitmask set the hairpin port mode.\n"
 	       "    0x10 - explicit Tx rule, 0x02 - hairpin ports paired\n"
 	       "    0x01 - hairpin ports loop, 0x00 - hairpin port self\n");
+	printf("  --nic-to-pmd-rx-metadata: let the NIC deliver per-packet Rx metadata to PMD\n");
 }
 
 #ifdef RTE_LIB_CMDLINE
@@ -710,6 +711,7 @@ launch_args_parse(int argc, char** argv)
 		{ "record-burst-stats",         0, 0, 0 },
 		{ PARAM_NUM_PROCS,              1, 0, 0 },
 		{ PARAM_PROC_ID,                1, 0, 0 },
+		{ "nic-to-pmd-rx-metadata",     0, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1510,6 +1512,8 @@ launch_args_parse(int argc, char** argv)
 				num_procs = atoi(optarg);
 			if (!strcmp(lgopts[opt_idx].name, PARAM_PROC_ID))
 				proc_id = atoi(optarg);
+			if (!strcmp(lgopts[opt_idx].name, "nic-to-pmd-rx-metadata"))
+				nic_to_pmd_rx_metadata = 1;
 			break;
 		case 'h':
 			usage(argv[0]);
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index addcbcac85..2b17d4f757 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -411,6 +411,9 @@ uint8_t clear_ptypes = true;
 /* Hairpin ports configuration mode. */
 uint16_t hairpin_mode;
 
+/* Send Rx metadata */
+uint8_t nic_to_pmd_rx_metadata;
+
 /* Pretty printing of ethdev events */
 static const char * const eth_event_desc[] = {
 	[RTE_ETH_EVENT_UNKNOWN] = "unknown",
@@ -1628,7 +1631,8 @@ init_config_port_offloads(portid_t pid, uint32_t socket_id)
 	int ret;
 	int i;
 
-	eth_rx_metadata_negotiate_mp(pid);
+	if (nic_to_pmd_rx_metadata)
+		eth_rx_metadata_negotiate_mp(pid);
 
 	port->dev_conf.txmode = tx_mode;
 	port->dev_conf.rxmode = rx_mode;
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index fb2f5195d3..294a9c8cf4 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -621,6 +621,8 @@ extern struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
 extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
 
+extern uint8_t nic_to_pmd_rx_metadata;
+
 #ifdef RTE_LIB_GRO
 #define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
 #define GRO_DEFAULT_FLOW_NUM (RTE_GRO_MAX_BURST_ITEM_NUM / \
-- 
2.25.1

[PATCH v2 1/1] app/testpmd: add command line argument 'nic-to-pmd-rx-metadata'

[Hanumanth Pothula]

Presently, rx metadata is sent to PMD by default, leading
to a performance drop as processing for the same in rx path
takes extra cycles.

Hence, introducing command line argument, 'nic-to-pmd-rx-metadata'
to control passing rx metadata to PMD. By default it’s disabled.

Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>

v2:
- taken cared alignment issues
- renamed command line argument from rx-metadata to nic-to-pmd-rx-metadata
- renamed variable name from rx-metadata to nic_to_pmd_rx_metadata
---
 app/test-pmd/parameters.c | 4 ++++
 app/test-pmd/testpmd.c    | 6 +++++-
 app/test-pmd/testpmd.h    | 2 ++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c
index e3c9757f3f..a381945492 100644
--- a/app/test-pmd/parameters.c
+++ b/app/test-pmd/parameters.c
@@ -213,6 +213,7 @@ usage(char* progname)
 	printf("  --hairpin-mode=0xXX: bitmask set the hairpin port mode.\n"
 	       "    0x10 - explicit Tx rule, 0x02 - hairpin ports paired\n"
 	       "    0x01 - hairpin ports loop, 0x00 - hairpin port self\n");
+	printf("  --nic-to-pmd-rx-metadata: let the NIC deliver per-packet Rx metadata to PMD\n");
 }
 
 #ifdef RTE_LIB_CMDLINE
@@ -710,6 +711,7 @@ launch_args_parse(int argc, char** argv)
 		{ "record-burst-stats",         0, 0, 0 },
 		{ PARAM_NUM_PROCS,              1, 0, 0 },
 		{ PARAM_PROC_ID,                1, 0, 0 },
+		{ "nic-to-pmd-rx-metadata",     0, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1510,6 +1512,8 @@ launch_args_parse(int argc, char** argv)
 				num_procs = atoi(optarg);
 			if (!strcmp(lgopts[opt_idx].name, PARAM_PROC_ID))
 				proc_id = atoi(optarg);
+			if (!strcmp(lgopts[opt_idx].name, "nic-to-pmd-rx-metadata"))
+				nic_to_pmd_rx_metadata = 1;
 			break;
 		case 'h':
 			usage(argv[0]);
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index addcbcac85..2b17d4f757 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -411,6 +411,9 @@ uint8_t clear_ptypes = true;
 /* Hairpin ports configuration mode. */
 uint16_t hairpin_mode;
 
+/* Send Rx metadata */
+uint8_t nic_to_pmd_rx_metadata;
+
 /* Pretty printing of ethdev events */
 static const char * const eth_event_desc[] = {
 	[RTE_ETH_EVENT_UNKNOWN] = "unknown",
@@ -1628,7 +1631,8 @@ init_config_port_offloads(portid_t pid, uint32_t socket_id)
 	int ret;
 	int i;
 
-	eth_rx_metadata_negotiate_mp(pid);
+	if (nic_to_pmd_rx_metadata)
+		eth_rx_metadata_negotiate_mp(pid);
 
 	port->dev_conf.txmode = tx_mode;
 	port->dev_conf.rxmode = rx_mode;
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index fb2f5195d3..294a9c8cf4 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -621,6 +621,8 @@ extern struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
 extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
 
+extern uint8_t nic_to_pmd_rx_metadata;
+
 #ifdef RTE_LIB_GRO
 #define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
 #define GRO_DEFAULT_FLOW_NUM (RTE_GRO_MAX_BURST_ITEM_NUM / \
-- 
2.25.1

RE: [PATCH v2 1/1] app/testpmd: add command line argument 'nic-to-pmd-rx-metadata'

[Hanumanth Reddy Pothula]

Ping

> -----Original Message-----
> From: Hanumanth Pothula <hpothula@marvell.com>
> Sent: Tuesday, August 2, 2022 11:22 PM
> To: Aman Singh <aman.deep.singh@intel.com>; Yuying Zhang
> <yuying.zhang@intel.com>
> Cc: dev@dpdk.org; Hanumanth Reddy Pothula <hpothula@marvell.com>
> Subject: [PATCH v2 1/1] app/testpmd: add command line argument 'nic-to-pmd-
> rx-metadata'
> 
> Presently, rx metadata is sent to PMD by default, leading to a performance drop
> as processing for the same in rx path takes extra cycles.
> 
> Hence, introducing command line argument, 'nic-to-pmd-rx-metadata'
> to control passing rx metadata to PMD. By default it’s disabled.
> 
> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> 
> v2:
> - taken cared alignment issues
> - renamed command line argument from rx-metadata to nic-to-pmd-rx-
> metadata
> - renamed variable name from rx-metadata to nic_to_pmd_rx_metadata
> ---
>  app/test-pmd/parameters.c | 4 ++++
>  app/test-pmd/testpmd.c    | 6 +++++-
>  app/test-pmd/testpmd.h    | 2 ++
>  3 files changed, 11 insertions(+), 1 deletion(-)
> 
> diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c index
> e3c9757f3f..a381945492 100644
> --- a/app/test-pmd/parameters.c
> +++ b/app/test-pmd/parameters.c
> @@ -213,6 +213,7 @@ usage(char* progname)
>  	printf("  --hairpin-mode=0xXX: bitmask set the hairpin port mode.\n"
>  	       "    0x10 - explicit Tx rule, 0x02 - hairpin ports paired\n"
>  	       "    0x01 - hairpin ports loop, 0x00 - hairpin port self\n");
> +	printf("  --nic-to-pmd-rx-metadata: let the NIC deliver per-packet Rx
> +metadata to PMD\n");
>  }
> 
>  #ifdef RTE_LIB_CMDLINE
> @@ -710,6 +711,7 @@ launch_args_parse(int argc, char** argv)
>  		{ "record-burst-stats",         0, 0, 0 },
>  		{ PARAM_NUM_PROCS,              1, 0, 0 },
>  		{ PARAM_PROC_ID,                1, 0, 0 },
> +		{ "nic-to-pmd-rx-metadata",     0, 0, 0 },
>  		{ 0, 0, 0, 0 },
>  	};
> 
> @@ -1510,6 +1512,8 @@ launch_args_parse(int argc, char** argv)
>  				num_procs = atoi(optarg);
>  			if (!strcmp(lgopts[opt_idx].name, PARAM_PROC_ID))
>  				proc_id = atoi(optarg);
> +			if (!strcmp(lgopts[opt_idx].name, "nic-to-pmd-rx-
> metadata"))
> +				nic_to_pmd_rx_metadata = 1;
>  			break;
>  		case 'h':
>  			usage(argv[0]);
> diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c index
> addcbcac85..2b17d4f757 100644
> --- a/app/test-pmd/testpmd.c
> +++ b/app/test-pmd/testpmd.c
> @@ -411,6 +411,9 @@ uint8_t clear_ptypes = true;
>  /* Hairpin ports configuration mode. */  uint16_t hairpin_mode;
> 
> +/* Send Rx metadata */
> +uint8_t nic_to_pmd_rx_metadata;
> +
>  /* Pretty printing of ethdev events */
>  static const char * const eth_event_desc[] = {
>  	[RTE_ETH_EVENT_UNKNOWN] = "unknown",
> @@ -1628,7 +1631,8 @@ init_config_port_offloads(portid_t pid, uint32_t
> socket_id)
>  	int ret;
>  	int i;
> 
> -	eth_rx_metadata_negotiate_mp(pid);
> +	if (nic_to_pmd_rx_metadata)
> +		eth_rx_metadata_negotiate_mp(pid);
> 
>  	port->dev_conf.txmode = tx_mode;
>  	port->dev_conf.rxmode = rx_mode;
> diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h index
> fb2f5195d3..294a9c8cf4 100644
> --- a/app/test-pmd/testpmd.h
> +++ b/app/test-pmd/testpmd.h
> @@ -621,6 +621,8 @@ extern struct rte_ether_addr
> peer_eth_addrs[RTE_MAX_ETHPORTS];  extern uint32_t burst_tx_delay_time;
> /**< Burst tx delay time(us) for mac-retry. */  extern uint32_t
> burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
> 
> +extern uint8_t nic_to_pmd_rx_metadata;
> +
>  #ifdef RTE_LIB_GRO
>  #define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32  #define
> GRO_DEFAULT_FLOW_NUM (RTE_GRO_MAX_BURST_ITEM_NUM / \
> --
> 2.25.1

Re: [PATCH v2 1/1] app/testpmd: add command line argument 'nic-to-pmd-rx-metadata'

[Singh, Aman Deep]

On 8/2/2022 11:21 PM, Hanumanth Pothula wrote:
> Presently, rx metadata is sent to PMD by default, leading
> to a performance drop as processing for the same in rx path
> takes extra cycles.
>
> Hence, introducing command line argument, 'nic-to-pmd-rx-metadata'
> to control passing rx metadata to PMD. By default it’s disabled.
>
> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>

Acked-by: Aman Singh <aman.deep.singh@intel.com>

> v2:
> - taken cared alignment issues
> - renamed command line argument from rx-metadata to nic-to-pmd-rx-metadata
> - renamed variable name from rx-metadata to nic_to_pmd_rx_metadata
> ---
>

Re: [PATCH v2 1/1] app/testpmd: add command line argument 'nic-to-pmd-rx-metadata'

[Andrew Rybchenko]

On 9/1/22 11:03, Singh, Aman Deep wrote:
> 
> 
> On 8/2/2022 11:21 PM, Hanumanth Pothula wrote:
>> Presently, rx metadata is sent to PMD by default, leading
>> to a performance drop as processing for the same in rx path
>> takes extra cycles.
>>
>> Hence, introducing command line argument, 'nic-to-pmd-rx-metadata'
>> to control passing rx metadata to PMD. By default it’s disabled.
>>
>> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> 
> Acked-by: Aman Singh <aman.deep.singh@intel.com>
> 
>> v2:
>> - taken cared alignment issues
>> - renamed command line argument from rx-metadata to 
>> nic-to-pmd-rx-metadata
>> - renamed variable name from rx-metadata to nic_to_pmd_rx_metadata
>> ---
>>
> 


Please, update doc/guides/testpmd_app_ug/testpmd_funcs.rst to
document the new command-line argument.

Also avoid argument name in the summary. It must be human-
readable.

[PATCH v3 1/1] app/testpmd: control passing Rx metadata to PMD

[Hanumanth Pothula]

Presently, Rx metadata is sent to PMD by default, leading
to a performance drop as processing for the same in rx path
takes extra cycles.

Hence, introducing command line argument, 'nic-to-pmd-rx-metadata'
to control passing rx metadata to PMD. By default it’s disabled.

Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>

v3:
- Updated run_app.rst with the new command line argument,
  nic-to-pmd-rx-metadata.
- Updated commit text.
v2:
- taken cared alignment issues
- renamed command line argument from rx-metadata to nic-to-pmd-rx-metadata
- renamed variable name from rx-metadata to nic_to_pmd_rx_metadata
---
 app/test-pmd/parameters.c             | 4 ++++
 app/test-pmd/testpmd.c                | 6 +++++-
 app/test-pmd/testpmd.h                | 2 ++
 doc/guides/testpmd_app_ug/run_app.rst | 3 +++
 4 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c
index cfd7cd1e50..2b1314771f 100644
--- a/app/test-pmd/parameters.c
+++ b/app/test-pmd/parameters.c
@@ -202,6 +202,7 @@ usage(char* progname)
 	printf("  --hairpin-mode=0xXX: bitmask set the hairpin port mode.\n"
 	       "    0x10 - explicit Tx rule, 0x02 - hairpin ports paired\n"
 	       "    0x01 - hairpin ports loop, 0x00 - hairpin port self\n");
+	printf("  --nic-to-pmd-rx-metadata: let the NIC deliver per-packet Rx metadata to PMD\n");
 }
 
 #ifdef RTE_LIB_CMDLINE
@@ -695,6 +696,7 @@ launch_args_parse(int argc, char** argv)
 		{ "record-burst-stats",         0, 0, 0 },
 		{ PARAM_NUM_PROCS,              1, 0, 0 },
 		{ PARAM_PROC_ID,                1, 0, 0 },
+		{ "nic-to-pmd-rx-metadata",     0, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1434,6 +1436,8 @@ launch_args_parse(int argc, char** argv)
 				num_procs = atoi(optarg);
 			if (!strcmp(lgopts[opt_idx].name, PARAM_PROC_ID))
 				proc_id = atoi(optarg);
+			if (!strcmp(lgopts[opt_idx].name, "nic-to-pmd-rx-metadata"))
+				nic_to_pmd_rx_metadata = 1;
 			break;
 		case 'h':
 			usage(argv[0]);
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index 77741fc41f..938072bb88 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -411,6 +411,9 @@ uint8_t clear_ptypes = true;
 /* Hairpin ports configuration mode. */
 uint16_t hairpin_mode;
 
+/* Send Rx metadata */
+uint8_t nic_to_pmd_rx_metadata;
+
 /* Pretty printing of ethdev events */
 static const char * const eth_event_desc[] = {
 	[RTE_ETH_EVENT_UNKNOWN] = "unknown",
@@ -1595,7 +1598,8 @@ init_config_port_offloads(portid_t pid, uint32_t socket_id)
 	int ret;
 	int i;
 
-	eth_rx_metadata_negotiate_mp(pid);
+	if (nic_to_pmd_rx_metadata)
+		eth_rx_metadata_negotiate_mp(pid);
 
 	port->dev_conf.txmode = tx_mode;
 	port->dev_conf.rxmode = rx_mode;
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index ddf5e21849..8522024364 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -615,6 +615,8 @@ extern struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
 extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
 
+extern uint8_t nic_to_pmd_rx_metadata;
+
 #ifdef RTE_LIB_GRO
 #define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
 #define GRO_DEFAULT_FLOW_NUM (RTE_GRO_MAX_BURST_ITEM_NUM / \
diff --git a/doc/guides/testpmd_app_ug/run_app.rst b/doc/guides/testpmd_app_ug/run_app.rst
index 8b41b960c8..24e2e8aa69 100644
--- a/doc/guides/testpmd_app_ug/run_app.rst
+++ b/doc/guides/testpmd_app_ug/run_app.rst
@@ -539,6 +539,9 @@ The command line options are:
 
     The default value is 0. Hairpin will use single port mode and implicit Tx flow mode.
 
+*   ``--nic_to_pmd_rx_metadata``
+
+    Enable passing Rx metadata to PMD.
 
 Testpmd Multi-Process Command-line Options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-- 
2.25.1

Re: [PATCH v3 1/1] app/testpmd: control passing Rx metadata to PMD

[Andrew Rybchenko]

On 10/6/22 21:35, Hanumanth Pothula wrote:
> Presently, Rx metadata is sent to PMD by default, leading
> to a performance drop as processing for the same in rx path
> takes extra cycles.
> 
> Hence, introducing command line argument, 'nic-to-pmd-rx-metadata'
> to control passing rx metadata to PMD. By default it’s disabled.
> 
> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>

Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>

> 
> v3:
> - Updated run_app.rst with the new command line argument,
>    nic-to-pmd-rx-metadata.
> - Updated commit text.
> v2:
> - taken cared alignment issues
> - renamed command line argument from rx-metadata to nic-to-pmd-rx-metadata
> - renamed variable name from rx-metadata to nic_to_pmd_rx_metadata

Applied to dpdk-next-net/main, thanks.

Re: [PATCH v3 1/1] app/testpmd: control passing Rx metadata to PMD

[Thomas Monjalon]

17/10/2022 10:32, Andrew Rybchenko:
> On 10/6/22 21:35, Hanumanth Pothula wrote:
> > Presently, Rx metadata is sent to PMD by default, leading
> > to a performance drop as processing for the same in rx path
> > takes extra cycles.
> > 
> > Hence, introducing command line argument, 'nic-to-pmd-rx-metadata'
> > to control passing rx metadata to PMD. By default it’s disabled.
> > 
> > Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> 
> Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> 
> Applied to dpdk-next-net/main, thanks.

I'm not sure this patch is really acceptable.
It is disabling Rx metadata by default just for benchmarking reason
because your driver is doing some processing even if metadata is not required.

From a user perspective, if a command requesting metadata is entered,
it won't work until we enable this new option on startup.
It looks terrible.

Please tell me I misunderstood something.

Re: [PATCH v3 1/1] app/testpmd: control passing Rx metadata to PMD

[Thomas Monjalon]

27/10/2022 09:34, Thomas Monjalon:
> 17/10/2022 10:32, Andrew Rybchenko:
> > On 10/6/22 21:35, Hanumanth Pothula wrote:
> > > Presently, Rx metadata is sent to PMD by default, leading
> > > to a performance drop as processing for the same in rx path
> > > takes extra cycles.
> > > 
> > > Hence, introducing command line argument, 'nic-to-pmd-rx-metadata'
> > > to control passing rx metadata to PMD. By default it’s disabled.
> > > 
> > > Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> > 
> > Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> > 
> > Applied to dpdk-next-net/main, thanks.
> 
> I'm not sure this patch is really acceptable.
> It is disabling Rx metadata by default just for benchmarking reason
> because your driver is doing some processing even if metadata is not required.
> 
> From a user perspective, if a command requesting metadata is entered,
> it won't work until we enable this new option on startup.
> It looks terrible.
> 
> Please tell me I misunderstood something.

While pulling, I see that the name is not compliant with others.
I think it should start with "enable-", use hyphens and be sorted.

I'll drop it from the pull for now, we can have it in -rc3.

RE: [EXT] Re: [PATCH v3 1/1] app/testpmd: control passing Rx metadata to PMD

[Hanumanth Reddy Pothula]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Thursday, October 27, 2022 6:25 PM
> To: Hanumanth Reddy Pothula <hpothula@marvell.com>; Andrew
> Rybchenko <andrew.rybchenko@oktetlabs.ru>
> Cc: dev@dpdk.org; Aman Singh <aman.deep.singh@intel.com>; Yuying
> Zhang <yuying.zhang@intel.com>; dev@dpdk.org; Jerin Jacob
> Kollanukkaran <jerinj@marvell.com>; Nithin Kumar Dabilpuram
> <ndabilpuram@marvell.com>; rasland@nvidia.com; orika@nvidia.com;
> viacheslavo@nvidia.com
> Subject: [EXT] Re: [PATCH v3 1/1] app/testpmd: control passing Rx
> metadata to PMD
> 
> External Email
> 
> ----------------------------------------------------------------------
> 27/10/2022 09:34, Thomas Monjalon:
> > 17/10/2022 10:32, Andrew Rybchenko:
> > > On 10/6/22 21:35, Hanumanth Pothula wrote:
> > > > Presently, Rx metadata is sent to PMD by default, leading to a
> > > > performance drop as processing for the same in rx path takes extra
> > > > cycles.
> > > >
> > > > Hence, introducing command line argument, 'nic-to-pmd-rx-metadata'
> > > > to control passing rx metadata to PMD. By default it’s disabled.
> > > >
> > > > Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> > >
> > > Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> > >
> > > Applied to dpdk-next-net/main, thanks.
> >
> > I'm not sure this patch is really acceptable.
> > It is disabling Rx metadata by default just for benchmarking reason
> > because your driver is doing some processing even if metadata is not
> required.
> >
> > From a user perspective, if a command requesting metadata is entered,
> > it won't work until we enable this new option on startup.
> > It looks terrible.
> >
> > Please tell me I misunderstood something.
> 
> While pulling, I see that the name is not compliant with others.
> I think it should start with "enable-", use hyphens and be sorted.
> 
> I'll drop it from the pull for now, we can have it in -rc3.
> 

@Thomas Monjalon I missed your comment, sorry for the delayed response.

Sending Rx metadata to PMD is added recently, which breaking our driver performance.  
Normally any feature added to testpmd will be disabled by default, to make sure it won't affect other code(PMD). 
Hence adding new testpmd command line argument to disable this feature by default. 

Presently, below three Rx meta features are sent to PMD by default.
/** The NIC is able to deliver flag (if set) with packets to the PMD. */
#define RTE_ETH_RX_METADATA_USER_FLAG RTE_BIT64(0)

/** The NIC is able to deliver mark ID with packets to the PMD. */
#define RTE_ETH_RX_METADATA_USER_MARK RTE_BIT64(1)

/** The NIC is able to deliver tunnel ID with packets to the PMD. */
#define RTE_ETH_RX_METADATA_TUNNEL_ID RTE_BIT64(2)

Re: [EXT] Re: [PATCH v3 1/1] app/testpmd: control passing Rx metadata to PMD

[Thomas Monjalon]

02/12/2022 17:14, Hanumanth Reddy Pothula:
> -----Original Message-----
> > 27/10/2022 09:34, Thomas Monjalon:
> > > 17/10/2022 10:32, Andrew Rybchenko:
> > > > On 10/6/22 21:35, Hanumanth Pothula wrote:
> > > > > Presently, Rx metadata is sent to PMD by default, leading to a
> > > > > performance drop as processing for the same in rx path takes extra
> > > > > cycles.
> > > > >
> > > > > Hence, introducing command line argument, 'nic-to-pmd-rx-metadata'
> > > > > to control passing rx metadata to PMD. By default it’s disabled.
> > > > >
> > > > > Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> > > >
> > > > Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> > > >
> > > > Applied to dpdk-next-net/main, thanks.
> > >
> > > I'm not sure this patch is really acceptable.
> > > It is disabling Rx metadata by default just for benchmarking reason
> > > because your driver is doing some processing even if metadata is not
> > required.
> > >
> > > From a user perspective, if a command requesting metadata is entered,
> > > it won't work until we enable this new option on startup.
> > > It looks terrible.
> > >
> > > Please tell me I misunderstood something.
> > 
> > While pulling, I see that the name is not compliant with others.
> > I think it should start with "enable-", use hyphens and be sorted.
> > 
> > I'll drop it from the pull for now, we can have it in -rc3.
> > 
> 
> @Thomas Monjalon I missed your comment, sorry for the delayed response.
> 
> Sending Rx metadata to PMD is added recently, which breaking our driver performance.  
> Normally any feature added to testpmd will be disabled by default, to make sure it won't affect other code(PMD). 
> Hence adding new testpmd command line argument to disable this feature by default. 

No, disabling by default doesn't mean you should enable with option.
It should be enabled if required by a command.

RE: [EXT] Re: [PATCH v3 1/1] app/testpmd: control passing Rx metadata to PMD

[Hanumanth Reddy Pothula]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Saturday, December 3, 2022 1:11 AM
> To: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>; Hanumanth
> Reddy Pothula <hpothula@marvell.com>
> Cc: dev@dpdk.org; Aman Singh <aman.deep.singh@intel.com>; Yuying
> Zhang <yuying.zhang@intel.com>; dev@dpdk.org; Jerin Jacob
> Kollanukkaran <jerinj@marvell.com>; Nithin Kumar Dabilpuram
> <ndabilpuram@marvell.com>; rasland@nvidia.com; orika@nvidia.com;
> viacheslavo@nvidia.com
> Subject: Re: [EXT] Re: [PATCH v3 1/1] app/testpmd: control passing Rx
> metadata to PMD
> 
> 02/12/2022 17:14, Hanumanth Reddy Pothula:
> > -----Original Message-----
> > > 27/10/2022 09:34, Thomas Monjalon:
> > > > 17/10/2022 10:32, Andrew Rybchenko:
> > > > > On 10/6/22 21:35, Hanumanth Pothula wrote:
> > > > > > Presently, Rx metadata is sent to PMD by default, leading to a
> > > > > > performance drop as processing for the same in rx path takes
> > > > > > extra cycles.
> > > > > >
> > > > > > Hence, introducing command line argument, 'nic-to-pmd-rx-
> metadata'
> > > > > > to control passing rx metadata to PMD. By default it’s disabled.
> > > > > >
> > > > > > Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> > > > >
> > > > > Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> > > > >
> > > > > Applied to dpdk-next-net/main, thanks.
> > > >
> > > > I'm not sure this patch is really acceptable.
> > > > It is disabling Rx metadata by default just for benchmarking
> > > > reason because your driver is doing some processing even if
> > > > metadata is not
> > > required.
> > > >
> > > > From a user perspective, if a command requesting metadata is
> > > > entered, it won't work until we enable this new option on startup.
> > > > It looks terrible.
> > > >
> > > > Please tell me I misunderstood something.
> > >
> > > While pulling, I see that the name is not compliant with others.
> > > I think it should start with "enable-", use hyphens and be sorted.
> > >
> > > I'll drop it from the pull for now, we can have it in -rc3.
> > >
> >
> > @Thomas Monjalon I missed your comment, sorry for the delayed
> response.
> >
> > Sending Rx metadata to PMD is added recently, which breaking our driver
> performance.
> > Normally any feature added to testpmd will be disabled by default, to
> make sure it won't affect other code(PMD).
> > Hence adding new testpmd command line argument to disable this
> feature by default.
> 
> No, disabling by default doesn't mean you should enable with option.
> It should be enabled if required by a command.
> 
Yeah, got it, user can enable/disable the feature on the fly without restarting the application(testpmd).
Will introduce new command, like below, 
  'port config rx-nic-to-pmd-metadata on/off'

Please let me know your thoughts on this command implementation.
If it's fine, will start implementing the command. Suggest if there is any other better way to implement the same.

Re: [EXT] Re: [PATCH v3 1/1] app/testpmd: control passing Rx metadata to PMD

[Thomas Monjalon]

05/12/2022 08:59, Hanumanth Reddy Pothula:
> From: Thomas Monjalon <thomas@monjalon.net>
> > 02/12/2022 17:14, Hanumanth Reddy Pothula:
> > > -----Original Message-----
> > > > 27/10/2022 09:34, Thomas Monjalon:
> > > > > 17/10/2022 10:32, Andrew Rybchenko:
> > > > > > On 10/6/22 21:35, Hanumanth Pothula wrote:
> > > > > > > Presently, Rx metadata is sent to PMD by default, leading to a
> > > > > > > performance drop as processing for the same in rx path takes
> > > > > > > extra cycles.
> > > > > > >
> > > > > > > Hence, introducing command line argument, 'nic-to-pmd-rx-
> > metadata'
> > > > > > > to control passing rx metadata to PMD. By default it’s disabled.
> > > > > > >
> > > > > > > Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> > > > > >
> > > > > > Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> > > > > >
> > > > > > Applied to dpdk-next-net/main, thanks.
> > > > >
> > > > > I'm not sure this patch is really acceptable.
> > > > > It is disabling Rx metadata by default just for benchmarking
> > > > > reason because your driver is doing some processing even if
> > > > > metadata is not
> > > > required.
> > > > >
> > > > > From a user perspective, if a command requesting metadata is
> > > > > entered, it won't work until we enable this new option on startup.
> > > > > It looks terrible.
> > > > >
> > > > > Please tell me I misunderstood something.
> > > >
> > > > While pulling, I see that the name is not compliant with others.
> > > > I think it should start with "enable-", use hyphens and be sorted.
> > > >
> > > > I'll drop it from the pull for now, we can have it in -rc3.
> > > >
> > >
> > > @Thomas Monjalon I missed your comment, sorry for the delayed
> > response.
> > >
> > > Sending Rx metadata to PMD is added recently, which breaking our driver
> > performance.
> > > Normally any feature added to testpmd will be disabled by default, to
> > make sure it won't affect other code(PMD).
> > > Hence adding new testpmd command line argument to disable this
> > feature by default.
> > 
> > No, disabling by default doesn't mean you should enable with option.
> > It should be enabled if required by a command.
> > 
> Yeah, got it, user can enable/disable the feature on the fly without restarting the application(testpmd).
> Will introduce new command, like below, 
>   'port config rx-nic-to-pmd-metadata on/off'
> 
> Please let me know your thoughts on this command implementation.
> If it's fine, will start implementing the command. Suggest if there is any other better way to implement the same.

What about enabling automatically when a flow command requests metadata?

RE: [EXT] Re: [PATCH v3 1/1] app/testpmd: control passing Rx metadata to PMD

[Slava Ovsiienko]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Monday, December 5, 2022 10:29
> To: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>; Hanumanth Reddy
> Pothula <hpothula@marvell.com>
> Cc: dev@dpdk.org; Aman Singh <aman.deep.singh@intel.com>; Yuying Zhang
> <yuying.zhang@intel.com>; dev@dpdk.org; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Nithin Kumar Dabilpuram <ndabilpuram@marvell.com>;
> Raslan Darawsheh <rasland@nvidia.com>; Ori Kam <orika@nvidia.com>; Slava
> Ovsiienko <viacheslavo@nvidia.com>
> Subject: Re: [EXT] Re: [PATCH v3 1/1] app/testpmd: control passing Rx
> metadata to PMD
> 
> 05/12/2022 08:59, Hanumanth Reddy Pothula:
> > From: Thomas Monjalon <thomas@monjalon.net>
> > > 02/12/2022 17:14, Hanumanth Reddy Pothula:
> > > > -----Original Message-----
> > > > > 27/10/2022 09:34, Thomas Monjalon:
> > > > > > 17/10/2022 10:32, Andrew Rybchenko:
> > > > > > > On 10/6/22 21:35, Hanumanth Pothula wrote:
> > > > > > > > Presently, Rx metadata is sent to PMD by default, leading
> > > > > > > > to a performance drop as processing for the same in rx
> > > > > > > > path takes extra cycles.
> > > > > > > >
> > > > > > > > Hence, introducing command line argument, 'nic-to-pmd-rx-
> > > metadata'
> > > > > > > > to control passing rx metadata to PMD. By default it’s
> disabled.
> > > > > > > >
> > > > > > > > Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> > > > > > >
> > > > > > > Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> > > > > > >
> > > > > > > Applied to dpdk-next-net/main, thanks.
> > > > > >
> > > > > > I'm not sure this patch is really acceptable.
> > > > > > It is disabling Rx metadata by default just for benchmarking
> > > > > > reason because your driver is doing some processing even if
> > > > > > metadata is not
> > > > > required.
> > > > > >
> > > > > > From a user perspective, if a command requesting metadata is
> > > > > > entered, it won't work until we enable this new option on startup.
> > > > > > It looks terrible.
> > > > > >
> > > > > > Please tell me I misunderstood something.
> > > > >
> > > > > While pulling, I see that the name is not compliant with others.
> > > > > I think it should start with "enable-", use hyphens and be sorted.
> > > > >
> > > > > I'll drop it from the pull for now, we can have it in -rc3.
> > > > >
> > > >
> > > > @Thomas Monjalon I missed your comment, sorry for the delayed
> > > response.
> > > >
> > > > Sending Rx metadata to PMD is added recently, which breaking our
> > > > driver
> > > performance.
> > > > Normally any feature added to testpmd will be disabled by default,
> > > > to
> > > make sure it won't affect other code(PMD).
> > > > Hence adding new testpmd command line argument to disable this
> > > feature by default.
> > >
> > > No, disabling by default doesn't mean you should enable with option.
> > > It should be enabled if required by a command.
> > >
> > Yeah, got it, user can enable/disable the feature on the fly without
> restarting the application(testpmd).
> > Will introduce new command, like below,
> >   'port config rx-nic-to-pmd-metadata on/off'
> >
> > Please let me know your thoughts on this command implementation.
> > If it's fine, will start implementing the command. Suggest if there is any
> other better way to implement the same.
> 
> What about enabling automatically when a flow command requests metadata?
> 
Another option - enable on defining metadata dynamic field/flag in mbuf.
mlx5 PMD, for example follows this approach.

Do we need disabling this in real application? If application is designed to use
metadata it is unlikely it stops using these ones.

With best regards,
Slava

[PATCH v4 1/2] ethdev: control Rx metadata negotiation

[Hanumanth Pothula]

Presently, Rx metadata is sent to PMD by default, leading
to a performance drop as processing for the same in Rx path
takes extra cycles.

Hence, adding a new eth port configuration filed,
'nic_to_pmd_rx_metadata', to control NIC to PMD Rx metadata
negotiation.

Also, reset dev_configured flag as part of device reset, this helps
in reconfiguring porting cleanly. Rx metadata negotiation,
rte_eth_rx_metadata_negotiate(), is allowed only when dev_configured
flag is reset.

Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
---
v4:
 - As per spec rte_eth_rx_metadata_negotiate() is processed only when
   dev_configured is set. Hence can't enable automatically when a flow
   command requests metadata.
 - Add new testpmd command to allow NIC to PMD Rx metadata negotiation.
v3:
 - Updated run_app.rst with the new command line argument,
   nic-to-pmd-rx-metadata.
 - Updated commit text.
v2:
 - taken cared alignment issues
 - renamed command line argument from rx-metadata to nic-to-pmd-rx-metadata
 - renamed variable name from rx-metadata to nic_to_pmd_rx_metadata
---
 lib/ethdev/rte_ethdev.c | 2 ++
 lib/ethdev/rte_ethdev.h | 1 +
 2 files changed, 3 insertions(+)

diff --git a/lib/ethdev/rte_ethdev.c b/lib/ethdev/rte_ethdev.c
index 5d5e18db1e..18c59044bc 100644
--- a/lib/ethdev/rte_ethdev.c
+++ b/lib/ethdev/rte_ethdev.c
@@ -1629,6 +1629,8 @@ rte_eth_dev_reset(uint16_t port_id)
 			port_id, rte_strerror(-ret));
 	}
 	ret = dev->dev_ops->dev_reset(dev);
+	if (!ret)
+		dev->data->dev_configured = 0;
 
 	return eth_err(port_id, ret);
 }
diff --git a/lib/ethdev/rte_ethdev.h b/lib/ethdev/rte_ethdev.h
index c129ca1eaf..f1160a8aca 100644
--- a/lib/ethdev/rte_ethdev.h
+++ b/lib/ethdev/rte_ethdev.h
@@ -1487,6 +1487,7 @@ struct rte_eth_conf {
 	    is needed,and the variable must be set RTE_ETH_DCB_PFC_SUPPORT. */
 	uint32_t dcb_capability_en;
 	struct rte_eth_intr_conf intr_conf; /**< Interrupt mode configuration. */
+	uint8_t nic_to_pmd_rx_metadata; /**< send rx metadata to PMD. */
 };
 
 /**
-- 
2.25.1

[PATCH v4 2/2] app/testpmd: add command to process Rx metadata negotiation

[Hanumanth Pothula]

Presently, Rx metadata is sent to PMD by default, leading
to a performance drop as processing for the same in Rx path
takes extra cycles.

Hence, add new testpmd command,
  'enable port <port_id> nic_to_pmd_rx_metadata'

This command helps in sending Rx metadata to PMD and thereby
Rx metadata flow command requests are processed.

Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
---
 app/test-pmd/cmdline.c                      | 58 +++++++++++++++++++++
 app/test-pmd/config.c                       |  9 ++++
 app/test-pmd/testpmd.c                      |  5 +-
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  6 +++
 4 files changed, 76 insertions(+), 2 deletions(-)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index b32dc8bfd4..56946b2520 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -610,6 +610,9 @@ static void cmd_help_long_parsed(void *parsed_result,
 			"set port (port_id) fec_mode auto|off|rs|baser\n"
 			"    set fec mode for a specific port\n\n"
 
+			"enable port <port_id> nic_to_pmd_rx_metadata"
+			"    Allow nic to pmd Rx metadata negotiation\n\n"
+
 			, list_pkt_forwarding_modes()
 		);
 	}
@@ -12621,6 +12624,60 @@ static cmdline_parse_inst_t cmd_show_port_flow_transfer_proxy = {
 	}
 };
 
+/* Allow negotiating Rx metadata between NIC and PMD */
+struct cmd_config_port_rx_metadata {
+	cmdline_fixed_string_t enable;
+	cmdline_fixed_string_t port;
+	uint16_t port_id;
+	cmdline_fixed_string_t nic_to_pmd_rx_metadata;
+};
+
+static void
+cmd_config_port_rx_metadata_parsed(void *parsed_result,
+				__rte_unused struct cmdline *cl,
+				__rte_unused void *data)
+{
+	struct cmd_config_port_rx_metadata *res = parsed_result;
+
+	if (port_id_is_invalid(res->port_id, ENABLED_WARN))
+		return;
+	if (!port_is_stopped(res->port_id)) {
+		fprintf(stderr, "Please stop port %u first\n", res->port_id);
+		return;
+	}
+
+	ports[res->port_id].dev_conf.nic_to_pmd_rx_metadata = 1;
+
+	reset_port(res->port_id);
+}
+
+
+static cmdline_parse_token_string_t cmd_config_port_rx_metadata_enable =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_port_rx_metadata, enable,
+								"enable");
+static cmdline_parse_token_string_t cmd_config_port_rx_metadata_port =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_port_rx_metadata, port,
+								"port");
+static cmdline_parse_token_num_t cmd_config_port_rx_metadata_id =
+	TOKEN_NUM_INITIALIZER(struct cmd_config_port_rx_metadata, port_id,
+								RTE_UINT16);
+static cmdline_parse_token_string_t cmd_config_port_rx_metadata_nic_to_pmd_rx_metadata =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_port_rx_metadata, nic_to_pmd_rx_metadata,
+								"nic_to_pmd_rx_metadata");
+
+static cmdline_parse_inst_t cmd_config_port_rx_metadata_parse = {
+	.f = cmd_config_port_rx_metadata_parsed,
+	.data = NULL,
+	.help_str = "enable port <port_id> nic_to_pmd_rx_metadata",
+	.tokens = {
+		(void *)&cmd_config_port_rx_metadata_enable,
+		(void *)&cmd_config_port_rx_metadata_port,
+		(void *)&cmd_config_port_rx_metadata_id,
+		(void *)&cmd_config_port_rx_metadata_nic_to_pmd_rx_metadata,
+		NULL,
+	},
+};
+
 /* ******************************************************************************** */
 
 /* list of instructions */
@@ -12851,6 +12908,7 @@ static cmdline_parse_ctx_t builtin_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_show_capability,
 	(cmdline_parse_inst_t *)&cmd_set_flex_is_pattern,
 	(cmdline_parse_inst_t *)&cmd_set_flex_spec_pattern,
+	(cmdline_parse_inst_t *)&cmd_config_port_rx_metadata_parse,
 	NULL,
 };
 
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index acccb6b035..47fce3accb 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -3249,6 +3249,15 @@ port_flow_create(portid_t port_id,
 		}
 		id = port->flow_list->id + 1;
 	}
+
+	if (port->dev_conf.nic_to_pmd_rx_metadata == 0 &&
+	   (actions->type == RTE_FLOW_ACTION_TYPE_MARK ||
+	    actions->type == RTE_FLOW_ACTION_TYPE_FLAG)) {
+		fprintf(stderr,
+			"rx metadata is not negotiated with PMD\n");
+		return -EINVAL;
+	}
+
 	if (tunnel_ops->enabled) {
 		pft = port_flow_tunnel_offload_cmd_prep(port_id, pattern,
 							actions, tunnel_ops);
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index 134d79a555..22e5e08fc1 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -1604,8 +1604,6 @@ init_config_port_offloads(portid_t pid, uint32_t socket_id)
 	int ret;
 	int i;
 
-	eth_rx_metadata_negotiate_mp(pid);
-
 	port->dev_conf.txmode = tx_mode;
 	port->dev_conf.rxmode = rx_mode;
 
@@ -2946,6 +2944,9 @@ start_port(portid_t pid)
 				port->update_conf = 0;
 			}
 
+			if (port->dev_conf.nic_to_pmd_rx_metadata)
+				eth_rx_metadata_negotiate_mp(pi);
+
 			/* configure port */
 			diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq,
 						     nb_txq + nb_hairpinq,
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 0037506a79..024dbf6012 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -1558,6 +1558,12 @@ Enable or disable a per port Rx offloading on all Rx queues of a port::
 
 This command should be run when the port is stopped, or else it will fail.
 
+Enable Rx metadata negotiation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Enable NIC to PMD Rx metadata negotiation::
+   testpmd> enable port <port_id> nic_to_pmd_rx_metadata
+
 config per queue Rx offloading
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- 
2.25.1

Re: [PATCH v4 2/2] app/testpmd: add command to process Rx metadata negotiation

[Stephen Hemminger]

On Wed, 21 Dec 2022 01:32:50 +0530
Hanumanth Pothula <hpothula@marvell.com> wrote:

> @@ -12851,6 +12908,7 @@ static cmdline_parse_ctx_t builtin_ctx[] = {
>  	(cmdline_parse_inst_t *)&cmd_show_capability,
>  	(cmdline_parse_inst_t *)&cmd_set_flex_is_pattern,
>  	(cmdline_parse_inst_t *)&cmd_set_flex_spec_pattern,
> +	(cmdline_parse_inst_t *)&cmd_config_port_rx_metadata_parse,
>  	NULL,
>  };
>  

Why are all these casts necessary? the function should match the prototype.

[PATCH v5 1/2] ethdev: fix ethdev configuration state on reset

[Hanumanth Pothula]

Presently, on device reset, ethdev configuration state,
dev_configured, is not reset.

On device reset, reset ethdev configuration state to make
sure device reconfiguration happens cleanly.

Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
---
v5:
 - Move nic-to-pmd-rx-metadata declaration to struct rte_port.
v4:
 - As per spec rte_eth_rx_metadata_negotiate() is processed only when
   dev_configured is set. Hence can't enable automatically when a flow
   command requests metadata.
 - Add new testpmd command to allow NIC to PMD Rx metadata negotiation.
v3:
 - Updated run_app.rst with the new command line argument,
   nic-to-pmd-rx-metadata.
 - Updated commit text.
v2:
 - taken cared alignment issues
 - renamed command line argument from rx-metadata to nic-to-pmd-rx-metadata
 - renamed variable name from rx-metadata to nic_to_pmd_rx_metadata
---
 lib/ethdev/rte_ethdev.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/ethdev/rte_ethdev.c b/lib/ethdev/rte_ethdev.c
index 5d5e18db1e..18c59044bc 100644
--- a/lib/ethdev/rte_ethdev.c
+++ b/lib/ethdev/rte_ethdev.c
@@ -1629,6 +1629,8 @@ rte_eth_dev_reset(uint16_t port_id)
 			port_id, rte_strerror(-ret));
 	}
 	ret = dev->dev_ops->dev_reset(dev);
+	if (!ret)
+		dev->data->dev_configured = 0;
 
 	return eth_err(port_id, ret);
 }
-- 
2.25.1

[PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Hanumanth Pothula]

Presently, Rx metadata is sent to PMD by default, leading
to a performance drop as processing for the same in Rx path
takes extra cycles.

Hence, add new testpmd command,
  'enable port <port_id> nic_to_pmd_rx_metadata'

This command helps in sending Rx metadata to PMD and thereby
Rx metadata flow command requests are processed.

Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
---
 app/test-pmd/cmdline.c                      | 58 +++++++++++++++++++++
 app/test-pmd/config.c                       |  9 ++++
 app/test-pmd/testpmd.c                      |  5 +-
 app/test-pmd/testpmd.h                      |  1 +
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  6 +++
 5 files changed, 77 insertions(+), 2 deletions(-)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index b32dc8bfd4..e3abc9e830 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -610,6 +610,9 @@ static void cmd_help_long_parsed(void *parsed_result,
 			"set port (port_id) fec_mode auto|off|rs|baser\n"
 			"    set fec mode for a specific port\n\n"
 
+			"enable port <port_id> nic_to_pmd_rx_metadata"
+			"    Allow nic to pmd Rx metadata negotiation\n\n"
+
 			, list_pkt_forwarding_modes()
 		);
 	}
@@ -12621,6 +12624,60 @@ static cmdline_parse_inst_t cmd_show_port_flow_transfer_proxy = {
 	}
 };
 
+/* Allow negotiating Rx metadata between NIC and PMD */
+struct cmd_config_port_rx_metadata {
+	cmdline_fixed_string_t enable;
+	cmdline_fixed_string_t port;
+	uint16_t port_id;
+	cmdline_fixed_string_t nic_to_pmd_rx_metadata;
+};
+
+static void
+cmd_config_port_rx_metadata_parsed(void *parsed_result,
+				__rte_unused struct cmdline *cl,
+				__rte_unused void *data)
+{
+	struct cmd_config_port_rx_metadata *res = parsed_result;
+
+	if (port_id_is_invalid(res->port_id, ENABLED_WARN))
+		return;
+	if (!port_is_stopped(res->port_id)) {
+		fprintf(stderr, "Please stop port %u first\n", res->port_id);
+		return;
+	}
+
+	ports[res->port_id].nic_to_pmd_rx_metadata = 1;
+
+	reset_port(res->port_id);
+}
+
+
+static cmdline_parse_token_string_t cmd_config_port_rx_metadata_enable =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_port_rx_metadata, enable,
+								"enable");
+static cmdline_parse_token_string_t cmd_config_port_rx_metadata_port =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_port_rx_metadata, port,
+								"port");
+static cmdline_parse_token_num_t cmd_config_port_rx_metadata_id =
+	TOKEN_NUM_INITIALIZER(struct cmd_config_port_rx_metadata, port_id,
+								RTE_UINT16);
+static cmdline_parse_token_string_t cmd_config_port_rx_metadata_nic_to_pmd_rx_metadata =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_port_rx_metadata, nic_to_pmd_rx_metadata,
+								"nic_to_pmd_rx_metadata");
+
+static cmdline_parse_inst_t cmd_config_port_rx_metadata_parse = {
+	.f = cmd_config_port_rx_metadata_parsed,
+	.data = NULL,
+	.help_str = "enable port <port_id> nic_to_pmd_rx_metadata",
+	.tokens = {
+		(void *)&cmd_config_port_rx_metadata_enable,
+		(void *)&cmd_config_port_rx_metadata_port,
+		(void *)&cmd_config_port_rx_metadata_id,
+		(void *)&cmd_config_port_rx_metadata_nic_to_pmd_rx_metadata,
+		NULL,
+	},
+};
+
 /* ******************************************************************************** */
 
 /* list of instructions */
@@ -12851,6 +12908,7 @@ static cmdline_parse_ctx_t builtin_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_show_capability,
 	(cmdline_parse_inst_t *)&cmd_set_flex_is_pattern,
 	(cmdline_parse_inst_t *)&cmd_set_flex_spec_pattern,
+	(cmdline_parse_inst_t *)&cmd_config_port_rx_metadata_parse,
 	NULL,
 };
 
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index acccb6b035..60df47407e 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -3249,6 +3249,15 @@ port_flow_create(portid_t port_id,
 		}
 		id = port->flow_list->id + 1;
 	}
+
+	if (port->nic_to_pmd_rx_metadata == 0 &&
+	   (actions->type == RTE_FLOW_ACTION_TYPE_MARK ||
+	    actions->type == RTE_FLOW_ACTION_TYPE_FLAG)) {
+		fprintf(stderr,
+			"rx metadata is not negotiated with PMD\n");
+		return -EINVAL;
+	}
+
 	if (tunnel_ops->enabled) {
 		pft = port_flow_tunnel_offload_cmd_prep(port_id, pattern,
 							actions, tunnel_ops);
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index 134d79a555..66d5a2634a 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -1604,8 +1604,6 @@ init_config_port_offloads(portid_t pid, uint32_t socket_id)
 	int ret;
 	int i;
 
-	eth_rx_metadata_negotiate_mp(pid);
-
 	port->dev_conf.txmode = tx_mode;
 	port->dev_conf.rxmode = rx_mode;
 
@@ -2946,6 +2944,9 @@ start_port(portid_t pid)
 				port->update_conf = 0;
 			}
 
+			if (port->nic_to_pmd_rx_metadata)
+				eth_rx_metadata_negotiate_mp(pi);
+
 			/* configure port */
 			diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq,
 						     nb_txq + nb_hairpinq,
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 7d24d25970..f44756173b 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -309,6 +309,7 @@ struct rte_port {
 	uint8_t                 need_setup;     /**< port just attached */
 	uint8_t                 need_reconfig;  /**< need reconfiguring port or not */
 	uint8_t                 need_reconfig_queues; /**< need reconfiguring queues or not */
+	uint8_t                 nic_to_pmd_rx_metadata; /**< send rx metadata to PMD. */
 	uint8_t                 rss_flag;   /**< enable rss or not */
 	uint8_t                 dcb_flag;   /**< enable dcb */
 	uint16_t                nb_rx_desc[RTE_MAX_QUEUES_PER_PORT+1]; /**< per queue rx desc number */
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 0037506a79..024dbf6012 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -1558,6 +1558,12 @@ Enable or disable a per port Rx offloading on all Rx queues of a port::
 
 This command should be run when the port is stopped, or else it will fail.
 
+Enable Rx metadata negotiation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Enable NIC to PMD Rx metadata negotiation::
+   testpmd> enable port <port_id> nic_to_pmd_rx_metadata
+
 config per queue Rx offloading
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- 
2.25.1

Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Thomas Monjalon]

21/12/2022 03:07, Hanumanth Pothula:
> Presently, Rx metadata is sent to PMD by default, leading
> to a performance drop as processing for the same in Rx path
> takes extra cycles.
> 
> Hence, add new testpmd command,
>   'enable port <port_id> nic_to_pmd_rx_metadata'
> 
> This command helps in sending Rx metadata to PMD and thereby
> Rx metadata flow command requests are processed.
> 
> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>

As said in previous versions, please don't add such option in testpmd.
The problem is somewhere else, probably in your driver.
You should make sure your driver is not sending metadata if not needed.
By the way you didn't reply to the last comments on v3 in December.

RE: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Hanumanth Reddy Pothula]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Wednesday, January 18, 2023 4:03 PM
> To: Hanumanth Reddy Pothula <hpothula@marvell.com>
> Cc: Aman Singh <aman.deep.singh@intel.com>; Yuying Zhang
> <yuying.zhang@intel.com>; dev@dpdk.org;
> andrew.rybchenko@oktetlabs.ru; viacheslavo@nvidia.com; Jerin Jacob
> Kollanukkaran <jerinj@marvell.com>; Nithin Kumar Dabilpuram
> <ndabilpuram@marvell.com>; david.marchand@redhat.com
> Subject: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process
> Rx metadata negotiation
> 
> External Email
> 
> ----------------------------------------------------------------------
> 21/12/2022 03:07, Hanumanth Pothula:
> > Presently, Rx metadata is sent to PMD by default, leading to a
> > performance drop as processing for the same in Rx path takes extra
> > cycles.
> >
> > Hence, add new testpmd command,
> >   'enable port <port_id> nic_to_pmd_rx_metadata'
> >
> > This command helps in sending Rx metadata to PMD and thereby Rx
> > metadata flow command requests are processed.
> >
> > Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> 
> As said in previous versions, please don't add such option in testpmd.
> The problem is somewhere else, probably in your driver.
> You should make sure your driver is not sending metadata if not needed.
> By the way you didn't reply to the last comments on v3 in December.
> 
>
I added following comment on V4,
- As per spec rte_eth_rx_metadata_negotiate() is processed only when dev_configured is set. Hence can't enable(Rx metadata negotiation) automatically when a flow command requests metadata.
- Add new testpmd command to allow NIC to PMD Rx metadata negotiation.

As we can't enable Rx metadata negotiation on fly, introduced new testpmd command to enable Rx metadata negotiation by resetting the ports. 
 

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Thomas Monjalon]

19/01/2023 11:33, Hanumanth Reddy Pothula:
> From: Thomas Monjalon <thomas@monjalon.net>
> > 21/12/2022 03:07, Hanumanth Pothula:
> > > Presently, Rx metadata is sent to PMD by default, leading to a
> > > performance drop as processing for the same in Rx path takes extra
> > > cycles.
> > >
> > > Hence, add new testpmd command,
> > >   'enable port <port_id> nic_to_pmd_rx_metadata'
> > >
> > > This command helps in sending Rx metadata to PMD and thereby Rx
> > > metadata flow command requests are processed.
> > >
> > > Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> > 
> > As said in previous versions, please don't add such option in testpmd.
> > The problem is somewhere else, probably in your driver.
> > You should make sure your driver is not sending metadata if not needed.
> > By the way you didn't reply to the last comments on v3 in December.
> > 
> I added following comment on V4,
> - As per spec rte_eth_rx_metadata_negotiate() is processed only
> when dev_configured is set.

Not exactly. It is said
"
Invoke this API before the first rte_eth_dev_configure() invocation
to let the PMD make preparations that are inconvenient to do later.
"

> Hence can't enable(Rx metadata negotiation)
> automatically when a flow command requests metadata.

This sentence does not make any sense.
First you negotiate to prepare the feature.
Later you enable per flow.

> - Add new testpmd command to allow NIC to PMD Rx metadata negotiation.
> 
> As we can't enable Rx metadata negotiation on fly,
> introduced new testpmd command to enable Rx metadata negotiation
> by resetting the ports. 

You can enable on the fly. Fix your driver.

Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ferruh Yigit]

On 12/21/2022 2:07 AM, Hanumanth Pothula wrote:
> Presently, Rx metadata is sent to PMD by default, leading
> to a performance drop as processing for the same in Rx path
> takes extra cycles.
> 
> Hence, add new testpmd command,
>   'enable port <port_id> nic_to_pmd_rx_metadata'
> 
> This command helps in sending Rx metadata to PMD and thereby
> Rx metadata flow command requests are processed.
> 
> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>

Hi Hanumanth,

I agree with Thomas for the patch.

'eth_rx_metadata_negotiate_mp()' requests all Rx metadata offloads to be
enabled, but at this stage if there is no flow rule for Rx metadata why
it is consuming extra cycles?

Can you update driver code to process Rx metadata when it is enabled by
application (via 'rte_eth_rx_metadata_negotiate()') AND there is at
least one flow rule for it?

RE: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Hanumanth Reddy Pothula]

++ Ivan Malov and Andrew Rybchenko

> -----Original Message-----
> From: Ferruh Yigit <ferruh.yigit@amd.com>
> Sent: Tuesday, January 24, 2023 11:34 PM
> To: Hanumanth Reddy Pothula <hpothula@marvell.com>; Aman Singh
> <aman.deep.singh@intel.com>; Yuying Zhang <yuying.zhang@intel.com>
> Cc: dev@dpdk.org; andrew.rybchenko@oktetlabs.ru;
> viacheslavo@nvidia.com; Jerin Jacob Kollanukkaran <jerinj@marvell.com>;
> Nithin Kumar Dabilpuram <ndabilpuram@marvell.com>
> Subject: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process
> Rx metadata negotiation
> 
> External Email
> 
> ----------------------------------------------------------------------
> On 12/21/2022 2:07 AM, Hanumanth Pothula wrote:
> > Presently, Rx metadata is sent to PMD by default, leading to a
> > performance drop as processing for the same in Rx path takes extra
> > cycles.
> >
> > Hence, add new testpmd command,
> >   'enable port <port_id> nic_to_pmd_rx_metadata'
> >
> > This command helps in sending Rx metadata to PMD and thereby Rx
> > metadata flow command requests are processed.
> >
> > Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> 
> Hi Hanumanth,
> 
> I agree with Thomas for the patch.
> 
> 'eth_rx_metadata_negotiate_mp()' requests all Rx metadata offloads to be
> enabled, but at this stage if there is no flow rule for Rx metadata why it is
> consuming extra cycles?
> 
> Can you update driver code to process Rx metadata when it is enabled by
> application (via 'rte_eth_rx_metadata_negotiate()') AND there is at least
> one flow rule for it?

#1 What is the purpose of rte_eth_rx_metadata_negotiate() API if it is always called by testpmd.
We thought it was added so that when that metadata is not needed, application need not call this
thereby saving cycles/bandwidth.

#2 We use this API similar to Rx/Tx offload flags so that we can set things up before device is
configured. We thought that is the purpose of having this negotiate API and avoid depleting offload flags.

#3 Generally any new offloads added to DPDK would be in disabled state in testpmd and we would have
an option to enable it. In this case, testpmd is by default calling this negotiation.

We can update the driver if the purpose of this API is clear.

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Thomas Monjalon]

25/01/2023 10:30, Hanumanth Reddy Pothula:
> ++ Ivan Malov and Andrew Rybchenko
> 
> From: Ferruh Yigit <ferruh.yigit@amd.com>
> > On 12/21/2022 2:07 AM, Hanumanth Pothula wrote:
> > > Presently, Rx metadata is sent to PMD by default, leading to a
> > > performance drop as processing for the same in Rx path takes extra
> > > cycles.
> > >
> > > Hence, add new testpmd command,
> > >   'enable port <port_id> nic_to_pmd_rx_metadata'
> > >
> > > This command helps in sending Rx metadata to PMD and thereby Rx
> > > metadata flow command requests are processed.
> > >
> > > Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> > 
> > Hi Hanumanth,
> > 
> > I agree with Thomas for the patch.
> > 
> > 'eth_rx_metadata_negotiate_mp()' requests all Rx metadata offloads to be
> > enabled, but at this stage if there is no flow rule for Rx metadata why it is
> > consuming extra cycles?
> > 
> > Can you update driver code to process Rx metadata when it is enabled by
> > application (via 'rte_eth_rx_metadata_negotiate()') AND there is at least
> > one flow rule for it?
> 
> #1 What is the purpose of rte_eth_rx_metadata_negotiate() API if it is always called by testpmd.
> We thought it was added so that when that metadata is not needed, application need not call this
> thereby saving cycles/bandwidth.

testpmd is for testing all features. That's why all is negotiated.
Cycles should be saved if you don't enable it until a flow rule requires it.

> #2 We use this API similar to Rx/Tx offload flags so that we can set things up before device is
> configured. We thought that is the purpose of having this negotiate API and avoid depleting offload flags.

It is just a configuration negotiation specific to metadata.

> #3 Generally any new offloads added to DPDK would be in disabled state in testpmd and we would have
> an option to enable it. In this case, testpmd is by default calling this negotiation.

Negotiating is not enabling.

> We can update the driver if the purpose of this API is clear.

Please do.

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ferruh Yigit]

On 1/25/2023 12:55 PM, Thomas Monjalon wrote:
> 25/01/2023 10:30, Hanumanth Reddy Pothula:
>> ++ Ivan Malov and Andrew Rybchenko
>>
>> From: Ferruh Yigit <ferruh.yigit@amd.com>
>>> On 12/21/2022 2:07 AM, Hanumanth Pothula wrote:
>>>> Presently, Rx metadata is sent to PMD by default, leading to a
>>>> performance drop as processing for the same in Rx path takes extra
>>>> cycles.
>>>>
>>>> Hence, add new testpmd command,
>>>>   'enable port <port_id> nic_to_pmd_rx_metadata'
>>>>
>>>> This command helps in sending Rx metadata to PMD and thereby Rx
>>>> metadata flow command requests are processed.
>>>>
>>>> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
>>>
>>> Hi Hanumanth,
>>>
>>> I agree with Thomas for the patch.
>>>
>>> 'eth_rx_metadata_negotiate_mp()' requests all Rx metadata offloads to be
>>> enabled, but at this stage if there is no flow rule for Rx metadata why it is
>>> consuming extra cycles?
>>>
>>> Can you update driver code to process Rx metadata when it is enabled by
>>> application (via 'rte_eth_rx_metadata_negotiate()') AND there is at least
>>> one flow rule for it?
>>
>> #1 What is the purpose of rte_eth_rx_metadata_negotiate() API if it is always called by testpmd.
>> We thought it was added so that when that metadata is not needed, application need not call this
>> thereby saving cycles/bandwidth.
> 
> testpmd is for testing all features. That's why all is negotiated.
> Cycles should be saved if you don't enable it until a flow rule requires it.
> 

Hi Thomas,

Not just for saving cycles, but from testing perspective too, do you
think does it work if a way to disable these Rx metadata added by
keeping default behavior as it is?

And new command can be in a consistent command syntax like:
"port config <port_id> ..."


>> #2 We use this API similar to Rx/Tx offload flags so that we can set things up before device is
>> configured. We thought that is the purpose of having this negotiate API and avoid depleting offload flags.
> 
> It is just a configuration negotiation specific to metadata.
> 
>> #3 Generally any new offloads added to DPDK would be in disabled state in testpmd and we would have
>> an option to enable it. In this case, testpmd is by default calling this negotiation.
> 
> Negotiating is not enabling.
> 
>> We can update the driver if the purpose of this API is clear.
> 
> Please do.
> 

Is following understanding correct?

     API        Flow Rule       Result
    -----    ------------     --------
    Enable    No Rule	       Feature Disabled
    Enable    Rule exist       Feature Enabled
    Disable     X              Feature Disabled

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Thomas Monjalon]

25/01/2023 14:55, Ferruh Yigit:
> On 1/25/2023 12:55 PM, Thomas Monjalon wrote:
> > 25/01/2023 10:30, Hanumanth Reddy Pothula:
> >> ++ Ivan Malov and Andrew Rybchenko
> >>
> >> From: Ferruh Yigit <ferruh.yigit@amd.com>
> >>> On 12/21/2022 2:07 AM, Hanumanth Pothula wrote:
> >>>> Presently, Rx metadata is sent to PMD by default, leading to a
> >>>> performance drop as processing for the same in Rx path takes extra
> >>>> cycles.
> >>>>
> >>>> Hence, add new testpmd command,
> >>>>   'enable port <port_id> nic_to_pmd_rx_metadata'
> >>>>
> >>>> This command helps in sending Rx metadata to PMD and thereby Rx
> >>>> metadata flow command requests are processed.
> >>>>
> >>>> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> >>>
> >>> Hi Hanumanth,
> >>>
> >>> I agree with Thomas for the patch.
> >>>
> >>> 'eth_rx_metadata_negotiate_mp()' requests all Rx metadata offloads to be
> >>> enabled, but at this stage if there is no flow rule for Rx metadata why it is
> >>> consuming extra cycles?
> >>>
> >>> Can you update driver code to process Rx metadata when it is enabled by
> >>> application (via 'rte_eth_rx_metadata_negotiate()') AND there is at least
> >>> one flow rule for it?
> >>
> >> #1 What is the purpose of rte_eth_rx_metadata_negotiate() API if it is always called by testpmd.
> >> We thought it was added so that when that metadata is not needed, application need not call this
> >> thereby saving cycles/bandwidth.
> > 
> > testpmd is for testing all features. That's why all is negotiated.
> > Cycles should be saved if you don't enable it until a flow rule requires it.
> > 
> 
> Hi Thomas,
> 
> Not just for saving cycles, but from testing perspective too, do you
> think does it work if a way to disable these Rx metadata added by
> keeping default behavior as it is?
> 
> And new command can be in a consistent command syntax like:
> "port config <port_id> ..."

Yes I agree it would be good to have a way to test different values.
And it would allow to completely disable metadata I suppose.

Note: I don't understand why we don't have
RTE_FLOW_ACTION_TYPE_SET_TAG and RTE_FLOW_ACTION_TYPE_SET_META
negotiated in this function. Probably something to add.


> >> #2 We use this API similar to Rx/Tx offload flags so that we can set things up before device is
> >> configured. We thought that is the purpose of having this negotiate API and avoid depleting offload flags.
> > 
> > It is just a configuration negotiation specific to metadata.
> > 
> >> #3 Generally any new offloads added to DPDK would be in disabled state in testpmd and we would have
> >> an option to enable it. In this case, testpmd is by default calling this negotiation.
> > 
> > Negotiating is not enabling.
> > 
> >> We can update the driver if the purpose of this API is clear.
> > 
> > Please do.
> 
> Is following understanding correct?
> 
>      API        Flow Rule       Result
>     -----    ------------     --------
>     Enable    No Rule	       Feature Disabled
>     Enable    Rule exist       Feature Enabled
>     Disable     X              Feature Disabled

In the API column, you should say "negotiated" instead of "Enable".

RE: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Nithin Kumar Dabilpuram]

> >Will it work to enable them all by default and add capability to disable
> >it in testpmd, which helps to run performance tests also to verify the
> > impact of the API?

The spirit of the negotiating features/Rx/Tx offloads upfront is to have it disabled by default and enable the feature only when needed. Having the features enabled by default is probably against that spirit.

We understand the concerns with drivers that didn't not implement that API.
Why not disable it by default(like other offloads) and modify rte_flow action creation in testpmd to check for if !ENOSUP and feature disabled and add print to enable. So for the PMD's that won't support rte_eth_rx_metadata_negotiate(), there won't be any difference and for very few PMD's that support this API, they need to enable it before using RTE_FLOW with MARK/FLAG.
Behavior change would be seen only with two PMD's(cnxk, sfc).

> Note: I don't understand why we don't have
> RTE_FLOW_ACTION_TYPE_SET_TAG and RTE_FLOW_ACTION_TYPE_SET_META
> negotiated in this function. Probably something to add.

The purpose of negotiate is to tell the PMD upfront so that PMD can prepare
HW appropriately.  Having these new actions would be very late to inform PMD and
I think won't solve the purpose.



> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Wednesday, January 25, 2023 7:30 PM
> To: Aman Singh <aman.deep.singh@intel.com>; Yuying Zhang <yuying.zhang@intel.com>;
> Ivan Malov <ivan.malov@oktetlabs.ru>; Andrew Rybchenko
> <andrew.rybchenko@oktetlabs.ru>; dev@dpdk.org; Hanumanth Reddy Pothula
> <hpothula@marvell.com>; Ferruh Yigit <ferruh.yigit@amd.com>
> Cc: viacheslavo@nvidia.com; Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Nithin Kumar
> Dabilpuram <ndabilpuram@marvell.com>; david.marchand@redhat.com
> Subject: Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata
> negotiation
> 
> 25/01/2023 14:55, Ferruh Yigit:
> > On 1/25/2023 12:55 PM, Thomas Monjalon wrote:
> > > 25/01/2023 10:30, Hanumanth Reddy Pothula:
> > >> ++ Ivan Malov and Andrew Rybchenko
> > >>
> > >> From: Ferruh Yigit <ferruh.yigit@amd.com>
> > >>> On 12/21/2022 2:07 AM, Hanumanth Pothula wrote:
> > >>>> Presently, Rx metadata is sent to PMD by default, leading to a
> > >>>> performance drop as processing for the same in Rx path takes extra
> > >>>> cycles.
> > >>>>
> > >>>> Hence, add new testpmd command,
> > >>>>   'enable port <port_id> nic_to_pmd_rx_metadata'
> > >>>>
> > >>>> This command helps in sending Rx metadata to PMD and thereby Rx
> > >>>> metadata flow command requests are processed.
> > >>>>
> > >>>> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> > >>>
> > >>> Hi Hanumanth,
> > >>>
> > >>> I agree with Thomas for the patch.
> > >>>
> > >>> 'eth_rx_metadata_negotiate_mp()' requests all Rx metadata offloads to be
> > >>> enabled, but at this stage if there is no flow rule for Rx metadata why it is
> > >>> consuming extra cycles?
> > >>>
> > >>> Can you update driver code to process Rx metadata when it is enabled by
> > >>> application (via 'rte_eth_rx_metadata_negotiate()') AND there is at least
> > >>> one flow rule for it?
> > >>
> > >> #1 What is the purpose of rte_eth_rx_metadata_negotiate() API if it is always called by
> testpmd.
> > >> We thought it was added so that when that metadata is not needed, application need
> not call this
> > >> thereby saving cycles/bandwidth.
> > >
> > > testpmd is for testing all features. That's why all is negotiated.
> > > Cycles should be saved if you don't enable it until a flow rule requires it.
> > >
> >
> > Hi Thomas,
> >
> > Not just for saving cycles, but from testing perspective too, do you
> > think does it work if a way to disable these Rx metadata added by
> > keeping default behavior as it is?
> >
> > And new command can be in a consistent command syntax like:
> > "port config <port_id> ..."
> 
> Yes I agree it would be good to have a way to test different values.
> And it would allow to completely disable metadata I suppose.
> 
> Note: I don't understand why we don't have
> RTE_FLOW_ACTION_TYPE_SET_TAG and RTE_FLOW_ACTION_TYPE_SET_META
> negotiated in this function. Probably something to add.
> 
> 
> > >> #2 We use this API similar to Rx/Tx offload flags so that we can set things up before
> device is
> > >> configured. We thought that is the purpose of having this negotiate API and avoid
> depleting offload flags.
> > >
> > > It is just a configuration negotiation specific to metadata.
> > >
> > >> #3 Generally any new offloads added to DPDK would be in disabled state in testpmd
> and we would have
> > >> an option to enable it. In this case, testpmd is by default calling this negotiation.
> > >
> > > Negotiating is not enabling.
> > >
> > >> We can update the driver if the purpose of this API is clear.
> > >
> > > Please do.
> >
> > Is following understanding correct?
> >
> >      API        Flow Rule       Result
> >     -----    ------------     --------
> >     Enable    No Rule	       Feature Disabled
> >     Enable    Rule exist       Feature Enabled
> >     Disable     X              Feature Disabled
> 
> In the API column, you should say "negotiated" instead of "Enable".
> 
> 

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Thomas Monjalon]

Please reply inline below instead of doing an incomplete copy
of the replies on top.


25/01/2023 15:42, Nithin Kumar Dabilpuram:
> > >Will it work to enable them all by default and add capability to disable
> > >it in testpmd, which helps to run performance tests also to verify the
> > > impact of the API?
> 
> The spirit of the negotiating features/Rx/Tx offloads upfront is to have it disabled by default and enable the feature only when needed. Having the features enabled by default is probably against that spirit.
> 
> We understand the concerns with drivers that didn't not implement that API.

There is no such concern I think.

> Why not disable it by default(like other offloads) and modify rte_flow action creation in testpmd to check for if !ENOSUP and feature disabled and add print to enable. So for the PMD's that won't support rte_eth_rx_metadata_negotiate(), there won't be any difference and for very few PMD's that support this API, they need to enable it before using RTE_FLOW with MARK/FLAG.
> Behavior change would be seen only with two PMD's(cnxk, sfc).

I think you missed the whole point.
Ferruh is proposing to have a command "port config <port_id> ..."
to configure the flags to negotiate.
Are you OK with this approach?

> > Note: I don't understand why we don't have
> > RTE_FLOW_ACTION_TYPE_SET_TAG and RTE_FLOW_ACTION_TYPE_SET_META
> > negotiated in this function. Probably something to add.
> 
> The purpose of negotiate is to tell the PMD upfront so that PMD can prepare
> HW appropriately.  Having these new actions would be very late to inform PMD and
> I think won't solve the purpose.

I am not talking about your problem here.
I am just saying that TAG and META should be negotiated as well
in rte_eth_rx_metadata_negotiate().

> From: Thomas Monjalon <thomas@monjalon.net>
> > 25/01/2023 14:55, Ferruh Yigit:
> > > On 1/25/2023 12:55 PM, Thomas Monjalon wrote:
> > > > 25/01/2023 10:30, Hanumanth Reddy Pothula:
> > > >> ++ Ivan Malov and Andrew Rybchenko
> > > >>
> > > >> From: Ferruh Yigit <ferruh.yigit@amd.com>
> > > >>> On 12/21/2022 2:07 AM, Hanumanth Pothula wrote:
> > > >>>> Presently, Rx metadata is sent to PMD by default, leading to a
> > > >>>> performance drop as processing for the same in Rx path takes extra
> > > >>>> cycles.
> > > >>>>
> > > >>>> Hence, add new testpmd command,
> > > >>>>   'enable port <port_id> nic_to_pmd_rx_metadata'
> > > >>>>
> > > >>>> This command helps in sending Rx metadata to PMD and thereby Rx
> > > >>>> metadata flow command requests are processed.
> > > >>>>
> > > >>>> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> > > >>>
> > > >>> Hi Hanumanth,
> > > >>>
> > > >>> I agree with Thomas for the patch.
> > > >>>
> > > >>> 'eth_rx_metadata_negotiate_mp()' requests all Rx metadata offloads to be
> > > >>> enabled, but at this stage if there is no flow rule for Rx metadata why it is
> > > >>> consuming extra cycles?
> > > >>>
> > > >>> Can you update driver code to process Rx metadata when it is enabled by
> > > >>> application (via 'rte_eth_rx_metadata_negotiate()') AND there is at least
> > > >>> one flow rule for it?
> > > >>
> > > >> #1 What is the purpose of rte_eth_rx_metadata_negotiate() API if it is always called by
> > testpmd.
> > > >> We thought it was added so that when that metadata is not needed, application need
> > not call this
> > > >> thereby saving cycles/bandwidth.
> > > >
> > > > testpmd is for testing all features. That's why all is negotiated.
> > > > Cycles should be saved if you don't enable it until a flow rule requires it.
> > > >
> > >
> > > Hi Thomas,
> > >
> > > Not just for saving cycles, but from testing perspective too, do you
> > > think does it work if a way to disable these Rx metadata added by
> > > keeping default behavior as it is?
> > >
> > > And new command can be in a consistent command syntax like:
> > > "port config <port_id> ..."
> > 
> > Yes I agree it would be good to have a way to test different values.
> > And it would allow to completely disable metadata I suppose.
> > 
> > Note: I don't understand why we don't have
> > RTE_FLOW_ACTION_TYPE_SET_TAG and RTE_FLOW_ACTION_TYPE_SET_META
> > negotiated in this function. Probably something to add.
> > 
> > 
> > > >> #2 We use this API similar to Rx/Tx offload flags so that we can set things up before
> > device is
> > > >> configured. We thought that is the purpose of having this negotiate API and avoid
> > depleting offload flags.
> > > >
> > > > It is just a configuration negotiation specific to metadata.
> > > >
> > > >> #3 Generally any new offloads added to DPDK would be in disabled state in testpmd
> > and we would have
> > > >> an option to enable it. In this case, testpmd is by default calling this negotiation.
> > > >
> > > > Negotiating is not enabling.
> > > >
> > > >> We can update the driver if the purpose of this API is clear.
> > > >
> > > > Please do.
> > >
> > > Is following understanding correct?
> > >
> > >      API        Flow Rule       Result
> > >     -----    ------------     --------
> > >     Enable    No Rule	       Feature Disabled
> > >     Enable    Rule exist       Feature Enabled
> > >     Disable     X              Feature Disabled
> > 
> > In the API column, you should say "negotiated" instead of "Enable".
> > 
> > 
> 
> 

RE: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Nithin Kumar Dabilpuram]

Please see inline.

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Thursday, January 26, 2023 4:33 PM
> To: Nithin Kumar Dabilpuram <ndabilpuram@marvell.com>
> Cc: Aman Singh <aman.deep.singh@intel.com>; Yuying Zhang <yuying.zhang@intel.com>;
> Ivan Malov <ivan.malov@oktetlabs.ru>; Andrew Rybchenko
> <andrew.rybchenko@oktetlabs.ru>; dev@dpdk.org; Hanumanth Reddy Pothula
> <hpothula@marvell.com>; Ferruh Yigit <ferruh.yigit@amd.com>; viacheslavo@nvidia.com;
> Jerin Jacob Kollanukkaran <jerinj@marvell.com>; david.marchand@redhat.com
> Subject: Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata
> negotiation
> 
> Please reply inline below instead of doing an incomplete copy
> of the replies on top.

Ack, was just trying converge threads.

> 
> 
> 25/01/2023 15:42, Nithin Kumar Dabilpuram:
> > > >Will it work to enable them all by default and add capability to disable
> > > >it in testpmd, which helps to run performance tests also to verify the
> > > > impact of the API?
> >
> > The spirit of the negotiating features/Rx/Tx offloads upfront is to have it disabled by
> default and enable the feature only when needed. Having the features enabled by default
> is probably against that spirit.
> >
> > We understand the concerns with drivers that didn't not implement that API.
> 
> There is no such concern I think.
> 
> > Why not disable it by default(like other offloads) and modify rte_flow action creation in
> testpmd to check for if !ENOSUP and feature disabled and add print to enable. So for the
> PMD's that won't support rte_eth_rx_metadata_negotiate(), there won't be any
> difference and for very few PMD's that support this API, they need to enable it before
> using RTE_FLOW with MARK/FLAG.
> > Behavior change would be seen only with two PMD's(cnxk, sfc).
> 
> I think you missed the whole point.
> Ferruh is proposing to have a command "port config <port_id> ..."
> to configure the flags to negotiate.
> Are you OK with this approach?
> 

Yes, we are fine to have such command to enable and disable the feature with default being it disabled if supported by PMD.
Is default being disabled fine if the feature is supported by a PMD ?

> > > Note: I don't understand why we don't have
> > > RTE_FLOW_ACTION_TYPE_SET_TAG and RTE_FLOW_ACTION_TYPE_SET_META
> > > negotiated in this function. Probably something to add.
> >
> > The purpose of negotiate is to tell the PMD upfront so that PMD can prepare
> > HW appropriately.  Having these new actions would be very late to inform PMD and
> > I think won't solve the purpose.
> 
> I am not talking about your problem here.
> I am just saying that TAG and META should be negotiated as well
> in rte_eth_rx_metadata_negotiate().
> 
> > From: Thomas Monjalon <thomas@monjalon.net>
> > > 25/01/2023 14:55, Ferruh Yigit:
> > > > On 1/25/2023 12:55 PM, Thomas Monjalon wrote:
> > > > > 25/01/2023 10:30, Hanumanth Reddy Pothula:
> > > > >> ++ Ivan Malov and Andrew Rybchenko
> > > > >>
> > > > >> From: Ferruh Yigit <ferruh.yigit@amd.com>
> > > > >>> On 12/21/2022 2:07 AM, Hanumanth Pothula wrote:
> > > > >>>> Presently, Rx metadata is sent to PMD by default, leading to a
> > > > >>>> performance drop as processing for the same in Rx path takes extra
> > > > >>>> cycles.
> > > > >>>>
> > > > >>>> Hence, add new testpmd command,
> > > > >>>>   'enable port <port_id> nic_to_pmd_rx_metadata'
> > > > >>>>
> > > > >>>> This command helps in sending Rx metadata to PMD and thereby Rx
> > > > >>>> metadata flow command requests are processed.
> > > > >>>>
> > > > >>>> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> > > > >>>
> > > > >>> Hi Hanumanth,
> > > > >>>
> > > > >>> I agree with Thomas for the patch.
> > > > >>>
> > > > >>> 'eth_rx_metadata_negotiate_mp()' requests all Rx metadata offloads to be
> > > > >>> enabled, but at this stage if there is no flow rule for Rx metadata why it is
> > > > >>> consuming extra cycles?
> > > > >>>
> > > > >>> Can you update driver code to process Rx metadata when it is enabled by
> > > > >>> application (via 'rte_eth_rx_metadata_negotiate()') AND there is at least
> > > > >>> one flow rule for it?
> > > > >>
> > > > >> #1 What is the purpose of rte_eth_rx_metadata_negotiate() API if it is always
> called by
> > > testpmd.
> > > > >> We thought it was added so that when that metadata is not needed, application
> need
> > > not call this
> > > > >> thereby saving cycles/bandwidth.
> > > > >
> > > > > testpmd is for testing all features. That's why all is negotiated.
> > > > > Cycles should be saved if you don't enable it until a flow rule requires it.
> > > > >
> > > >
> > > > Hi Thomas,
> > > >
> > > > Not just for saving cycles, but from testing perspective too, do you
> > > > think does it work if a way to disable these Rx metadata added by
> > > > keeping default behavior as it is?
> > > >
> > > > And new command can be in a consistent command syntax like:
> > > > "port config <port_id> ..."
> > >
> > > Yes I agree it would be good to have a way to test different values.
> > > And it would allow to completely disable metadata I suppose.
> > >
> > > Note: I don't understand why we don't have
> > > RTE_FLOW_ACTION_TYPE_SET_TAG and RTE_FLOW_ACTION_TYPE_SET_META
> > > negotiated in this function. Probably something to add.
> > >
> > >
> > > > >> #2 We use this API similar to Rx/Tx offload flags so that we can set things up before
> > > device is
> > > > >> configured. We thought that is the purpose of having this negotiate API and avoid
> > > depleting offload flags.
> > > > >
> > > > > It is just a configuration negotiation specific to metadata.
> > > > >
> > > > >> #3 Generally any new offloads added to DPDK would be in disabled state in
> testpmd
> > > and we would have
> > > > >> an option to enable it. In this case, testpmd is by default calling this negotiation.
> > > > >
> > > > > Negotiating is not enabling.
> > > > >
> > > > >> We can update the driver if the purpose of this API is clear.
> > > > >
> > > > > Please do.
> > > >
> > > > Is following understanding correct?
> > > >
> > > >      API        Flow Rule       Result
> > > >     -----    ------------     --------
> > > >     Enable    No Rule	       Feature Disabled
> > > >     Enable    Rule exist       Feature Enabled
> > > >     Disable     X              Feature Disabled
> > >
> > > In the API column, you should say "negotiated" instead of "Enable".
> > >
> > >
> >
> >
> 
> 
> 
> 

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Thomas Monjalon]

27/01/2023 06:02, Nithin Kumar Dabilpuram:
> From: Thomas Monjalon <thomas@monjalon.net>
> > Ferruh is proposing to have a command "port config <port_id> ..."
> > to configure the flags to negotiate.
> > Are you OK with this approach?
> 
> Yes, we are fine to have such command to enable and disable the feature
> with default being it disabled if supported by PMD.
> Is default being disabled fine if the feature is supported by a PMD ?

I think the default should be enabled for ease of use.
Do we have similar features disables by default?
I mean do we know features in testpmd which require a "double enablement"
like one configuration command + one rte_flow rule?

RE: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Nithin Kumar Dabilpuram]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Friday, January 27, 2023 2:25 PM
> To: Nithin Kumar Dabilpuram <ndabilpuram@marvell.com>
> Cc: Aman Singh <aman.deep.singh@intel.com>; Yuying Zhang <yuying.zhang@intel.com>;
> Ivan Malov <ivan.malov@oktetlabs.ru>; Andrew Rybchenko
> <andrew.rybchenko@oktetlabs.ru>; dev@dpdk.org; Hanumanth Reddy Pothula
> <hpothula@marvell.com>; Ferruh Yigit <ferruh.yigit@amd.com>; viacheslavo@nvidia.com;
> Jerin Jacob Kollanukkaran <jerinj@marvell.com>; david.marchand@redhat.com
> Subject: Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata
> negotiation
> 
> 27/01/2023 06:02, Nithin Kumar Dabilpuram:
> > From: Thomas Monjalon <thomas@monjalon.net>
> > > Ferruh is proposing to have a command "port config <port_id> ..."
> > > to configure the flags to negotiate.
> > > Are you OK with this approach?
> >
> > Yes, we are fine to have such command to enable and disable the feature
> > with default being it disabled if supported by PMD.
> > Is default being disabled fine if the feature is supported by a PMD ?
> 
> I think the default should be enabled for ease of use.

Since testpmd is used extensively for benchmarking purposes, we thought it should have minimum features
enabled by default. The default testpmd doesn't have any Rx/Tx offloads enabled(except for FAST FREE),  default
fwd mode being "iofwd" and the Rx metadata is only referenced when dumping packets.


> Do we have similar features disables by default?
> I mean do we know features in testpmd which require a "double enablement"
> like one configuration command + one rte_flow rule?

Spec itself is that way i.e "RTE_FLOW_RULE + RX_METADATA_NEGOTIATE(once)"

Isn't it enough if 

#1 We have enough print when rte_flow is being create without negotiation done and ask user to enable rx metadata using
"port config <port_id>..."
#2 Provide testpmd app arg to enable Rx metadata(for example " --rx-metadata") like other features to avoid calling another
command before rte flow create.

> 
> 

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Thomas Monjalon]

27/01/2023 11:42, Nithin Kumar Dabilpuram:
> From: Thomas Monjalon <thomas@monjalon.net>
> > 27/01/2023 06:02, Nithin Kumar Dabilpuram:
> > > From: Thomas Monjalon <thomas@monjalon.net>
> > > > Ferruh is proposing to have a command "port config <port_id> ..."
> > > > to configure the flags to negotiate.
> > > > Are you OK with this approach?
> > >
> > > Yes, we are fine to have such command to enable and disable the feature
> > > with default being it disabled if supported by PMD.
> > > Is default being disabled fine if the feature is supported by a PMD ?
> > 
> > I think the default should be enabled for ease of use.
> 
> Since testpmd is used extensively for benchmarking purposes, we thought it should have minimum features
> enabled by default. The default testpmd doesn't have any Rx/Tx offloads enabled(except for FAST FREE),  default
> fwd mode being "iofwd" and the Rx metadata is only referenced when dumping packets.
> 
> 
> > Do we have similar features disables by default?
> > I mean do we know features in testpmd which require a "double enablement"
> > like one configuration command + one rte_flow rule?
> 
> Spec itself is that way i.e "RTE_FLOW_RULE + RX_METADATA_NEGOTIATE(once)"
> 
> Isn't it enough if 
> 
> #1 We have enough print when rte_flow is being create without negotiation done and ask user to enable rx metadata using
> "port config <port_id>..."
> #2 Provide testpmd app arg to enable Rx metadata(for example " --rx-metadata") like other features to avoid calling another
> command before rte flow create.

I'm not sure what is best.
I will let others discuss this part.

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Jerin Jacob]

On Fri, Jan 27, 2023 at 8:31 PM Thomas Monjalon <thomas@monjalon.net> wrote:
>
> 27/01/2023 11:42, Nithin Kumar Dabilpuram:
> > From: Thomas Monjalon <thomas@monjalon.net>
> > > 27/01/2023 06:02, Nithin Kumar Dabilpuram:
> > > > From: Thomas Monjalon <thomas@monjalon.net>
> > > > > Ferruh is proposing to have a command "port config <port_id> ..."
> > > > > to configure the flags to negotiate.
> > > > > Are you OK with this approach?
> > > >
> > > > Yes, we are fine to have such command to enable and disable the feature
> > > > with default being it disabled if supported by PMD.
> > > > Is default being disabled fine if the feature is supported by a PMD ?
> > >
> > > I think the default should be enabled for ease of use.
> >
> > Since testpmd is used extensively for benchmarking purposes, we thought it should have minimum features
> > enabled by default. The default testpmd doesn't have any Rx/Tx offloads enabled(except for FAST FREE),  default
> > fwd mode being "iofwd" and the Rx metadata is only referenced when dumping packets.
> >
> >
> > > Do we have similar features disables by default?
> > > I mean do we know features in testpmd which require a "double enablement"
> > > like one configuration command + one rte_flow rule?
> >
> > Spec itself is that way i.e "RTE_FLOW_RULE + RX_METADATA_NEGOTIATE(once)"
> >
> > Isn't it enough if
> >
> > #1 We have enough print when rte_flow is being create without negotiation done and ask user to enable rx metadata using
> > "port config <port_id>..."
> > #2 Provide testpmd app arg to enable Rx metadata(for example " --rx-metadata") like other features to avoid calling another
> > command before rte flow create.
>
> I'm not sure what is best.
> I will let others discuss this part.

IMO, enabling something always defeat the purpose to negotiate. IMO,
someone needs to negotiate
for a feature if the feature is needed. I think, the double enablement
is part of the spec itself. In case, The PMD
drivers won't like double enablement, no need to implement the PMD
callback. That way, there is no change in the existing flow.

The reason why cnxk driver thought of leveraging negotiate() feature
so that it helps for optimization. e.s.p
function template for multiprocess case as we know what features
needed in fastpath upfront.

If there still concerns with patch we can take up this to TB decide
one way or another to make forward progress. Let us know.



>
>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Thomas Monjalon]

31/01/2023 17:17, Jerin Jacob:
> On Fri, Jan 27, 2023 at 8:31 PM Thomas Monjalon <thomas@monjalon.net> wrote:
> >
> > 27/01/2023 11:42, Nithin Kumar Dabilpuram:
> > > From: Thomas Monjalon <thomas@monjalon.net>
> > > > 27/01/2023 06:02, Nithin Kumar Dabilpuram:
> > > > > From: Thomas Monjalon <thomas@monjalon.net>
> > > > > > Ferruh is proposing to have a command "port config <port_id> ..."
> > > > > > to configure the flags to negotiate.
> > > > > > Are you OK with this approach?
> > > > >
> > > > > Yes, we are fine to have such command to enable and disable the feature
> > > > > with default being it disabled if supported by PMD.
> > > > > Is default being disabled fine if the feature is supported by a PMD ?
> > > >
> > > > I think the default should be enabled for ease of use.
> > >
> > > Since testpmd is used extensively for benchmarking purposes, we thought it should have minimum features
> > > enabled by default. The default testpmd doesn't have any Rx/Tx offloads enabled(except for FAST FREE),  default
> > > fwd mode being "iofwd" and the Rx metadata is only referenced when dumping packets.
> > >
> > >
> > > > Do we have similar features disables by default?
> > > > I mean do we know features in testpmd which require a "double enablement"
> > > > like one configuration command + one rte_flow rule?
> > >
> > > Spec itself is that way i.e "RTE_FLOW_RULE + RX_METADATA_NEGOTIATE(once)"
> > >
> > > Isn't it enough if
> > >
> > > #1 We have enough print when rte_flow is being create without negotiation done and ask user to enable rx metadata using
> > > "port config <port_id>..."
> > > #2 Provide testpmd app arg to enable Rx metadata(for example " --rx-metadata") like other features to avoid calling another
> > > command before rte flow create.
> >
> > I'm not sure what is best.
> > I will let others discuss this part.
> 
> IMO, enabling something always defeat the purpose to negotiate. IMO,
> someone needs to negotiate
> for a feature if the feature is needed. I think, the double enablement
> is part of the spec itself. In case, The PMD
> drivers won't like double enablement, no need to implement the PMD
> callback. That way, there is no change in the existing flow.
> 
> The reason why cnxk driver thought of leveraging negotiate() feature
> so that it helps for optimization. e.s.p
> function template for multiprocess case as we know what features
> needed in fastpath upfront.
> 
> If there still concerns with patch we can take up this to TB decide
> one way or another to make forward progress. Let us know.

Ferruh, Andrew, Ori, Ivan, what is your opinion?
Let's progress with this patch to make it in -rc1.

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ivan Malov]

Hello everyone,

Since making automatic, or implicit, offload decisions does
not belong in testpmd responsibility domain, it should be
safer to avoid calling the "negotiate metadata delivery"
API with some default selection unless the user asks to
do so explicitly, via internal CLI or app options.

With that in mind, port config <port_id> ... sounds OK.

PMDs that support flow primitives which can generate metadata
but, if in started state, can't enable its delivery may pass
appropriate rte_error messages to the user suggesting
they enable delivery of such metadata from NIC to PMD
first. This way, if the person operating testpmd
enters a flow create command and that fails,
they can figure out the inconsistency, stop
the port, negotiate, start and try again.

As for non-debug applications, their developers shall
be properly informed about the problem of enabling
delivery of metadata from NIC to PMD. This way,
they will invoke the negotiate API by default
in their apps, with the feature selection (eg.
MARK) as per nature of the app's business.

This API should indeed be helpful to some PMDs with
regard to collecting upfront knowledge like this.
At the same time, should be benign to those PMDs
who do not need this knowledge and can enable
delivery of metadata right when inserting the
flow rules. So I hope the API does not create
too much discomfort to vendors not needing it.

Thank you.

On Wed, 1 Feb 2023, Thomas Monjalon wrote:

> 31/01/2023 17:17, Jerin Jacob:
>> On Fri, Jan 27, 2023 at 8:31 PM Thomas Monjalon <thomas@monjalon.net> wrote:
>>>
>>> 27/01/2023 11:42, Nithin Kumar Dabilpuram:
>>>> From: Thomas Monjalon <thomas@monjalon.net>
>>>>> 27/01/2023 06:02, Nithin Kumar Dabilpuram:
>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
>>>>>>> Ferruh is proposing to have a command "port config <port_id> ..."
>>>>>>> to configure the flags to negotiate.
>>>>>>> Are you OK with this approach?
>>>>>>
>>>>>> Yes, we are fine to have such command to enable and disable the feature
>>>>>> with default being it disabled if supported by PMD.
>>>>>> Is default being disabled fine if the feature is supported by a PMD ?
>>>>>
>>>>> I think the default should be enabled for ease of use.
>>>>
>>>> Since testpmd is used extensively for benchmarking purposes, we thought it should have minimum features
>>>> enabled by default. The default testpmd doesn't have any Rx/Tx offloads enabled(except for FAST FREE),  default
>>>> fwd mode being "iofwd" and the Rx metadata is only referenced when dumping packets.
>>>>
>>>>
>>>>> Do we have similar features disables by default?
>>>>> I mean do we know features in testpmd which require a "double enablement"
>>>>> like one configuration command + one rte_flow rule?
>>>>
>>>> Spec itself is that way i.e "RTE_FLOW_RULE + RX_METADATA_NEGOTIATE(once)"
>>>>
>>>> Isn't it enough if
>>>>
>>>> #1 We have enough print when rte_flow is being create without negotiation done and ask user to enable rx metadata using
>>>> "port config <port_id>..."
>>>> #2 Provide testpmd app arg to enable Rx metadata(for example " --rx-metadata") like other features to avoid calling another
>>>> command before rte flow create.
>>>
>>> I'm not sure what is best.
>>> I will let others discuss this part.
>>
>> IMO, enabling something always defeat the purpose to negotiate. IMO,
>> someone needs to negotiate
>> for a feature if the feature is needed. I think, the double enablement
>> is part of the spec itself. In case, The PMD
>> drivers won't like double enablement, no need to implement the PMD
>> callback. That way, there is no change in the existing flow.
>>
>> The reason why cnxk driver thought of leveraging negotiate() feature
>> so that it helps for optimization. e.s.p
>> function template for multiprocess case as we know what features
>> needed in fastpath upfront.
>>
>> If there still concerns with patch we can take up this to TB decide
>> one way or another to make forward progress. Let us know.
>
> Ferruh, Andrew, Ori, Ivan, what is your opinion?
> Let's progress with this patch to make it in -rc1.
>
>
>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Andrew Rybchenko]

On 2/1/23 09:10, Ivan Malov wrote:
> Hello everyone,
> 
> Since making automatic, or implicit, offload decisions does
> not belong in testpmd responsibility domain, it should be
> safer to avoid calling the "negotiate metadata delivery"
> API with some default selection unless the user asks to
> do so explicitly, via internal CLI or app options.
> 
> With that in mind, port config <port_id> ... sounds OK.
> 
> PMDs that support flow primitives which can generate metadata
> but, if in started state, can't enable its delivery may pass
> appropriate rte_error messages to the user suggesting
> they enable delivery of such metadata from NIC to PMD
> first. This way, if the person operating testpmd
> enters a flow create command and that fails,
> they can figure out the inconsistency, stop
> the port, negotiate, start and try again.
> 
> As for non-debug applications, their developers shall
> be properly informed about the problem of enabling
> delivery of metadata from NIC to PMD. This way,
> they will invoke the negotiate API by default
> in their apps, with the feature selection (eg.
> MARK) as per nature of the app's business.
> 
> This API should indeed be helpful to some PMDs with
> regard to collecting upfront knowledge like this.
> At the same time, should be benign to those PMDs
> who do not need this knowledge and can enable
> delivery of metadata right when inserting the
> flow rules. So I hope the API does not create
> too much discomfort to vendors not needing it.
> 
> Thank you.
> 
> On Wed, 1 Feb 2023, Thomas Monjalon wrote:
> 
>> 31/01/2023 17:17, Jerin Jacob:
>>> On Fri, Jan 27, 2023 at 8:31 PM Thomas Monjalon <thomas@monjalon.net> 
>>> wrote:
>>>>
>>>> 27/01/2023 11:42, Nithin Kumar Dabilpuram:
>>>>> From: Thomas Monjalon <thomas@monjalon.net>
>>>>>> 27/01/2023 06:02, Nithin Kumar Dabilpuram:
>>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
>>>>>>>> Ferruh is proposing to have a command "port config <port_id> ..."
>>>>>>>> to configure the flags to negotiate.
>>>>>>>> Are you OK with this approach?
>>>>>>>
>>>>>>> Yes, we are fine to have such command to enable and disable the 
>>>>>>> feature
>>>>>>> with default being it disabled if supported by PMD.
>>>>>>> Is default being disabled fine if the feature is supported by a 
>>>>>>> PMD ?
>>>>>>
>>>>>> I think the default should be enabled for ease of use.
>>>>>
>>>>> Since testpmd is used extensively for benchmarking purposes, we 
>>>>> thought it should have minimum features
>>>>> enabled by default. The default testpmd doesn't have any Rx/Tx 
>>>>> offloads enabled(except for FAST FREE),  default
>>>>> fwd mode being "iofwd" and the Rx metadata is only referenced when 
>>>>> dumping packets.
>>>>>
>>>>>
>>>>>> Do we have similar features disables by default?
>>>>>> I mean do we know features in testpmd which require a "double 
>>>>>> enablement"
>>>>>> like one configuration command + one rte_flow rule?
>>>>>
>>>>> Spec itself is that way i.e "RTE_FLOW_RULE + 
>>>>> RX_METADATA_NEGOTIATE(once)"
>>>>>
>>>>> Isn't it enough if
>>>>>
>>>>> #1 We have enough print when rte_flow is being create without 
>>>>> negotiation done and ask user to enable rx metadata using
>>>>> "port config <port_id>..."
>>>>> #2 Provide testpmd app arg to enable Rx metadata(for example " 
>>>>> --rx-metadata") like other features to avoid calling another
>>>>> command before rte flow create.
>>>>
>>>> I'm not sure what is best.
>>>> I will let others discuss this part.
>>>
>>> IMO, enabling something always defeat the purpose to negotiate. IMO,
>>> someone needs to negotiate
>>> for a feature if the feature is needed. I think, the double enablement
>>> is part of the spec itself. In case, The PMD
>>> drivers won't like double enablement, no need to implement the PMD
>>> callback. That way, there is no change in the existing flow.
>>>
>>> The reason why cnxk driver thought of leveraging negotiate() feature
>>> so that it helps for optimization. e.s.p
>>> function template for multiprocess case as we know what features
>>> needed in fastpath upfront.
>>>
>>> If there still concerns with patch we can take up this to TB decide
>>> one way or another to make forward progress. Let us know.
>>
>> Ferruh, Andrew, Ori, Ivan, what is your opinion?
>> Let's progress with this patch to make it in -rc1.

As I understand all agreed that we need testpmd command to
control negotiated Rx metadata. May be even command-line
option would be useful.

So, remaining question is what should be the default value in
testpmd. Note that it is just testpmd question since default
value in an abstract application is nothing negotiated
(if I'm not mistaken).

The key advantage of the current behaviour is to avoid
"double-enabling" in testpmd. It preserves behaviour which
we had before before the API addition. It is a strong
argument. Basically it puts the feature into the same
basket as FAST_FREE - need an action to run faster.
I see no problem in such approach.

The key disadvantage is the difference in testpmd and
other applications default behaviour.

I'd look at the feature in the following way:
if an application theoretically wants to use
USER_FLAG, USER_MARK or TUNNEL_ID it must negotiate
corresponding Rx metadata to ensure that the feature is
available and HW is informed that application may need it.
Since testpmd supports corresponding flow API actions and
flow tunnels, it tries to negotiate it by default, but do
not fail if the negotiation fails.

So, I'd would vote to keeping the default value as is.

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Jerin Jacob]

On Wed, Feb 1, 2023 at 12:46 PM Andrew Rybchenko
<andrew.rybchenko@oktetlabs.ru> wrote:
>
> On 2/1/23 09:10, Ivan Malov wrote:
> > Hello everyone,
> >
> > Since making automatic, or implicit, offload decisions does
> > not belong in testpmd responsibility domain, it should be
> > safer to avoid calling the "negotiate metadata delivery"
> > API with some default selection unless the user asks to
> > do so explicitly, via internal CLI or app options.
> >
> > With that in mind, port config <port_id> ... sounds OK.
> >
> > PMDs that support flow primitives which can generate metadata
> > but, if in started state, can't enable its delivery may pass
> > appropriate rte_error messages to the user suggesting
> > they enable delivery of such metadata from NIC to PMD
> > first. This way, if the person operating testpmd
> > enters a flow create command and that fails,
> > they can figure out the inconsistency, stop
> > the port, negotiate, start and try again.
> >
> > As for non-debug applications, their developers shall
> > be properly informed about the problem of enabling
> > delivery of metadata from NIC to PMD. This way,
> > they will invoke the negotiate API by default
> > in their apps, with the feature selection (eg.
> > MARK) as per nature of the app's business.
> >
> > This API should indeed be helpful to some PMDs with
> > regard to collecting upfront knowledge like this.
> > At the same time, should be benign to those PMDs
> > who do not need this knowledge and can enable
> > delivery of metadata right when inserting the
> > flow rules. So I hope the API does not create
> > too much discomfort to vendors not needing it.
> >
> > Thank you.
> >
> > On Wed, 1 Feb 2023, Thomas Monjalon wrote:
> >
> >> 31/01/2023 17:17, Jerin Jacob:
> >>> On Fri, Jan 27, 2023 at 8:31 PM Thomas Monjalon <thomas@monjalon.net>
> >>> wrote:
> >>>>
> >>>> 27/01/2023 11:42, Nithin Kumar Dabilpuram:
> >>>>> From: Thomas Monjalon <thomas@monjalon.net>
> >>>>>> 27/01/2023 06:02, Nithin Kumar Dabilpuram:
> >>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
> >>>>>>>> Ferruh is proposing to have a command "port config <port_id> ..."
> >>>>>>>> to configure the flags to negotiate.
> >>>>>>>> Are you OK with this approach?
> >>>>>>>
> >>>>>>> Yes, we are fine to have such command to enable and disable the
> >>>>>>> feature
> >>>>>>> with default being it disabled if supported by PMD.
> >>>>>>> Is default being disabled fine if the feature is supported by a
> >>>>>>> PMD ?
> >>>>>>
> >>>>>> I think the default should be enabled for ease of use.
> >>>>>
> >>>>> Since testpmd is used extensively for benchmarking purposes, we
> >>>>> thought it should have minimum features
> >>>>> enabled by default. The default testpmd doesn't have any Rx/Tx
> >>>>> offloads enabled(except for FAST FREE),  default
> >>>>> fwd mode being "iofwd" and the Rx metadata is only referenced when
> >>>>> dumping packets.
> >>>>>
> >>>>>
> >>>>>> Do we have similar features disables by default?
> >>>>>> I mean do we know features in testpmd which require a "double
> >>>>>> enablement"
> >>>>>> like one configuration command + one rte_flow rule?
> >>>>>
> >>>>> Spec itself is that way i.e "RTE_FLOW_RULE +
> >>>>> RX_METADATA_NEGOTIATE(once)"
> >>>>>
> >>>>> Isn't it enough if
> >>>>>
> >>>>> #1 We have enough print when rte_flow is being create without
> >>>>> negotiation done and ask user to enable rx metadata using
> >>>>> "port config <port_id>..."
> >>>>> #2 Provide testpmd app arg to enable Rx metadata(for example "
> >>>>> --rx-metadata") like other features to avoid calling another
> >>>>> command before rte flow create.
> >>>>
> >>>> I'm not sure what is best.
> >>>> I will let others discuss this part.
> >>>
> >>> IMO, enabling something always defeat the purpose to negotiate. IMO,
> >>> someone needs to negotiate
> >>> for a feature if the feature is needed. I think, the double enablement
> >>> is part of the spec itself. In case, The PMD
> >>> drivers won't like double enablement, no need to implement the PMD
> >>> callback. That way, there is no change in the existing flow.
> >>>
> >>> The reason why cnxk driver thought of leveraging negotiate() feature
> >>> so that it helps for optimization. e.s.p
> >>> function template for multiprocess case as we know what features
> >>> needed in fastpath upfront.
> >>>
> >>> If there still concerns with patch we can take up this to TB decide
> >>> one way or another to make forward progress. Let us know.
> >>
> >> Ferruh, Andrew, Ori, Ivan, what is your opinion?
> >> Let's progress with this patch to make it in -rc1.
>
> As I understand all agreed that we need testpmd command to
> control negotiated Rx metadata. May be even command-line
> option would be useful.
>
> So, remaining question is what should be the default value in
> testpmd. Note that it is just testpmd question since default
> value in an abstract application is nothing negotiated
> (if I'm not mistaken).
>
> The key advantage of the current behaviour is to avoid
> "double-enabling" in testpmd. It preserves behaviour which
> we had before before the API addition. It is a strong
> argument. Basically it puts the feature into the same
> basket as FAST_FREE - need an action to run faster.

I think, there is a disconnect here. FAST_FREE is enabled by default.
i.e We don't need any specific action to run faster. To align with performance
test application, by default the configuration should be run faster. User
needs to give explicit configuration to allow more offload or the one causes
the mpps drops. IMO, That is the theme followed in testpmd.


> I see no problem in such approach.
>
> The key disadvantage is the difference in testpmd and
> other applications default behaviour.
>
> I'd look at the feature in the following way:
> if an application theoretically wants to use
> USER_FLAG, USER_MARK or TUNNEL_ID it must negotiate
> corresponding Rx metadata to ensure that the feature is
> available and HW is informed that application may need it.
> Since testpmd supports corresponding flow API actions and
> flow tunnels, it tries to negotiate it by default, but do
> not fail if the negotiation fails.
>
> So, I'd would vote to keeping the default value as is.
>

RE: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ori Kam]

Hi all,

Sorry for jumping in late,

> -----Original Message-----
> From: Jerin Jacob <jerinjacobk@gmail.com>
> Sent: Wednesday, 1 February 2023 10:53
> 
> On Wed, Feb 1, 2023 at 12:46 PM Andrew Rybchenko
> <andrew.rybchenko@oktetlabs.ru> wrote:
> >
> > On 2/1/23 09:10, Ivan Malov wrote:
> > > Hello everyone,
> > >
> > > Since making automatic, or implicit, offload decisions does
> > > not belong in testpmd responsibility domain, it should be
> > > safer to avoid calling the "negotiate metadata delivery"
> > > API with some default selection unless the user asks to
> > > do so explicitly, via internal CLI or app options.
> > >
> > > With that in mind, port config <port_id> ... sounds OK.
> > >
> > > PMDs that support flow primitives which can generate metadata
> > > but, if in started state, can't enable its delivery may pass
> > > appropriate rte_error messages to the user suggesting
> > > they enable delivery of such metadata from NIC to PMD
> > > first. This way, if the person operating testpmd
> > > enters a flow create command and that fails,
> > > they can figure out the inconsistency, stop
> > > the port, negotiate, start and try again.
> > >
> > > As for non-debug applications, their developers shall
> > > be properly informed about the problem of enabling
> > > delivery of metadata from NIC to PMD. This way,
> > > they will invoke the negotiate API by default
> > > in their apps, with the feature selection (eg.
> > > MARK) as per nature of the app's business.
> > >
> > > This API should indeed be helpful to some PMDs with
> > > regard to collecting upfront knowledge like this.
> > > At the same time, should be benign to those PMDs
> > > who do not need this knowledge and can enable
> > > delivery of metadata right when inserting the
> > > flow rules. So I hope the API does not create
> > > too much discomfort to vendors not needing it.
> > >
> > > Thank you.
> > >
> > > On Wed, 1 Feb 2023, Thomas Monjalon wrote:
> > >
> > >> 31/01/2023 17:17, Jerin Jacob:
> > >>> On Fri, Jan 27, 2023 at 8:31 PM Thomas Monjalon
> <thomas@monjalon.net>
> > >>> wrote:
> > >>>>
> > >>>> 27/01/2023 11:42, Nithin Kumar Dabilpuram:
> > >>>>> From: Thomas Monjalon <thomas@monjalon.net>
> > >>>>>> 27/01/2023 06:02, Nithin Kumar Dabilpuram:
> > >>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
> > >>>>>>>> Ferruh is proposing to have a command "port config <port_id>
> ..."
> > >>>>>>>> to configure the flags to negotiate.
> > >>>>>>>> Are you OK with this approach?
> > >>>>>>>
> > >>>>>>> Yes, we are fine to have such command to enable and disable the
> > >>>>>>> feature
> > >>>>>>> with default being it disabled if supported by PMD.
> > >>>>>>> Is default being disabled fine if the feature is supported by a
> > >>>>>>> PMD ?
> > >>>>>>
> > >>>>>> I think the default should be enabled for ease of use.
> > >>>>>
> > >>>>> Since testpmd is used extensively for benchmarking purposes, we
> > >>>>> thought it should have minimum features
> > >>>>> enabled by default. The default testpmd doesn't have any Rx/Tx
> > >>>>> offloads enabled(except for FAST FREE),  default
> > >>>>> fwd mode being "iofwd" and the Rx metadata is only referenced
> when
> > >>>>> dumping packets.
> > >>>>>
> > >>>>>
> > >>>>>> Do we have similar features disables by default?
> > >>>>>> I mean do we know features in testpmd which require a "double
> > >>>>>> enablement"
> > >>>>>> like one configuration command + one rte_flow rule?
> > >>>>>
> > >>>>> Spec itself is that way i.e "RTE_FLOW_RULE +
> > >>>>> RX_METADATA_NEGOTIATE(once)"
> > >>>>>
> > >>>>> Isn't it enough if
> > >>>>>
> > >>>>> #1 We have enough print when rte_flow is being create without
> > >>>>> negotiation done and ask user to enable rx metadata using
> > >>>>> "port config <port_id>..."
> > >>>>> #2 Provide testpmd app arg to enable Rx metadata(for example "
> > >>>>> --rx-metadata") like other features to avoid calling another
> > >>>>> command before rte flow create.
> > >>>>
> > >>>> I'm not sure what is best.
> > >>>> I will let others discuss this part.
> > >>>
> > >>> IMO, enabling something always defeat the purpose to negotiate. IMO,
> > >>> someone needs to negotiate
> > >>> for a feature if the feature is needed. I think, the double enablement
> > >>> is part of the spec itself. In case, The PMD
> > >>> drivers won't like double enablement, no need to implement the PMD
> > >>> callback. That way, there is no change in the existing flow.
> > >>>
> > >>> The reason why cnxk driver thought of leveraging negotiate() feature
> > >>> so that it helps for optimization. e.s.p
> > >>> function template for multiprocess case as we know what features
> > >>> needed in fastpath upfront.
> > >>>
> > >>> If there still concerns with patch we can take up this to TB decide
> > >>> one way or another to make forward progress. Let us know.
> > >>
> > >> Ferruh, Andrew, Ori, Ivan, what is your opinion?
> > >> Let's progress with this patch to make it in -rc1.
> >
> > As I understand all agreed that we need testpmd command to
> > control negotiated Rx metadata. May be even command-line
> > option would be useful.
> >
> > So, remaining question is what should be the default value in
> > testpmd. Note that it is just testpmd question since default
> > value in an abstract application is nothing negotiated
> > (if I'm not mistaken).
> >
> > The key advantaan ge of the current behaviour is to avoid
> > "double-enabling" in testpmd. It preserves behaviour which
> > we had before before the API addition. It is a strong
> > argument. Basically it puts the feature into the same
> > basket as FAST_FREE - need an action to run faster.
> 
> I think, there is a disconnect here. FAST_FREE is enabled by default.
> i.e We don't need any specific action to run faster. To align with performance
> test application, by default the configuration should be run faster. User
> needs to give explicit configuration to allow more offload or the one causes
> the mpps drops. IMO, That is the theme followed in testpmd.
> 
> 
I agree with Andrew, the default should stay the same, as now, PMD may already implement
logic to only enable the feature if there is a flow rule.
Changing the default will result in breaking applications.

I want to suggest new approach for this feature, 
maybe we can use the rte_flow_configure, and add a new bit that says if those
actions are going to be used.
What do you think?



> > I see no problem in such approach.
> >
> > The key disadvantage is the difference in testpmd and
> > other applications default behaviour.
> >
> > I'd look at the feature in the following way:
> > if an application theoretically wants to use
> > USER_FLAG, USER_MARK or TUNNEL_ID it must negotiate
> > corresponding Rx metadata to ensure that the feature is
> > available and HW is informed that application may need it.
> > Since testpmd supports corresponding flow API actions and
> > flow tunnels, it tries to negotiate it by default, but do
> > not fail if the negotiation fails.
> >
> > So, I'd would vote to keeping the default value as is.
> >

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Thomas Monjalon]

01/02/2023 10:00, Ori Kam:
> Hi all,
> 
> Sorry for jumping in late,
> 
> > -----Original Message-----
> > From: Jerin Jacob <jerinjacobk@gmail.com>
> > Sent: Wednesday, 1 February 2023 10:53
> > 
> > On Wed, Feb 1, 2023 at 12:46 PM Andrew Rybchenko
> > <andrew.rybchenko@oktetlabs.ru> wrote:
> > >
> > > On 2/1/23 09:10, Ivan Malov wrote:
> > > > Hello everyone,
> > > >
> > > > Since making automatic, or implicit, offload decisions does
> > > > not belong in testpmd responsibility domain, it should be
> > > > safer to avoid calling the "negotiate metadata delivery"
> > > > API with some default selection unless the user asks to
> > > > do so explicitly, via internal CLI or app options.
> > > >
> > > > With that in mind, port config <port_id> ... sounds OK.
> > > >
> > > > PMDs that support flow primitives which can generate metadata
> > > > but, if in started state, can't enable its delivery may pass
> > > > appropriate rte_error messages to the user suggesting
> > > > they enable delivery of such metadata from NIC to PMD
> > > > first. This way, if the person operating testpmd
> > > > enters a flow create command and that fails,
> > > > they can figure out the inconsistency, stop
> > > > the port, negotiate, start and try again.
> > > >
> > > > As for non-debug applications, their developers shall
> > > > be properly informed about the problem of enabling
> > > > delivery of metadata from NIC to PMD. This way,
> > > > they will invoke the negotiate API by default
> > > > in their apps, with the feature selection (eg.
> > > > MARK) as per nature of the app's business.
> > > >
> > > > This API should indeed be helpful to some PMDs with
> > > > regard to collecting upfront knowledge like this.
> > > > At the same time, should be benign to those PMDs
> > > > who do not need this knowledge and can enable
> > > > delivery of metadata right when inserting the
> > > > flow rules. So I hope the API does not create
> > > > too much discomfort to vendors not needing it.
> > > >
> > > > Thank you.
> > > >
> > > > On Wed, 1 Feb 2023, Thomas Monjalon wrote:
> > > >
> > > >> 31/01/2023 17:17, Jerin Jacob:
> > > >>> On Fri, Jan 27, 2023 at 8:31 PM Thomas Monjalon
> > <thomas@monjalon.net>
> > > >>> wrote:
> > > >>>>
> > > >>>> 27/01/2023 11:42, Nithin Kumar Dabilpuram:
> > > >>>>> From: Thomas Monjalon <thomas@monjalon.net>
> > > >>>>>> 27/01/2023 06:02, Nithin Kumar Dabilpuram:
> > > >>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
> > > >>>>>>>> Ferruh is proposing to have a command "port config <port_id>
> > ..."
> > > >>>>>>>> to configure the flags to negotiate.
> > > >>>>>>>> Are you OK with this approach?
> > > >>>>>>>
> > > >>>>>>> Yes, we are fine to have such command to enable and disable the
> > > >>>>>>> feature
> > > >>>>>>> with default being it disabled if supported by PMD.
> > > >>>>>>> Is default being disabled fine if the feature is supported by a
> > > >>>>>>> PMD ?
> > > >>>>>>
> > > >>>>>> I think the default should be enabled for ease of use.
> > > >>>>>
> > > >>>>> Since testpmd is used extensively for benchmarking purposes, we
> > > >>>>> thought it should have minimum features
> > > >>>>> enabled by default. The default testpmd doesn't have any Rx/Tx
> > > >>>>> offloads enabled(except for FAST FREE),  default
> > > >>>>> fwd mode being "iofwd" and the Rx metadata is only referenced
> > when
> > > >>>>> dumping packets.
> > > >>>>>
> > > >>>>>
> > > >>>>>> Do we have similar features disables by default?
> > > >>>>>> I mean do we know features in testpmd which require a "double
> > > >>>>>> enablement"
> > > >>>>>> like one configuration command + one rte_flow rule?
> > > >>>>>
> > > >>>>> Spec itself is that way i.e "RTE_FLOW_RULE +
> > > >>>>> RX_METADATA_NEGOTIATE(once)"
> > > >>>>>
> > > >>>>> Isn't it enough if
> > > >>>>>
> > > >>>>> #1 We have enough print when rte_flow is being create without
> > > >>>>> negotiation done and ask user to enable rx metadata using
> > > >>>>> "port config <port_id>..."
> > > >>>>> #2 Provide testpmd app arg to enable Rx metadata(for example "
> > > >>>>> --rx-metadata") like other features to avoid calling another
> > > >>>>> command before rte flow create.
> > > >>>>
> > > >>>> I'm not sure what is best.
> > > >>>> I will let others discuss this part.
> > > >>>
> > > >>> IMO, enabling something always defeat the purpose to negotiate. IMO,
> > > >>> someone needs to negotiate
> > > >>> for a feature if the feature is needed. I think, the double enablement
> > > >>> is part of the spec itself. In case, The PMD
> > > >>> drivers won't like double enablement, no need to implement the PMD
> > > >>> callback. That way, there is no change in the existing flow.
> > > >>>
> > > >>> The reason why cnxk driver thought of leveraging negotiate() feature
> > > >>> so that it helps for optimization. e.s.p
> > > >>> function template for multiprocess case as we know what features
> > > >>> needed in fastpath upfront.
> > > >>>
> > > >>> If there still concerns with patch we can take up this to TB decide
> > > >>> one way or another to make forward progress. Let us know.
> > > >>
> > > >> Ferruh, Andrew, Ori, Ivan, what is your opinion?
> > > >> Let's progress with this patch to make it in -rc1.
> > >
> > > As I understand all agreed that we need testpmd command to
> > > control negotiated Rx metadata. May be even command-line
> > > option would be useful.
> > >
> > > So, remaining question is what should be the default value in
> > > testpmd. Note that it is just testpmd question since default
> > > value in an abstract application is nothing negotiated
> > > (if I'm not mistaken).
> > >
> > > The key advantaan ge of the current behaviour is to avoid
> > > "double-enabling" in testpmd. It preserves behaviour which
> > > we had before before the API addition. It is a strong
> > > argument. Basically it puts the feature into the same
> > > basket as FAST_FREE - need an action to run faster.
> > 
> > I think, there is a disconnect here. FAST_FREE is enabled by default.
> > i.e We don't need any specific action to run faster. To align with performance
> > test application, by default the configuration should be run faster. User
> > needs to give explicit configuration to allow more offload or the one causes
> > the mpps drops. IMO, That is the theme followed in testpmd.
> > 
> > 
> I agree with Andrew, the default should stay the same, as now, PMD may already implement
> logic to only enable the feature if there is a flow rule.
> Changing the default will result in breaking applications.

That's not what is discussed here.
We are talking only about testpmd default.

> I want to suggest new approach for this feature, 
> maybe we can use the rte_flow_configure, and add a new bit that says if those
> actions are going to be used.
> What do you think?

Let's not change the API please.


> > > I see no problem in such approach.
> > >
> > > The key disadvantage is the difference in testpmd and
> > > other applications default behaviour.
> > >
> > > I'd look at the feature in the following way:
> > > if an application theoretically wants to use
> > > USER_FLAG, USER_MARK or TUNNEL_ID it must negotiate
> > > corresponding Rx metadata to ensure that the feature is
> > > available and HW is informed that application may need it.
> > > Since testpmd supports corresponding flow API actions and
> > > flow tunnels, it tries to negotiate it by default, but do
> > > not fail if the negotiation fails.
> > >
> > > So, I'd would vote to keeping the default value as is.
> > >
> 

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Andrew Rybchenko]

On 2/1/23 12:05, Thomas Monjalon wrote:
> 01/02/2023 10:00, Ori Kam:
>> Hi all,
>>
>> Sorry for jumping in late,
>>
>>> -----Original Message-----
>>> From: Jerin Jacob <jerinjacobk@gmail.com>
>>> Sent: Wednesday, 1 February 2023 10:53
>>>
>>> On Wed, Feb 1, 2023 at 12:46 PM Andrew Rybchenko
>>> <andrew.rybchenko@oktetlabs.ru> wrote:
>>>>
>>>> On 2/1/23 09:10, Ivan Malov wrote:
>>>>> Hello everyone,
>>>>>
>>>>> Since making automatic, or implicit, offload decisions does
>>>>> not belong in testpmd responsibility domain, it should be
>>>>> safer to avoid calling the "negotiate metadata delivery"
>>>>> API with some default selection unless the user asks to
>>>>> do so explicitly, via internal CLI or app options.
>>>>>
>>>>> With that in mind, port config <port_id> ... sounds OK.
>>>>>
>>>>> PMDs that support flow primitives which can generate metadata
>>>>> but, if in started state, can't enable its delivery may pass
>>>>> appropriate rte_error messages to the user suggesting
>>>>> they enable delivery of such metadata from NIC to PMD
>>>>> first. This way, if the person operating testpmd
>>>>> enters a flow create command and that fails,
>>>>> they can figure out the inconsistency, stop
>>>>> the port, negotiate, start and try again.
>>>>>
>>>>> As for non-debug applications, their developers shall
>>>>> be properly informed about the problem of enabling
>>>>> delivery of metadata from NIC to PMD. This way,
>>>>> they will invoke the negotiate API by default
>>>>> in their apps, with the feature selection (eg.
>>>>> MARK) as per nature of the app's business.
>>>>>
>>>>> This API should indeed be helpful to some PMDs with
>>>>> regard to collecting upfront knowledge like this.
>>>>> At the same time, should be benign to those PMDs
>>>>> who do not need this knowledge and can enable
>>>>> delivery of metadata right when inserting the
>>>>> flow rules. So I hope the API does not create
>>>>> too much discomfort to vendors not needing it.
>>>>>
>>>>> Thank you.
>>>>>
>>>>> On Wed, 1 Feb 2023, Thomas Monjalon wrote:
>>>>>
>>>>>> 31/01/2023 17:17, Jerin Jacob:
>>>>>>> On Fri, Jan 27, 2023 at 8:31 PM Thomas Monjalon
>>> <thomas@monjalon.net>
>>>>>>> wrote:
>>>>>>>>
>>>>>>>> 27/01/2023 11:42, Nithin Kumar Dabilpuram:
>>>>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
>>>>>>>>>> 27/01/2023 06:02, Nithin Kumar Dabilpuram:
>>>>>>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
>>>>>>>>>>>> Ferruh is proposing to have a command "port config <port_id>
>>> ..."
>>>>>>>>>>>> to configure the flags to negotiate.
>>>>>>>>>>>> Are you OK with this approach?
>>>>>>>>>>>
>>>>>>>>>>> Yes, we are fine to have such command to enable and disable the
>>>>>>>>>>> feature
>>>>>>>>>>> with default being it disabled if supported by PMD.
>>>>>>>>>>> Is default being disabled fine if the feature is supported by a
>>>>>>>>>>> PMD ?
>>>>>>>>>>
>>>>>>>>>> I think the default should be enabled for ease of use.
>>>>>>>>>
>>>>>>>>> Since testpmd is used extensively for benchmarking purposes, we
>>>>>>>>> thought it should have minimum features
>>>>>>>>> enabled by default. The default testpmd doesn't have any Rx/Tx
>>>>>>>>> offloads enabled(except for FAST FREE),  default
>>>>>>>>> fwd mode being "iofwd" and the Rx metadata is only referenced
>>> when
>>>>>>>>> dumping packets.
>>>>>>>>>
>>>>>>>>>
>>>>>>>>>> Do we have similar features disables by default?
>>>>>>>>>> I mean do we know features in testpmd which require a "double
>>>>>>>>>> enablement"
>>>>>>>>>> like one configuration command + one rte_flow rule?
>>>>>>>>>
>>>>>>>>> Spec itself is that way i.e "RTE_FLOW_RULE +
>>>>>>>>> RX_METADATA_NEGOTIATE(once)"
>>>>>>>>>
>>>>>>>>> Isn't it enough if
>>>>>>>>>
>>>>>>>>> #1 We have enough print when rte_flow is being create without
>>>>>>>>> negotiation done and ask user to enable rx metadata using
>>>>>>>>> "port config <port_id>..."
>>>>>>>>> #2 Provide testpmd app arg to enable Rx metadata(for example "
>>>>>>>>> --rx-metadata") like other features to avoid calling another
>>>>>>>>> command before rte flow create.
>>>>>>>>
>>>>>>>> I'm not sure what is best.
>>>>>>>> I will let others discuss this part.
>>>>>>>
>>>>>>> IMO, enabling something always defeat the purpose to negotiate. IMO,
>>>>>>> someone needs to negotiate
>>>>>>> for a feature if the feature is needed. I think, the double enablement
>>>>>>> is part of the spec itself. In case, The PMD
>>>>>>> drivers won't like double enablement, no need to implement the PMD
>>>>>>> callback. That way, there is no change in the existing flow.
>>>>>>>
>>>>>>> The reason why cnxk driver thought of leveraging negotiate() feature
>>>>>>> so that it helps for optimization. e.s.p
>>>>>>> function template for multiprocess case as we know what features
>>>>>>> needed in fastpath upfront.
>>>>>>>
>>>>>>> If there still concerns with patch we can take up this to TB decide
>>>>>>> one way or another to make forward progress. Let us know.
>>>>>>
>>>>>> Ferruh, Andrew, Ori, Ivan, what is your opinion?
>>>>>> Let's progress with this patch to make it in -rc1.
>>>>
>>>> As I understand all agreed that we need testpmd command to
>>>> control negotiated Rx metadata. May be even command-line
>>>> option would be useful.
>>>>
>>>> So, remaining question is what should be the default value in
>>>> testpmd. Note that it is just testpmd question since default
>>>> value in an abstract application is nothing negotiated
>>>> (if I'm not mistaken).
>>>>
>>>> The key advantaan ge of the current behaviour is to avoid
>>>> "double-enabling" in testpmd. It preserves behaviour which
>>>> we had before before the API addition. It is a strong
>>>> argument. Basically it puts the feature into the same
>>>> basket as FAST_FREE - need an action to run faster.
>>>
>>> I think, there is a disconnect here. FAST_FREE is enabled by default.

Sorry, I'm lost here. Don't we need to enable FAST_FREE
offload? As far as I know all offloads are disabled by default.

>>> i.e We don't need any specific action to run faster. To align with performance
>>> test application, by default the configuration should be run faster. User
>>> needs to give explicit configuration to allow more offload or the one causes
>>> the mpps drops. IMO, That is the theme followed in testpmd.
>>>
>>>
>> I agree with Andrew, the default should stay the same, as now, PMD may already implement
>> logic to only enable the feature if there is a flow rule.
>> Changing the default will result in breaking applications.
> 
> That's not what is discussed here.
> We are talking only about testpmd default.
> 
>> I want to suggest new approach for this feature,
>> maybe we can use the rte_flow_configure, and add a new bit that says if those
>> actions are going to be used.
>> What do you think?
> 
> Let's not change the API please.
> 
> 
>>>> I see no problem in such approach.
>>>>
>>>> The key disadvantage is the difference in testpmd and
>>>> other applications default behaviour.
>>>>
>>>> I'd look at the feature in the following way:
>>>> if an application theoretically wants to use
>>>> USER_FLAG, USER_MARK or TUNNEL_ID it must negotiate
>>>> corresponding Rx metadata to ensure that the feature is
>>>> available and HW is informed that application may need it.
>>>> Since testpmd supports corresponding flow API actions and
>>>> flow tunnels, it tries to negotiate it by default, but do
>>>> not fail if the negotiation fails.
>>>>
>>>> So, I'd would vote to keeping the default value as is.
>>>>
>>
> 
> 
> 
> 
> 

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Jerin Jacob]

On Wed, Feb 1, 2023 at 2:37 PM Andrew Rybchenko
<andrew.rybchenko@oktetlabs.ru> wrote:
>
> On 2/1/23 12:05, Thomas Monjalon wrote:
> > 01/02/2023 10:00, Ori Kam:
> >> Hi all,
> >>
> >> Sorry for jumping in late,
> >>
> >>> -----Original Message-----
> >>> From: Jerin Jacob <jerinjacobk@gmail.com>
> >>> Sent: Wednesday, 1 February 2023 10:53
> >>>
> >>> On Wed, Feb 1, 2023 at 12:46 PM Andrew Rybchenko
> >>> <andrew.rybchenko@oktetlabs.ru> wrote:
> >>>>
> >>>> On 2/1/23 09:10, Ivan Malov wrote:
> >>>>> Hello everyone,
> >>>>>
> >>>>> Since making automatic, or implicit, offload decisions does
> >>>>> not belong in testpmd responsibility domain, it should be
> >>>>> safer to avoid calling the "negotiate metadata delivery"
> >>>>> API with some default selection unless the user asks to
> >>>>> do so explicitly, via internal CLI or app options.
> >>>>>
> >>>>> With that in mind, port config <port_id> ... sounds OK.
> >>>>>
> >>>>> PMDs that support flow primitives which can generate metadata
> >>>>> but, if in started state, can't enable its delivery may pass
> >>>>> appropriate rte_error messages to the user suggesting
> >>>>> they enable delivery of such metadata from NIC to PMD
> >>>>> first. This way, if the person operating testpmd
> >>>>> enters a flow create command and that fails,
> >>>>> they can figure out the inconsistency, stop
> >>>>> the port, negotiate, start and try again.
> >>>>>
> >>>>> As for non-debug applications, their developers shall
> >>>>> be properly informed about the problem of enabling
> >>>>> delivery of metadata from NIC to PMD. This way,
> >>>>> they will invoke the negotiate API by default
> >>>>> in their apps, with the feature selection (eg.
> >>>>> MARK) as per nature of the app's business.
> >>>>>
> >>>>> This API should indeed be helpful to some PMDs with
> >>>>> regard to collecting upfront knowledge like this.
> >>>>> At the same time, should be benign to those PMDs
> >>>>> who do not need this knowledge and can enable
> >>>>> delivery of metadata right when inserting the
> >>>>> flow rules. So I hope the API does not create
> >>>>> too much discomfort to vendors not needing it.
> >>>>>
> >>>>> Thank you.
> >>>>>
> >>>>> On Wed, 1 Feb 2023, Thomas Monjalon wrote:
> >>>>>
> >>>>>> 31/01/2023 17:17, Jerin Jacob:
> >>>>>>> On Fri, Jan 27, 2023 at 8:31 PM Thomas Monjalon
> >>> <thomas@monjalon.net>
> >>>>>>> wrote:
> >>>>>>>>
> >>>>>>>> 27/01/2023 11:42, Nithin Kumar Dabilpuram:
> >>>>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
> >>>>>>>>>> 27/01/2023 06:02, Nithin Kumar Dabilpuram:
> >>>>>>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
> >>>>>>>>>>>> Ferruh is proposing to have a command "port config <port_id>
> >>> ..."
> >>>>>>>>>>>> to configure the flags to negotiate.
> >>>>>>>>>>>> Are you OK with this approach?
> >>>>>>>>>>>
> >>>>>>>>>>> Yes, we are fine to have such command to enable and disable the
> >>>>>>>>>>> feature
> >>>>>>>>>>> with default being it disabled if supported by PMD.
> >>>>>>>>>>> Is default being disabled fine if the feature is supported by a
> >>>>>>>>>>> PMD ?
> >>>>>>>>>>
> >>>>>>>>>> I think the default should be enabled for ease of use.
> >>>>>>>>>
> >>>>>>>>> Since testpmd is used extensively for benchmarking purposes, we
> >>>>>>>>> thought it should have minimum features
> >>>>>>>>> enabled by default. The default testpmd doesn't have any Rx/Tx
> >>>>>>>>> offloads enabled(except for FAST FREE),  default
> >>>>>>>>> fwd mode being "iofwd" and the Rx metadata is only referenced
> >>> when
> >>>>>>>>> dumping packets.
> >>>>>>>>>
> >>>>>>>>>
> >>>>>>>>>> Do we have similar features disables by default?
> >>>>>>>>>> I mean do we know features in testpmd which require a "double
> >>>>>>>>>> enablement"
> >>>>>>>>>> like one configuration command + one rte_flow rule?
> >>>>>>>>>
> >>>>>>>>> Spec itself is that way i.e "RTE_FLOW_RULE +
> >>>>>>>>> RX_METADATA_NEGOTIATE(once)"
> >>>>>>>>>
> >>>>>>>>> Isn't it enough if
> >>>>>>>>>
> >>>>>>>>> #1 We have enough print when rte_flow is being create without
> >>>>>>>>> negotiation done and ask user to enable rx metadata using
> >>>>>>>>> "port config <port_id>..."
> >>>>>>>>> #2 Provide testpmd app arg to enable Rx metadata(for example "
> >>>>>>>>> --rx-metadata") like other features to avoid calling another
> >>>>>>>>> command before rte flow create.
> >>>>>>>>
> >>>>>>>> I'm not sure what is best.
> >>>>>>>> I will let others discuss this part.
> >>>>>>>
> >>>>>>> IMO, enabling something always defeat the purpose to negotiate. IMO,
> >>>>>>> someone needs to negotiate
> >>>>>>> for a feature if the feature is needed. I think, the double enablement
> >>>>>>> is part of the spec itself. In case, The PMD
> >>>>>>> drivers won't like double enablement, no need to implement the PMD
> >>>>>>> callback. That way, there is no change in the existing flow.
> >>>>>>>
> >>>>>>> The reason why cnxk driver thought of leveraging negotiate() feature
> >>>>>>> so that it helps for optimization. e.s.p
> >>>>>>> function template for multiprocess case as we know what features
> >>>>>>> needed in fastpath upfront.
> >>>>>>>
> >>>>>>> If there still concerns with patch we can take up this to TB decide
> >>>>>>> one way or another to make forward progress. Let us know.
> >>>>>>
> >>>>>> Ferruh, Andrew, Ori, Ivan, what is your opinion?
> >>>>>> Let's progress with this patch to make it in -rc1.
> >>>>
> >>>> As I understand all agreed that we need testpmd command to
> >>>> control negotiated Rx metadata. May be even command-line
> >>>> option would be useful.
> >>>>
> >>>> So, remaining question is what should be the default value in
> >>>> testpmd. Note that it is just testpmd question since default
> >>>> value in an abstract application is nothing negotiated
> >>>> (if I'm not mistaken).
> >>>>
> >>>> The key advantaan ge of the current behaviour is to avoid
> >>>> "double-enabling" in testpmd. It preserves behaviour which
> >>>> we had before before the API addition. It is a strong
> >>>> argument. Basically it puts the feature into the same
> >>>> basket as FAST_FREE - need an action to run faster.
> >>>
> >>> I think, there is a disconnect here. FAST_FREE is enabled by default.
>
> Sorry, I'm lost here. Don't we need to enable FAST_FREE
> offload? As far as I know all offloads are disabled by default.

Not the case for FAST_FREE as disabling needs "more cycles on processor" side.

See app/test-pmd/testpmd.c
/*
 * Ethernet device configuration.
 */
struct rte_eth_rxmode rx_mode;

struct rte_eth_txmode tx_mode = {
        .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE,
};

>
> >>> i.e We don't need any specific action to run faster. To align with performance
> >>> test application, by default the configuration should be run faster. User
> >>> needs to give explicit configuration to allow more offload or the one causes
> >>> the mpps drops. IMO, That is the theme followed in testpmd.
> >>>
> >>>
> >> I agree with Andrew, the default should stay the same, as now, PMD may already implement
> >> logic to only enable the feature if there is a flow rule.
> >> Changing the default will result in breaking applications.
> >
> > That's not what is discussed here.
> > We are talking only about testpmd default.
> >
> >> I want to suggest new approach for this feature,
> >> maybe we can use the rte_flow_configure, and add a new bit that says if those
> >> actions are going to be used.
> >> What do you think?
> >
> > Let's not change the API please.
> >
> >
> >>>> I see no problem in such approach.
> >>>>
> >>>> The key disadvantage is the difference in testpmd and
> >>>> other applications default behaviour.
> >>>>
> >>>> I'd look at the feature in the following way:
> >>>> if an application theoretically wants to use
> >>>> USER_FLAG, USER_MARK or TUNNEL_ID it must negotiate
> >>>> corresponding Rx metadata to ensure that the feature is
> >>>> available and HW is informed that application may need it.
> >>>> Since testpmd supports corresponding flow API actions and
> >>>> flow tunnels, it tries to negotiate it by default, but do
> >>>> not fail if the negotiation fails.
> >>>>
> >>>> So, I'd would vote to keeping the default value as is.
> >>>>
> >>
> >
> >
> >
> >
> >
>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Andrew Rybchenko]

On 2/1/23 12:14, Jerin Jacob wrote:
> On Wed, Feb 1, 2023 at 2:37 PM Andrew Rybchenko
> <andrew.rybchenko@oktetlabs.ru> wrote:
>>
>> On 2/1/23 12:05, Thomas Monjalon wrote:
>>> 01/02/2023 10:00, Ori Kam:
>>>> Hi all,
>>>>
>>>> Sorry for jumping in late,
>>>>
>>>>> -----Original Message-----
>>>>> From: Jerin Jacob <jerinjacobk@gmail.com>
>>>>> Sent: Wednesday, 1 February 2023 10:53
>>>>>
>>>>> On Wed, Feb 1, 2023 at 12:46 PM Andrew Rybchenko
>>>>> <andrew.rybchenko@oktetlabs.ru> wrote:
>>>>>>
>>>>>> On 2/1/23 09:10, Ivan Malov wrote:
>>>>>>> Hello everyone,
>>>>>>>
>>>>>>> Since making automatic, or implicit, offload decisions does
>>>>>>> not belong in testpmd responsibility domain, it should be
>>>>>>> safer to avoid calling the "negotiate metadata delivery"
>>>>>>> API with some default selection unless the user asks to
>>>>>>> do so explicitly, via internal CLI or app options.
>>>>>>>
>>>>>>> With that in mind, port config <port_id> ... sounds OK.
>>>>>>>
>>>>>>> PMDs that support flow primitives which can generate metadata
>>>>>>> but, if in started state, can't enable its delivery may pass
>>>>>>> appropriate rte_error messages to the user suggesting
>>>>>>> they enable delivery of such metadata from NIC to PMD
>>>>>>> first. This way, if the person operating testpmd
>>>>>>> enters a flow create command and that fails,
>>>>>>> they can figure out the inconsistency, stop
>>>>>>> the port, negotiate, start and try again.
>>>>>>>
>>>>>>> As for non-debug applications, their developers shall
>>>>>>> be properly informed about the problem of enabling
>>>>>>> delivery of metadata from NIC to PMD. This way,
>>>>>>> they will invoke the negotiate API by default
>>>>>>> in their apps, with the feature selection (eg.
>>>>>>> MARK) as per nature of the app's business.
>>>>>>>
>>>>>>> This API should indeed be helpful to some PMDs with
>>>>>>> regard to collecting upfront knowledge like this.
>>>>>>> At the same time, should be benign to those PMDs
>>>>>>> who do not need this knowledge and can enable
>>>>>>> delivery of metadata right when inserting the
>>>>>>> flow rules. So I hope the API does not create
>>>>>>> too much discomfort to vendors not needing it.
>>>>>>>
>>>>>>> Thank you.
>>>>>>>
>>>>>>> On Wed, 1 Feb 2023, Thomas Monjalon wrote:
>>>>>>>
>>>>>>>> 31/01/2023 17:17, Jerin Jacob:
>>>>>>>>> On Fri, Jan 27, 2023 at 8:31 PM Thomas Monjalon
>>>>> <thomas@monjalon.net>
>>>>>>>>> wrote:
>>>>>>>>>>
>>>>>>>>>> 27/01/2023 11:42, Nithin Kumar Dabilpuram:
>>>>>>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
>>>>>>>>>>>> 27/01/2023 06:02, Nithin Kumar Dabilpuram:
>>>>>>>>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
>>>>>>>>>>>>>> Ferruh is proposing to have a command "port config <port_id>
>>>>> ..."
>>>>>>>>>>>>>> to configure the flags to negotiate.
>>>>>>>>>>>>>> Are you OK with this approach?
>>>>>>>>>>>>>
>>>>>>>>>>>>> Yes, we are fine to have such command to enable and disable the
>>>>>>>>>>>>> feature
>>>>>>>>>>>>> with default being it disabled if supported by PMD.
>>>>>>>>>>>>> Is default being disabled fine if the feature is supported by a
>>>>>>>>>>>>> PMD ?
>>>>>>>>>>>>
>>>>>>>>>>>> I think the default should be enabled for ease of use.
>>>>>>>>>>>
>>>>>>>>>>> Since testpmd is used extensively for benchmarking purposes, we
>>>>>>>>>>> thought it should have minimum features
>>>>>>>>>>> enabled by default. The default testpmd doesn't have any Rx/Tx
>>>>>>>>>>> offloads enabled(except for FAST FREE),  default
>>>>>>>>>>> fwd mode being "iofwd" and the Rx metadata is only referenced
>>>>> when
>>>>>>>>>>> dumping packets.
>>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>>> Do we have similar features disables by default?
>>>>>>>>>>>> I mean do we know features in testpmd which require a "double
>>>>>>>>>>>> enablement"
>>>>>>>>>>>> like one configuration command + one rte_flow rule?
>>>>>>>>>>>
>>>>>>>>>>> Spec itself is that way i.e "RTE_FLOW_RULE +
>>>>>>>>>>> RX_METADATA_NEGOTIATE(once)"
>>>>>>>>>>>
>>>>>>>>>>> Isn't it enough if
>>>>>>>>>>>
>>>>>>>>>>> #1 We have enough print when rte_flow is being create without
>>>>>>>>>>> negotiation done and ask user to enable rx metadata using
>>>>>>>>>>> "port config <port_id>..."
>>>>>>>>>>> #2 Provide testpmd app arg to enable Rx metadata(for example "
>>>>>>>>>>> --rx-metadata") like other features to avoid calling another
>>>>>>>>>>> command before rte flow create.
>>>>>>>>>>
>>>>>>>>>> I'm not sure what is best.
>>>>>>>>>> I will let others discuss this part.
>>>>>>>>>
>>>>>>>>> IMO, enabling something always defeat the purpose to negotiate. IMO,
>>>>>>>>> someone needs to negotiate
>>>>>>>>> for a feature if the feature is needed. I think, the double enablement
>>>>>>>>> is part of the spec itself. In case, The PMD
>>>>>>>>> drivers won't like double enablement, no need to implement the PMD
>>>>>>>>> callback. That way, there is no change in the existing flow.
>>>>>>>>>
>>>>>>>>> The reason why cnxk driver thought of leveraging negotiate() feature
>>>>>>>>> so that it helps for optimization. e.s.p
>>>>>>>>> function template for multiprocess case as we know what features
>>>>>>>>> needed in fastpath upfront.
>>>>>>>>>
>>>>>>>>> If there still concerns with patch we can take up this to TB decide
>>>>>>>>> one way or another to make forward progress. Let us know.
>>>>>>>>
>>>>>>>> Ferruh, Andrew, Ori, Ivan, what is your opinion?
>>>>>>>> Let's progress with this patch to make it in -rc1.
>>>>>>
>>>>>> As I understand all agreed that we need testpmd command to
>>>>>> control negotiated Rx metadata. May be even command-line
>>>>>> option would be useful.
>>>>>>
>>>>>> So, remaining question is what should be the default value in
>>>>>> testpmd. Note that it is just testpmd question since default
>>>>>> value in an abstract application is nothing negotiated
>>>>>> (if I'm not mistaken).
>>>>>>
>>>>>> The key advantaan ge of the current behaviour is to avoid
>>>>>> "double-enabling" in testpmd. It preserves behaviour which
>>>>>> we had before before the API addition. It is a strong
>>>>>> argument. Basically it puts the feature into the same
>>>>>> basket as FAST_FREE - need an action to run faster.
>>>>>
>>>>> I think, there is a disconnect here. FAST_FREE is enabled by default.
>>
>> Sorry, I'm lost here. Don't we need to enable FAST_FREE
>> offload? As far as I know all offloads are disabled by default.
> 
> Not the case for FAST_FREE as disabling needs "more cycles on processor" side.
> 
> See app/test-pmd/testpmd.c
> /*
>   * Ethernet device configuration.
>   */
> struct rte_eth_rxmode rx_mode;
> 
> struct rte_eth_txmode tx_mode = {
>          .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE,
> };
> 

Surprised, thanks. So, it one more difference of the testpmd
defaults from an abstract application.

>>
>>>>> i.e We don't need any specific action to run faster. To align with performance
>>>>> test application, by default the configuration should be run faster. User
>>>>> needs to give explicit configuration to allow more offload or the one causes
>>>>> the mpps drops. IMO, That is the theme followed in testpmd.
>>>>>
>>>>>
>>>> I agree with Andrew, the default should stay the same, as now, PMD may already implement
>>>> logic to only enable the feature if there is a flow rule.
>>>> Changing the default will result in breaking applications.
>>>
>>> That's not what is discussed here.
>>> We are talking only about testpmd default.
>>>
>>>> I want to suggest new approach for this feature,
>>>> maybe we can use the rte_flow_configure, and add a new bit that says if those
>>>> actions are going to be used.
>>>> What do you think?
>>>
>>> Let's not change the API please.
>>>
>>>
>>>>>> I see no problem in such approach.
>>>>>>
>>>>>> The key disadvantage is the difference in testpmd and
>>>>>> other applications default behaviour.
>>>>>>
>>>>>> I'd look at the feature in the following way:
>>>>>> if an application theoretically wants to use
>>>>>> USER_FLAG, USER_MARK or TUNNEL_ID it must negotiate
>>>>>> corresponding Rx metadata to ensure that the feature is
>>>>>> available and HW is informed that application may need it.
>>>>>> Since testpmd supports corresponding flow API actions and
>>>>>> flow tunnels, it tries to negotiate it by default, but do
>>>>>> not fail if the negotiation fails.
>>>>>>
>>>>>> So, I'd would vote to keeping the default value as is.

Two above paragraphs still stand.

Frankly speaking I don't understand why default value is so
important if we have a way to change it. Reasons should be
really strong to change existing defaults.

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Jerin Jacob]

On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
<andrew.rybchenko@oktetlabs.ru> wrote:
>
> On 2/1/23 12:14, Jerin Jacob wrote:
> > On Wed, Feb 1, 2023 at 2:37 PM Andrew Rybchenko
> > <andrew.rybchenko@oktetlabs.ru> wrote:
> >>
> >> On 2/1/23 12:05, Thomas Monjalon wrote:
> >>> 01/02/2023 10:00, Ori Kam:
> >>>> Hi all,
> >>>>
> >>>> Sorry for jumping in late,
> >>>>
> >>>>> -----Original Message-----
> >>>>> From: Jerin Jacob <jerinjacobk@gmail.com>
> >>>>> Sent: Wednesday, 1 February 2023 10:53
> >>>>>
> >>>>> On Wed, Feb 1, 2023 at 12:46 PM Andrew Rybchenko
> >>>>> <andrew.rybchenko@oktetlabs.ru> wrote:
> >>>>>>
> >>>>>> On 2/1/23 09:10, Ivan Malov wrote:
> >>>>>>> Hello everyone,
> >>>>>>>
> >>>>>>> Since making automatic, or implicit, offload decisions does
> >>>>>>> not belong in testpmd responsibility domain, it should be
> >>>>>>> safer to avoid calling the "negotiate metadata delivery"
> >>>>>>> API with some default selection unless the user asks to
> >>>>>>> do so explicitly, via internal CLI or app options.
> >>>>>>>
> >>>>>>> With that in mind, port config <port_id> ... sounds OK.
> >>>>>>>
> >>>>>>> PMDs that support flow primitives which can generate metadata
> >>>>>>> but, if in started state, can't enable its delivery may pass
> >>>>>>> appropriate rte_error messages to the user suggesting
> >>>>>>> they enable delivery of such metadata from NIC to PMD
> >>>>>>> first. This way, if the person operating testpmd
> >>>>>>> enters a flow create command and that fails,
> >>>>>>> they can figure out the inconsistency, stop
> >>>>>>> the port, negotiate, start and try again.
> >>>>>>>
> >>>>>>> As for non-debug applications, their developers shall
> >>>>>>> be properly informed about the problem of enabling
> >>>>>>> delivery of metadata from NIC to PMD. This way,
> >>>>>>> they will invoke the negotiate API by default
> >>>>>>> in their apps, with the feature selection (eg.
> >>>>>>> MARK) as per nature of the app's business.
> >>>>>>>
> >>>>>>> This API should indeed be helpful to some PMDs with
> >>>>>>> regard to collecting upfront knowledge like this.
> >>>>>>> At the same time, should be benign to those PMDs
> >>>>>>> who do not need this knowledge and can enable
> >>>>>>> delivery of metadata right when inserting the
> >>>>>>> flow rules. So I hope the API does not create
> >>>>>>> too much discomfort to vendors not needing it.
> >>>>>>>
> >>>>>>> Thank you.
> >>>>>>>
> >>>>>>> On Wed, 1 Feb 2023, Thomas Monjalon wrote:
> >>>>>>>
> >>>>>>>> 31/01/2023 17:17, Jerin Jacob:
> >>>>>>>>> On Fri, Jan 27, 2023 at 8:31 PM Thomas Monjalon
> >>>>> <thomas@monjalon.net>
> >>>>>>>>> wrote:
> >>>>>>>>>>
> >>>>>>>>>> 27/01/2023 11:42, Nithin Kumar Dabilpuram:
> >>>>>>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
> >>>>>>>>>>>> 27/01/2023 06:02, Nithin Kumar Dabilpuram:
> >>>>>>>>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
> >>>>>>>>>>>>>> Ferruh is proposing to have a command "port config <port_id>
> >>>>> ..."
> >>>>>>>>>>>>>> to configure the flags to negotiate.
> >>>>>>>>>>>>>> Are you OK with this approach?
> >>>>>>>>>>>>>
> >>>>>>>>>>>>> Yes, we are fine to have such command to enable and disable the
> >>>>>>>>>>>>> feature
> >>>>>>>>>>>>> with default being it disabled if supported by PMD.
> >>>>>>>>>>>>> Is default being disabled fine if the feature is supported by a
> >>>>>>>>>>>>> PMD ?
> >>>>>>>>>>>>
> >>>>>>>>>>>> I think the default should be enabled for ease of use.
> >>>>>>>>>>>
> >>>>>>>>>>> Since testpmd is used extensively for benchmarking purposes, we
> >>>>>>>>>>> thought it should have minimum features
> >>>>>>>>>>> enabled by default. The default testpmd doesn't have any Rx/Tx
> >>>>>>>>>>> offloads enabled(except for FAST FREE),  default
> >>>>>>>>>>> fwd mode being "iofwd" and the Rx metadata is only referenced
> >>>>> when
> >>>>>>>>>>> dumping packets.
> >>>>>>>>>>>
> >>>>>>>>>>>
> >>>>>>>>>>>> Do we have similar features disables by default?
> >>>>>>>>>>>> I mean do we know features in testpmd which require a "double
> >>>>>>>>>>>> enablement"
> >>>>>>>>>>>> like one configuration command + one rte_flow rule?
> >>>>>>>>>>>
> >>>>>>>>>>> Spec itself is that way i.e "RTE_FLOW_RULE +
> >>>>>>>>>>> RX_METADATA_NEGOTIATE(once)"
> >>>>>>>>>>>
> >>>>>>>>>>> Isn't it enough if
> >>>>>>>>>>>
> >>>>>>>>>>> #1 We have enough print when rte_flow is being create without
> >>>>>>>>>>> negotiation done and ask user to enable rx metadata using
> >>>>>>>>>>> "port config <port_id>..."
> >>>>>>>>>>> #2 Provide testpmd app arg to enable Rx metadata(for example "
> >>>>>>>>>>> --rx-metadata") like other features to avoid calling another
> >>>>>>>>>>> command before rte flow create.
> >>>>>>>>>>
> >>>>>>>>>> I'm not sure what is best.
> >>>>>>>>>> I will let others discuss this part.
> >>>>>>>>>
> >>>>>>>>> IMO, enabling something always defeat the purpose to negotiate. IMO,
> >>>>>>>>> someone needs to negotiate
> >>>>>>>>> for a feature if the feature is needed. I think, the double enablement
> >>>>>>>>> is part of the spec itself. In case, The PMD
> >>>>>>>>> drivers won't like double enablement, no need to implement the PMD
> >>>>>>>>> callback. That way, there is no change in the existing flow.
> >>>>>>>>>
> >>>>>>>>> The reason why cnxk driver thought of leveraging negotiate() feature
> >>>>>>>>> so that it helps for optimization. e.s.p
> >>>>>>>>> function template for multiprocess case as we know what features
> >>>>>>>>> needed in fastpath upfront.
> >>>>>>>>>
> >>>>>>>>> If there still concerns with patch we can take up this to TB decide
> >>>>>>>>> one way or another to make forward progress. Let us know.
> >>>>>>>>
> >>>>>>>> Ferruh, Andrew, Ori, Ivan, what is your opinion?
> >>>>>>>> Let's progress with this patch to make it in -rc1.
> >>>>>>
> >>>>>> As I understand all agreed that we need testpmd command to
> >>>>>> control negotiated Rx metadata. May be even command-line
> >>>>>> option would be useful.
> >>>>>>
> >>>>>> So, remaining question is what should be the default value in
> >>>>>> testpmd. Note that it is just testpmd question since default
> >>>>>> value in an abstract application is nothing negotiated
> >>>>>> (if I'm not mistaken).
> >>>>>>
> >>>>>> The key advantaan ge of the current behaviour is to avoid
> >>>>>> "double-enabling" in testpmd. It preserves behaviour which
> >>>>>> we had before before the API addition. It is a strong
> >>>>>> argument. Basically it puts the feature into the same
> >>>>>> basket as FAST_FREE - need an action to run faster.
> >>>>>
> >>>>> I think, there is a disconnect here. FAST_FREE is enabled by default.
> >>
> >> Sorry, I'm lost here. Don't we need to enable FAST_FREE
> >> offload? As far as I know all offloads are disabled by default.
> >
> > Not the case for FAST_FREE as disabling needs "more cycles on processor" side.
> >
> > See app/test-pmd/testpmd.c
> > /*
> >   * Ethernet device configuration.
> >   */
> > struct rte_eth_rxmode rx_mode;
> >
> > struct rte_eth_txmode tx_mode = {
> >          .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE,
> > };
> >
>
> Surprised, thanks. So, it one more difference of the testpmd
> defaults from an abstract application.

It is not, even l2fwd is same as testpmd. In my view, the ideal
application just to select what is
really needed.

>
> >>
> >>>>> i.e We don't need any specific action to run faster. To align with performance
> >>>>> test application, by default the configuration should be run faster. User
> >>>>> needs to give explicit configuration to allow more offload or the one causes
> >>>>> the mpps drops. IMO, That is the theme followed in testpmd.
> >>>>>
> >>>>>
> >>>> I agree with Andrew, the default should stay the same, as now, PMD may already implement
> >>>> logic to only enable the feature if there is a flow rule.
> >>>> Changing the default will result in breaking applications.
> >>>
> >>> That's not what is discussed here.
> >>> We are talking only about testpmd default.
> >>>
> >>>> I want to suggest new approach for this feature,
> >>>> maybe we can use the rte_flow_configure, and add a new bit that says if those
> >>>> actions are going to be used.
> >>>> What do you think?
> >>>
> >>> Let's not change the API please.
> >>>
> >>>
> >>>>>> I see no problem in such approach.
> >>>>>>
> >>>>>> The key disadvantage is the difference in testpmd and
> >>>>>> other applications default behaviour.

But none of the other applications in not negotiating by default.

> >>>>>>
> >>>>>> I'd look at the feature in the following way:
> >>>>>> if an application theoretically wants to use
> >>>>>> USER_FLAG, USER_MARK or TUNNEL_ID it must negotiate
> >>>>>> corresponding Rx metadata to ensure that the feature is
> >>>>>> available and HW is informed that application may need it.
> >>>>>> Since testpmd supports corresponding flow API actions and
> >>>>>> flow tunnels, it tries to negotiate it by default, but do
> >>>>>> not fail if the negotiation fails.
> >>>>>>
> >>>>>> So, I'd would vote to keeping the default value as is.
>
> Two above paragraphs still stand.
>
> Frankly speaking I don't understand why default value is so
> important if we have a way to change it. Reasons should be
> really strong to change existing defaults.

The only reason is, typically testpmd will be used performance
benchmarking as an industry standard. It is difficult to tell/educate
the QA or customers
that, "BTW if you need to get better performance add more flag to
testpmd command line".
To make that worst, only some PMD needs to give the additional
parameter to get better number.
And also, testpmd usage will be treated as application modeling.

Since this feature only used on sfc and cnxk driver, What is the
situation with sfc driver?
Keeping it as negotiated and not use the feature, will impact the per
core performance of sfc or
is it just PCI bandwidth thing which really dont show any difference in testpmd?

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Andrew Rybchenko]

On 2/1/23 13:48, Jerin Jacob wrote:
> On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
> <andrew.rybchenko@oktetlabs.ru> wrote:
>>
>> On 2/1/23 12:14, Jerin Jacob wrote:
>>> On Wed, Feb 1, 2023 at 2:37 PM Andrew Rybchenko
>>> <andrew.rybchenko@oktetlabs.ru> wrote:
>>>>
>>>> On 2/1/23 12:05, Thomas Monjalon wrote:
>>>>> 01/02/2023 10:00, Ori Kam:
>>>>>> Hi all,
>>>>>>
>>>>>> Sorry for jumping in late,
>>>>>>
>>>>>>> -----Original Message-----
>>>>>>> From: Jerin Jacob <jerinjacobk@gmail.com>
>>>>>>> Sent: Wednesday, 1 February 2023 10:53
>>>>>>>
>>>>>>> On Wed, Feb 1, 2023 at 12:46 PM Andrew Rybchenko
>>>>>>> <andrew.rybchenko@oktetlabs.ru> wrote:
>>>>>>>>
>>>>>>>> On 2/1/23 09:10, Ivan Malov wrote:
>>>>>>>>> Hello everyone,
>>>>>>>>>
>>>>>>>>> Since making automatic, or implicit, offload decisions does
>>>>>>>>> not belong in testpmd responsibility domain, it should be
>>>>>>>>> safer to avoid calling the "negotiate metadata delivery"
>>>>>>>>> API with some default selection unless the user asks to
>>>>>>>>> do so explicitly, via internal CLI or app options.
>>>>>>>>>
>>>>>>>>> With that in mind, port config <port_id> ... sounds OK.
>>>>>>>>>
>>>>>>>>> PMDs that support flow primitives which can generate metadata
>>>>>>>>> but, if in started state, can't enable its delivery may pass
>>>>>>>>> appropriate rte_error messages to the user suggesting
>>>>>>>>> they enable delivery of such metadata from NIC to PMD
>>>>>>>>> first. This way, if the person operating testpmd
>>>>>>>>> enters a flow create command and that fails,
>>>>>>>>> they can figure out the inconsistency, stop
>>>>>>>>> the port, negotiate, start and try again.
>>>>>>>>>
>>>>>>>>> As for non-debug applications, their developers shall
>>>>>>>>> be properly informed about the problem of enabling
>>>>>>>>> delivery of metadata from NIC to PMD. This way,
>>>>>>>>> they will invoke the negotiate API by default
>>>>>>>>> in their apps, with the feature selection (eg.
>>>>>>>>> MARK) as per nature of the app's business.
>>>>>>>>>
>>>>>>>>> This API should indeed be helpful to some PMDs with
>>>>>>>>> regard to collecting upfront knowledge like this.
>>>>>>>>> At the same time, should be benign to those PMDs
>>>>>>>>> who do not need this knowledge and can enable
>>>>>>>>> delivery of metadata right when inserting the
>>>>>>>>> flow rules. So I hope the API does not create
>>>>>>>>> too much discomfort to vendors not needing it.
>>>>>>>>>
>>>>>>>>> Thank you.
>>>>>>>>>
>>>>>>>>> On Wed, 1 Feb 2023, Thomas Monjalon wrote:
>>>>>>>>>
>>>>>>>>>> 31/01/2023 17:17, Jerin Jacob:
>>>>>>>>>>> On Fri, Jan 27, 2023 at 8:31 PM Thomas Monjalon
>>>>>>> <thomas@monjalon.net>
>>>>>>>>>>> wrote:
>>>>>>>>>>>>
>>>>>>>>>>>> 27/01/2023 11:42, Nithin Kumar Dabilpuram:
>>>>>>>>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
>>>>>>>>>>>>>> 27/01/2023 06:02, Nithin Kumar Dabilpuram:
>>>>>>>>>>>>>>> From: Thomas Monjalon <thomas@monjalon.net>
>>>>>>>>>>>>>>>> Ferruh is proposing to have a command "port config <port_id>
>>>>>>> ..."
>>>>>>>>>>>>>>>> to configure the flags to negotiate.
>>>>>>>>>>>>>>>> Are you OK with this approach?
>>>>>>>>>>>>>>>
>>>>>>>>>>>>>>> Yes, we are fine to have such command to enable and disable the
>>>>>>>>>>>>>>> feature
>>>>>>>>>>>>>>> with default being it disabled if supported by PMD.
>>>>>>>>>>>>>>> Is default being disabled fine if the feature is supported by a
>>>>>>>>>>>>>>> PMD ?
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> I think the default should be enabled for ease of use.
>>>>>>>>>>>>>
>>>>>>>>>>>>> Since testpmd is used extensively for benchmarking purposes, we
>>>>>>>>>>>>> thought it should have minimum features
>>>>>>>>>>>>> enabled by default. The default testpmd doesn't have any Rx/Tx
>>>>>>>>>>>>> offloads enabled(except for FAST FREE),  default
>>>>>>>>>>>>> fwd mode being "iofwd" and the Rx metadata is only referenced
>>>>>>> when
>>>>>>>>>>>>> dumping packets.
>>>>>>>>>>>>>
>>>>>>>>>>>>>
>>>>>>>>>>>>>> Do we have similar features disables by default?
>>>>>>>>>>>>>> I mean do we know features in testpmd which require a "double
>>>>>>>>>>>>>> enablement"
>>>>>>>>>>>>>> like one configuration command + one rte_flow rule?
>>>>>>>>>>>>>
>>>>>>>>>>>>> Spec itself is that way i.e "RTE_FLOW_RULE +
>>>>>>>>>>>>> RX_METADATA_NEGOTIATE(once)"
>>>>>>>>>>>>>
>>>>>>>>>>>>> Isn't it enough if
>>>>>>>>>>>>>
>>>>>>>>>>>>> #1 We have enough print when rte_flow is being create without
>>>>>>>>>>>>> negotiation done and ask user to enable rx metadata using
>>>>>>>>>>>>> "port config <port_id>..."
>>>>>>>>>>>>> #2 Provide testpmd app arg to enable Rx metadata(for example "
>>>>>>>>>>>>> --rx-metadata") like other features to avoid calling another
>>>>>>>>>>>>> command before rte flow create.
>>>>>>>>>>>>
>>>>>>>>>>>> I'm not sure what is best.
>>>>>>>>>>>> I will let others discuss this part.
>>>>>>>>>>>
>>>>>>>>>>> IMO, enabling something always defeat the purpose to negotiate. IMO,
>>>>>>>>>>> someone needs to negotiate
>>>>>>>>>>> for a feature if the feature is needed. I think, the double enablement
>>>>>>>>>>> is part of the spec itself. In case, The PMD
>>>>>>>>>>> drivers won't like double enablement, no need to implement the PMD
>>>>>>>>>>> callback. That way, there is no change in the existing flow.
>>>>>>>>>>>
>>>>>>>>>>> The reason why cnxk driver thought of leveraging negotiate() feature
>>>>>>>>>>> so that it helps for optimization. e.s.p
>>>>>>>>>>> function template for multiprocess case as we know what features
>>>>>>>>>>> needed in fastpath upfront.
>>>>>>>>>>>
>>>>>>>>>>> If there still concerns with patch we can take up this to TB decide
>>>>>>>>>>> one way or another to make forward progress. Let us know.
>>>>>>>>>>
>>>>>>>>>> Ferruh, Andrew, Ori, Ivan, what is your opinion?
>>>>>>>>>> Let's progress with this patch to make it in -rc1.
>>>>>>>>
>>>>>>>> As I understand all agreed that we need testpmd command to
>>>>>>>> control negotiated Rx metadata. May be even command-line
>>>>>>>> option would be useful.
>>>>>>>>
>>>>>>>> So, remaining question is what should be the default value in
>>>>>>>> testpmd. Note that it is just testpmd question since default
>>>>>>>> value in an abstract application is nothing negotiated
>>>>>>>> (if I'm not mistaken).
>>>>>>>>
>>>>>>>> The key advantaan ge of the current behaviour is to avoid
>>>>>>>> "double-enabling" in testpmd. It preserves behaviour which
>>>>>>>> we had before before the API addition. It is a strong
>>>>>>>> argument. Basically it puts the feature into the same
>>>>>>>> basket as FAST_FREE - need an action to run faster.
>>>>>>>
>>>>>>> I think, there is a disconnect here. FAST_FREE is enabled by default.
>>>>
>>>> Sorry, I'm lost here. Don't we need to enable FAST_FREE
>>>> offload? As far as I know all offloads are disabled by default.
>>>
>>> Not the case for FAST_FREE as disabling needs "more cycles on processor" side.
>>>
>>> See app/test-pmd/testpmd.c
>>> /*
>>>    * Ethernet device configuration.
>>>    */
>>> struct rte_eth_rxmode rx_mode;
>>>
>>> struct rte_eth_txmode tx_mode = {
>>>           .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE,
>>> };
>>>
>>
>> Surprised, thanks. So, it one more difference of the testpmd
>> defaults from an abstract application.
> 
> It is not, even l2fwd is same as testpmd. In my view, the ideal
> application just to select what is
> really needed.
> 
>>
>>>>
>>>>>>> i.e We don't need any specific action to run faster. To align with performance
>>>>>>> test application, by default the configuration should be run faster. User
>>>>>>> needs to give explicit configuration to allow more offload or the one causes
>>>>>>> the mpps drops. IMO, That is the theme followed in testpmd.
>>>>>>>
>>>>>>>
>>>>>> I agree with Andrew, the default should stay the same, as now, PMD may already implement
>>>>>> logic to only enable the feature if there is a flow rule.
>>>>>> Changing the default will result in breaking applications.
>>>>>
>>>>> That's not what is discussed here.
>>>>> We are talking only about testpmd default.
>>>>>
>>>>>> I want to suggest new approach for this feature,
>>>>>> maybe we can use the rte_flow_configure, and add a new bit that says if those
>>>>>> actions are going to be used.
>>>>>> What do you think?
>>>>>
>>>>> Let's not change the API please.
>>>>>
>>>>>
>>>>>>>> I see no problem in such approach.
>>>>>>>>
>>>>>>>> The key disadvantage is the difference in testpmd and
>>>>>>>> other applications default behaviour.
> 
> But none of the other applications in not negotiating by default.
> 
>>>>>>>>
>>>>>>>> I'd look at the feature in the following way:
>>>>>>>> if an application theoretically wants to use
>>>>>>>> USER_FLAG, USER_MARK or TUNNEL_ID it must negotiate
>>>>>>>> corresponding Rx metadata to ensure that the feature is
>>>>>>>> available and HW is informed that application may need it.
>>>>>>>> Since testpmd supports corresponding flow API actions and
>>>>>>>> flow tunnels, it tries to negotiate it by default, but do
>>>>>>>> not fail if the negotiation fails.
>>>>>>>>
>>>>>>>> So, I'd would vote to keeping the default value as is.
>>
>> Two above paragraphs still stand.
>>
>> Frankly speaking I don't understand why default value is so
>> important if we have a way to change it. Reasons should be
>> really strong to change existing defaults.
> 
> The only reason is, typically testpmd will be used performance
> benchmarking as an industry standard. It is difficult to tell/educate
> the QA or customers
> that, "BTW if you need to get better performance add more flag to
> testpmd command line".
> To make that worst, only some PMD needs to give the additional
> parameter to get better number.
> And also, testpmd usage will be treated as application modeling.
> 
> Since this feature only used on sfc and cnxk driver, What is the
> situation with sfc driver?
> Keeping it as negotiated and not use the feature, will impact the per
> core performance of sfc or
> is it just PCI bandwidth thing which really dont show any difference in testpmd?

Yes, sfc could run faster if no Rx metadata are negotiated. So,
it is better to negotiate nothing by default. But it is always
painful to change defaults. You need to explain that now you
need to negotiate Rx metadata to use mark, flag and tunnel offloads. 
Yes, it will be required on sfc and cnxk only.
As an sfc maintainer I don't mind to change testpmd defaults.

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Thomas Monjalon]

01/02/2023 11:58, Andrew Rybchenko:
> On 2/1/23 13:48, Jerin Jacob wrote:
> > On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
> > <andrew.rybchenko@oktetlabs.ru> wrote:
> >> Frankly speaking I don't understand why default value is so
> >> important if we have a way to change it. Reasons should be
> >> really strong to change existing defaults.
> > 
> > The only reason is, typically testpmd will be used performance
> > benchmarking as an industry standard. It is difficult to tell/educate
> > the QA or customers
> > that, "BTW if you need to get better performance add more flag to
> > testpmd command line".

I disagree.
When you do performance benchmark, you tune settings accordingly.

> > To make that worst, only some PMD needs to give the additional
> > parameter to get better number.
> > And also, testpmd usage will be treated as application modeling.
> > 
> > Since this feature only used on sfc and cnxk driver, What is the
> > situation with sfc driver?
> > Keeping it as negotiated and not use the feature, will impact the per
> > core performance of sfc or
> > is it just PCI bandwidth thing which really dont show any difference in testpmd?
> 
> Yes, sfc could run faster if no Rx metadata are negotiated. So,
> it is better to negotiate nothing by default. But it is always
> painful to change defaults. You need to explain that now you
> need to negotiate Rx metadata to use mark, flag and tunnel offloads. 
> Yes, it will be required on sfc and cnxk only.
> As an sfc maintainer I don't mind to change testpmd defaults.

If we change testpmd defaults to "do nothing",
then we should disable MBUF_FAST_FREE as well.

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Jerin Jacob]

On Wed, Feb 1, 2023 at 4:35 PM Thomas Monjalon <thomas@monjalon.net> wrote:
>
> 01/02/2023 11:58, Andrew Rybchenko:
> > On 2/1/23 13:48, Jerin Jacob wrote:
> > > On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
> > > <andrew.rybchenko@oktetlabs.ru> wrote:
> > >> Frankly speaking I don't understand why default value is so
> > >> important if we have a way to change it. Reasons should be
> > >> really strong to change existing defaults.
> > >
> > > The only reason is, typically testpmd will be used performance
> > > benchmarking as an industry standard. It is difficult to tell/educate
> > > the QA or customers
> > > that, "BTW if you need to get better performance add more flag to
> > > testpmd command line".
>
> I disagree.
> When you do performance benchmark, you tune settings accordingly.

IMO, We tune the system resources like queue depth not the disabling
features for raw performance.
queue depth etc people know to tune so it is obvious. What is not
obvious is, testpmd only
negotiated some features by default.I am not using that feature, hence
I need to explicitly
disable it.


>
> > > To make that worst, only some PMD needs to give the additional
> > > parameter to get better number.
> > > And also, testpmd usage will be treated as application modeling.
> > >
> > > Since this feature only used on sfc and cnxk driver, What is the
> > > situation with sfc driver?
> > > Keeping it as negotiated and not use the feature, will impact the per
> > > core performance of sfc or
> > > is it just PCI bandwidth thing which really dont show any difference in testpmd?
> >
> > Yes, sfc could run faster if no Rx metadata are negotiated. So,
> > it is better to negotiate nothing by default. But it is always
> > painful to change defaults. You need to explain that now you
> > need to negotiate Rx metadata to use mark, flag and tunnel offloads.
> > Yes, it will be required on sfc and cnxk only.
> > As an sfc maintainer I don't mind to change testpmd defaults.
>
> If we change testpmd defaults to "do nothing",
> then we should disable MBUF_FAST_FREE as well.

if you see MBUF_FAST_FREE, it does nothing. Actually,
!MBUF_FAST_FREE is doing more work.


>
>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ferruh Yigit]

On 2/1/2023 11:15 AM, Jerin Jacob wrote:
> On Wed, Feb 1, 2023 at 4:35 PM Thomas Monjalon <thomas@monjalon.net> wrote:
>>
>> 01/02/2023 11:58, Andrew Rybchenko:
>>> On 2/1/23 13:48, Jerin Jacob wrote:
>>>> On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
>>>> <andrew.rybchenko@oktetlabs.ru> wrote:
>>>>> Frankly speaking I don't understand why default value is so
>>>>> important if we have a way to change it. Reasons should be
>>>>> really strong to change existing defaults.
>>>>
>>>> The only reason is, typically testpmd will be used performance
>>>> benchmarking as an industry standard. It is difficult to tell/educate
>>>> the QA or customers
>>>> that, "BTW if you need to get better performance add more flag to
>>>> testpmd command line".
>>
>> I disagree.
>> When you do performance benchmark, you tune settings accordingly.
> 
> IMO, We tune the system resources like queue depth not the disabling
> features for raw performance.
> queue depth etc people know to tune so it is obvious. What is not
> obvious is, testpmd only
> negotiated some features by default.I am not using that feature, hence
> I need to explicitly
> disable it.
> 

When 'rte_eth_rx_metadata_negotiate()' API is NOT used at all, and I
believe that is the case for almost all applications since API is a
relatively new one, PMD default behavior should be to enable Rx metadata
flow rules, in case user requests them later.

So, enabling all in application is same with not calling the API at all.

In this perspective, disabling Rx metadata is additional
optimization/tuning that application can do if it is sure that Rx
metadata flow rules won't be used at all.
And API is more meaningful when it is used to disable Rx metadata.

I think it is reasonable to enable all Rx metadata by default in testpmd
with a capability to disable it when wanted.

OR

May be we don't call 'rte_eth_rx_metadata_negotiate()' API by default in
testpmd, it is only called when it is requested explicitly from user,
enable or disable.

> 
>>
>>>> To make that worst, only some PMD needs to give the additional
>>>> parameter to get better number.
>>>> And also, testpmd usage will be treated as application modeling.
>>>>
>>>> Since this feature only used on sfc and cnxk driver, What is the
>>>> situation with sfc driver?
>>>> Keeping it as negotiated and not use the feature, will impact the per
>>>> core performance of sfc or
>>>> is it just PCI bandwidth thing which really dont show any difference in testpmd?
>>>
>>> Yes, sfc could run faster if no Rx metadata are negotiated. So,
>>> it is better to negotiate nothing by default. But it is always
>>> painful to change defaults. You need to explain that now you
>>> need to negotiate Rx metadata to use mark, flag and tunnel offloads.
>>> Yes, it will be required on sfc and cnxk only.
>>> As an sfc maintainer I don't mind to change testpmd defaults.
>>
>> If we change testpmd defaults to "do nothing",
>> then we should disable MBUF_FAST_FREE as well.
> 
> if you see MBUF_FAST_FREE, it does nothing. Actually,
> !MBUF_FAST_FREE is doing more work.
> 
> 
>>
>>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Jerin Jacob]

On Wed, Feb 1, 2023 at 5:06 PM Ferruh Yigit <ferruh.yigit@amd.com> wrote:
>
> On 2/1/2023 11:15 AM, Jerin Jacob wrote:
> > On Wed, Feb 1, 2023 at 4:35 PM Thomas Monjalon <thomas@monjalon.net> wrote:
> >>
> >> 01/02/2023 11:58, Andrew Rybchenko:
> >>> On 2/1/23 13:48, Jerin Jacob wrote:
> >>>> On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
> >>>> <andrew.rybchenko@oktetlabs.ru> wrote:
> >>>>> Frankly speaking I don't understand why default value is so
> >>>>> important if we have a way to change it. Reasons should be
> >>>>> really strong to change existing defaults.
> >>>>
> >>>> The only reason is, typically testpmd will be used performance
> >>>> benchmarking as an industry standard. It is difficult to tell/educate
> >>>> the QA or customers
> >>>> that, "BTW if you need to get better performance add more flag to
> >>>> testpmd command line".
> >>
> >> I disagree.
> >> When you do performance benchmark, you tune settings accordingly.
> >
> > IMO, We tune the system resources like queue depth not the disabling
> > features for raw performance.
> > queue depth etc people know to tune so it is obvious. What is not
> > obvious is, testpmd only
> > negotiated some features by default.I am not using that feature, hence
> > I need to explicitly
> > disable it.
> >
>
> When 'rte_eth_rx_metadata_negotiate()' API is NOT used at all, and I
> believe that is the case for almost all applications since API is a
> relatively new one, PMD default behavior should be to enable Rx metadata
> flow rules, in case user requests them later.
>
> So, enabling all in application is same with not calling the API at all.
>
> In this perspective, disabling Rx metadata is additional
> optimization/tuning that application can do if it is sure that Rx
> metadata flow rules won't be used at all.
> And API is more meaningful when it is used to disable Rx metadata.
>
> I think it is reasonable to enable all Rx metadata by default in testpmd
> with a capability to disable it when wanted.
>
> OR
>
> May be we don't call 'rte_eth_rx_metadata_negotiate()' API by default in
> testpmd, it is only called when it is requested explicitly from user,
> enable or disable.

Second option looks good to me.
When
1) user request for action which is needed negotiate(),
AND
2) rte_eth_rx_metadata_negotiate() != ENOSUP
then, testpmd print a warning that need to enable
rte_eth_rx_metadata_negotiate().


> >
> >>
> >>>> To make that worst, only some PMD needs to give the additional
> >>>> parameter to get better number.
> >>>> And also, testpmd usage will be treated as application modeling.
> >>>>
> >>>> Since this feature only used on sfc and cnxk driver, What is the
> >>>> situation with sfc driver?
> >>>> Keeping it as negotiated and not use the feature, will impact the per
> >>>> core performance of sfc or
> >>>> is it just PCI bandwidth thing which really dont show any difference in testpmd?
> >>>
> >>> Yes, sfc could run faster if no Rx metadata are negotiated. So,
> >>> it is better to negotiate nothing by default. But it is always
> >>> painful to change defaults. You need to explain that now you
> >>> need to negotiate Rx metadata to use mark, flag and tunnel offloads.
> >>> Yes, it will be required on sfc and cnxk only.
> >>> As an sfc maintainer I don't mind to change testpmd defaults.
> >>
> >> If we change testpmd defaults to "do nothing",
> >> then we should disable MBUF_FAST_FREE as well.
> >
> > if you see MBUF_FAST_FREE, it does nothing. Actually,
> > !MBUF_FAST_FREE is doing more work.
> >
> >
> >>
> >>
>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ferruh Yigit]

On 2/1/2023 1:48 PM, Jerin Jacob wrote:
> On Wed, Feb 1, 2023 at 5:06 PM Ferruh Yigit <ferruh.yigit@amd.com> wrote:
>>
>> On 2/1/2023 11:15 AM, Jerin Jacob wrote:
>>> On Wed, Feb 1, 2023 at 4:35 PM Thomas Monjalon <thomas@monjalon.net> wrote:
>>>>
>>>> 01/02/2023 11:58, Andrew Rybchenko:
>>>>> On 2/1/23 13:48, Jerin Jacob wrote:
>>>>>> On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
>>>>>> <andrew.rybchenko@oktetlabs.ru> wrote:
>>>>>>> Frankly speaking I don't understand why default value is so
>>>>>>> important if we have a way to change it. Reasons should be
>>>>>>> really strong to change existing defaults.
>>>>>>
>>>>>> The only reason is, typically testpmd will be used performance
>>>>>> benchmarking as an industry standard. It is difficult to tell/educate
>>>>>> the QA or customers
>>>>>> that, "BTW if you need to get better performance add more flag to
>>>>>> testpmd command line".
>>>>
>>>> I disagree.
>>>> When you do performance benchmark, you tune settings accordingly.
>>>
>>> IMO, We tune the system resources like queue depth not the disabling
>>> features for raw performance.
>>> queue depth etc people know to tune so it is obvious. What is not
>>> obvious is, testpmd only
>>> negotiated some features by default.I am not using that feature, hence
>>> I need to explicitly
>>> disable it.
>>>
>>
>> When 'rte_eth_rx_metadata_negotiate()' API is NOT used at all, and I
>> believe that is the case for almost all applications since API is a
>> relatively new one, PMD default behavior should be to enable Rx metadata
>> flow rules, in case user requests them later.
>>
>> So, enabling all in application is same with not calling the API at all.
>>
>> In this perspective, disabling Rx metadata is additional
>> optimization/tuning that application can do if it is sure that Rx
>> metadata flow rules won't be used at all.
>> And API is more meaningful when it is used to disable Rx metadata.
>>
>> I think it is reasonable to enable all Rx metadata by default in testpmd
>> with a capability to disable it when wanted.
>>
>> OR
>>
>> May be we don't call 'rte_eth_rx_metadata_negotiate()' API by default in
>> testpmd, it is only called when it is requested explicitly from user,
>> enable or disable.
> 
> Second option looks good to me.
> When
> 1) user request for action which is needed negotiate(),
> AND
> 2) rte_eth_rx_metadata_negotiate() != ENOSUP
> then, testpmd print a warning that need to enable
> rte_eth_rx_metadata_negotiate().
> 

We are not suggesting same thing.

What you described above assumes PMD disabled Rx metadata flow rule
support by default, and it needs to be enabled explicitly by
'rte_eth_rx_metadata_negotiate()' API. This API becomes mandatory for
functionality.

As far as I understand PMD wants to disable this flow rule by default
because of performance concerns. But this creates inconsistency between
PMDs, because rest of them will enable this flow rule by default (if it
is supported) and be ready to use it when proper flow rule created.

With this approach some PMDs will need 'rte_eth_rx_metadata_negotiate()'
to enable Rx metadata flow rules, some won't. This can be confusing for
applications that *some* PMDs require double enabling with specific API
call.


Instead what I was trying to suggest is reverse,
all PMDs enable the Rx metadata flow rule by default, and don't require
double enabling.
But if application knows that it won't use Rx metadata flow rule, it can
disable it to optimize the performance.
This makes 'rte_eth_rx_metadata_negotiate()' functionally optional, and
for testpmd context it can be called via a command on demand by user for
optimization purpose.




> 
>>>
>>>>
>>>>>> To make that worst, only some PMD needs to give the additional
>>>>>> parameter to get better number.
>>>>>> And also, testpmd usage will be treated as application modeling.
>>>>>>
>>>>>> Since this feature only used on sfc and cnxk driver, What is the
>>>>>> situation with sfc driver?
>>>>>> Keeping it as negotiated and not use the feature, will impact the per
>>>>>> core performance of sfc or
>>>>>> is it just PCI bandwidth thing which really dont show any difference in testpmd?
>>>>>
>>>>> Yes, sfc could run faster if no Rx metadata are negotiated. So,
>>>>> it is better to negotiate nothing by default. But it is always
>>>>> painful to change defaults. You need to explain that now you
>>>>> need to negotiate Rx metadata to use mark, flag and tunnel offloads.
>>>>> Yes, it will be required on sfc and cnxk only.
>>>>> As an sfc maintainer I don't mind to change testpmd defaults.
>>>>
>>>> If we change testpmd defaults to "do nothing",
>>>> then we should disable MBUF_FAST_FREE as well.
>>>
>>> if you see MBUF_FAST_FREE, it does nothing. Actually,
>>> !MBUF_FAST_FREE is doing more work.
>>>
>>>
>>>>
>>>>
>>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Jerin Jacob]

On Wed, Feb 1, 2023 at 8:20 PM Ferruh Yigit <ferruh.yigit@amd.com> wrote:
>
> On 2/1/2023 1:48 PM, Jerin Jacob wrote:
> > On Wed, Feb 1, 2023 at 5:06 PM Ferruh Yigit <ferruh.yigit@amd.com> wrote:
> >>
> >> On 2/1/2023 11:15 AM, Jerin Jacob wrote:
> >>> On Wed, Feb 1, 2023 at 4:35 PM Thomas Monjalon <thomas@monjalon.net> wrote:
> >>>>
> >>>> 01/02/2023 11:58, Andrew Rybchenko:
> >>>>> On 2/1/23 13:48, Jerin Jacob wrote:
> >>>>>> On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
> >>>>>> <andrew.rybchenko@oktetlabs.ru> wrote:
> >>>>>>> Frankly speaking I don't understand why default value is so
> >>>>>>> important if we have a way to change it. Reasons should be
> >>>>>>> really strong to change existing defaults.
> >>>>>>
> >>>>>> The only reason is, typically testpmd will be used performance
> >>>>>> benchmarking as an industry standard. It is difficult to tell/educate
> >>>>>> the QA or customers
> >>>>>> that, "BTW if you need to get better performance add more flag to
> >>>>>> testpmd command line".
> >>>>
> >>>> I disagree.
> >>>> When you do performance benchmark, you tune settings accordingly.
> >>>
> >>> IMO, We tune the system resources like queue depth not the disabling
> >>> features for raw performance.
> >>> queue depth etc people know to tune so it is obvious. What is not
> >>> obvious is, testpmd only
> >>> negotiated some features by default.I am not using that feature, hence
> >>> I need to explicitly
> >>> disable it.
> >>>
> >>
> >> When 'rte_eth_rx_metadata_negotiate()' API is NOT used at all, and I
> >> believe that is the case for almost all applications since API is a
> >> relatively new one, PMD default behavior should be to enable Rx metadata
> >> flow rules, in case user requests them later.
> >>
> >> So, enabling all in application is same with not calling the API at all.
> >>
> >> In this perspective, disabling Rx metadata is additional
> >> optimization/tuning that application can do if it is sure that Rx
> >> metadata flow rules won't be used at all.
> >> And API is more meaningful when it is used to disable Rx metadata.
> >>
> >> I think it is reasonable to enable all Rx metadata by default in testpmd
> >> with a capability to disable it when wanted.
> >>
> >> OR
> >>
> >> May be we don't call 'rte_eth_rx_metadata_negotiate()' API by default in
> >> testpmd, it is only called when it is requested explicitly from user,
> >> enable or disable.
> >
> > Second option looks good to me.
> > When
> > 1) user request for action which is needed negotiate(),
> > AND
> > 2) rte_eth_rx_metadata_negotiate() != ENOSUP
> > then, testpmd print a warning that need to enable
> > rte_eth_rx_metadata_negotiate().
> >
>
> We are not suggesting same thing.
>
> What you described above assumes PMD disabled Rx metadata flow rule
> support by default, and it needs to be enabled explicitly by
> 'rte_eth_rx_metadata_negotiate()' API. This API becomes mandatory for
> functionality.
>
> As far as I understand PMD wants to disable this flow rule by default
> because of performance concerns. But this creates inconsistency between
> PMDs, because rest of them will enable this flow rule by default (if it
> is supported) and be ready to use it when proper flow rule created.
>
> With this approach some PMDs will need 'rte_eth_rx_metadata_negotiate()'
> to enable Rx metadata flow rules, some won't. This can be confusing for
> applications that *some* PMDs require double enabling with specific API
> call.
>
>
> Instead what I was trying to suggest is reverse,
> all PMDs enable the Rx metadata flow rule by default, and don't require
> double enabling.
> But if application knows that it won't use Rx metadata flow rule, it can
> disable it to optimize the performance.
> This makes 'rte_eth_rx_metadata_negotiate()' functionally optional, and
> for testpmd context it can be called via a command on demand by user for
> optimization purpose.

This won't solve concern I have outlined earlier[1].

I think, The part of the problem there is no enough adaption of
rte_eth_rx_metadata_negotiate(),

The view is total different from PMD maintainer PoV vs testpmd application PoV.

Just to avoid back and forth. We will call off this patch and remove
rte_eth_rx_metadata_negotiate()
PMD callback from cnxk driver. Keep it as old behavior, so we don't need to care
about rte_eth_rx_metadata_negotiate().

[1]
The only reason is, typically testpmd will be used performance
benchmarking as an industry standard. It is difficult to tell/educate
the QA or customers
that, "BTW if you need to get better performance add more flag to
testpmd command line".
To make that worst, only some PMD needs to give the additional
parameter to get better number.
And also, testpmd usage will be treated as application modeling.

>
>
>
>
> >
> >>>
> >>>>
> >>>>>> To make that worst, only some PMD needs to give the additional
> >>>>>> parameter to get better number.
> >>>>>> And also, testpmd usage will be treated as application modeling.
> >>>>>>
> >>>>>> Since this feature only used on sfc and cnxk driver, What is the
> >>>>>> situation with sfc driver?
> >>>>>> Keeping it as negotiated and not use the feature, will impact the per
> >>>>>> core performance of sfc or
> >>>>>> is it just PCI bandwidth thing which really dont show any difference in testpmd?
> >>>>>
> >>>>> Yes, sfc could run faster if no Rx metadata are negotiated. So,
> >>>>> it is better to negotiate nothing by default. But it is always
> >>>>> painful to change defaults. You need to explain that now you
> >>>>> need to negotiate Rx metadata to use mark, flag and tunnel offloads.
> >>>>> Yes, it will be required on sfc and cnxk only.
> >>>>> As an sfc maintainer I don't mind to change testpmd defaults.
> >>>>
> >>>> If we change testpmd defaults to "do nothing",
> >>>> then we should disable MBUF_FAST_FREE as well.
> >>>
> >>> if you see MBUF_FAST_FREE, it does nothing. Actually,
> >>> !MBUF_FAST_FREE is doing more work.
> >>>
> >>>
> >>>>
> >>>>
> >>
>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ferruh Yigit]

On 2/1/2023 3:22 PM, Jerin Jacob wrote:
> On Wed, Feb 1, 2023 at 8:20 PM Ferruh Yigit <ferruh.yigit@amd.com> wrote:
>>
>> On 2/1/2023 1:48 PM, Jerin Jacob wrote:
>>> On Wed, Feb 1, 2023 at 5:06 PM Ferruh Yigit <ferruh.yigit@amd.com> wrote:
>>>>
>>>> On 2/1/2023 11:15 AM, Jerin Jacob wrote:
>>>>> On Wed, Feb 1, 2023 at 4:35 PM Thomas Monjalon <thomas@monjalon.net> wrote:
>>>>>>
>>>>>> 01/02/2023 11:58, Andrew Rybchenko:
>>>>>>> On 2/1/23 13:48, Jerin Jacob wrote:
>>>>>>>> On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
>>>>>>>> <andrew.rybchenko@oktetlabs.ru> wrote:
>>>>>>>>> Frankly speaking I don't understand why default value is so
>>>>>>>>> important if we have a way to change it. Reasons should be
>>>>>>>>> really strong to change existing defaults.
>>>>>>>>
>>>>>>>> The only reason is, typically testpmd will be used performance
>>>>>>>> benchmarking as an industry standard. It is difficult to tell/educate
>>>>>>>> the QA or customers
>>>>>>>> that, "BTW if you need to get better performance add more flag to
>>>>>>>> testpmd command line".
>>>>>>
>>>>>> I disagree.
>>>>>> When you do performance benchmark, you tune settings accordingly.
>>>>>
>>>>> IMO, We tune the system resources like queue depth not the disabling
>>>>> features for raw performance.
>>>>> queue depth etc people know to tune so it is obvious. What is not
>>>>> obvious is, testpmd only
>>>>> negotiated some features by default.I am not using that feature, hence
>>>>> I need to explicitly
>>>>> disable it.
>>>>>
>>>>
>>>> When 'rte_eth_rx_metadata_negotiate()' API is NOT used at all, and I
>>>> believe that is the case for almost all applications since API is a
>>>> relatively new one, PMD default behavior should be to enable Rx metadata
>>>> flow rules, in case user requests them later.
>>>>
>>>> So, enabling all in application is same with not calling the API at all.
>>>>
>>>> In this perspective, disabling Rx metadata is additional
>>>> optimization/tuning that application can do if it is sure that Rx
>>>> metadata flow rules won't be used at all.
>>>> And API is more meaningful when it is used to disable Rx metadata.
>>>>
>>>> I think it is reasonable to enable all Rx metadata by default in testpmd
>>>> with a capability to disable it when wanted.
>>>>
>>>> OR
>>>>
>>>> May be we don't call 'rte_eth_rx_metadata_negotiate()' API by default in
>>>> testpmd, it is only called when it is requested explicitly from user,
>>>> enable or disable.
>>>
>>> Second option looks good to me.
>>> When
>>> 1) user request for action which is needed negotiate(),
>>> AND
>>> 2) rte_eth_rx_metadata_negotiate() != ENOSUP
>>> then, testpmd print a warning that need to enable
>>> rte_eth_rx_metadata_negotiate().
>>>
>>
>> We are not suggesting same thing.
>>
>> What you described above assumes PMD disabled Rx metadata flow rule
>> support by default, and it needs to be enabled explicitly by
>> 'rte_eth_rx_metadata_negotiate()' API. This API becomes mandatory for
>> functionality.
>>
>> As far as I understand PMD wants to disable this flow rule by default
>> because of performance concerns. But this creates inconsistency between
>> PMDs, because rest of them will enable this flow rule by default (if it
>> is supported) and be ready to use it when proper flow rule created.
>>
>> With this approach some PMDs will need 'rte_eth_rx_metadata_negotiate()'
>> to enable Rx metadata flow rules, some won't. This can be confusing for
>> applications that *some* PMDs require double enabling with specific API
>> call.
>>
>>
>> Instead what I was trying to suggest is reverse,
>> all PMDs enable the Rx metadata flow rule by default, and don't require
>> double enabling.
>> But if application knows that it won't use Rx metadata flow rule, it can
>> disable it to optimize the performance.
>> This makes 'rte_eth_rx_metadata_negotiate()' functionally optional, and
>> for testpmd context it can be called via a command on demand by user for
>> optimization purpose.
> 
> This won't solve concern I have outlined earlier[1].
> 

Yes, it won't.

> I think, The part of the problem there is no enough adaption of
> rte_eth_rx_metadata_negotiate(),
> 
> The view is total different from PMD maintainer PoV vs testpmd application PoV.
> 

Agree,
and I assume it is different for user application too, which may
prioritize consistency and portability.

Overall, I am not fan of the 'rte_eth_rx_metadata_negotiate()' API, I
think it is confusing.

> Just to avoid back and forth. We will call off this patch and remove
> rte_eth_rx_metadata_negotiate()
> PMD callback from cnxk driver. Keep it as old behavior, so we don't need to care
> about rte_eth_rx_metadata_negotiate().
> 

When you remove 'rx_metadata_negotiate' callback, what will be the PMD
behavior? I assume PMD will do the required preparations as if all Rx
metadata is enabled.
And what is the performance impact, is removing callback improve the
performance?


> [1]
> The only reason is, typically testpmd will be used performance
> benchmarking as an industry standard. It is difficult to tell/educate
> the QA or customers
> that, "BTW if you need to get better performance add more flag to
> testpmd command line".
> To make that worst, only some PMD needs to give the additional
> parameter to get better number.
> And also, testpmd usage will be treated as application modeling.
> 
>>
>>
>>
>>
>>>
>>>>>
>>>>>>
>>>>>>>> To make that worst, only some PMD needs to give the additional
>>>>>>>> parameter to get better number.
>>>>>>>> And also, testpmd usage will be treated as application modeling.
>>>>>>>>
>>>>>>>> Since this feature only used on sfc and cnxk driver, What is the
>>>>>>>> situation with sfc driver?
>>>>>>>> Keeping it as negotiated and not use the feature, will impact the per
>>>>>>>> core performance of sfc or
>>>>>>>> is it just PCI bandwidth thing which really dont show any difference in testpmd?
>>>>>>>
>>>>>>> Yes, sfc could run faster if no Rx metadata are negotiated. So,
>>>>>>> it is better to negotiate nothing by default. But it is always
>>>>>>> painful to change defaults. You need to explain that now you
>>>>>>> need to negotiate Rx metadata to use mark, flag and tunnel offloads.
>>>>>>> Yes, it will be required on sfc and cnxk only.
>>>>>>> As an sfc maintainer I don't mind to change testpmd defaults.
>>>>>>
>>>>>> If we change testpmd defaults to "do nothing",
>>>>>> then we should disable MBUF_FAST_FREE as well.
>>>>>
>>>>> if you see MBUF_FAST_FREE, it does nothing. Actually,
>>>>> !MBUF_FAST_FREE is doing more work.
>>>>>
>>>>>
>>>>>>
>>>>>>
>>>>
>>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ivan Malov]

On Thu, 2 Feb 2023, Ferruh Yigit wrote:

> On 2/1/2023 3:22 PM, Jerin Jacob wrote:
>> On Wed, Feb 1, 2023 at 8:20 PM Ferruh Yigit <ferruh.yigit@amd.com> wrote:
>>>
>>> On 2/1/2023 1:48 PM, Jerin Jacob wrote:
>>>> On Wed, Feb 1, 2023 at 5:06 PM Ferruh Yigit <ferruh.yigit@amd.com> wrote:
>>>>>
>>>>> On 2/1/2023 11:15 AM, Jerin Jacob wrote:
>>>>>> On Wed, Feb 1, 2023 at 4:35 PM Thomas Monjalon <thomas@monjalon.net> wrote:
>>>>>>>
>>>>>>> 01/02/2023 11:58, Andrew Rybchenko:
>>>>>>>> On 2/1/23 13:48, Jerin Jacob wrote:
>>>>>>>>> On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
>>>>>>>>> <andrew.rybchenko@oktetlabs.ru> wrote:
>>>>>>>>>> Frankly speaking I don't understand why default value is so
>>>>>>>>>> important if we have a way to change it. Reasons should be
>>>>>>>>>> really strong to change existing defaults.
>>>>>>>>>
>>>>>>>>> The only reason is, typically testpmd will be used performance
>>>>>>>>> benchmarking as an industry standard. It is difficult to tell/educate
>>>>>>>>> the QA or customers
>>>>>>>>> that, "BTW if you need to get better performance add more flag to
>>>>>>>>> testpmd command line".
>>>>>>>
>>>>>>> I disagree.
>>>>>>> When you do performance benchmark, you tune settings accordingly.
>>>>>>
>>>>>> IMO, We tune the system resources like queue depth not the disabling
>>>>>> features for raw performance.
>>>>>> queue depth etc people know to tune so it is obvious. What is not
>>>>>> obvious is, testpmd only
>>>>>> negotiated some features by default.I am not using that feature, hence
>>>>>> I need to explicitly
>>>>>> disable it.
>>>>>>
>>>>>
>>>>> When 'rte_eth_rx_metadata_negotiate()' API is NOT used at all, and I
>>>>> believe that is the case for almost all applications since API is a
>>>>> relatively new one, PMD default behavior should be to enable Rx metadata
>>>>> flow rules, in case user requests them later.
>>>>>
>>>>> So, enabling all in application is same with not calling the API at all.
>>>>>
>>>>> In this perspective, disabling Rx metadata is additional
>>>>> optimization/tuning that application can do if it is sure that Rx
>>>>> metadata flow rules won't be used at all.
>>>>> And API is more meaningful when it is used to disable Rx metadata.
>>>>>
>>>>> I think it is reasonable to enable all Rx metadata by default in testpmd
>>>>> with a capability to disable it when wanted.
>>>>>
>>>>> OR
>>>>>
>>>>> May be we don't call 'rte_eth_rx_metadata_negotiate()' API by default in
>>>>> testpmd, it is only called when it is requested explicitly from user,
>>>>> enable or disable.
>>>>
>>>> Second option looks good to me.
>>>> When
>>>> 1) user request for action which is needed negotiate(),
>>>> AND
>>>> 2) rte_eth_rx_metadata_negotiate() != ENOSUP
>>>> then, testpmd print a warning that need to enable
>>>> rte_eth_rx_metadata_negotiate().
>>>>
>>>
>>> We are not suggesting same thing.
>>>
>>> What you described above assumes PMD disabled Rx metadata flow rule
>>> support by default, and it needs to be enabled explicitly by
>>> 'rte_eth_rx_metadata_negotiate()' API. This API becomes mandatory for
>>> functionality.
>>>
>>> As far as I understand PMD wants to disable this flow rule by default
>>> because of performance concerns. But this creates inconsistency between
>>> PMDs, because rest of them will enable this flow rule by default (if it
>>> is supported) and be ready to use it when proper flow rule created.
>>>
>>> With this approach some PMDs will need 'rte_eth_rx_metadata_negotiate()'
>>> to enable Rx metadata flow rules, some won't. This can be confusing for
>>> applications that *some* PMDs require double enabling with specific API
>>> call.
>>>
>>>
>>> Instead what I was trying to suggest is reverse,
>>> all PMDs enable the Rx metadata flow rule by default, and don't require
>>> double enabling.
>>> But if application knows that it won't use Rx metadata flow rule, it can
>>> disable it to optimize the performance.
>>> This makes 'rte_eth_rx_metadata_negotiate()' functionally optional, and
>>> for testpmd context it can be called via a command on demand by user for
>>> optimization purpose.
>>
>> This won't solve concern I have outlined earlier[1].
>>
>
> Yes, it won't.
>
>> I think, The part of the problem there is no enough adaption of
>> rte_eth_rx_metadata_negotiate(),
>>
>> The view is total different from PMD maintainer PoV vs testpmd application PoV.
>>
>
> Agree,
> and I assume it is different for user application too, which may
> prioritize consistency and portability.
>
> Overall, I am not fan of the 'rte_eth_rx_metadata_negotiate()' API, I
> think it is confusing.

Forgive me, in which way is it confusing?

>
>> Just to avoid back and forth. We will call off this patch and remove
>> rte_eth_rx_metadata_negotiate()
>> PMD callback from cnxk driver. Keep it as old behavior, so we don't need to care
>> about rte_eth_rx_metadata_negotiate().
>>
>
> When you remove 'rx_metadata_negotiate' callback, what will be the PMD
> behavior? I assume PMD will do the required preparations as if all Rx
> metadata is enabled.
> And what is the performance impact, is removing callback improve the
> performance?
>
>
>> [1]
>> The only reason is, typically testpmd will be used performance
>> benchmarking as an industry standard. It is difficult to tell/educate
>> the QA or customers
>> that, "BTW if you need to get better performance add more flag to
>> testpmd command line".
>> To make that worst, only some PMD needs to give the additional
>> parameter to get better number.
>> And also, testpmd usage will be treated as application modeling.
>>
>>>
>>>
>>>
>>>
>>>>
>>>>>>
>>>>>>>
>>>>>>>>> To make that worst, only some PMD needs to give the additional
>>>>>>>>> parameter to get better number.
>>>>>>>>> And also, testpmd usage will be treated as application modeling.
>>>>>>>>>
>>>>>>>>> Since this feature only used on sfc and cnxk driver, What is the
>>>>>>>>> situation with sfc driver?
>>>>>>>>> Keeping it as negotiated and not use the feature, will impact the per
>>>>>>>>> core performance of sfc or
>>>>>>>>> is it just PCI bandwidth thing which really dont show any difference in testpmd?
>>>>>>>>
>>>>>>>> Yes, sfc could run faster if no Rx metadata are negotiated. So,
>>>>>>>> it is better to negotiate nothing by default. But it is always
>>>>>>>> painful to change defaults. You need to explain that now you
>>>>>>>> need to negotiate Rx metadata to use mark, flag and tunnel offloads.
>>>>>>>> Yes, it will be required on sfc and cnxk only.
>>>>>>>> As an sfc maintainer I don't mind to change testpmd defaults.
>>>>>>>
>>>>>>> If we change testpmd defaults to "do nothing",
>>>>>>> then we should disable MBUF_FAST_FREE as well.
>>>>>>
>>>>>> if you see MBUF_FAST_FREE, it does nothing. Actually,
>>>>>> !MBUF_FAST_FREE is doing more work.
>>>>>>
>>>>>>
>>>>>>>
>>>>>>>
>>>>>
>>>
>
>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ferruh Yigit]

On 2/2/2023 8:50 AM, Ivan Malov wrote:
> On Thu, 2 Feb 2023, Ferruh Yigit wrote:
> 
>> On 2/1/2023 3:22 PM, Jerin Jacob wrote:
>>> On Wed, Feb 1, 2023 at 8:20 PM Ferruh Yigit <ferruh.yigit@amd.com>
>>> wrote:
>>>>
>>>> On 2/1/2023 1:48 PM, Jerin Jacob wrote:
>>>>> On Wed, Feb 1, 2023 at 5:06 PM Ferruh Yigit <ferruh.yigit@amd.com>
>>>>> wrote:
>>>>>>
>>>>>> On 2/1/2023 11:15 AM, Jerin Jacob wrote:
>>>>>>> On Wed, Feb 1, 2023 at 4:35 PM Thomas Monjalon
>>>>>>> <thomas@monjalon.net> wrote:
>>>>>>>>
>>>>>>>> 01/02/2023 11:58, Andrew Rybchenko:
>>>>>>>>> On 2/1/23 13:48, Jerin Jacob wrote:
>>>>>>>>>> On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
>>>>>>>>>> <andrew.rybchenko@oktetlabs.ru> wrote:
>>>>>>>>>>> Frankly speaking I don't understand why default value is so
>>>>>>>>>>> important if we have a way to change it. Reasons should be
>>>>>>>>>>> really strong to change existing defaults.
>>>>>>>>>>
>>>>>>>>>> The only reason is, typically testpmd will be used performance
>>>>>>>>>> benchmarking as an industry standard. It is difficult to
>>>>>>>>>> tell/educate
>>>>>>>>>> the QA or customers
>>>>>>>>>> that, "BTW if you need to get better performance add more flag to
>>>>>>>>>> testpmd command line".
>>>>>>>>
>>>>>>>> I disagree.
>>>>>>>> When you do performance benchmark, you tune settings accordingly.
>>>>>>>
>>>>>>> IMO, We tune the system resources like queue depth not the disabling
>>>>>>> features for raw performance.
>>>>>>> queue depth etc people know to tune so it is obvious. What is not
>>>>>>> obvious is, testpmd only
>>>>>>> negotiated some features by default.I am not using that feature,
>>>>>>> hence
>>>>>>> I need to explicitly
>>>>>>> disable it.
>>>>>>>
>>>>>>
>>>>>> When 'rte_eth_rx_metadata_negotiate()' API is NOT used at all, and I
>>>>>> believe that is the case for almost all applications since API is a
>>>>>> relatively new one, PMD default behavior should be to enable Rx
>>>>>> metadata
>>>>>> flow rules, in case user requests them later.
>>>>>>
>>>>>> So, enabling all in application is same with not calling the API
>>>>>> at all.
>>>>>>
>>>>>> In this perspective, disabling Rx metadata is additional
>>>>>> optimization/tuning that application can do if it is sure that Rx
>>>>>> metadata flow rules won't be used at all.
>>>>>> And API is more meaningful when it is used to disable Rx metadata.
>>>>>>
>>>>>> I think it is reasonable to enable all Rx metadata by default in
>>>>>> testpmd
>>>>>> with a capability to disable it when wanted.
>>>>>>
>>>>>> OR
>>>>>>
>>>>>> May be we don't call 'rte_eth_rx_metadata_negotiate()' API by
>>>>>> default in
>>>>>> testpmd, it is only called when it is requested explicitly from user,
>>>>>> enable or disable.
>>>>>
>>>>> Second option looks good to me.
>>>>> When
>>>>> 1) user request for action which is needed negotiate(),
>>>>> AND
>>>>> 2) rte_eth_rx_metadata_negotiate() != ENOSUP
>>>>> then, testpmd print a warning that need to enable
>>>>> rte_eth_rx_metadata_negotiate().
>>>>>
>>>>
>>>> We are not suggesting same thing.
>>>>
>>>> What you described above assumes PMD disabled Rx metadata flow rule
>>>> support by default, and it needs to be enabled explicitly by
>>>> 'rte_eth_rx_metadata_negotiate()' API. This API becomes mandatory for
>>>> functionality.
>>>>
>>>> As far as I understand PMD wants to disable this flow rule by default
>>>> because of performance concerns. But this creates inconsistency between
>>>> PMDs, because rest of them will enable this flow rule by default (if it
>>>> is supported) and be ready to use it when proper flow rule created.
>>>>
>>>> With this approach some PMDs will need
>>>> 'rte_eth_rx_metadata_negotiate()'
>>>> to enable Rx metadata flow rules, some won't. This can be confusing for
>>>> applications that *some* PMDs require double enabling with specific API
>>>> call.
>>>>
>>>>
>>>> Instead what I was trying to suggest is reverse,
>>>> all PMDs enable the Rx metadata flow rule by default, and don't require
>>>> double enabling.
>>>> But if application knows that it won't use Rx metadata flow rule, it
>>>> can
>>>> disable it to optimize the performance.
>>>> This makes 'rte_eth_rx_metadata_negotiate()' functionally optional, and
>>>> for testpmd context it can be called via a command on demand by user
>>>> for
>>>> optimization purpose.
>>>
>>> This won't solve concern I have outlined earlier[1].
>>>
>>
>> Yes, it won't.
>>
>>> I think, The part of the problem there is no enough adaption of
>>> rte_eth_rx_metadata_negotiate(),
>>>
>>> The view is total different from PMD maintainer PoV vs testpmd
>>> application PoV.
>>>
>>
>> Agree,
>> and I assume it is different for user application too, which may
>> prioritize consistency and portability.
>>
>> Overall, I am not fan of the 'rte_eth_rx_metadata_negotiate()' API, I
>> think it is confusing.
> 
> Forgive me, in which way is it confusing?
> 

All other flow rules enabled by creating flow rule, but this one
requires an API to enable it, so I believe it is inconsistent in that way.

From application perspective, assume that it doesn't know NIC details,
should it call this API or not? Without API call should application
assume Rx metadata flow rules are enabled or disabled?


As I understand intention is to get hint from application if it will
require Rx metadata flow rules so that PMD can optimize better, but if
PMD doesn't enable Rx metadata flow rules when this API call is missing
than it becomes a mandatory API to configure the device. But I think it
should be optional for optimization.

Also if application not sure to use this flow rule or not, it will by
default enable all in any case, this will reduce the benefit. As done in
testpmd.


API works in both ways, it request to enable some Rx metadata flow rule,
but based on what PMD returns application can know what device supports,
this also inconsistent with how other flow rules work, we don't have API
to get capability but detect them via flow create/validate.
Can there be a case API returns a flow rule negotiated, but it fails
when tried to create the flow rule, isn't this confusing for application


I think if we continue this approach there can be multiple enable and
capability learning APIs for various flow rules or flow rule groups, and
this can make flow API much more harder to use for applications.



>>
>>> Just to avoid back and forth. We will call off this patch and remove
>>> rte_eth_rx_metadata_negotiate()
>>> PMD callback from cnxk driver. Keep it as old behavior, so we don't
>>> need to care
>>> about rte_eth_rx_metadata_negotiate().
>>>
>>
>> When you remove 'rx_metadata_negotiate' callback, what will be the PMD
>> behavior? I assume PMD will do the required preparations as if all Rx
>> metadata is enabled.
>> And what is the performance impact, is removing callback improve the
>> performance?
>>
>>
>>> [1]
>>> The only reason is, typically testpmd will be used performance
>>> benchmarking as an industry standard. It is difficult to tell/educate
>>> the QA or customers
>>> that, "BTW if you need to get better performance add more flag to
>>> testpmd command line".
>>> To make that worst, only some PMD needs to give the additional
>>> parameter to get better number.
>>> And also, testpmd usage will be treated as application modeling.
>>>
>>>>
>>>>
>>>>
>>>>
>>>>>
>>>>>>>
>>>>>>>>
>>>>>>>>>> To make that worst, only some PMD needs to give the additional
>>>>>>>>>> parameter to get better number.
>>>>>>>>>> And also, testpmd usage will be treated as application modeling.
>>>>>>>>>>
>>>>>>>>>> Since this feature only used on sfc and cnxk driver, What is the
>>>>>>>>>> situation with sfc driver?
>>>>>>>>>> Keeping it as negotiated and not use the feature, will impact
>>>>>>>>>> the per
>>>>>>>>>> core performance of sfc or
>>>>>>>>>> is it just PCI bandwidth thing which really dont show any
>>>>>>>>>> difference in testpmd?
>>>>>>>>>
>>>>>>>>> Yes, sfc could run faster if no Rx metadata are negotiated. So,
>>>>>>>>> it is better to negotiate nothing by default. But it is always
>>>>>>>>> painful to change defaults. You need to explain that now you
>>>>>>>>> need to negotiate Rx metadata to use mark, flag and tunnel
>>>>>>>>> offloads.
>>>>>>>>> Yes, it will be required on sfc and cnxk only.
>>>>>>>>> As an sfc maintainer I don't mind to change testpmd defaults.
>>>>>>>>
>>>>>>>> If we change testpmd defaults to "do nothing",
>>>>>>>> then we should disable MBUF_FAST_FREE as well.
>>>>>>>
>>>>>>> if you see MBUF_FAST_FREE, it does nothing. Actually,
>>>>>>> !MBUF_FAST_FREE is doing more work.
>>>>>>>
>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>
>>>>
>>
>>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ivan Malov]

On Thu, 2 Feb 2023, Ferruh Yigit wrote:

> On 2/2/2023 8:50 AM, Ivan Malov wrote:
>> On Thu, 2 Feb 2023, Ferruh Yigit wrote:
>>
>>> On 2/1/2023 3:22 PM, Jerin Jacob wrote:
>>>> On Wed, Feb 1, 2023 at 8:20 PM Ferruh Yigit <ferruh.yigit@amd.com>
>>>> wrote:
>>>>>
>>>>> On 2/1/2023 1:48 PM, Jerin Jacob wrote:
>>>>>> On Wed, Feb 1, 2023 at 5:06 PM Ferruh Yigit <ferruh.yigit@amd.com>
>>>>>> wrote:
>>>>>>>
>>>>>>> On 2/1/2023 11:15 AM, Jerin Jacob wrote:
>>>>>>>> On Wed, Feb 1, 2023 at 4:35 PM Thomas Monjalon
>>>>>>>> <thomas@monjalon.net> wrote:
>>>>>>>>>
>>>>>>>>> 01/02/2023 11:58, Andrew Rybchenko:
>>>>>>>>>> On 2/1/23 13:48, Jerin Jacob wrote:
>>>>>>>>>>> On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
>>>>>>>>>>> <andrew.rybchenko@oktetlabs.ru> wrote:
>>>>>>>>>>>> Frankly speaking I don't understand why default value is so
>>>>>>>>>>>> important if we have a way to change it. Reasons should be
>>>>>>>>>>>> really strong to change existing defaults.
>>>>>>>>>>>
>>>>>>>>>>> The only reason is, typically testpmd will be used performance
>>>>>>>>>>> benchmarking as an industry standard. It is difficult to
>>>>>>>>>>> tell/educate
>>>>>>>>>>> the QA or customers
>>>>>>>>>>> that, "BTW if you need to get better performance add more flag to
>>>>>>>>>>> testpmd command line".
>>>>>>>>>
>>>>>>>>> I disagree.
>>>>>>>>> When you do performance benchmark, you tune settings accordingly.
>>>>>>>>
>>>>>>>> IMO, We tune the system resources like queue depth not the disabling
>>>>>>>> features for raw performance.
>>>>>>>> queue depth etc people know to tune so it is obvious. What is not
>>>>>>>> obvious is, testpmd only
>>>>>>>> negotiated some features by default.I am not using that feature,
>>>>>>>> hence
>>>>>>>> I need to explicitly
>>>>>>>> disable it.
>>>>>>>>
>>>>>>>
>>>>>>> When 'rte_eth_rx_metadata_negotiate()' API is NOT used at all, and I
>>>>>>> believe that is the case for almost all applications since API is a
>>>>>>> relatively new one, PMD default behavior should be to enable Rx
>>>>>>> metadata
>>>>>>> flow rules, in case user requests them later.
>>>>>>>
>>>>>>> So, enabling all in application is same with not calling the API
>>>>>>> at all.
>>>>>>>
>>>>>>> In this perspective, disabling Rx metadata is additional
>>>>>>> optimization/tuning that application can do if it is sure that Rx
>>>>>>> metadata flow rules won't be used at all.
>>>>>>> And API is more meaningful when it is used to disable Rx metadata.
>>>>>>>
>>>>>>> I think it is reasonable to enable all Rx metadata by default in
>>>>>>> testpmd
>>>>>>> with a capability to disable it when wanted.
>>>>>>>
>>>>>>> OR
>>>>>>>
>>>>>>> May be we don't call 'rte_eth_rx_metadata_negotiate()' API by
>>>>>>> default in
>>>>>>> testpmd, it is only called when it is requested explicitly from user,
>>>>>>> enable or disable.
>>>>>>
>>>>>> Second option looks good to me.
>>>>>> When
>>>>>> 1) user request for action which is needed negotiate(),
>>>>>> AND
>>>>>> 2) rte_eth_rx_metadata_negotiate() != ENOSUP
>>>>>> then, testpmd print a warning that need to enable
>>>>>> rte_eth_rx_metadata_negotiate().
>>>>>>
>>>>>
>>>>> We are not suggesting same thing.
>>>>>
>>>>> What you described above assumes PMD disabled Rx metadata flow rule
>>>>> support by default, and it needs to be enabled explicitly by
>>>>> 'rte_eth_rx_metadata_negotiate()' API. This API becomes mandatory for
>>>>> functionality.
>>>>>
>>>>> As far as I understand PMD wants to disable this flow rule by default
>>>>> because of performance concerns. But this creates inconsistency between
>>>>> PMDs, because rest of them will enable this flow rule by default (if it
>>>>> is supported) and be ready to use it when proper flow rule created.
>>>>>
>>>>> With this approach some PMDs will need
>>>>> 'rte_eth_rx_metadata_negotiate()'
>>>>> to enable Rx metadata flow rules, some won't. This can be confusing for
>>>>> applications that *some* PMDs require double enabling with specific API
>>>>> call.
>>>>>
>>>>>
>>>>> Instead what I was trying to suggest is reverse,
>>>>> all PMDs enable the Rx metadata flow rule by default, and don't require
>>>>> double enabling.
>>>>> But if application knows that it won't use Rx metadata flow rule, it
>>>>> can
>>>>> disable it to optimize the performance.
>>>>> This makes 'rte_eth_rx_metadata_negotiate()' functionally optional, and
>>>>> for testpmd context it can be called via a command on demand by user
>>>>> for
>>>>> optimization purpose.
>>>>
>>>> This won't solve concern I have outlined earlier[1].
>>>>
>>>
>>> Yes, it won't.
>>>
>>>> I think, The part of the problem there is no enough adaption of
>>>> rte_eth_rx_metadata_negotiate(),
>>>>
>>>> The view is total different from PMD maintainer PoV vs testpmd
>>>> application PoV.
>>>>
>>>
>>> Agree,
>>> and I assume it is different for user application too, which may
>>> prioritize consistency and portability.
>>>
>>> Overall, I am not fan of the 'rte_eth_rx_metadata_negotiate()' API, I
>>> think it is confusing.
>>
>> Forgive me, in which way is it confusing?
>>
>
> All other flow rules enabled by creating flow rule, but this one

Hold on.. Did you just say "flow rules"? But this API is *not* about
flow rules. I suggest you please re-read description of the API.
It reads: "Negotiate the NIC's ability to deliver specific
kinds of metadata to the PMD". Nothing about flows there.

Why is it decoupled from flow library this way? Because there is
a drastic difference between generating and plumbing metadata
INSIDE the NIC's flow engine, on the one hand, and delivering
these data from the NIC to the host driver, on the other.

Let me explain. Say, one creates the following flow rules:

(a) flow create 0 transfer group 0 pattern [A] / end \
     actions mark id XXX jump group 1 / end
(b) flow create 0 transfer group 0 pattern [B] / end \
     actions mark id YYY jump group 1 / end

(c) flow create 0 transfer group 1 pattern mark id is XXX / end \
     actions represented_port ethdev_id 1 / end
(d) flow create 0 transfer group 1 pattern mark id is YYY / end \
     actions drop / end

In this example, metadata of type "MARK" is used to partition
flow group (table) 1 into multiple lookup sections. So the
mark value is GENERATED by the flow engine and then it is
CONSUMED by this very flow engine. The application may
NOT necessarily want to receive the mark with mbufs...

And it is only when the application wants these metadata
DELIVERED to it that it has to call the negotiate API.

The short of it, nothing prevents the driver from accepting
flow create requests that leverage some meta items/actions.
Drivers do not need the negotiate API to *configure flow*.
They only need this API in order to let the application
choose whether metadata will be DELIVERED (!) or not.

> requires an API to enable it, so I believe it is inconsistent in that way.

Please see above. Everything is consistent as *flow library*
and *negotiate metadata delivery* API are totally decoupled.

>
> From application perspective, assume that it doesn't know NIC details,
> should it call this API or not? Without API call should application
> assume Rx metadata flow rules are enabled or disabled?

Frankly, applications like testpmd need never call this API.
Simply because seeing, for example, MARK values in mbufs is
useless to it. This API is needed by other applications.
For example, OvS has partial MARK+RSS offload. It adds
flows that distribute packets across multiple queues
and set some MARK values for them. OvS is interested
in getting this values with mbufs since they affect
further lookups in software... So, OvS, knowing it
wants these metadata DELIVERED (!), should invoke
this metadata negotiate API to ensure that.

>
>
> As I understand intention is to get hint from application if it will
> require Rx metadata flow rules so that PMD can optimize better, but if

No, nothing about flow rules. Just delivery of metadata with mbufs.

> PMD doesn't enable Rx metadata flow rules when this API call is missing

Again, PMD shall not make decisions on whether to enable or disable
support for some FLOW primitives based on interactions via this API.
This API exists solely to let PMD configure delivery of metadata,
i.e. not the way it is generated in the first instance.

> than it becomes a mandatory API to configure the device. But I think it
> should be optional for optimization.
>
> Also if application not sure to use this flow rule or not, it will by
> default enable all in any case, this will reduce the benefit. As done in
> testpmd.

Please see above. PMD does not need this API, I take it.

>
>
> API works in both ways, it request to enable some Rx metadata flow rule,
> but based on what PMD returns application can know what device supports,
> this also inconsistent with how other flow rules work, we don't have API
> to get capability but detect them via flow create/validate.
> Can there be a case API returns a flow rule negotiated, but it fails
> when tried to create the flow rule, isn't this confusing for application
>
>
> I think if we continue this approach there can be multiple enable and
> capability learning APIs for various flow rules or flow rule groups, and
> this can make flow API much more harder to use for applications.

See my explanations above. This API is not about flows. Period.

>
>
>
>>>
>>>> Just to avoid back and forth. We will call off this patch and remove
>>>> rte_eth_rx_metadata_negotiate()
>>>> PMD callback from cnxk driver. Keep it as old behavior, so we don't
>>>> need to care
>>>> about rte_eth_rx_metadata_negotiate().
>>>>
>>>
>>> When you remove 'rx_metadata_negotiate' callback, what will be the PMD
>>> behavior? I assume PMD will do the required preparations as if all Rx
>>> metadata is enabled.
>>> And what is the performance impact, is removing callback improve the
>>> performance?
>>>
>>>
>>>> [1]
>>>> The only reason is, typically testpmd will be used performance
>>>> benchmarking as an industry standard. It is difficult to tell/educate
>>>> the QA or customers
>>>> that, "BTW if you need to get better performance add more flag to
>>>> testpmd command line".
>>>> To make that worst, only some PMD needs to give the additional
>>>> parameter to get better number.
>>>> And also, testpmd usage will be treated as application modeling.
>>>>
>>>>>
>>>>>
>>>>>
>>>>>
>>>>>>
>>>>>>>>
>>>>>>>>>
>>>>>>>>>>> To make that worst, only some PMD needs to give the additional
>>>>>>>>>>> parameter to get better number.
>>>>>>>>>>> And also, testpmd usage will be treated as application modeling.
>>>>>>>>>>>
>>>>>>>>>>> Since this feature only used on sfc and cnxk driver, What is the
>>>>>>>>>>> situation with sfc driver?
>>>>>>>>>>> Keeping it as negotiated and not use the feature, will impact
>>>>>>>>>>> the per
>>>>>>>>>>> core performance of sfc or
>>>>>>>>>>> is it just PCI bandwidth thing which really dont show any
>>>>>>>>>>> difference in testpmd?
>>>>>>>>>>
>>>>>>>>>> Yes, sfc could run faster if no Rx metadata are negotiated. So,
>>>>>>>>>> it is better to negotiate nothing by default. But it is always
>>>>>>>>>> painful to change defaults. You need to explain that now you
>>>>>>>>>> need to negotiate Rx metadata to use mark, flag and tunnel
>>>>>>>>>> offloads.
>>>>>>>>>> Yes, it will be required on sfc and cnxk only.
>>>>>>>>>> As an sfc maintainer I don't mind to change testpmd defaults.
>>>>>>>>>
>>>>>>>>> If we change testpmd defaults to "do nothing",
>>>>>>>>> then we should disable MBUF_FAST_FREE as well.
>>>>>>>>
>>>>>>>> if you see MBUF_FAST_FREE, it does nothing. Actually,
>>>>>>>> !MBUF_FAST_FREE is doing more work.
>>>>>>>>
>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>
>>>>>
>>>
>>>
>
>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ivan Malov]

I apologise, there was a typo in the previous mail: "PMD does
not need this API". Should read as "TESTPMD does not need it".

Thank you.

On Thu, 2 Feb 2023, Ivan Malov wrote:

> On Thu, 2 Feb 2023, Ferruh Yigit wrote:
>
>> On 2/2/2023 8:50 AM, Ivan Malov wrote:
>>> On Thu, 2 Feb 2023, Ferruh Yigit wrote:
>>> 
>>>> On 2/1/2023 3:22 PM, Jerin Jacob wrote:
>>>>> On Wed, Feb 1, 2023 at 8:20 PM Ferruh Yigit <ferruh.yigit@amd.com>
>>>>> wrote:
>>>>>> 
>>>>>> On 2/1/2023 1:48 PM, Jerin Jacob wrote:
>>>>>>> On Wed, Feb 1, 2023 at 5:06 PM Ferruh Yigit <ferruh.yigit@amd.com>
>>>>>>> wrote:
>>>>>>>> 
>>>>>>>> On 2/1/2023 11:15 AM, Jerin Jacob wrote:
>>>>>>>>> On Wed, Feb 1, 2023 at 4:35 PM Thomas Monjalon
>>>>>>>>> <thomas@monjalon.net> wrote:
>>>>>>>>>> 
>>>>>>>>>> 01/02/2023 11:58, Andrew Rybchenko:
>>>>>>>>>>> On 2/1/23 13:48, Jerin Jacob wrote:
>>>>>>>>>>>> On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
>>>>>>>>>>>> <andrew.rybchenko@oktetlabs.ru> wrote:
>>>>>>>>>>>>> Frankly speaking I don't understand why default value is so
>>>>>>>>>>>>> important if we have a way to change it. Reasons should be
>>>>>>>>>>>>> really strong to change existing defaults.
>>>>>>>>>>>> 
>>>>>>>>>>>> The only reason is, typically testpmd will be used performance
>>>>>>>>>>>> benchmarking as an industry standard. It is difficult to
>>>>>>>>>>>> tell/educate
>>>>>>>>>>>> the QA or customers
>>>>>>>>>>>> that, "BTW if you need to get better performance add more flag to
>>>>>>>>>>>> testpmd command line".
>>>>>>>>>> 
>>>>>>>>>> I disagree.
>>>>>>>>>> When you do performance benchmark, you tune settings accordingly.
>>>>>>>>> 
>>>>>>>>> IMO, We tune the system resources like queue depth not the disabling
>>>>>>>>> features for raw performance.
>>>>>>>>> queue depth etc people know to tune so it is obvious. What is not
>>>>>>>>> obvious is, testpmd only
>>>>>>>>> negotiated some features by default.I am not using that feature,
>>>>>>>>> hence
>>>>>>>>> I need to explicitly
>>>>>>>>> disable it.
>>>>>>>>> 
>>>>>>>> 
>>>>>>>> When 'rte_eth_rx_metadata_negotiate()' API is NOT used at all, and I
>>>>>>>> believe that is the case for almost all applications since API is a
>>>>>>>> relatively new one, PMD default behavior should be to enable Rx
>>>>>>>> metadata
>>>>>>>> flow rules, in case user requests them later.
>>>>>>>> 
>>>>>>>> So, enabling all in application is same with not calling the API
>>>>>>>> at all.
>>>>>>>> 
>>>>>>>> In this perspective, disabling Rx metadata is additional
>>>>>>>> optimization/tuning that application can do if it is sure that Rx
>>>>>>>> metadata flow rules won't be used at all.
>>>>>>>> And API is more meaningful when it is used to disable Rx metadata.
>>>>>>>> 
>>>>>>>> I think it is reasonable to enable all Rx metadata by default in
>>>>>>>> testpmd
>>>>>>>> with a capability to disable it when wanted.
>>>>>>>> 
>>>>>>>> OR
>>>>>>>> 
>>>>>>>> May be we don't call 'rte_eth_rx_metadata_negotiate()' API by
>>>>>>>> default in
>>>>>>>> testpmd, it is only called when it is requested explicitly from user,
>>>>>>>> enable or disable.
>>>>>>> 
>>>>>>> Second option looks good to me.
>>>>>>> When
>>>>>>> 1) user request for action which is needed negotiate(),
>>>>>>> AND
>>>>>>> 2) rte_eth_rx_metadata_negotiate() != ENOSUP
>>>>>>> then, testpmd print a warning that need to enable
>>>>>>> rte_eth_rx_metadata_negotiate().
>>>>>>> 
>>>>>> 
>>>>>> We are not suggesting same thing.
>>>>>> 
>>>>>> What you described above assumes PMD disabled Rx metadata flow rule
>>>>>> support by default, and it needs to be enabled explicitly by
>>>>>> 'rte_eth_rx_metadata_negotiate()' API. This API becomes mandatory for
>>>>>> functionality.
>>>>>> 
>>>>>> As far as I understand PMD wants to disable this flow rule by default
>>>>>> because of performance concerns. But this creates inconsistency between
>>>>>> PMDs, because rest of them will enable this flow rule by default (if it
>>>>>> is supported) and be ready to use it when proper flow rule created.
>>>>>> 
>>>>>> With this approach some PMDs will need
>>>>>> 'rte_eth_rx_metadata_negotiate()'
>>>>>> to enable Rx metadata flow rules, some won't. This can be confusing for
>>>>>> applications that *some* PMDs require double enabling with specific API
>>>>>> call.
>>>>>> 
>>>>>> 
>>>>>> Instead what I was trying to suggest is reverse,
>>>>>> all PMDs enable the Rx metadata flow rule by default, and don't require
>>>>>> double enabling.
>>>>>> But if application knows that it won't use Rx metadata flow rule, it
>>>>>> can
>>>>>> disable it to optimize the performance.
>>>>>> This makes 'rte_eth_rx_metadata_negotiate()' functionally optional, and
>>>>>> for testpmd context it can be called via a command on demand by user
>>>>>> for
>>>>>> optimization purpose.
>>>>> 
>>>>> This won't solve concern I have outlined earlier[1].
>>>>> 
>>>> 
>>>> Yes, it won't.
>>>> 
>>>>> I think, The part of the problem there is no enough adaption of
>>>>> rte_eth_rx_metadata_negotiate(),
>>>>> 
>>>>> The view is total different from PMD maintainer PoV vs testpmd
>>>>> application PoV.
>>>>> 
>>>> 
>>>> Agree,
>>>> and I assume it is different for user application too, which may
>>>> prioritize consistency and portability.
>>>> 
>>>> Overall, I am not fan of the 'rte_eth_rx_metadata_negotiate()' API, I
>>>> think it is confusing.
>>> 
>>> Forgive me, in which way is it confusing?
>>> 
>> 
>> All other flow rules enabled by creating flow rule, but this one
>
> Hold on.. Did you just say "flow rules"? But this API is *not* about
> flow rules. I suggest you please re-read description of the API.
> It reads: "Negotiate the NIC's ability to deliver specific
> kinds of metadata to the PMD". Nothing about flows there.
>
> Why is it decoupled from flow library this way? Because there is
> a drastic difference between generating and plumbing metadata
> INSIDE the NIC's flow engine, on the one hand, and delivering
> these data from the NIC to the host driver, on the other.
>
> Let me explain. Say, one creates the following flow rules:
>
> (a) flow create 0 transfer group 0 pattern [A] / end \
>    actions mark id XXX jump group 1 / end
> (b) flow create 0 transfer group 0 pattern [B] / end \
>    actions mark id YYY jump group 1 / end
>
> (c) flow create 0 transfer group 1 pattern mark id is XXX / end \
>    actions represented_port ethdev_id 1 / end
> (d) flow create 0 transfer group 1 pattern mark id is YYY / end \
>    actions drop / end
>
> In this example, metadata of type "MARK" is used to partition
> flow group (table) 1 into multiple lookup sections. So the
> mark value is GENERATED by the flow engine and then it is
> CONSUMED by this very flow engine. The application may
> NOT necessarily want to receive the mark with mbufs...
>
> And it is only when the application wants these metadata
> DELIVERED to it that it has to call the negotiate API.
>
> The short of it, nothing prevents the driver from accepting
> flow create requests that leverage some meta items/actions.
> Drivers do not need the negotiate API to *configure flow*.
> They only need this API in order to let the application
> choose whether metadata will be DELIVERED (!) or not.
>
>> requires an API to enable it, so I believe it is inconsistent in that way.
>
> Please see above. Everything is consistent as *flow library*
> and *negotiate metadata delivery* API are totally decoupled.
>
>> 
>> From application perspective, assume that it doesn't know NIC details,
>> should it call this API or not? Without API call should application
>> assume Rx metadata flow rules are enabled or disabled?
>
> Frankly, applications like testpmd need never call this API.
> Simply because seeing, for example, MARK values in mbufs is
> useless to it. This API is needed by other applications.
> For example, OvS has partial MARK+RSS offload. It adds
> flows that distribute packets across multiple queues
> and set some MARK values for them. OvS is interested
> in getting this values with mbufs since they affect
> further lookups in software... So, OvS, knowing it
> wants these metadata DELIVERED (!), should invoke
> this metadata negotiate API to ensure that.
>
>> 
>> 
>> As I understand intention is to get hint from application if it will
>> require Rx metadata flow rules so that PMD can optimize better, but if
>
> No, nothing about flow rules. Just delivery of metadata with mbufs.
>
>> PMD doesn't enable Rx metadata flow rules when this API call is missing
>
> Again, PMD shall not make decisions on whether to enable or disable
> support for some FLOW primitives based on interactions via this API.
> This API exists solely to let PMD configure delivery of metadata,
> i.e. not the way it is generated in the first instance.
>
>> than it becomes a mandatory API to configure the device. But I think it
>> should be optional for optimization.
>> 
>> Also if application not sure to use this flow rule or not, it will by
>> default enable all in any case, this will reduce the benefit. As done in
>> testpmd.
>
> Please see above. PMD does not need this API, I take it.
>
>> 
>> 
>> API works in both ways, it request to enable some Rx metadata flow rule,
>> but based on what PMD returns application can know what device supports,
>> this also inconsistent with how other flow rules work, we don't have API
>> to get capability but detect them via flow create/validate.
>> Can there be a case API returns a flow rule negotiated, but it fails
>> when tried to create the flow rule, isn't this confusing for application
>> 
>> 
>> I think if we continue this approach there can be multiple enable and
>> capability learning APIs for various flow rules or flow rule groups, and
>> this can make flow API much more harder to use for applications.
>
> See my explanations above. This API is not about flows. Period.
>
>> 
>> 
>> 
>>>> 
>>>>> Just to avoid back and forth. We will call off this patch and remove
>>>>> rte_eth_rx_metadata_negotiate()
>>>>> PMD callback from cnxk driver. Keep it as old behavior, so we don't
>>>>> need to care
>>>>> about rte_eth_rx_metadata_negotiate().
>>>>> 
>>>> 
>>>> When you remove 'rx_metadata_negotiate' callback, what will be the PMD
>>>> behavior? I assume PMD will do the required preparations as if all Rx
>>>> metadata is enabled.
>>>> And what is the performance impact, is removing callback improve the
>>>> performance?
>>>> 
>>>> 
>>>>> [1]
>>>>> The only reason is, typically testpmd will be used performance
>>>>> benchmarking as an industry standard. It is difficult to tell/educate
>>>>> the QA or customers
>>>>> that, "BTW if you need to get better performance add more flag to
>>>>> testpmd command line".
>>>>> To make that worst, only some PMD needs to give the additional
>>>>> parameter to get better number.
>>>>> And also, testpmd usage will be treated as application modeling.
>>>>> 
>>>>>> 
>>>>>> 
>>>>>> 
>>>>>> 
>>>>>>> 
>>>>>>>>> 
>>>>>>>>>> 
>>>>>>>>>>>> To make that worst, only some PMD needs to give the additional
>>>>>>>>>>>> parameter to get better number.
>>>>>>>>>>>> And also, testpmd usage will be treated as application modeling.
>>>>>>>>>>>> 
>>>>>>>>>>>> Since this feature only used on sfc and cnxk driver, What is the
>>>>>>>>>>>> situation with sfc driver?
>>>>>>>>>>>> Keeping it as negotiated and not use the feature, will impact
>>>>>>>>>>>> the per
>>>>>>>>>>>> core performance of sfc or
>>>>>>>>>>>> is it just PCI bandwidth thing which really dont show any
>>>>>>>>>>>> difference in testpmd?
>>>>>>>>>>> 
>>>>>>>>>>> Yes, sfc could run faster if no Rx metadata are negotiated. So,
>>>>>>>>>>> it is better to negotiate nothing by default. But it is always
>>>>>>>>>>> painful to change defaults. You need to explain that now you
>>>>>>>>>>> need to negotiate Rx metadata to use mark, flag and tunnel
>>>>>>>>>>> offloads.
>>>>>>>>>>> Yes, it will be required on sfc and cnxk only.
>>>>>>>>>>> As an sfc maintainer I don't mind to change testpmd defaults.
>>>>>>>>>> 
>>>>>>>>>> If we change testpmd defaults to "do nothing",
>>>>>>>>>> then we should disable MBUF_FAST_FREE as well.
>>>>>>>>> 
>>>>>>>>> if you see MBUF_FAST_FREE, it does nothing. Actually,
>>>>>>>>> !MBUF_FAST_FREE is doing more work.
>>>>>>>>> 
>>>>>>>>> 
>>>>>>>>>> 
>>>>>>>>>> 
>>>>>>>> 
>>>>>> 
>>>> 
>>>> 
>> 
>> 
>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Thomas Monjalon]

OK we are progressing on this topic :)

02/02/2023 11:48, Ivan Malov:
> I apologise, there was a typo in the previous mail: "PMD does
> not need this API". Should read as "TESTPMD does not need it".

testpmd needs all ethdev API,
because its purpose is to test the whole all ethdev API.

Maybe the use of this function is misplaced in testpmd.
It should be a specific command.
By the way, what is the driver default if negotiation is not done?


> On Thu, 2 Feb 2023, Ivan Malov wrote:
> > On Thu, 2 Feb 2023, Ferruh Yigit wrote:
> >
> >> On 2/2/2023 8:50 AM, Ivan Malov wrote:
> >>> On Thu, 2 Feb 2023, Ferruh Yigit wrote:
> >>> 
> >>>> On 2/1/2023 3:22 PM, Jerin Jacob wrote:
> >>>>> On Wed, Feb 1, 2023 at 8:20 PM Ferruh Yigit <ferruh.yigit@amd.com>
> >>>>> wrote:
> >>>>>> 
> >>>>>> On 2/1/2023 1:48 PM, Jerin Jacob wrote:
> >>>>>>> On Wed, Feb 1, 2023 at 5:06 PM Ferruh Yigit <ferruh.yigit@amd.com>
> >>>>>>> wrote:
> >>>>>>>> 
> >>>>>>>> On 2/1/2023 11:15 AM, Jerin Jacob wrote:
> >>>>>>>>> On Wed, Feb 1, 2023 at 4:35 PM Thomas Monjalon
> >>>>>>>>> <thomas@monjalon.net> wrote:
> >>>>>>>>>> 
> >>>>>>>>>> 01/02/2023 11:58, Andrew Rybchenko:
> >>>>>>>>>>> On 2/1/23 13:48, Jerin Jacob wrote:
> >>>>>>>>>>>> On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
> >>>>>>>>>>>> <andrew.rybchenko@oktetlabs.ru> wrote:
> >>>>>>>>>>>>> Frankly speaking I don't understand why default value is so
> >>>>>>>>>>>>> important if we have a way to change it. Reasons should be
> >>>>>>>>>>>>> really strong to change existing defaults.
> >>>>>>>>>>>> 
> >>>>>>>>>>>> The only reason is, typically testpmd will be used performance
> >>>>>>>>>>>> benchmarking as an industry standard. It is difficult to
> >>>>>>>>>>>> tell/educate
> >>>>>>>>>>>> the QA or customers
> >>>>>>>>>>>> that, "BTW if you need to get better performance add more flag to
> >>>>>>>>>>>> testpmd command line".
> >>>>>>>>>> 
> >>>>>>>>>> I disagree.
> >>>>>>>>>> When you do performance benchmark, you tune settings accordingly.
> >>>>>>>>> 
> >>>>>>>>> IMO, We tune the system resources like queue depth not the disabling
> >>>>>>>>> features for raw performance.
> >>>>>>>>> queue depth etc people know to tune so it is obvious. What is not
> >>>>>>>>> obvious is, testpmd only
> >>>>>>>>> negotiated some features by default.I am not using that feature,
> >>>>>>>>> hence
> >>>>>>>>> I need to explicitly
> >>>>>>>>> disable it.
> >>>>>>>>> 
> >>>>>>>> 
> >>>>>>>> When 'rte_eth_rx_metadata_negotiate()' API is NOT used at all, and I
> >>>>>>>> believe that is the case for almost all applications since API is a
> >>>>>>>> relatively new one, PMD default behavior should be to enable Rx
> >>>>>>>> metadata
> >>>>>>>> flow rules, in case user requests them later.
> >>>>>>>> 
> >>>>>>>> So, enabling all in application is same with not calling the API
> >>>>>>>> at all.
> >>>>>>>> 
> >>>>>>>> In this perspective, disabling Rx metadata is additional
> >>>>>>>> optimization/tuning that application can do if it is sure that Rx
> >>>>>>>> metadata flow rules won't be used at all.
> >>>>>>>> And API is more meaningful when it is used to disable Rx metadata.
> >>>>>>>> 
> >>>>>>>> I think it is reasonable to enable all Rx metadata by default in
> >>>>>>>> testpmd
> >>>>>>>> with a capability to disable it when wanted.
> >>>>>>>> 
> >>>>>>>> OR
> >>>>>>>> 
> >>>>>>>> May be we don't call 'rte_eth_rx_metadata_negotiate()' API by
> >>>>>>>> default in
> >>>>>>>> testpmd, it is only called when it is requested explicitly from user,
> >>>>>>>> enable or disable.
> >>>>>>> 
> >>>>>>> Second option looks good to me.
> >>>>>>> When
> >>>>>>> 1) user request for action which is needed negotiate(),
> >>>>>>> AND
> >>>>>>> 2) rte_eth_rx_metadata_negotiate() != ENOSUP
> >>>>>>> then, testpmd print a warning that need to enable
> >>>>>>> rte_eth_rx_metadata_negotiate().
> >>>>>>> 
> >>>>>> 
> >>>>>> We are not suggesting same thing.
> >>>>>> 
> >>>>>> What you described above assumes PMD disabled Rx metadata flow rule
> >>>>>> support by default, and it needs to be enabled explicitly by
> >>>>>> 'rte_eth_rx_metadata_negotiate()' API. This API becomes mandatory for
> >>>>>> functionality.
> >>>>>> 
> >>>>>> As far as I understand PMD wants to disable this flow rule by default
> >>>>>> because of performance concerns. But this creates inconsistency between
> >>>>>> PMDs, because rest of them will enable this flow rule by default (if it
> >>>>>> is supported) and be ready to use it when proper flow rule created.
> >>>>>> 
> >>>>>> With this approach some PMDs will need
> >>>>>> 'rte_eth_rx_metadata_negotiate()'
> >>>>>> to enable Rx metadata flow rules, some won't. This can be confusing for
> >>>>>> applications that *some* PMDs require double enabling with specific API
> >>>>>> call.
> >>>>>> 
> >>>>>> 
> >>>>>> Instead what I was trying to suggest is reverse,
> >>>>>> all PMDs enable the Rx metadata flow rule by default, and don't require
> >>>>>> double enabling.
> >>>>>> But if application knows that it won't use Rx metadata flow rule, it
> >>>>>> can
> >>>>>> disable it to optimize the performance.
> >>>>>> This makes 'rte_eth_rx_metadata_negotiate()' functionally optional, and
> >>>>>> for testpmd context it can be called via a command on demand by user
> >>>>>> for
> >>>>>> optimization purpose.
> >>>>> 
> >>>>> This won't solve concern I have outlined earlier[1].
> >>>>> 
> >>>> 
> >>>> Yes, it won't.
> >>>> 
> >>>>> I think, The part of the problem there is no enough adaption of
> >>>>> rte_eth_rx_metadata_negotiate(),
> >>>>> 
> >>>>> The view is total different from PMD maintainer PoV vs testpmd
> >>>>> application PoV.
> >>>>> 
> >>>> 
> >>>> Agree,
> >>>> and I assume it is different for user application too, which may
> >>>> prioritize consistency and portability.
> >>>> 
> >>>> Overall, I am not fan of the 'rte_eth_rx_metadata_negotiate()' API, I
> >>>> think it is confusing.
> >>> 
> >>> Forgive me, in which way is it confusing?
> >>> 
> >> 
> >> All other flow rules enabled by creating flow rule, but this one
> >
> > Hold on.. Did you just say "flow rules"? But this API is *not* about
> > flow rules. I suggest you please re-read description of the API.
> > It reads: "Negotiate the NIC's ability to deliver specific
> > kinds of metadata to the PMD". Nothing about flows there.
> >
> > Why is it decoupled from flow library this way? Because there is
> > a drastic difference between generating and plumbing metadata
> > INSIDE the NIC's flow engine, on the one hand, and delivering
> > these data from the NIC to the host driver, on the other.
> >
> > Let me explain. Say, one creates the following flow rules:
> >
> > (a) flow create 0 transfer group 0 pattern [A] / end \
> >    actions mark id XXX jump group 1 / end
> > (b) flow create 0 transfer group 0 pattern [B] / end \
> >    actions mark id YYY jump group 1 / end
> >
> > (c) flow create 0 transfer group 1 pattern mark id is XXX / end \
> >    actions represented_port ethdev_id 1 / end
> > (d) flow create 0 transfer group 1 pattern mark id is YYY / end \
> >    actions drop / end
> >
> > In this example, metadata of type "MARK" is used to partition
> > flow group (table) 1 into multiple lookup sections. So the
> > mark value is GENERATED by the flow engine and then it is
> > CONSUMED by this very flow engine. The application may
> > NOT necessarily want to receive the mark with mbufs...
> >
> > And it is only when the application wants these metadata
> > DELIVERED to it that it has to call the negotiate API.
> >
> > The short of it, nothing prevents the driver from accepting
> > flow create requests that leverage some meta items/actions.
> > Drivers do not need the negotiate API to *configure flow*.
> > They only need this API in order to let the application
> > choose whether metadata will be DELIVERED (!) or not.
> >
> >> requires an API to enable it, so I believe it is inconsistent in that way.
> >
> > Please see above. Everything is consistent as *flow library*
> > and *negotiate metadata delivery* API are totally decoupled.
> >
> >> 
> >> From application perspective, assume that it doesn't know NIC details,
> >> should it call this API or not? Without API call should application
> >> assume Rx metadata flow rules are enabled or disabled?
> >
> > Frankly, applications like testpmd need never call this API.
> > Simply because seeing, for example, MARK values in mbufs is
> > useless to it. This API is needed by other applications.
> > For example, OvS has partial MARK+RSS offload. It adds
> > flows that distribute packets across multiple queues
> > and set some MARK values for them. OvS is interested
> > in getting this values with mbufs since they affect
> > further lookups in software... So, OvS, knowing it
> > wants these metadata DELIVERED (!), should invoke
> > this metadata negotiate API to ensure that.
> >
> >> 
> >> 
> >> As I understand intention is to get hint from application if it will
> >> require Rx metadata flow rules so that PMD can optimize better, but if
> >
> > No, nothing about flow rules. Just delivery of metadata with mbufs.
> >
> >> PMD doesn't enable Rx metadata flow rules when this API call is missing
> >
> > Again, PMD shall not make decisions on whether to enable or disable
> > support for some FLOW primitives based on interactions via this API.
> > This API exists solely to let PMD configure delivery of metadata,
> > i.e. not the way it is generated in the first instance.
> >
> >> than it becomes a mandatory API to configure the device. But I think it
> >> should be optional for optimization.
> >> 
> >> Also if application not sure to use this flow rule or not, it will by
> >> default enable all in any case, this will reduce the benefit. As done in
> >> testpmd.
> >
> > Please see above. PMD does not need this API, I take it.
> >
> >> 
> >> 
> >> API works in both ways, it request to enable some Rx metadata flow rule,
> >> but based on what PMD returns application can know what device supports,
> >> this also inconsistent with how other flow rules work, we don't have API
> >> to get capability but detect them via flow create/validate.
> >> Can there be a case API returns a flow rule negotiated, but it fails
> >> when tried to create the flow rule, isn't this confusing for application
> >> 
> >> 
> >> I think if we continue this approach there can be multiple enable and
> >> capability learning APIs for various flow rules or flow rule groups, and
> >> this can make flow API much more harder to use for applications.
> >
> > See my explanations above. This API is not about flows. Period.
> >
> >> 
> >> 
> >> 
> >>>> 
> >>>>> Just to avoid back and forth. We will call off this patch and remove
> >>>>> rte_eth_rx_metadata_negotiate()
> >>>>> PMD callback from cnxk driver. Keep it as old behavior, so we don't
> >>>>> need to care
> >>>>> about rte_eth_rx_metadata_negotiate().
> >>>>> 
> >>>> 
> >>>> When you remove 'rx_metadata_negotiate' callback, what will be the PMD
> >>>> behavior? I assume PMD will do the required preparations as if all Rx
> >>>> metadata is enabled.
> >>>> And what is the performance impact, is removing callback improve the
> >>>> performance?
> >>>> 
> >>>> 
> >>>>> [1]
> >>>>> The only reason is, typically testpmd will be used performance
> >>>>> benchmarking as an industry standard. It is difficult to tell/educate
> >>>>> the QA or customers
> >>>>> that, "BTW if you need to get better performance add more flag to
> >>>>> testpmd command line".
> >>>>> To make that worst, only some PMD needs to give the additional
> >>>>> parameter to get better number.
> >>>>> And also, testpmd usage will be treated as application modeling.
> >>>>> 
> >>>>>> 
> >>>>>> 
> >>>>>> 
> >>>>>> 
> >>>>>>> 
> >>>>>>>>> 
> >>>>>>>>>> 
> >>>>>>>>>>>> To make that worst, only some PMD needs to give the additional
> >>>>>>>>>>>> parameter to get better number.
> >>>>>>>>>>>> And also, testpmd usage will be treated as application modeling.
> >>>>>>>>>>>> 
> >>>>>>>>>>>> Since this feature only used on sfc and cnxk driver, What is the
> >>>>>>>>>>>> situation with sfc driver?
> >>>>>>>>>>>> Keeping it as negotiated and not use the feature, will impact
> >>>>>>>>>>>> the per
> >>>>>>>>>>>> core performance of sfc or
> >>>>>>>>>>>> is it just PCI bandwidth thing which really dont show any
> >>>>>>>>>>>> difference in testpmd?
> >>>>>>>>>>> 
> >>>>>>>>>>> Yes, sfc could run faster if no Rx metadata are negotiated. So,
> >>>>>>>>>>> it is better to negotiate nothing by default. But it is always
> >>>>>>>>>>> painful to change defaults. You need to explain that now you
> >>>>>>>>>>> need to negotiate Rx metadata to use mark, flag and tunnel
> >>>>>>>>>>> offloads.
> >>>>>>>>>>> Yes, it will be required on sfc and cnxk only.
> >>>>>>>>>>> As an sfc maintainer I don't mind to change testpmd defaults.
> >>>>>>>>>> 
> >>>>>>>>>> If we change testpmd defaults to "do nothing",
> >>>>>>>>>> then we should disable MBUF_FAST_FREE as well.
> >>>>>>>>> 
> >>>>>>>>> if you see MBUF_FAST_FREE, it does nothing. Actually,
> >>>>>>>>> !MBUF_FAST_FREE is doing more work.
> >>>>>>>>> 
> >>>>>>>>> 
> >>>>>>>>>> 
> >>>>>>>>>> 
> >>>>>>>> 
> >>>>>> 
> >>>> 
> >>>> 
> >> 
> >> 
> >
> 

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ivan Malov]

[-- Attachment #1: Type: text/plain, Size: 13671 bytes --]

On Thu, 2 Feb 2023, Thomas Monjalon wrote:

> OK we are progressing on this topic :)

Indeed we are.

>
> 02/02/2023 11:48, Ivan Malov:
>> I apologise, there was a typo in the previous mail: "PMD does
>> not need this API". Should read as "TESTPMD does not need it".
>
> testpmd needs all ethdev API,
> because its purpose is to test the whole all ethdev API.

Touché.

>
> Maybe the use of this function is misplaced in testpmd.
> It should be a specific command.

So.. indeed. A specific command which, as I said, is
invoked explicitly by the person operating testpmd.

> By the way, what is the driver default if negotiation is not done?

The answer is in the question. It's the driver's default.
If the driver believes it shall NOT deliver metadata for
the sake of improved performance, default = all disabled.
If this delivery is a don't care to performance, then
the driver might want to enable everything by default.

This decision is made by a PMD maintainer, I take it.

>
>
>> On Thu, 2 Feb 2023, Ivan Malov wrote:
>>> On Thu, 2 Feb 2023, Ferruh Yigit wrote:
>>>
>>>> On 2/2/2023 8:50 AM, Ivan Malov wrote:
>>>>> On Thu, 2 Feb 2023, Ferruh Yigit wrote:
>>>>>
>>>>>> On 2/1/2023 3:22 PM, Jerin Jacob wrote:
>>>>>>> On Wed, Feb 1, 2023 at 8:20 PM Ferruh Yigit <ferruh.yigit@amd.com>
>>>>>>> wrote:
>>>>>>>>
>>>>>>>> On 2/1/2023 1:48 PM, Jerin Jacob wrote:
>>>>>>>>> On Wed, Feb 1, 2023 at 5:06 PM Ferruh Yigit <ferruh.yigit@amd.com>
>>>>>>>>> wrote:
>>>>>>>>>>
>>>>>>>>>> On 2/1/2023 11:15 AM, Jerin Jacob wrote:
>>>>>>>>>>> On Wed, Feb 1, 2023 at 4:35 PM Thomas Monjalon
>>>>>>>>>>> <thomas@monjalon.net> wrote:
>>>>>>>>>>>>
>>>>>>>>>>>> 01/02/2023 11:58, Andrew Rybchenko:
>>>>>>>>>>>>> On 2/1/23 13:48, Jerin Jacob wrote:
>>>>>>>>>>>>>> On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
>>>>>>>>>>>>>> <andrew.rybchenko@oktetlabs.ru> wrote:
>>>>>>>>>>>>>>> Frankly speaking I don't understand why default value is so
>>>>>>>>>>>>>>> important if we have a way to change it. Reasons should be
>>>>>>>>>>>>>>> really strong to change existing defaults.
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> The only reason is, typically testpmd will be used performance
>>>>>>>>>>>>>> benchmarking as an industry standard. It is difficult to
>>>>>>>>>>>>>> tell/educate
>>>>>>>>>>>>>> the QA or customers
>>>>>>>>>>>>>> that, "BTW if you need to get better performance add more flag to
>>>>>>>>>>>>>> testpmd command line".
>>>>>>>>>>>>
>>>>>>>>>>>> I disagree.
>>>>>>>>>>>> When you do performance benchmark, you tune settings accordingly.
>>>>>>>>>>>
>>>>>>>>>>> IMO, We tune the system resources like queue depth not the disabling
>>>>>>>>>>> features for raw performance.
>>>>>>>>>>> queue depth etc people know to tune so it is obvious. What is not
>>>>>>>>>>> obvious is, testpmd only
>>>>>>>>>>> negotiated some features by default.I am not using that feature,
>>>>>>>>>>> hence
>>>>>>>>>>> I need to explicitly
>>>>>>>>>>> disable it.
>>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>> When 'rte_eth_rx_metadata_negotiate()' API is NOT used at all, and I
>>>>>>>>>> believe that is the case for almost all applications since API is a
>>>>>>>>>> relatively new one, PMD default behavior should be to enable Rx
>>>>>>>>>> metadata
>>>>>>>>>> flow rules, in case user requests them later.
>>>>>>>>>>
>>>>>>>>>> So, enabling all in application is same with not calling the API
>>>>>>>>>> at all.
>>>>>>>>>>
>>>>>>>>>> In this perspective, disabling Rx metadata is additional
>>>>>>>>>> optimization/tuning that application can do if it is sure that Rx
>>>>>>>>>> metadata flow rules won't be used at all.
>>>>>>>>>> And API is more meaningful when it is used to disable Rx metadata.
>>>>>>>>>>
>>>>>>>>>> I think it is reasonable to enable all Rx metadata by default in
>>>>>>>>>> testpmd
>>>>>>>>>> with a capability to disable it when wanted.
>>>>>>>>>>
>>>>>>>>>> OR
>>>>>>>>>>
>>>>>>>>>> May be we don't call 'rte_eth_rx_metadata_negotiate()' API by
>>>>>>>>>> default in
>>>>>>>>>> testpmd, it is only called when it is requested explicitly from user,
>>>>>>>>>> enable or disable.
>>>>>>>>>
>>>>>>>>> Second option looks good to me.
>>>>>>>>> When
>>>>>>>>> 1) user request for action which is needed negotiate(),
>>>>>>>>> AND
>>>>>>>>> 2) rte_eth_rx_metadata_negotiate() != ENOSUP
>>>>>>>>> then, testpmd print a warning that need to enable
>>>>>>>>> rte_eth_rx_metadata_negotiate().
>>>>>>>>>
>>>>>>>>
>>>>>>>> We are not suggesting same thing.
>>>>>>>>
>>>>>>>> What you described above assumes PMD disabled Rx metadata flow rule
>>>>>>>> support by default, and it needs to be enabled explicitly by
>>>>>>>> 'rte_eth_rx_metadata_negotiate()' API. This API becomes mandatory for
>>>>>>>> functionality.
>>>>>>>>
>>>>>>>> As far as I understand PMD wants to disable this flow rule by default
>>>>>>>> because of performance concerns. But this creates inconsistency between
>>>>>>>> PMDs, because rest of them will enable this flow rule by default (if it
>>>>>>>> is supported) and be ready to use it when proper flow rule created.
>>>>>>>>
>>>>>>>> With this approach some PMDs will need
>>>>>>>> 'rte_eth_rx_metadata_negotiate()'
>>>>>>>> to enable Rx metadata flow rules, some won't. This can be confusing for
>>>>>>>> applications that *some* PMDs require double enabling with specific API
>>>>>>>> call.
>>>>>>>>
>>>>>>>>
>>>>>>>> Instead what I was trying to suggest is reverse,
>>>>>>>> all PMDs enable the Rx metadata flow rule by default, and don't require
>>>>>>>> double enabling.
>>>>>>>> But if application knows that it won't use Rx metadata flow rule, it
>>>>>>>> can
>>>>>>>> disable it to optimize the performance.
>>>>>>>> This makes 'rte_eth_rx_metadata_negotiate()' functionally optional, and
>>>>>>>> for testpmd context it can be called via a command on demand by user
>>>>>>>> for
>>>>>>>> optimization purpose.
>>>>>>>
>>>>>>> This won't solve concern I have outlined earlier[1].
>>>>>>>
>>>>>>
>>>>>> Yes, it won't.
>>>>>>
>>>>>>> I think, The part of the problem there is no enough adaption of
>>>>>>> rte_eth_rx_metadata_negotiate(),
>>>>>>>
>>>>>>> The view is total different from PMD maintainer PoV vs testpmd
>>>>>>> application PoV.
>>>>>>>
>>>>>>
>>>>>> Agree,
>>>>>> and I assume it is different for user application too, which may
>>>>>> prioritize consistency and portability.
>>>>>>
>>>>>> Overall, I am not fan of the 'rte_eth_rx_metadata_negotiate()' API, I
>>>>>> think it is confusing.
>>>>>
>>>>> Forgive me, in which way is it confusing?
>>>>>
>>>>
>>>> All other flow rules enabled by creating flow rule, but this one
>>>
>>> Hold on.. Did you just say "flow rules"? But this API is *not* about
>>> flow rules. I suggest you please re-read description of the API.
>>> It reads: "Negotiate the NIC's ability to deliver specific
>>> kinds of metadata to the PMD". Nothing about flows there.
>>>
>>> Why is it decoupled from flow library this way? Because there is
>>> a drastic difference between generating and plumbing metadata
>>> INSIDE the NIC's flow engine, on the one hand, and delivering
>>> these data from the NIC to the host driver, on the other.
>>>
>>> Let me explain. Say, one creates the following flow rules:
>>>
>>> (a) flow create 0 transfer group 0 pattern [A] / end \
>>>    actions mark id XXX jump group 1 / end
>>> (b) flow create 0 transfer group 0 pattern [B] / end \
>>>    actions mark id YYY jump group 1 / end
>>>
>>> (c) flow create 0 transfer group 1 pattern mark id is XXX / end \
>>>    actions represented_port ethdev_id 1 / end
>>> (d) flow create 0 transfer group 1 pattern mark id is YYY / end \
>>>    actions drop / end
>>>
>>> In this example, metadata of type "MARK" is used to partition
>>> flow group (table) 1 into multiple lookup sections. So the
>>> mark value is GENERATED by the flow engine and then it is
>>> CONSUMED by this very flow engine. The application may
>>> NOT necessarily want to receive the mark with mbufs...
>>>
>>> And it is only when the application wants these metadata
>>> DELIVERED to it that it has to call the negotiate API.
>>>
>>> The short of it, nothing prevents the driver from accepting
>>> flow create requests that leverage some meta items/actions.
>>> Drivers do not need the negotiate API to *configure flow*.
>>> They only need this API in order to let the application
>>> choose whether metadata will be DELIVERED (!) or not.
>>>
>>>> requires an API to enable it, so I believe it is inconsistent in that way.
>>>
>>> Please see above. Everything is consistent as *flow library*
>>> and *negotiate metadata delivery* API are totally decoupled.
>>>
>>>>
>>>> From application perspective, assume that it doesn't know NIC details,
>>>> should it call this API or not? Without API call should application
>>>> assume Rx metadata flow rules are enabled or disabled?
>>>
>>> Frankly, applications like testpmd need never call this API.
>>> Simply because seeing, for example, MARK values in mbufs is
>>> useless to it. This API is needed by other applications.
>>> For example, OvS has partial MARK+RSS offload. It adds
>>> flows that distribute packets across multiple queues
>>> and set some MARK values for them. OvS is interested
>>> in getting this values with mbufs since they affect
>>> further lookups in software... So, OvS, knowing it
>>> wants these metadata DELIVERED (!), should invoke
>>> this metadata negotiate API to ensure that.
>>>
>>>>
>>>>
>>>> As I understand intention is to get hint from application if it will
>>>> require Rx metadata flow rules so that PMD can optimize better, but if
>>>
>>> No, nothing about flow rules. Just delivery of metadata with mbufs.
>>>
>>>> PMD doesn't enable Rx metadata flow rules when this API call is missing
>>>
>>> Again, PMD shall not make decisions on whether to enable or disable
>>> support for some FLOW primitives based on interactions via this API.
>>> This API exists solely to let PMD configure delivery of metadata,
>>> i.e. not the way it is generated in the first instance.
>>>
>>>> than it becomes a mandatory API to configure the device. But I think it
>>>> should be optional for optimization.
>>>>
>>>> Also if application not sure to use this flow rule or not, it will by
>>>> default enable all in any case, this will reduce the benefit. As done in
>>>> testpmd.
>>>
>>> Please see above. PMD does not need this API, I take it.
>>>
>>>>
>>>>
>>>> API works in both ways, it request to enable some Rx metadata flow rule,
>>>> but based on what PMD returns application can know what device supports,
>>>> this also inconsistent with how other flow rules work, we don't have API
>>>> to get capability but detect them via flow create/validate.
>>>> Can there be a case API returns a flow rule negotiated, but it fails
>>>> when tried to create the flow rule, isn't this confusing for application
>>>>
>>>>
>>>> I think if we continue this approach there can be multiple enable and
>>>> capability learning APIs for various flow rules or flow rule groups, and
>>>> this can make flow API much more harder to use for applications.
>>>
>>> See my explanations above. This API is not about flows. Period.
>>>
>>>>
>>>>
>>>>
>>>>>>
>>>>>>> Just to avoid back and forth. We will call off this patch and remove
>>>>>>> rte_eth_rx_metadata_negotiate()
>>>>>>> PMD callback from cnxk driver. Keep it as old behavior, so we don't
>>>>>>> need to care
>>>>>>> about rte_eth_rx_metadata_negotiate().
>>>>>>>
>>>>>>
>>>>>> When you remove 'rx_metadata_negotiate' callback, what will be the PMD
>>>>>> behavior? I assume PMD will do the required preparations as if all Rx
>>>>>> metadata is enabled.
>>>>>> And what is the performance impact, is removing callback improve the
>>>>>> performance?
>>>>>>
>>>>>>
>>>>>>> [1]
>>>>>>> The only reason is, typically testpmd will be used performance
>>>>>>> benchmarking as an industry standard. It is difficult to tell/educate
>>>>>>> the QA or customers
>>>>>>> that, "BTW if you need to get better performance add more flag to
>>>>>>> testpmd command line".
>>>>>>> To make that worst, only some PMD needs to give the additional
>>>>>>> parameter to get better number.
>>>>>>> And also, testpmd usage will be treated as application modeling.
>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>>>
>>>>>>>>>>>>>> To make that worst, only some PMD needs to give the additional
>>>>>>>>>>>>>> parameter to get better number.
>>>>>>>>>>>>>> And also, testpmd usage will be treated as application modeling.
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> Since this feature only used on sfc and cnxk driver, What is the
>>>>>>>>>>>>>> situation with sfc driver?
>>>>>>>>>>>>>> Keeping it as negotiated and not use the feature, will impact
>>>>>>>>>>>>>> the per
>>>>>>>>>>>>>> core performance of sfc or
>>>>>>>>>>>>>> is it just PCI bandwidth thing which really dont show any
>>>>>>>>>>>>>> difference in testpmd?
>>>>>>>>>>>>>
>>>>>>>>>>>>> Yes, sfc could run faster if no Rx metadata are negotiated. So,
>>>>>>>>>>>>> it is better to negotiate nothing by default. But it is always
>>>>>>>>>>>>> painful to change defaults. You need to explain that now you
>>>>>>>>>>>>> need to negotiate Rx metadata to use mark, flag and tunnel
>>>>>>>>>>>>> offloads.
>>>>>>>>>>>>> Yes, it will be required on sfc and cnxk only.
>>>>>>>>>>>>> As an sfc maintainer I don't mind to change testpmd defaults.
>>>>>>>>>>>>
>>>>>>>>>>>> If we change testpmd defaults to "do nothing",
>>>>>>>>>>>> then we should disable MBUF_FAST_FREE as well.
>>>>>>>>>>>
>>>>>>>>>>> if you see MBUF_FAST_FREE, it does nothing. Actually,
>>>>>>>>>>> !MBUF_FAST_FREE is doing more work.
>>>>>>>>>>>
>>>>>>>>>>>
>>>>>>>>>>>>
>>>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>
>>>>>>
>>>>>>
>>>>
>>>>
>>>
>>
>
>
>
>
>
>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Thomas Monjalon]

02/02/2023 12:55, Ivan Malov:
> On Thu, 2 Feb 2023, Thomas Monjalon wrote:
> 
> > OK we are progressing on this topic :)
> 
> Indeed we are.
> 
> >
> > 02/02/2023 11:48, Ivan Malov:
> >> I apologise, there was a typo in the previous mail: "PMD does
> >> not need this API". Should read as "TESTPMD does not need it".
> >
> > testpmd needs all ethdev API,
> > because its purpose is to test the whole all ethdev API.
> 
> Touché.
> 
> >
> > Maybe the use of this function is misplaced in testpmd.
> > It should be a specific command.
> 
> So.. indeed. A specific command which, as I said, is
> invoked explicitly by the person operating testpmd.
> 
> > By the way, what is the driver default if negotiation is not done?
> 
> The answer is in the question. It's the driver's default.
> If the driver believes it shall NOT deliver metadata for
> the sake of improved performance, default = all disabled.
> If this delivery is a don't care to performance, then
> the driver might want to enable everything by default.
> 
> This decision is made by a PMD maintainer, I take it.

I think it is better to have the same default for all drivers.
What others think?

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Andrew Rybchenko]

On 2/2/23 15:03, Thomas Monjalon wrote:
> 02/02/2023 12:55, Ivan Malov:
>> On Thu, 2 Feb 2023, Thomas Monjalon wrote:
>>
>>> OK we are progressing on this topic :)
>>
>> Indeed we are.
>>
>>>
>>> 02/02/2023 11:48, Ivan Malov:
>>>> I apologise, there was a typo in the previous mail: "PMD does
>>>> not need this API". Should read as "TESTPMD does not need it".
>>>
>>> testpmd needs all ethdev API,
>>> because its purpose is to test the whole all ethdev API.
>>
>> Touché.
>>
>>>
>>> Maybe the use of this function is misplaced in testpmd.
>>> It should be a specific command.
>>
>> So.. indeed. A specific command which, as I said, is
>> invoked explicitly by the person operating testpmd.
>>
>>> By the way, what is the driver default if negotiation is not done?
>>
>> The answer is in the question. It's the driver's default.
>> If the driver believes it shall NOT deliver metadata for
>> the sake of improved performance, default = all disabled.
>> If this delivery is a don't care to performance, then
>> the driver might want to enable everything by default.
>>
>> This decision is made by a PMD maintainer, I take it.
> 
> I think it is better to have the same default for all drivers.
> What others think?

Of course, it would be better since it would simplify
porting apps from one NIC to another.
The problem is that some NICs do not require negotiation
since corresponding data is always available.
If so, the same default should be to provide all Rx metadata
by default. However, it would decrease performance for NIC
NICs with default settings.

(Similar situation is with Rx checksum offload. Some NICs
always calculate Rx checksum and always provide results,
however, it is not a good reason to require it for all NICs.)

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ivan Malov]

Hi Thomas,

On Wed, 1 Feb 2023, Thomas Monjalon wrote:

> 01/02/2023 11:58, Andrew Rybchenko:
>> On 2/1/23 13:48, Jerin Jacob wrote:
>>> On Wed, Feb 1, 2023 at 2:59 PM Andrew Rybchenko
>>> <andrew.rybchenko@oktetlabs.ru> wrote:
>>>> Frankly speaking I don't understand why default value is so
>>>> important if we have a way to change it. Reasons should be
>>>> really strong to change existing defaults.
>>>
>>> The only reason is, typically testpmd will be used performance
>>> benchmarking as an industry standard. It is difficult to tell/educate
>>> the QA or customers
>>> that, "BTW if you need to get better performance add more flag to
>>> testpmd command line".
>
> I disagree.
> When you do performance benchmark, you tune settings accordingly.
>
>>> To make that worst, only some PMD needs to give the additional
>>> parameter to get better number.
>>> And also, testpmd usage will be treated as application modeling.
>>>
>>> Since this feature only used on sfc and cnxk driver, What is the
>>> situation with sfc driver?
>>> Keeping it as negotiated and not use the feature, will impact the per
>>> core performance of sfc or
>>> is it just PCI bandwidth thing which really dont show any difference in testpmd?
>>
>> Yes, sfc could run faster if no Rx metadata are negotiated. So,
>> it is better to negotiate nothing by default. But it is always
>> painful to change defaults. You need to explain that now you
>> need to negotiate Rx metadata to use mark, flag and tunnel offloads.
>> Yes, it will be required on sfc and cnxk only.
>> As an sfc maintainer I don't mind to change testpmd defaults.
>
> If we change testpmd defaults to "do nothing",
> then we should disable MBUF_FAST_FREE as well.

No. These are completely different. MBUF_FAST_FREE is an
optimisation technique, it's in the offload namespace.
Whilst "negotiate metadata" does not offload anything.
It just tells the PMD to enable *delivery* of some
data *when it is present*. And whether it will be
present or not, - this is decided by flow actions.
And, flow actions, in turn, belong in the domain
of decisions made by a specific person operating
the application. So no need to remove FAST_FREE.

>
>
>

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ferruh Yigit]

On 1/25/2023 9:30 AM, Hanumanth Reddy Pothula wrote:
> ++ Ivan Malov and Andrew Rybchenko
> 
>> -----Original Message-----
>> From: Ferruh Yigit <ferruh.yigit@amd.com>
>> Sent: Tuesday, January 24, 2023 11:34 PM
>> To: Hanumanth Reddy Pothula <hpothula@marvell.com>; Aman Singh
>> <aman.deep.singh@intel.com>; Yuying Zhang <yuying.zhang@intel.com>
>> Cc: dev@dpdk.org; andrew.rybchenko@oktetlabs.ru;
>> viacheslavo@nvidia.com; Jerin Jacob Kollanukkaran <jerinj@marvell.com>;
>> Nithin Kumar Dabilpuram <ndabilpuram@marvell.com>
>> Subject: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process
>> Rx metadata negotiation
>>
>> External Email
>>
>> ----------------------------------------------------------------------
>> On 12/21/2022 2:07 AM, Hanumanth Pothula wrote:
>>> Presently, Rx metadata is sent to PMD by default, leading to a
>>> performance drop as processing for the same in Rx path takes extra
>>> cycles.
>>>
>>> Hence, add new testpmd command,
>>>   'enable port <port_id> nic_to_pmd_rx_metadata'
>>>
>>> This command helps in sending Rx metadata to PMD and thereby Rx
>>> metadata flow command requests are processed.
>>>
>>> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
>>
>> Hi Hanumanth,
>>
>> I agree with Thomas for the patch.
>>
>> 'eth_rx_metadata_negotiate_mp()' requests all Rx metadata offloads to be
>> enabled, but at this stage if there is no flow rule for Rx metadata why it is
>> consuming extra cycles?
>>
>> Can you update driver code to process Rx metadata when it is enabled by
>> application (via 'rte_eth_rx_metadata_negotiate()') AND there is at least
>> one flow rule for it?
> 
> #1 What is the purpose of rte_eth_rx_metadata_negotiate() API if it is always called by testpmd.
> We thought it was added so that when that metadata is not needed, application need not call this
> thereby saving cycles/bandwidth.
> 

Purpose looks like to 'negotiate', so it is both ways:
a) To learn Rx Meta capability of HW
b) To configure Rx Meta feature for HW

So it is both to help application to learn what flow rule is supported
and to help driver to improve performance.

Before this API application assumed all Rx metadata flow rules are
supported, so why enabling all causing performance drop, that was the
default without this API.

But other-way around can be true, to disable some offloads can improve
driver performance.

Looking again,


> #2 We use this API similar to Rx/Tx offload flags so that we can set things up before device is
> configured. We thought that is the purpose of having this negotiate API and avoid depleting offload flags.
> 
> #3 Generally any new offloads added to DPDK would be in disabled state in testpmd and we would have
> an option to enable it. In this case, testpmd is by default calling this negotiation.
> 
> We can update the driver if the purpose of this API is clear.


Hi Hanumanth,

After looking the history of the API again, you may be right.

One of the previous version commit log describes the intention better
[1], because of negative performance impact of enabling Rx metadata
offload by default and difficulty to switch configuration dynamically,
there is a desire to learn application intention before configuration.

Although I have some concerns with this API [2] it is already there as
stable API.

So it sounds reasonable to make this configurable for a test
application, indeed intention of the API is to get this configuration
from application and operate based on it.

Next question is what should be the default value, I am not sure about
it, there are only a few drivers impacted from this overall.
For the Thomas' point, it helps to test the feature of its impact if it
is enabled by default.

Will it work to enable them all by default and add capability to disable
it in testpmd, which helps to run performance tests also to verify the
impact of the API?

Thanks,
ferruh



[1]
https://inbox.dpdk.org/dev/20210902142359.28138-2-ivan.malov@oktetlabs.ru/


[2]
API does two things:
a) Learn Rx Meta capability of HW
b) Configure Rx Meta feature for HW

Functionality (a) conflicts with rest of the flow rules that capability
checked via `rte_flow_validate()` API.

Functionality (b) conflicts with configuring flow actions, this
configuration should be controlled by flow rule not with a specific API,
although I understand the reasoning behind the API.

RSS_HASH offload seems given example in the discussions but it is still
controlled via offload flag, there is no specific API to configure RSS
hash functionality, same could be done here.
[/2]

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ferruh Yigit]

On 1/25/2023 1:17 PM, Ferruh Yigit wrote:
> On 1/25/2023 9:30 AM, Hanumanth Reddy Pothula wrote:
>> ++ Ivan Malov and Andrew Rybchenko
>>
>>> -----Original Message-----
>>> From: Ferruh Yigit <ferruh.yigit@amd.com>
>>> Sent: Tuesday, January 24, 2023 11:34 PM
>>> To: Hanumanth Reddy Pothula <hpothula@marvell.com>; Aman Singh
>>> <aman.deep.singh@intel.com>; Yuying Zhang <yuying.zhang@intel.com>
>>> Cc: dev@dpdk.org; andrew.rybchenko@oktetlabs.ru;
>>> viacheslavo@nvidia.com; Jerin Jacob Kollanukkaran <jerinj@marvell.com>;
>>> Nithin Kumar Dabilpuram <ndabilpuram@marvell.com>
>>> Subject: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process
>>> Rx metadata negotiation
>>>
>>> External Email
>>>
>>> ----------------------------------------------------------------------
>>> On 12/21/2022 2:07 AM, Hanumanth Pothula wrote:
>>>> Presently, Rx metadata is sent to PMD by default, leading to a
>>>> performance drop as processing for the same in Rx path takes extra
>>>> cycles.
>>>>
>>>> Hence, add new testpmd command,
>>>>   'enable port <port_id> nic_to_pmd_rx_metadata'
>>>>
>>>> This command helps in sending Rx metadata to PMD and thereby Rx
>>>> metadata flow command requests are processed.
>>>>
>>>> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
>>>
>>> Hi Hanumanth,
>>>
>>> I agree with Thomas for the patch.
>>>
>>> 'eth_rx_metadata_negotiate_mp()' requests all Rx metadata offloads to be
>>> enabled, but at this stage if there is no flow rule for Rx metadata why it is
>>> consuming extra cycles?
>>>
>>> Can you update driver code to process Rx metadata when it is enabled by
>>> application (via 'rte_eth_rx_metadata_negotiate()') AND there is at least
>>> one flow rule for it?
>>
>> #1 What is the purpose of rte_eth_rx_metadata_negotiate() API if it is always called by testpmd.
>> We thought it was added so that when that metadata is not needed, application need not call this
>> thereby saving cycles/bandwidth.
>>
> 
> Purpose looks like to 'negotiate', so it is both ways:
> a) To learn Rx Meta capability of HW
> b) To configure Rx Meta feature for HW
> 
> So it is both to help application to learn what flow rule is supported
> and to help driver to improve performance.
> 
> Before this API application assumed all Rx metadata flow rules are
> supported, so why enabling all causing performance drop, that was the
> default without this API.
> 
> But other-way around can be true, to disable some offloads can improve
> driver performance.
> 
> Looking again,
> 

please ignore this email, it has unfinished draft reply by mistake, I
will send a new one.

> 
>> #2 We use this API similar to Rx/Tx offload flags so that we can set things up before device is
>> configured. We thought that is the purpose of having this negotiate API and avoid depleting offload flags.
>>
>> #3 Generally any new offloads added to DPDK would be in disabled state in testpmd and we would have
>> an option to enable it. In this case, testpmd is by default calling this negotiation.
>>
>> We can update the driver if the purpose of this API is clear.
> 
> 
> Hi Hanumanth,
> 
> After looking the history of the API again, you may be right.
> 
> One of the previous version commit log describes the intention better
> [1], because of negative performance impact of enabling Rx metadata
> offload by default and difficulty to switch configuration dynamically,
> there is a desire to learn application intention before configuration.
> 
> Although I have some concerns with this API [2] it is already there as
> stable API.
> 
> So it sounds reasonable to make this configurable for a test
> application, indeed intention of the API is to get this configuration
> from application and operate based on it.
> 
> Next question is what should be the default value, I am not sure about
> it, there are only a few drivers impacted from this overall.
> For the Thomas' point, it helps to test the feature of its impact if it
> is enabled by default.
> 
> Will it work to enable them all by default and add capability to disable
> it in testpmd, which helps to run performance tests also to verify the
> impact of the API?
> 
> Thanks,
> ferruh
> 
> 
> 
> [1]
> https://inbox.dpdk.org/dev/20210902142359.28138-2-ivan.malov@oktetlabs.ru/
> 
> 
> [2]
> API does two things:
> a) Learn Rx Meta capability of HW
> b) Configure Rx Meta feature for HW
> 
> Functionality (a) conflicts with rest of the flow rules that capability
> checked via `rte_flow_validate()` API.
> 
> Functionality (b) conflicts with configuring flow actions, this
> configuration should be controlled by flow rule not with a specific API,
> although I understand the reasoning behind the API.
> 
> RSS_HASH offload seems given example in the discussions but it is still
> controlled via offload flag, there is no specific API to configure RSS
> hash functionality, same could be done here.
> [/2]

Re: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process Rx metadata negotiation

[Ferruh Yigit]

On 1/25/2023 9:30 AM, Hanumanth Reddy Pothula wrote:
> ++ Ivan Malov and Andrew Rybchenko
> 
>> -----Original Message-----
>> From: Ferruh Yigit <ferruh.yigit@amd.com>
>> Sent: Tuesday, January 24, 2023 11:34 PM
>> To: Hanumanth Reddy Pothula <hpothula@marvell.com>; Aman Singh
>> <aman.deep.singh@intel.com>; Yuying Zhang <yuying.zhang@intel.com>
>> Cc: dev@dpdk.org; andrew.rybchenko@oktetlabs.ru;
>> viacheslavo@nvidia.com; Jerin Jacob Kollanukkaran <jerinj@marvell.com>;
>> Nithin Kumar Dabilpuram <ndabilpuram@marvell.com>
>> Subject: [EXT] Re: [PATCH v5 2/2] app/testpmd: add command to process
>> Rx metadata negotiation
>>
>> External Email
>>
>> ----------------------------------------------------------------------
>> On 12/21/2022 2:07 AM, Hanumanth Pothula wrote:
>>> Presently, Rx metadata is sent to PMD by default, leading to a
>>> performance drop as processing for the same in Rx path takes extra
>>> cycles.
>>>
>>> Hence, add new testpmd command,
>>>   'enable port <port_id> nic_to_pmd_rx_metadata'
>>>
>>> This command helps in sending Rx metadata to PMD and thereby Rx
>>> metadata flow command requests are processed.
>>>
>>> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
>>
>> Hi Hanumanth,
>>
>> I agree with Thomas for the patch.
>>
>> 'eth_rx_metadata_negotiate_mp()' requests all Rx metadata offloads to be
>> enabled, but at this stage if there is no flow rule for Rx metadata why it is
>> consuming extra cycles?
>>
>> Can you update driver code to process Rx metadata when it is enabled by
>> application (via 'rte_eth_rx_metadata_negotiate()') AND there is at least
>> one flow rule for it?
> 
> #1 What is the purpose of rte_eth_rx_metadata_negotiate() API if it is always called by testpmd.
> We thought it was added so that when that metadata is not needed, application need not call this
> thereby saving cycles/bandwidth.
> 
> #2 We use this API similar to Rx/Tx offload flags so that we can set things up before device is
> configured. We thought that is the purpose of having this negotiate API and avoid depleting offload flags.
> 
> #3 Generally any new offloads added to DPDK would be in disabled state in testpmd and we would have
> an option to enable it. In this case, testpmd is by default calling this negotiation.
> 
> We can update the driver if the purpose of this API is clear.


Hi Hanumanth,

After looking the history of the API again, you may be right.

One of the previous version commit log describes the intention better
[1], because of negative performance impact of enabling Rx metadata
offload by default and difficulty to switch configuration dynamically,
there is a desire to learn application intention before configuration.

Although I have some concerns with this API [2] it is already there as
stable API.

So it sounds reasonable to make this configurable for a test
application, indeed intention of the API is to get this configuration
from application and operate based on it.

Next question is what should be the default value, I am not sure about
it, there are only a few drivers impacted from this overall.
For the Thomas' point, it helps to test the feature of its impact if it
is enabled by default.

Will it work to enable them all by default and add capability to disable
it in testpmd, which helps to run performance tests also to verify the
impact of the API?

Thanks,
ferruh



[1]
https://inbox.dpdk.org/dev/20210902142359.28138-2-ivan.malov@oktetlabs.ru/


[2]
API does two things:
a) Learn Rx Meta capability of HW
b) Configure Rx Meta feature for HW

Functionality (a) conflicts with rest of the flow rules that capability
checked via `rte_flow_validate()` API.

Functionality (b) conflicts with configuring flow actions, this
configuration should be controlled by flow rule not with a specific API,
although I understand the reasoning behind the API.

RSS_HASH offload seems given example in the discussions but it is still
controlled via offload flag, there is no specific API to configure RSS
hash functionality, same could be done here.
[/2]

RE: [PATCH v5 1/2] ethdev: fix ethdev configuration state on reset

[Hanumanth Reddy Pothula]

Ping

> -----Original Message-----
> From: Hanumanth Pothula <hpothula@marvell.com>
> Sent: Wednesday, December 21, 2022 7:37 AM
> To: Thomas Monjalon <thomas@monjalon.net>; Ferruh Yigit
> <ferruh.yigit@amd.com>; Andrew Rybchenko
> <andrew.rybchenko@oktetlabs.ru>
> Cc: dev@dpdk.org; viacheslavo@nvidia.com; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Nithin Kumar Dabilpuram
> <ndabilpuram@marvell.com>; Hanumanth Reddy Pothula
> <hpothula@marvell.com>
> Subject: [PATCH v5 1/2] ethdev: fix ethdev configuration state on reset
> 
> Presently, on device reset, ethdev configuration state, dev_configured, is
> not reset.
> 
> On device reset, reset ethdev configuration state to make sure device
> reconfiguration happens cleanly.
> 
> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> ---
> v5:
>  - Move nic-to-pmd-rx-metadata declaration to struct rte_port.
> v4:
>  - As per spec rte_eth_rx_metadata_negotiate() is processed only when
>    dev_configured is set. Hence can't enable automatically when a flow
>    command requests metadata.
>  - Add new testpmd command to allow NIC to PMD Rx metadata
> negotiation.
> v3:
>  - Updated run_app.rst with the new command line argument,
>    nic-to-pmd-rx-metadata.
>  - Updated commit text.
> v2:
>  - taken cared alignment issues
>  - renamed command line argument from rx-metadata to nic-to-pmd-rx-
> metadata
>  - renamed variable name from rx-metadata to nic_to_pmd_rx_metadata
> ---
>  lib/ethdev/rte_ethdev.c | 2 ++
>  1 file changed, 2 insertions(+)
> 
> diff --git a/lib/ethdev/rte_ethdev.c b/lib/ethdev/rte_ethdev.c index
> 5d5e18db1e..18c59044bc 100644
> --- a/lib/ethdev/rte_ethdev.c
> +++ b/lib/ethdev/rte_ethdev.c
> @@ -1629,6 +1629,8 @@ rte_eth_dev_reset(uint16_t port_id)
>  			port_id, rte_strerror(-ret));
>  	}
>  	ret = dev->dev_ops->dev_reset(dev);
> +	if (!ret)
> +		dev->data->dev_configured = 0;
> 
>  	return eth_err(port_id, ret);
>  }
> --
> 2.25.1

Re: [PATCH v5 1/2] ethdev: fix ethdev configuration state on reset

[Thomas Monjalon]

21/12/2022 03:07, Hanumanth Pothula:
> Presently, on device reset, ethdev configuration state,
> dev_configured, is not reset.
> 
> On device reset, reset ethdev configuration state to make
> sure device reconfiguration happens cleanly.
> 
> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>

A "Fixes" line is missing to show the root cause and help backports.

> --- a/lib/ethdev/rte_ethdev.c
> +++ b/lib/ethdev/rte_ethdev.c
> @@ -1629,6 +1629,8 @@ rte_eth_dev_reset(uint16_t port_id)
>  			port_id, rte_strerror(-ret));
>  	}
>  	ret = dev->dev_ops->dev_reset(dev);
> +	if (!ret)

Should be if (ret == 0)

> +		dev->data->dev_configured = 0;

Re: [PATCH v5 1/2] ethdev: fix ethdev configuration state on reset

[Ferruh Yigit]

On 1/18/2023 10:29 AM, Thomas Monjalon wrote:
> 21/12/2022 03:07, Hanumanth Pothula:
>> Presently, on device reset, ethdev configuration state,
>> dev_configured, is not reset.
>>
>> On device reset, reset ethdev configuration state to make
>> sure device reconfiguration happens cleanly.
>>
>> Signed-off-by: Hanumanth Pothula <hpothula@marvell.com>
> 
> A "Fixes" line is missing to show the root cause and help backports.
> 

It should be following, that is when 'dev_configured' flag is introduced
but not updated in 'rte_eth_dev_reset()':

Fixes: 02edbfab1e7d ("ethdev: add dev configured flag")
Cc: stable@dpdk.org
Cc: lihuisong@huawei.com


@Hanumanth, can you please send a new version with above fixes line and
below syntax fix?


>> --- a/lib/ethdev/rte_ethdev.c
>> +++ b/lib/ethdev/rte_ethdev.c
>> @@ -1629,6 +1629,8 @@ rte_eth_dev_reset(uint16_t port_id)
>>  			port_id, rte_strerror(-ret));
>>  	}
>>  	ret = dev->dev_ops->dev_reset(dev);
>> +	if (!ret)
> 
> Should be if (ret == 0)
> 
>> +		dev->data->dev_configured = 0;
> 
> 
>