[dpdk-dev] [PATCH v1] net/ice: use dynamic mbuf API to handle protocol extraction

[dpdk-dev] [PATCH v1] net/ice: use dynamic mbuf API to handle protocol extraction

[Haiyue Wang]

The original design is to occupy rte_mbuf::udata64 to save the metadata
which combines the network protocol fields and related type, then use
private API to decode this metadata. It is no so generic.

Use the new dynamic mbuf field and flags to handle protocol extraction.
Now the new metadata is very clean, just network protocol fields in the
specified offset, and its type will be indicated by related ol_flags.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 doc/guides/nics/ice.rst       |  22 +++--
 drivers/net/ice/Makefile      |   3 -
 drivers/net/ice/ice_ethdev.c  |  62 +++++++++++++-
 drivers/net/ice/ice_ethdev.h  |  18 ++++
 drivers/net/ice/ice_rxtx.c    |  82 +++++++++++-------
 drivers/net/ice/ice_rxtx.h    |   1 -
 drivers/net/ice/meson.build   |   2 -
 drivers/net/ice/rte_pmd_ice.h | 152 ----------------------------------
 8 files changed, 147 insertions(+), 195 deletions(-)
 delete mode 100644 drivers/net/ice/rte_pmd_ice.h

diff --git a/doc/guides/nics/ice.rst b/doc/guides/nics/ice.rst
index 933f63480..f460d1cfd 100644
--- a/doc/guides/nics/ice.rst
+++ b/doc/guides/nics/ice.rst
@@ -82,8 +82,8 @@ Runtime Config Options
 
 - ``Protocol extraction for per queue``
 
-  Configure the RX queues to do protocol extraction into ``rte_mbuf::udata64``
-  for protocol handling acceleration, like checking the TCP SYN packets quickly.
+  Configure the RX queues to do protocol extraction into mbuf for protocol
+  handling acceleration, like checking the TCP SYN packets quickly.
 
   The argument format is::
 
@@ -111,7 +111,9 @@ Runtime Config Options
   This setting means queues 1, 2-3, 8-9 are TCP extraction, queues 10-23 are
   IPv6 extraction, other queues use the default VLAN extraction.
 
-  The extraction will be copied into the lower 32 bit of ``rte_mbuf::udata64``.
+  The extraction metadata will be copied into the registered dynamic mbuf field
+  with name: "proto-xtr-metadata", size: 4B, align: __alignof__(uint32_t), and
+  the related dynamic mbuf flags in ``rte_mbuf::ol_flags`` will be set.
 
   .. table:: Protocol extraction : ``vlan``
 
@@ -125,6 +127,8 @@ Runtime Config Options
 
   VLAN2 - C-VLAN (second for QinQ).
 
+  Dynamic mbuf flag : "proto-xtr-ol-vlan"
+
   .. table:: Protocol extraction : ``ipv4``
 
    +----------------------------+----------------------------+
@@ -137,6 +141,8 @@ Runtime Config Options
 
   IPHDR2 - IPv4 header word 0, "Ver", "Hdr Len" and "Type of Service" fields.
 
+  Dynamic mbuf flag : "proto-xtr-ol-ipv4"
+
   .. table:: Protocol extraction : ``ipv6``
 
    +----------------------------+----------------------------+
@@ -150,6 +156,8 @@ Runtime Config Options
   IPHDR2 - IPv6 header word 0, "Ver", "Traffic class" and high 4 bits of
   "Flow Label" fields.
 
+  Dynamic mbuf flag : "proto-xtr-ol-ipv6"
+
   .. table:: Protocol extraction : ``ipv6_flow``
 
    +----------------------------+----------------------------+
@@ -163,6 +171,8 @@ Runtime Config Options
   IPHDR2 - IPv6 header word 0, "Ver", "Traffic class" and high 4 bits of
   "Flow Label" fields.
 
+  Dynamic mbuf flag : "proto-xtr-ol-ipv6-flow"
+
   .. table:: Protocol extraction : ``tcp``
 
    +----------------------------+----------------------------+
@@ -175,11 +185,7 @@ Runtime Config Options
 
   TCPHDR2 - Reserved
 
-  Use ``get_proto_xtr_flds(struct rte_mbuf *mb)`` to access the protocol
-  extraction, do not use ``rte_mbuf::udata64`` directly.
-
-  The ``dump_proto_xtr_flds(struct rte_mbuf *mb)`` routine shows how to
-  access the protocol extraction result in ``struct rte_mbuf``.
+  Dynamic mbuf flag : "proto-xtr-ol-tcp"
 
 Driver compilation and testing
 ------------------------------
diff --git a/drivers/net/ice/Makefile b/drivers/net/ice/Makefile
index 7c3b6a7ff..a1817e14c 100644
--- a/drivers/net/ice/Makefile
+++ b/drivers/net/ice/Makefile
@@ -84,7 +84,4 @@ ifeq ($(CC_AVX2_SUPPORT), 1)
 endif
 SRCS-$(CONFIG_RTE_LIBRTE_ICE_PMD) += ice_generic_flow.c
 
-# install this header file
-SYMLINK-$(CONFIG_RTE_LIBRTE_ICE_PMD)-include := rte_pmd_ice.h
-
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index d74675842..fc0124cf4 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -30,6 +30,26 @@ static const char * const ice_valid_args[] = {
 	NULL
 };
 
+static const struct rte_mbuf_dynfield proto_xtr_metadata_param = {
+	.name = "proto-xtr-metadata",
+	.size = sizeof(uint32_t),
+	.align = __alignof__(uint32_t),
+	.flags = 0,
+};
+
+struct proto_xtr_ol_flag {
+	const struct rte_mbuf_dynflag param;
+	uint64_t *flag;
+};
+
+static const struct proto_xtr_ol_flag proto_xtr_ol_flag_params[] = {
+	{ { "proto-xtr-ol-vlan", 0 },      &proto_xtr_ol_vlan },
+	{ { "proto-xtr-ol-ipv4",  0 },     &proto_xtr_ol_ipv4 },
+	{ { "proto-xtr-ol-ipv6", 0 },      &proto_xtr_ol_ipv6 },
+	{ { "proto-xtr-ol-ipv6-flow", 0 }, &proto_xtr_ol_ipv6_flow },
+	{ { "proto-xtr-ol-tcp", 0 },       &proto_xtr_ol_tcp },
+};
+
 #define ICE_DFLT_OUTER_TAG_TYPE ICE_AQ_VSI_OUTER_TAG_VLAN_9100
 
 /* DDP package search path */
@@ -1385,6 +1405,9 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 			ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 	struct ice_hw *hw = ICE_PF_TO_HW(pf);
+	const struct proto_xtr_ol_flag *ol_flag;
+	bool proto_xtr_enable = false;
+	int offset;
 	uint16_t i;
 
 	if (!ice_proto_xtr_support(hw)) {
@@ -1398,10 +1421,47 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 		return;
 	}
 
-	for (i = 0; i < pf->lan_nb_qps; i++)
+	for (i = 0; i < pf->lan_nb_qps; i++) {
 		pf->proto_xtr[i] = ad->devargs.proto_xtr[i] != PROTO_XTR_NONE ?
 				   ad->devargs.proto_xtr[i] :
 				   ad->devargs.proto_xtr_dflt;
+
+		if (pf->proto_xtr[i] != PROTO_XTR_NONE && !proto_xtr_enable)
+			proto_xtr_enable = true;
+	}
+
+	if (likely(!proto_xtr_enable))
+		return;
+
+	offset = rte_mbuf_dynfield_register(&proto_xtr_metadata_param);
+	if (unlikely(offset == -1)) {
+		PMD_DRV_LOG(ERR,
+			    "Protocol extraction metadata is disabled in mbuf with error %d",
+			    -rte_errno);
+		return;
+	}
+
+	PMD_DRV_LOG(DEBUG,
+		    "Protocol extraction metadata offset in mbuf is : %d",
+		    offset);
+	proto_xtr_metadata = offset;
+
+	for (i = 0; i < RTE_DIM(proto_xtr_ol_flag_params); i++) {
+		ol_flag = &proto_xtr_ol_flag_params[i];
+
+		offset = rte_mbuf_dynflag_register(&ol_flag->param);
+		if (unlikely(offset == -1)) {
+			PMD_DRV_LOG(ERR,
+				    "Protocol extraction offload '%s' is disabled in mbuf with error %d",
+				    ol_flag->param.name, -rte_errno);
+			continue;
+		}
+
+		PMD_DRV_LOG(DEBUG,
+			    "Protocol extraction offload '%s' offset in mbuf is : %d",
+			    ol_flag->param.name, offset);
+		*ol_flag->flag = 1ULL << offset;
+	}
 }
 
 /*  Initialize SW parameters of PF */
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index de67e5934..1ca00bda4 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -8,6 +8,7 @@
 #include <rte_kvargs.h>
 
 #include <rte_ethdev_driver.h>
+#include <rte_mbuf_dyn.h>
 
 #include "base/ice_common.h"
 #include "base/ice_adminq_cmd.h"
@@ -241,6 +242,23 @@ struct ice_vsi {
 	bool offset_loaded;
 };
 
+enum proto_xtr_type {
+	PROTO_XTR_NONE,
+	PROTO_XTR_VLAN,
+	PROTO_XTR_IPV4,
+	PROTO_XTR_IPV6,
+	PROTO_XTR_IPV6_FLOW,
+	PROTO_XTR_TCP,
+};
+
+extern int proto_xtr_metadata;
+
+extern uint64_t proto_xtr_ol_vlan;
+extern uint64_t proto_xtr_ol_ipv4;
+extern uint64_t proto_xtr_ol_ipv6;
+extern uint64_t proto_xtr_ol_ipv6_flow;
+extern uint64_t proto_xtr_ol_tcp;
+
 enum ice_fdir_tunnel_type {
 	ICE_FDIR_TUNNEL_TYPE_NONE = 0,
 	ICE_FDIR_TUNNEL_TYPE_VXLAN,
diff --git a/drivers/net/ice/ice_rxtx.c b/drivers/net/ice/ice_rxtx.c
index 8d4820d3c..6493aa700 100644
--- a/drivers/net/ice/ice_rxtx.c
+++ b/drivers/net/ice/ice_rxtx.c
@@ -13,18 +13,31 @@
 		PKT_TX_TCP_SEG |		 \
 		PKT_TX_OUTER_IP_CKSUM)
 
-static inline uint8_t
-ice_rxdid_to_proto_xtr_type(uint8_t rxdid)
-{
-	static uint8_t xtr_map[] = {
-		[ICE_RXDID_COMMS_AUX_VLAN]      = PROTO_XTR_VLAN,
-		[ICE_RXDID_COMMS_AUX_IPV4]      = PROTO_XTR_IPV4,
-		[ICE_RXDID_COMMS_AUX_IPV6]      = PROTO_XTR_IPV6,
-		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = PROTO_XTR_IPV6_FLOW,
-		[ICE_RXDID_COMMS_AUX_TCP]       = PROTO_XTR_TCP,
+/* Offset of mbuf dynamic field for protocol extraction data */
+int proto_xtr_metadata = -1;
+
+/* Mask of mbuf dynamic flags for protocol extraction type */
+uint64_t proto_xtr_ol_vlan;
+uint64_t proto_xtr_ol_ipv4;
+uint64_t proto_xtr_ol_ipv6;
+uint64_t proto_xtr_ol_ipv6_flow;
+uint64_t proto_xtr_ol_tcp;
+
+static inline uint64_t
+ice_rxdid_to_proto_xtr_ol_flag(uint8_t rxdid)
+{
+	static uint64_t *ol_flag_map[] = {
+		[ICE_RXDID_COMMS_AUX_VLAN]      = &proto_xtr_ol_vlan,
+		[ICE_RXDID_COMMS_AUX_IPV4]      = &proto_xtr_ol_ipv4,
+		[ICE_RXDID_COMMS_AUX_IPV6]      = &proto_xtr_ol_ipv6,
+		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = &proto_xtr_ol_ipv6_flow,
+		[ICE_RXDID_COMMS_AUX_TCP]       = &proto_xtr_ol_tcp,
 	};
+	uint64_t *ol_flag;
 
-	return rxdid < RTE_DIM(xtr_map) ? xtr_map[rxdid] : PROTO_XTR_NONE;
+	ol_flag = rxdid < RTE_DIM(ol_flag_map) ? ol_flag_map[rxdid] : NULL;
+
+	return ol_flag != NULL ? *ol_flag : 0;
 }
 
 static inline uint8_t
@@ -1325,10 +1338,38 @@ ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc *rxdp)
 		   mb->vlan_tci, mb->vlan_tci_outer);
 }
 
+#ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 #define ICE_RX_PROTO_XTR_VALID \
 	((1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) | \
 	 (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
 
+static void
+ice_rxd_to_proto_xtr(struct rte_mbuf *mb,
+		     volatile struct ice_32b_rx_flex_desc_comms *desc)
+{
+	uint16_t stat_err = rte_le_to_cpu_16(desc->status_error1);
+	uint32_t metadata;
+	uint64_t ol_flag;
+
+	if (unlikely(!(stat_err & ICE_RX_PROTO_XTR_VALID)))
+		return;
+
+	ol_flag = ice_rxdid_to_proto_xtr_ol_flag(desc->rxdid);
+	if (unlikely(!ol_flag))
+		return;
+
+	mb->ol_flags |= ol_flag;
+
+	metadata = stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) ?
+				rte_le_to_cpu_16(desc->flex_ts.flex.aux0) : 0;
+
+	if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S)))
+		metadata |= rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
+
+	*RTE_MBUF_DYNFIELD(mb, proto_xtr_metadata, uint32_t *) = metadata;
+}
+#endif
+
 static inline void
 ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 		      volatile union ice_rx_flex_desc *rxdp)
@@ -1344,28 +1385,13 @@ ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 	}
 
 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
-	init_proto_xtr_flds(mb);
-
-	stat_err = rte_le_to_cpu_16(desc->status_error1);
-	if (stat_err & ICE_RX_PROTO_XTR_VALID) {
-		struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
-			xtr->u.raw.data0 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
-			xtr->u.raw.data1 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
-
-		xtr->type = ice_rxdid_to_proto_xtr_type(desc->rxdid);
-		xtr->magic = PROTO_XTR_MAGIC_ID;
-	}
-
 	if (desc->flow_id != 0xFFFFFFFF) {
 		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 	}
+
+	if (unlikely(proto_xtr_metadata != -1))
+		ice_rxd_to_proto_xtr(mb, desc);
 #endif
 }
 
diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h
index 5de618976..9e3d2cd07 100644
--- a/drivers/net/ice/ice_rxtx.h
+++ b/drivers/net/ice/ice_rxtx.h
@@ -5,7 +5,6 @@
 #ifndef _ICE_RXTX_H_
 #define _ICE_RXTX_H_
 
-#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 
 #define ICE_ALIGN_RING_DESC  32
diff --git a/drivers/net/ice/meson.build b/drivers/net/ice/meson.build
index f9e897bbc..45e760063 100644
--- a/drivers/net/ice/meson.build
+++ b/drivers/net/ice/meson.build
@@ -36,5 +36,3 @@ if arch_subdir == 'x86'
 		objs += ice_avx2_lib.extract_objects('ice_rxtx_vec_avx2.c')
 	endif
 endif
-
-install_headers('rte_pmd_ice.h')
diff --git a/drivers/net/ice/rte_pmd_ice.h b/drivers/net/ice/rte_pmd_ice.h
deleted file mode 100644
index 719487e1e..000000000
--- a/drivers/net/ice/rte_pmd_ice.h
+++ /dev/null
@@ -1,152 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019 Intel Corporation
- */
-
-#ifndef _RTE_PMD_ICE_H_
-#define _RTE_PMD_ICE_H_
-
-#include <stdio.h>
-#include <rte_mbuf.h>
-#include <rte_ethdev.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-enum proto_xtr_type {
-	PROTO_XTR_NONE,
-	PROTO_XTR_VLAN,
-	PROTO_XTR_IPV4,
-	PROTO_XTR_IPV6,
-	PROTO_XTR_IPV6_FLOW,
-	PROTO_XTR_TCP,
-};
-
-struct proto_xtr_flds {
-	union {
-		struct {
-			uint16_t data0;
-			uint16_t data1;
-		} raw;
-		struct {
-			uint16_t stag_vid:12,
-				 stag_dei:1,
-				 stag_pcp:3;
-			uint16_t ctag_vid:12,
-				 ctag_dei:1,
-				 ctag_pcp:3;
-		} vlan;
-		struct {
-			uint16_t protocol:8,
-				 ttl:8;
-			uint16_t tos:8,
-				 ihl:4,
-				 version:4;
-		} ipv4;
-		struct {
-			uint16_t hoplimit:8,
-				 nexthdr:8;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6;
-		struct {
-			uint16_t flowlo16;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6_flow;
-		struct {
-			uint16_t fin:1,
-				 syn:1,
-				 rst:1,
-				 psh:1,
-				 ack:1,
-				 urg:1,
-				 ece:1,
-				 cwr:1,
-				 res1:4,
-				 doff:4;
-			uint16_t rsvd;
-		} tcp;
-	} u;
-
-	uint16_t rsvd;
-
-	uint8_t type;
-
-#define PROTO_XTR_MAGIC_ID	0xCE
-	uint8_t magic;
-};
-
-static inline void
-init_proto_xtr_flds(struct rte_mbuf *mb)
-{
-	mb->udata64 = 0;
-}
-
-static inline struct proto_xtr_flds *
-get_proto_xtr_flds(struct rte_mbuf *mb)
-{
-	RTE_BUILD_BUG_ON(sizeof(struct proto_xtr_flds) > sizeof(mb->udata64));
-
-	return (struct proto_xtr_flds *)&mb->udata64;
-}
-
-static inline void
-dump_proto_xtr_flds(struct rte_mbuf *mb)
-{
-	struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
-
-	if (xtr->magic != PROTO_XTR_MAGIC_ID || xtr->type == PROTO_XTR_NONE)
-		return;
-
-	printf(" - Protocol Extraction:[0x%04x:0x%04x],",
-	       xtr->u.raw.data0, xtr->u.raw.data1);
-
-	if (xtr->type == PROTO_XTR_VLAN)
-		printf("vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
-		       xtr->u.vlan.stag_pcp,
-		       xtr->u.vlan.stag_dei,
-		       xtr->u.vlan.stag_vid,
-		       xtr->u.vlan.ctag_pcp,
-		       xtr->u.vlan.ctag_dei,
-		       xtr->u.vlan.ctag_vid);
-	else if (xtr->type == PROTO_XTR_IPV4)
-		printf("ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
-		       xtr->u.ipv4.version,
-		       xtr->u.ipv4.ihl,
-		       xtr->u.ipv4.tos,
-		       xtr->u.ipv4.ttl,
-		       xtr->u.ipv4.protocol);
-	else if (xtr->type == PROTO_XTR_IPV6)
-		printf("ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
-		       xtr->u.ipv6.version,
-		       xtr->u.ipv6.tc,
-		       xtr->u.ipv6.flowhi4,
-		       xtr->u.ipv6.nexthdr,
-		       xtr->u.ipv6.hoplimit);
-	else if (xtr->type == PROTO_XTR_IPV6_FLOW)
-		printf("ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
-		       xtr->u.ipv6_flow.version,
-		       xtr->u.ipv6_flow.tc,
-		       xtr->u.ipv6_flow.flowhi4,
-		       xtr->u.ipv6_flow.flowlo16);
-	else if (xtr->type == PROTO_XTR_TCP)
-		printf("tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
-		       xtr->u.tcp.doff,
-		       xtr->u.tcp.cwr ? "C" : "",
-		       xtr->u.tcp.ece ? "E" : "",
-		       xtr->u.tcp.urg ? "U" : "",
-		       xtr->u.tcp.ack ? "A" : "",
-		       xtr->u.tcp.psh ? "P" : "",
-		       xtr->u.tcp.rst ? "R" : "",
-		       xtr->u.tcp.syn ? "S" : "",
-		       xtr->u.tcp.fin ? "F" : "");
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_PMD_ICE_H_ */
-- 
2.17.1

[dpdk-dev] [PATCH v2] net/ice: optimize protocol extraction by dynamic mbuf API

[Haiyue Wang]

The original design is to use rte_mbuf::udata64 to save the metadata of
protocol extraction which has network protocol data fields and type, an
private API is used to decode this metadata. It is not so generic.

Use the new dynamic mbuf field and flags to handle protocol extraction.
Then the metadata is very clean, only network protocol fields extracted
by defined offset, and its type will be indicated by related ol_flags.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---

v2: - disable the protocol extraction if failed to register some ol_flags
    - rewrite the commit message

 doc/guides/nics/ice.rst       |  22 +++--
 drivers/net/ice/Makefile      |   3 -
 drivers/net/ice/ice_ethdev.c  |  67 ++++++++++++++-
 drivers/net/ice/ice_ethdev.h  |  18 ++++
 drivers/net/ice/ice_rxtx.c    |  82 +++++++++++-------
 drivers/net/ice/ice_rxtx.h    |   1 -
 drivers/net/ice/meson.build   |   2 -
 drivers/net/ice/rte_pmd_ice.h | 152 ----------------------------------
 8 files changed, 152 insertions(+), 195 deletions(-)
 delete mode 100644 drivers/net/ice/rte_pmd_ice.h

diff --git a/doc/guides/nics/ice.rst b/doc/guides/nics/ice.rst
index 933f63480..f460d1cfd 100644
--- a/doc/guides/nics/ice.rst
+++ b/doc/guides/nics/ice.rst
@@ -82,8 +82,8 @@ Runtime Config Options
 
 - ``Protocol extraction for per queue``
 
-  Configure the RX queues to do protocol extraction into ``rte_mbuf::udata64``
-  for protocol handling acceleration, like checking the TCP SYN packets quickly.
+  Configure the RX queues to do protocol extraction into mbuf for protocol
+  handling acceleration, like checking the TCP SYN packets quickly.
 
   The argument format is::
 
@@ -111,7 +111,9 @@ Runtime Config Options
   This setting means queues 1, 2-3, 8-9 are TCP extraction, queues 10-23 are
   IPv6 extraction, other queues use the default VLAN extraction.
 
-  The extraction will be copied into the lower 32 bit of ``rte_mbuf::udata64``.
+  The extraction metadata will be copied into the registered dynamic mbuf field
+  with name: "proto-xtr-metadata", size: 4B, align: __alignof__(uint32_t), and
+  the related dynamic mbuf flags in ``rte_mbuf::ol_flags`` will be set.
 
   .. table:: Protocol extraction : ``vlan``
 
@@ -125,6 +127,8 @@ Runtime Config Options
 
   VLAN2 - C-VLAN (second for QinQ).
 
+  Dynamic mbuf flag : "proto-xtr-ol-vlan"
+
   .. table:: Protocol extraction : ``ipv4``
 
    +----------------------------+----------------------------+
@@ -137,6 +141,8 @@ Runtime Config Options
 
   IPHDR2 - IPv4 header word 0, "Ver", "Hdr Len" and "Type of Service" fields.
 
+  Dynamic mbuf flag : "proto-xtr-ol-ipv4"
+
   .. table:: Protocol extraction : ``ipv6``
 
    +----------------------------+----------------------------+
@@ -150,6 +156,8 @@ Runtime Config Options
   IPHDR2 - IPv6 header word 0, "Ver", "Traffic class" and high 4 bits of
   "Flow Label" fields.
 
+  Dynamic mbuf flag : "proto-xtr-ol-ipv6"
+
   .. table:: Protocol extraction : ``ipv6_flow``
 
    +----------------------------+----------------------------+
@@ -163,6 +171,8 @@ Runtime Config Options
   IPHDR2 - IPv6 header word 0, "Ver", "Traffic class" and high 4 bits of
   "Flow Label" fields.
 
+  Dynamic mbuf flag : "proto-xtr-ol-ipv6-flow"
+
   .. table:: Protocol extraction : ``tcp``
 
    +----------------------------+----------------------------+
@@ -175,11 +185,7 @@ Runtime Config Options
 
   TCPHDR2 - Reserved
 
-  Use ``get_proto_xtr_flds(struct rte_mbuf *mb)`` to access the protocol
-  extraction, do not use ``rte_mbuf::udata64`` directly.
-
-  The ``dump_proto_xtr_flds(struct rte_mbuf *mb)`` routine shows how to
-  access the protocol extraction result in ``struct rte_mbuf``.
+  Dynamic mbuf flag : "proto-xtr-ol-tcp"
 
 Driver compilation and testing
 ------------------------------
diff --git a/drivers/net/ice/Makefile b/drivers/net/ice/Makefile
index 7c3b6a7ff..a1817e14c 100644
--- a/drivers/net/ice/Makefile
+++ b/drivers/net/ice/Makefile
@@ -84,7 +84,4 @@ ifeq ($(CC_AVX2_SUPPORT), 1)
 endif
 SRCS-$(CONFIG_RTE_LIBRTE_ICE_PMD) += ice_generic_flow.c
 
-# install this header file
-SYMLINK-$(CONFIG_RTE_LIBRTE_ICE_PMD)-include := rte_pmd_ice.h
-
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index d74675842..c68acdef0 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -30,6 +30,26 @@ static const char * const ice_valid_args[] = {
 	NULL
 };
 
+static const struct rte_mbuf_dynfield proto_xtr_metadata_param = {
+	.name = "proto-xtr-metadata",
+	.size = sizeof(uint32_t),
+	.align = __alignof__(uint32_t),
+	.flags = 0,
+};
+
+struct proto_xtr_ol_flag {
+	const struct rte_mbuf_dynflag param;
+	uint64_t *flag;
+};
+
+static const struct proto_xtr_ol_flag proto_xtr_ol_flag_params[] = {
+	{ { "proto-xtr-ol-vlan", 0 },      &proto_xtr_ol_vlan },
+	{ { "proto-xtr-ol-ipv4",  0 },     &proto_xtr_ol_ipv4 },
+	{ { "proto-xtr-ol-ipv6", 0 },      &proto_xtr_ol_ipv6 },
+	{ { "proto-xtr-ol-ipv6-flow", 0 }, &proto_xtr_ol_ipv6_flow },
+	{ { "proto-xtr-ol-tcp", 0 },       &proto_xtr_ol_tcp },
+};
+
 #define ICE_DFLT_OUTER_TAG_TYPE ICE_AQ_VSI_OUTER_TAG_VLAN_9100
 
 /* DDP package search path */
@@ -1385,6 +1405,9 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 			ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 	struct ice_hw *hw = ICE_PF_TO_HW(pf);
+	const struct proto_xtr_ol_flag *ol_flag;
+	bool proto_xtr_enable = false;
+	int offset;
 	uint16_t i;
 
 	if (!ice_proto_xtr_support(hw)) {
@@ -1398,10 +1421,52 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 		return;
 	}
 
-	for (i = 0; i < pf->lan_nb_qps; i++)
+	for (i = 0; i < pf->lan_nb_qps; i++) {
 		pf->proto_xtr[i] = ad->devargs.proto_xtr[i] != PROTO_XTR_NONE ?
 				   ad->devargs.proto_xtr[i] :
 				   ad->devargs.proto_xtr_dflt;
+
+		if (pf->proto_xtr[i] != PROTO_XTR_NONE && !proto_xtr_enable)
+			proto_xtr_enable = true;
+	}
+
+	if (likely(!proto_xtr_enable))
+		return;
+
+	offset = rte_mbuf_dynfield_register(&proto_xtr_metadata_param);
+	if (unlikely(offset == -1)) {
+		PMD_DRV_LOG(ERR,
+			    "Protocol extraction metadata is disabled in mbuf with error %d",
+			    -rte_errno);
+		return;
+	}
+
+	PMD_DRV_LOG(DEBUG,
+		    "Protocol extraction metadata offset in mbuf is : %d",
+		    offset);
+	proto_xtr_metadata = offset;
+
+	for (i = 0; i < RTE_DIM(proto_xtr_ol_flag_params); i++) {
+		ol_flag = &proto_xtr_ol_flag_params[i];
+
+		offset = rte_mbuf_dynflag_register(&ol_flag->param);
+		if (unlikely(offset == -1)) {
+			PMD_DRV_LOG(ERR,
+				    "Protocol extraction offload '%s' failed to register with error %d",
+				    ol_flag->param.name, -rte_errno);
+			break;
+		}
+
+		PMD_DRV_LOG(DEBUG,
+			    "Protocol extraction offload '%s' offset in mbuf is : %d",
+			    ol_flag->param.name, offset);
+		*ol_flag->flag = 1ULL << offset;
+	}
+
+	if (i != RTE_DIM(proto_xtr_ol_flag_params)) {
+		PMD_DRV_LOG(ERR, "Protocol extraction metadata is disabled");
+		proto_xtr_metadata = -1;
+	}
 }
 
 /*  Initialize SW parameters of PF */
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index de67e5934..1ca00bda4 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -8,6 +8,7 @@
 #include <rte_kvargs.h>
 
 #include <rte_ethdev_driver.h>
+#include <rte_mbuf_dyn.h>
 
 #include "base/ice_common.h"
 #include "base/ice_adminq_cmd.h"
@@ -241,6 +242,23 @@ struct ice_vsi {
 	bool offset_loaded;
 };
 
+enum proto_xtr_type {
+	PROTO_XTR_NONE,
+	PROTO_XTR_VLAN,
+	PROTO_XTR_IPV4,
+	PROTO_XTR_IPV6,
+	PROTO_XTR_IPV6_FLOW,
+	PROTO_XTR_TCP,
+};
+
+extern int proto_xtr_metadata;
+
+extern uint64_t proto_xtr_ol_vlan;
+extern uint64_t proto_xtr_ol_ipv4;
+extern uint64_t proto_xtr_ol_ipv6;
+extern uint64_t proto_xtr_ol_ipv6_flow;
+extern uint64_t proto_xtr_ol_tcp;
+
 enum ice_fdir_tunnel_type {
 	ICE_FDIR_TUNNEL_TYPE_NONE = 0,
 	ICE_FDIR_TUNNEL_TYPE_VXLAN,
diff --git a/drivers/net/ice/ice_rxtx.c b/drivers/net/ice/ice_rxtx.c
index 8d4820d3c..6493aa700 100644
--- a/drivers/net/ice/ice_rxtx.c
+++ b/drivers/net/ice/ice_rxtx.c
@@ -13,18 +13,31 @@
 		PKT_TX_TCP_SEG |		 \
 		PKT_TX_OUTER_IP_CKSUM)
 
-static inline uint8_t
-ice_rxdid_to_proto_xtr_type(uint8_t rxdid)
-{
-	static uint8_t xtr_map[] = {
-		[ICE_RXDID_COMMS_AUX_VLAN]      = PROTO_XTR_VLAN,
-		[ICE_RXDID_COMMS_AUX_IPV4]      = PROTO_XTR_IPV4,
-		[ICE_RXDID_COMMS_AUX_IPV6]      = PROTO_XTR_IPV6,
-		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = PROTO_XTR_IPV6_FLOW,
-		[ICE_RXDID_COMMS_AUX_TCP]       = PROTO_XTR_TCP,
+/* Offset of mbuf dynamic field for protocol extraction data */
+int proto_xtr_metadata = -1;
+
+/* Mask of mbuf dynamic flags for protocol extraction type */
+uint64_t proto_xtr_ol_vlan;
+uint64_t proto_xtr_ol_ipv4;
+uint64_t proto_xtr_ol_ipv6;
+uint64_t proto_xtr_ol_ipv6_flow;
+uint64_t proto_xtr_ol_tcp;
+
+static inline uint64_t
+ice_rxdid_to_proto_xtr_ol_flag(uint8_t rxdid)
+{
+	static uint64_t *ol_flag_map[] = {
+		[ICE_RXDID_COMMS_AUX_VLAN]      = &proto_xtr_ol_vlan,
+		[ICE_RXDID_COMMS_AUX_IPV4]      = &proto_xtr_ol_ipv4,
+		[ICE_RXDID_COMMS_AUX_IPV6]      = &proto_xtr_ol_ipv6,
+		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = &proto_xtr_ol_ipv6_flow,
+		[ICE_RXDID_COMMS_AUX_TCP]       = &proto_xtr_ol_tcp,
 	};
+	uint64_t *ol_flag;
 
-	return rxdid < RTE_DIM(xtr_map) ? xtr_map[rxdid] : PROTO_XTR_NONE;
+	ol_flag = rxdid < RTE_DIM(ol_flag_map) ? ol_flag_map[rxdid] : NULL;
+
+	return ol_flag != NULL ? *ol_flag : 0;
 }
 
 static inline uint8_t
@@ -1325,10 +1338,38 @@ ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc *rxdp)
 		   mb->vlan_tci, mb->vlan_tci_outer);
 }
 
+#ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 #define ICE_RX_PROTO_XTR_VALID \
 	((1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) | \
 	 (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
 
+static void
+ice_rxd_to_proto_xtr(struct rte_mbuf *mb,
+		     volatile struct ice_32b_rx_flex_desc_comms *desc)
+{
+	uint16_t stat_err = rte_le_to_cpu_16(desc->status_error1);
+	uint32_t metadata;
+	uint64_t ol_flag;
+
+	if (unlikely(!(stat_err & ICE_RX_PROTO_XTR_VALID)))
+		return;
+
+	ol_flag = ice_rxdid_to_proto_xtr_ol_flag(desc->rxdid);
+	if (unlikely(!ol_flag))
+		return;
+
+	mb->ol_flags |= ol_flag;
+
+	metadata = stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) ?
+				rte_le_to_cpu_16(desc->flex_ts.flex.aux0) : 0;
+
+	if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S)))
+		metadata |= rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
+
+	*RTE_MBUF_DYNFIELD(mb, proto_xtr_metadata, uint32_t *) = metadata;
+}
+#endif
+
 static inline void
 ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 		      volatile union ice_rx_flex_desc *rxdp)
@@ -1344,28 +1385,13 @@ ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 	}
 
 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
-	init_proto_xtr_flds(mb);
-
-	stat_err = rte_le_to_cpu_16(desc->status_error1);
-	if (stat_err & ICE_RX_PROTO_XTR_VALID) {
-		struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
-			xtr->u.raw.data0 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
-			xtr->u.raw.data1 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
-
-		xtr->type = ice_rxdid_to_proto_xtr_type(desc->rxdid);
-		xtr->magic = PROTO_XTR_MAGIC_ID;
-	}
-
 	if (desc->flow_id != 0xFFFFFFFF) {
 		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 	}
+
+	if (unlikely(proto_xtr_metadata != -1))
+		ice_rxd_to_proto_xtr(mb, desc);
 #endif
 }
 
diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h
index 5de618976..9e3d2cd07 100644
--- a/drivers/net/ice/ice_rxtx.h
+++ b/drivers/net/ice/ice_rxtx.h
@@ -5,7 +5,6 @@
 #ifndef _ICE_RXTX_H_
 #define _ICE_RXTX_H_
 
-#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 
 #define ICE_ALIGN_RING_DESC  32
diff --git a/drivers/net/ice/meson.build b/drivers/net/ice/meson.build
index f9e897bbc..45e760063 100644
--- a/drivers/net/ice/meson.build
+++ b/drivers/net/ice/meson.build
@@ -36,5 +36,3 @@ if arch_subdir == 'x86'
 		objs += ice_avx2_lib.extract_objects('ice_rxtx_vec_avx2.c')
 	endif
 endif
-
-install_headers('rte_pmd_ice.h')
diff --git a/drivers/net/ice/rte_pmd_ice.h b/drivers/net/ice/rte_pmd_ice.h
deleted file mode 100644
index 719487e1e..000000000
--- a/drivers/net/ice/rte_pmd_ice.h
+++ /dev/null
@@ -1,152 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019 Intel Corporation
- */
-
-#ifndef _RTE_PMD_ICE_H_
-#define _RTE_PMD_ICE_H_
-
-#include <stdio.h>
-#include <rte_mbuf.h>
-#include <rte_ethdev.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-enum proto_xtr_type {
-	PROTO_XTR_NONE,
-	PROTO_XTR_VLAN,
-	PROTO_XTR_IPV4,
-	PROTO_XTR_IPV6,
-	PROTO_XTR_IPV6_FLOW,
-	PROTO_XTR_TCP,
-};
-
-struct proto_xtr_flds {
-	union {
-		struct {
-			uint16_t data0;
-			uint16_t data1;
-		} raw;
-		struct {
-			uint16_t stag_vid:12,
-				 stag_dei:1,
-				 stag_pcp:3;
-			uint16_t ctag_vid:12,
-				 ctag_dei:1,
-				 ctag_pcp:3;
-		} vlan;
-		struct {
-			uint16_t protocol:8,
-				 ttl:8;
-			uint16_t tos:8,
-				 ihl:4,
-				 version:4;
-		} ipv4;
-		struct {
-			uint16_t hoplimit:8,
-				 nexthdr:8;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6;
-		struct {
-			uint16_t flowlo16;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6_flow;
-		struct {
-			uint16_t fin:1,
-				 syn:1,
-				 rst:1,
-				 psh:1,
-				 ack:1,
-				 urg:1,
-				 ece:1,
-				 cwr:1,
-				 res1:4,
-				 doff:4;
-			uint16_t rsvd;
-		} tcp;
-	} u;
-
-	uint16_t rsvd;
-
-	uint8_t type;
-
-#define PROTO_XTR_MAGIC_ID	0xCE
-	uint8_t magic;
-};
-
-static inline void
-init_proto_xtr_flds(struct rte_mbuf *mb)
-{
-	mb->udata64 = 0;
-}
-
-static inline struct proto_xtr_flds *
-get_proto_xtr_flds(struct rte_mbuf *mb)
-{
-	RTE_BUILD_BUG_ON(sizeof(struct proto_xtr_flds) > sizeof(mb->udata64));
-
-	return (struct proto_xtr_flds *)&mb->udata64;
-}
-
-static inline void
-dump_proto_xtr_flds(struct rte_mbuf *mb)
-{
-	struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
-
-	if (xtr->magic != PROTO_XTR_MAGIC_ID || xtr->type == PROTO_XTR_NONE)
-		return;
-
-	printf(" - Protocol Extraction:[0x%04x:0x%04x],",
-	       xtr->u.raw.data0, xtr->u.raw.data1);
-
-	if (xtr->type == PROTO_XTR_VLAN)
-		printf("vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
-		       xtr->u.vlan.stag_pcp,
-		       xtr->u.vlan.stag_dei,
-		       xtr->u.vlan.stag_vid,
-		       xtr->u.vlan.ctag_pcp,
-		       xtr->u.vlan.ctag_dei,
-		       xtr->u.vlan.ctag_vid);
-	else if (xtr->type == PROTO_XTR_IPV4)
-		printf("ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
-		       xtr->u.ipv4.version,
-		       xtr->u.ipv4.ihl,
-		       xtr->u.ipv4.tos,
-		       xtr->u.ipv4.ttl,
-		       xtr->u.ipv4.protocol);
-	else if (xtr->type == PROTO_XTR_IPV6)
-		printf("ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
-		       xtr->u.ipv6.version,
-		       xtr->u.ipv6.tc,
-		       xtr->u.ipv6.flowhi4,
-		       xtr->u.ipv6.nexthdr,
-		       xtr->u.ipv6.hoplimit);
-	else if (xtr->type == PROTO_XTR_IPV6_FLOW)
-		printf("ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
-		       xtr->u.ipv6_flow.version,
-		       xtr->u.ipv6_flow.tc,
-		       xtr->u.ipv6_flow.flowhi4,
-		       xtr->u.ipv6_flow.flowlo16);
-	else if (xtr->type == PROTO_XTR_TCP)
-		printf("tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
-		       xtr->u.tcp.doff,
-		       xtr->u.tcp.cwr ? "C" : "",
-		       xtr->u.tcp.ece ? "E" : "",
-		       xtr->u.tcp.urg ? "U" : "",
-		       xtr->u.tcp.ack ? "A" : "",
-		       xtr->u.tcp.psh ? "P" : "",
-		       xtr->u.tcp.rst ? "R" : "",
-		       xtr->u.tcp.syn ? "S" : "",
-		       xtr->u.tcp.fin ? "F" : "");
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_PMD_ICE_H_ */
-- 
2.17.1

Re: [dpdk-dev] [PATCH v2] net/ice: optimize protocol extraction by dynamic mbuf API

[Olivier Matz]

Hi Haiyue,

Please see some comments below.

On Tue, Oct 29, 2019 at 03:34:48PM +0800, Haiyue Wang wrote:
> The original design is to use rte_mbuf::udata64 to save the metadata of
> protocol extraction which has network protocol data fields and type, an

nit: an -> a

> private API is used to decode this metadata. It is not so generic.
> 
> Use the new dynamic mbuf field and flags to handle protocol extraction.
> Then the metadata is very clean, only network protocol fields extracted
> by defined offset, and its type will be indicated by related ol_flags.
> 
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> ---

Before the patch, the result of a specific hw parsing was stored in
m->udata on Rx and this information is supposed to used by an
application or a library, is that correct?


> v2: - disable the protocol extraction if failed to register some ol_flags
>     - rewrite the commit message
> 
>  doc/guides/nics/ice.rst       |  22 +++--
>  drivers/net/ice/Makefile      |   3 -
>  drivers/net/ice/ice_ethdev.c  |  67 ++++++++++++++-
>  drivers/net/ice/ice_ethdev.h  |  18 ++++
>  drivers/net/ice/ice_rxtx.c    |  82 +++++++++++-------
>  drivers/net/ice/ice_rxtx.h    |   1 -
>  drivers/net/ice/meson.build   |   2 -
>  drivers/net/ice/rte_pmd_ice.h | 152 ----------------------------------
>  8 files changed, 152 insertions(+), 195 deletions(-)
>  delete mode 100644 drivers/net/ice/rte_pmd_ice.h
> 
> diff --git a/doc/guides/nics/ice.rst b/doc/guides/nics/ice.rst
> index 933f63480..f460d1cfd 100644
> --- a/doc/guides/nics/ice.rst
> +++ b/doc/guides/nics/ice.rst
> @@ -82,8 +82,8 @@ Runtime Config Options
>  
>  - ``Protocol extraction for per queue``
>  
> -  Configure the RX queues to do protocol extraction into ``rte_mbuf::udata64``
> -  for protocol handling acceleration, like checking the TCP SYN packets quickly.
> +  Configure the RX queues to do protocol extraction into mbuf for protocol
> +  handling acceleration, like checking the TCP SYN packets quickly.
>  
>    The argument format is::
>  
> @@ -111,7 +111,9 @@ Runtime Config Options
>    This setting means queues 1, 2-3, 8-9 are TCP extraction, queues 10-23 are
>    IPv6 extraction, other queues use the default VLAN extraction.
>  
> -  The extraction will be copied into the lower 32 bit of ``rte_mbuf::udata64``.
> +  The extraction metadata will be copied into the registered dynamic mbuf field
> +  with name: "proto-xtr-metadata", size: 4B, align: __alignof__(uint32_t), and
> +  the related dynamic mbuf flags in ``rte_mbuf::ol_flags`` will be set.
>  
>    .. table:: Protocol extraction : ``vlan``
>  
> @@ -125,6 +127,8 @@ Runtime Config Options
>  
>    VLAN2 - C-VLAN (second for QinQ).
>  
> +  Dynamic mbuf flag : "proto-xtr-ol-vlan"
> +
>    .. table:: Protocol extraction : ``ipv4``
>  
>     +----------------------------+----------------------------+
> @@ -137,6 +141,8 @@ Runtime Config Options
>  
>    IPHDR2 - IPv4 header word 0, "Ver", "Hdr Len" and "Type of Service" fields.
>  
> +  Dynamic mbuf flag : "proto-xtr-ol-ipv4"
> +
>    .. table:: Protocol extraction : ``ipv6``
>  
>     +----------------------------+----------------------------+
> @@ -150,6 +156,8 @@ Runtime Config Options
>    IPHDR2 - IPv6 header word 0, "Ver", "Traffic class" and high 4 bits of
>    "Flow Label" fields.
>  
> +  Dynamic mbuf flag : "proto-xtr-ol-ipv6"
> +
>    .. table:: Protocol extraction : ``ipv6_flow``
>  
>     +----------------------------+----------------------------+
> @@ -163,6 +171,8 @@ Runtime Config Options
>    IPHDR2 - IPv6 header word 0, "Ver", "Traffic class" and high 4 bits of
>    "Flow Label" fields.
>  
> +  Dynamic mbuf flag : "proto-xtr-ol-ipv6-flow"
> +
>    .. table:: Protocol extraction : ``tcp``
>  
>     +----------------------------+----------------------------+
> @@ -175,11 +185,7 @@ Runtime Config Options
>  
>    TCPHDR2 - Reserved
>  
> -  Use ``get_proto_xtr_flds(struct rte_mbuf *mb)`` to access the protocol
> -  extraction, do not use ``rte_mbuf::udata64`` directly.
> -
> -  The ``dump_proto_xtr_flds(struct rte_mbuf *mb)`` routine shows how to
> -  access the protocol extraction result in ``struct rte_mbuf``.
> +  Dynamic mbuf flag : "proto-xtr-ol-tcp"
>  
>  Driver compilation and testing
>  ------------------------------
> diff --git a/drivers/net/ice/Makefile b/drivers/net/ice/Makefile
> index 7c3b6a7ff..a1817e14c 100644
> --- a/drivers/net/ice/Makefile
> +++ b/drivers/net/ice/Makefile
> @@ -84,7 +84,4 @@ ifeq ($(CC_AVX2_SUPPORT), 1)
>  endif
>  SRCS-$(CONFIG_RTE_LIBRTE_ICE_PMD) += ice_generic_flow.c
>  
> -# install this header file
> -SYMLINK-$(CONFIG_RTE_LIBRTE_ICE_PMD)-include := rte_pmd_ice.h
> -
>  include $(RTE_SDK)/mk/rte.lib.mk
> diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
> index d74675842..c68acdef0 100644
> --- a/drivers/net/ice/ice_ethdev.c
> +++ b/drivers/net/ice/ice_ethdev.c
> @@ -30,6 +30,26 @@ static const char * const ice_valid_args[] = {
>  	NULL
>  };
>  
> +static const struct rte_mbuf_dynfield proto_xtr_metadata_param = {
> +	.name = "proto-xtr-metadata",
> +	.size = sizeof(uint32_t),
> +	.align = __alignof__(uint32_t),
> +	.flags = 0,
> +};
> +
> +struct proto_xtr_ol_flag {
> +	const struct rte_mbuf_dynflag param;
> +	uint64_t *flag;
> +};
> +
> +static const struct proto_xtr_ol_flag proto_xtr_ol_flag_params[] = {
> +	{ { "proto-xtr-ol-vlan", 0 },      &proto_xtr_ol_vlan },
> +	{ { "proto-xtr-ol-ipv4",  0 },     &proto_xtr_ol_ipv4 },
> +	{ { "proto-xtr-ol-ipv6", 0 },      &proto_xtr_ol_ipv6 },
> +	{ { "proto-xtr-ol-ipv6-flow", 0 }, &proto_xtr_ol_ipv6_flow },
> +	{ { "proto-xtr-ol-tcp", 0 },       &proto_xtr_ol_tcp },
> +};

If this fields are going to be used by an application, I think you need
to provide an api to access to the fields.

Solution 1, do the same than in Slava's patch (flow metadata):

- provide a function to do the registration (something like
  rte_pmd_ice_proto_xtr_register()), that sets global offset and masks.
- export these global offset and masks
- provide static inline helpers to access to the dyn fields/flags

Solution 2, without global variable export and helpers:

- define the rte_mbuf_dynfield and rte_mbuf_dynflag structures in
  the .h as static const, so they can be used by the application
- to access to the dynamic fields/flags, the application calls
  rte_mbuf_dynfield_register(&rte_pmd_ice_proto_xtr_metadata_param) to get
  the offset, and store it privately. Then it uses the generic macros
  to access to the field.


>  #define ICE_DFLT_OUTER_TAG_TYPE ICE_AQ_VSI_OUTER_TAG_VLAN_9100
>  
>  /* DDP package search path */
> @@ -1385,6 +1405,9 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
>  			ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
>  	struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
>  	struct ice_hw *hw = ICE_PF_TO_HW(pf);
> +	const struct proto_xtr_ol_flag *ol_flag;
> +	bool proto_xtr_enable = false;
> +	int offset;
>  	uint16_t i;
>  
>  	if (!ice_proto_xtr_support(hw)) {
> @@ -1398,10 +1421,52 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
>  		return;
>  	}
>  
> -	for (i = 0; i < pf->lan_nb_qps; i++)
> +	for (i = 0; i < pf->lan_nb_qps; i++) {
>  		pf->proto_xtr[i] = ad->devargs.proto_xtr[i] != PROTO_XTR_NONE ?
>  				   ad->devargs.proto_xtr[i] :
>  				   ad->devargs.proto_xtr_dflt;
> +
> +		if (pf->proto_xtr[i] != PROTO_XTR_NONE && !proto_xtr_enable)
> +			proto_xtr_enable = true;
> +	}
> +
> +	if (likely(!proto_xtr_enable))
> +		return;
> +
> +	offset = rte_mbuf_dynfield_register(&proto_xtr_metadata_param);
> +	if (unlikely(offset == -1)) {
> +		PMD_DRV_LOG(ERR,
> +			    "Protocol extraction metadata is disabled in mbuf with error %d",
> +			    -rte_errno);
> +		return;
> +	}

I think it is quite dangerous not to return an error in case the
registration fails. If the application makes use of a helper (in case of
solution 1), it may crash if there is no verification in the helper.

Another thing that can be a problem: from what I see, the registration
of the dynamic field will always be done as soon as the ice port is
configured. But maybe the user does not want to use this feature, and
prefers instead to keep space in mbuf for another purpose. I think this
should be an option.

A possibility is to let the application do the registration through the
registration helper (still solution 1). If the application wants to use
this feature, it will first register the fields/flags, and they will be
filled by the PMD on rx if they are registered.

> +
> +	PMD_DRV_LOG(DEBUG,
> +		    "Protocol extraction metadata offset in mbuf is : %d",
> +		    offset);
> +	proto_xtr_metadata = offset;
> +
> +	for (i = 0; i < RTE_DIM(proto_xtr_ol_flag_params); i++) {
> +		ol_flag = &proto_xtr_ol_flag_params[i];
> +
> +		offset = rte_mbuf_dynflag_register(&ol_flag->param);
> +		if (unlikely(offset == -1)) {
> +			PMD_DRV_LOG(ERR,
> +				    "Protocol extraction offload '%s' failed to register with error %d",
> +				    ol_flag->param.name, -rte_errno);
> +			break;
> +		}
> +
> +		PMD_DRV_LOG(DEBUG,
> +			    "Protocol extraction offload '%s' offset in mbuf is : %d",
> +			    ol_flag->param.name, offset);
> +		*ol_flag->flag = 1ULL << offset;
> +	}
> +
> +	if (i != RTE_DIM(proto_xtr_ol_flag_params)) {
> +		PMD_DRV_LOG(ERR, "Protocol extraction metadata is disabled");
> +		proto_xtr_metadata = -1;
> +	}
>  }
>  
>  /*  Initialize SW parameters of PF */
> diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
> index de67e5934..1ca00bda4 100644
> --- a/drivers/net/ice/ice_ethdev.h
> +++ b/drivers/net/ice/ice_ethdev.h
> @@ -8,6 +8,7 @@
>  #include <rte_kvargs.h>
>  
>  #include <rte_ethdev_driver.h>
> +#include <rte_mbuf_dyn.h>
>  
>  #include "base/ice_common.h"
>  #include "base/ice_adminq_cmd.h"
> @@ -241,6 +242,23 @@ struct ice_vsi {
>  	bool offset_loaded;
>  };
>  
> +enum proto_xtr_type {
> +	PROTO_XTR_NONE,
> +	PROTO_XTR_VLAN,
> +	PROTO_XTR_IPV4,
> +	PROTO_XTR_IPV6,
> +	PROTO_XTR_IPV6_FLOW,
> +	PROTO_XTR_TCP,
> +};
> +
> +extern int proto_xtr_metadata;
> +
> +extern uint64_t proto_xtr_ol_vlan;
> +extern uint64_t proto_xtr_ol_ipv4;
> +extern uint64_t proto_xtr_ol_ipv6;
> +extern uint64_t proto_xtr_ol_ipv6_flow;
> +extern uint64_t proto_xtr_ol_tcp;
> +
>  enum ice_fdir_tunnel_type {
>  	ICE_FDIR_TUNNEL_TYPE_NONE = 0,
>  	ICE_FDIR_TUNNEL_TYPE_VXLAN,
> diff --git a/drivers/net/ice/ice_rxtx.c b/drivers/net/ice/ice_rxtx.c
> index 8d4820d3c..6493aa700 100644
> --- a/drivers/net/ice/ice_rxtx.c
> +++ b/drivers/net/ice/ice_rxtx.c
> @@ -13,18 +13,31 @@
>  		PKT_TX_TCP_SEG |		 \
>  		PKT_TX_OUTER_IP_CKSUM)
>  
> -static inline uint8_t
> -ice_rxdid_to_proto_xtr_type(uint8_t rxdid)
> -{
> -	static uint8_t xtr_map[] = {
> -		[ICE_RXDID_COMMS_AUX_VLAN]      = PROTO_XTR_VLAN,
> -		[ICE_RXDID_COMMS_AUX_IPV4]      = PROTO_XTR_IPV4,
> -		[ICE_RXDID_COMMS_AUX_IPV6]      = PROTO_XTR_IPV6,
> -		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = PROTO_XTR_IPV6_FLOW,
> -		[ICE_RXDID_COMMS_AUX_TCP]       = PROTO_XTR_TCP,
> +/* Offset of mbuf dynamic field for protocol extraction data */
> +int proto_xtr_metadata = -1;
> +
> +/* Mask of mbuf dynamic flags for protocol extraction type */
> +uint64_t proto_xtr_ol_vlan;
> +uint64_t proto_xtr_ol_ipv4;
> +uint64_t proto_xtr_ol_ipv6;
> +uint64_t proto_xtr_ol_ipv6_flow;
> +uint64_t proto_xtr_ol_tcp;
> +
> +static inline uint64_t
> +ice_rxdid_to_proto_xtr_ol_flag(uint8_t rxdid)
> +{
> +	static uint64_t *ol_flag_map[] = {
> +		[ICE_RXDID_COMMS_AUX_VLAN]      = &proto_xtr_ol_vlan,
> +		[ICE_RXDID_COMMS_AUX_IPV4]      = &proto_xtr_ol_ipv4,
> +		[ICE_RXDID_COMMS_AUX_IPV6]      = &proto_xtr_ol_ipv6,
> +		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = &proto_xtr_ol_ipv6_flow,
> +		[ICE_RXDID_COMMS_AUX_TCP]       = &proto_xtr_ol_tcp,
>  	};
> +	uint64_t *ol_flag;
>  
> -	return rxdid < RTE_DIM(xtr_map) ? xtr_map[rxdid] : PROTO_XTR_NONE;
> +	ol_flag = rxdid < RTE_DIM(ol_flag_map) ? ol_flag_map[rxdid] : NULL;
> +
> +	return ol_flag != NULL ? *ol_flag : 0;

Just an observation: I wonder if it wouldn't be more efficient to
register a 3 consecutive flags, and use it as a value: 0=none, 1=vlan,
2=ipv4, 3=ipv6, ...
Unfortunately, there is no 'count' parameter to the
rte_mbuf_dynflag_register() function, but I think is doable easily.


>  }
>  
>  static inline uint8_t
> @@ -1325,10 +1338,38 @@ ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc *rxdp)
>  		   mb->vlan_tci, mb->vlan_tci_outer);
>  }
>  
> +#ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
>  #define ICE_RX_PROTO_XTR_VALID \
>  	((1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) | \
>  	 (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
>  
> +static void
> +ice_rxd_to_proto_xtr(struct rte_mbuf *mb,
> +		     volatile struct ice_32b_rx_flex_desc_comms *desc)
> +{
> +	uint16_t stat_err = rte_le_to_cpu_16(desc->status_error1);
> +	uint32_t metadata;
> +	uint64_t ol_flag;
> +
> +	if (unlikely(!(stat_err & ICE_RX_PROTO_XTR_VALID)))
> +		return;
> +
> +	ol_flag = ice_rxdid_to_proto_xtr_ol_flag(desc->rxdid);
> +	if (unlikely(!ol_flag))
> +		return;
> +
> +	mb->ol_flags |= ol_flag;
> +
> +	metadata = stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) ?
> +				rte_le_to_cpu_16(desc->flex_ts.flex.aux0) : 0;
> +
> +	if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S)))
> +		metadata |= rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
> +
> +	*RTE_MBUF_DYNFIELD(mb, proto_xtr_metadata, uint32_t *) = metadata;
> +}
> +#endif
> +
>  static inline void
>  ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
>  		      volatile union ice_rx_flex_desc *rxdp)
> @@ -1344,28 +1385,13 @@ ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
>  	}
>  
>  #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
> -	init_proto_xtr_flds(mb);
> -
> -	stat_err = rte_le_to_cpu_16(desc->status_error1);
> -	if (stat_err & ICE_RX_PROTO_XTR_VALID) {
> -		struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
> -
> -		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
> -			xtr->u.raw.data0 =
> -				rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
> -
> -		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
> -			xtr->u.raw.data1 =
> -				rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
> -
> -		xtr->type = ice_rxdid_to_proto_xtr_type(desc->rxdid);
> -		xtr->magic = PROTO_XTR_MAGIC_ID;
> -	}
> -
>  	if (desc->flow_id != 0xFFFFFFFF) {
>  		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
>  		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
>  	}
> +
> +	if (unlikely(proto_xtr_metadata != -1))
> +		ice_rxd_to_proto_xtr(mb, desc);
>  #endif
>  }
>  
> diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h
> index 5de618976..9e3d2cd07 100644
> --- a/drivers/net/ice/ice_rxtx.h
> +++ b/drivers/net/ice/ice_rxtx.h
> @@ -5,7 +5,6 @@
>  #ifndef _ICE_RXTX_H_
>  #define _ICE_RXTX_H_
>  
> -#include "rte_pmd_ice.h"
>  #include "ice_ethdev.h"
>  
>  #define ICE_ALIGN_RING_DESC  32
> diff --git a/drivers/net/ice/meson.build b/drivers/net/ice/meson.build
> index f9e897bbc..45e760063 100644
> --- a/drivers/net/ice/meson.build
> +++ b/drivers/net/ice/meson.build
> @@ -36,5 +36,3 @@ if arch_subdir == 'x86'
>  		objs += ice_avx2_lib.extract_objects('ice_rxtx_vec_avx2.c')
>  	endif
>  endif
> -
> -install_headers('rte_pmd_ice.h')
> diff --git a/drivers/net/ice/rte_pmd_ice.h b/drivers/net/ice/rte_pmd_ice.h
> deleted file mode 100644
> index 719487e1e..000000000
> --- a/drivers/net/ice/rte_pmd_ice.h
> +++ /dev/null
> @@ -1,152 +0,0 @@
> -/* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright(c) 2019 Intel Corporation
> - */
> -
> -#ifndef _RTE_PMD_ICE_H_
> -#define _RTE_PMD_ICE_H_
> -
> -#include <stdio.h>
> -#include <rte_mbuf.h>
> -#include <rte_ethdev.h>
> -
> -#ifdef __cplusplus
> -extern "C" {
> -#endif
> -
> -enum proto_xtr_type {
> -	PROTO_XTR_NONE,
> -	PROTO_XTR_VLAN,
> -	PROTO_XTR_IPV4,
> -	PROTO_XTR_IPV6,
> -	PROTO_XTR_IPV6_FLOW,
> -	PROTO_XTR_TCP,
> -};
> -
> -struct proto_xtr_flds {
> -	union {
> -		struct {
> -			uint16_t data0;
> -			uint16_t data1;
> -		} raw;
> -		struct {
> -			uint16_t stag_vid:12,
> -				 stag_dei:1,
> -				 stag_pcp:3;
> -			uint16_t ctag_vid:12,
> -				 ctag_dei:1,
> -				 ctag_pcp:3;
> -		} vlan;
> -		struct {
> -			uint16_t protocol:8,
> -				 ttl:8;
> -			uint16_t tos:8,
> -				 ihl:4,
> -				 version:4;
> -		} ipv4;
> -		struct {
> -			uint16_t hoplimit:8,
> -				 nexthdr:8;
> -			uint16_t flowhi4:4,
> -				 tc:8,
> -				 version:4;
> -		} ipv6;
> -		struct {
> -			uint16_t flowlo16;
> -			uint16_t flowhi4:4,
> -				 tc:8,
> -				 version:4;
> -		} ipv6_flow;
> -		struct {
> -			uint16_t fin:1,
> -				 syn:1,
> -				 rst:1,
> -				 psh:1,
> -				 ack:1,
> -				 urg:1,
> -				 ece:1,
> -				 cwr:1,
> -				 res1:4,
> -				 doff:4;
> -			uint16_t rsvd;
> -		} tcp;
> -	} u;
> -
> -	uint16_t rsvd;
> -
> -	uint8_t type;
> -
> -#define PROTO_XTR_MAGIC_ID	0xCE
> -	uint8_t magic;
> -};
> -
> -static inline void
> -init_proto_xtr_flds(struct rte_mbuf *mb)
> -{
> -	mb->udata64 = 0;
> -}
> -
> -static inline struct proto_xtr_flds *
> -get_proto_xtr_flds(struct rte_mbuf *mb)
> -{
> -	RTE_BUILD_BUG_ON(sizeof(struct proto_xtr_flds) > sizeof(mb->udata64));
> -
> -	return (struct proto_xtr_flds *)&mb->udata64;
> -}
> -
> -static inline void
> -dump_proto_xtr_flds(struct rte_mbuf *mb)
> -{
> -	struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
> -
> -	if (xtr->magic != PROTO_XTR_MAGIC_ID || xtr->type == PROTO_XTR_NONE)
> -		return;
> -
> -	printf(" - Protocol Extraction:[0x%04x:0x%04x],",
> -	       xtr->u.raw.data0, xtr->u.raw.data1);
> -
> -	if (xtr->type == PROTO_XTR_VLAN)
> -		printf("vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
> -		       xtr->u.vlan.stag_pcp,
> -		       xtr->u.vlan.stag_dei,
> -		       xtr->u.vlan.stag_vid,
> -		       xtr->u.vlan.ctag_pcp,
> -		       xtr->u.vlan.ctag_dei,
> -		       xtr->u.vlan.ctag_vid);
> -	else if (xtr->type == PROTO_XTR_IPV4)
> -		printf("ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
> -		       xtr->u.ipv4.version,
> -		       xtr->u.ipv4.ihl,
> -		       xtr->u.ipv4.tos,
> -		       xtr->u.ipv4.ttl,
> -		       xtr->u.ipv4.protocol);
> -	else if (xtr->type == PROTO_XTR_IPV6)
> -		printf("ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
> -		       xtr->u.ipv6.version,
> -		       xtr->u.ipv6.tc,
> -		       xtr->u.ipv6.flowhi4,
> -		       xtr->u.ipv6.nexthdr,
> -		       xtr->u.ipv6.hoplimit);
> -	else if (xtr->type == PROTO_XTR_IPV6_FLOW)
> -		printf("ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
> -		       xtr->u.ipv6_flow.version,
> -		       xtr->u.ipv6_flow.tc,
> -		       xtr->u.ipv6_flow.flowhi4,
> -		       xtr->u.ipv6_flow.flowlo16);
> -	else if (xtr->type == PROTO_XTR_TCP)
> -		printf("tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
> -		       xtr->u.tcp.doff,
> -		       xtr->u.tcp.cwr ? "C" : "",
> -		       xtr->u.tcp.ece ? "E" : "",
> -		       xtr->u.tcp.urg ? "U" : "",
> -		       xtr->u.tcp.ack ? "A" : "",
> -		       xtr->u.tcp.psh ? "P" : "",
> -		       xtr->u.tcp.rst ? "R" : "",
> -		       xtr->u.tcp.syn ? "S" : "",
> -		       xtr->u.tcp.fin ? "F" : "");
> -}
> -
> -#ifdef __cplusplus
> -}
> -#endif
> -
> -#endif /* _RTE_PMD_ICE_H_ */
> -- 
> 2.17.1
> 

Re: [dpdk-dev] [PATCH v2] net/ice: optimize protocol extraction by dynamic mbuf API

[Wang, Haiyue]

Hi Olivier,

Very appreciate your response, please see reply below.

> -----Original Message-----
> From: Olivier Matz <olivier.matz@6wind.com>
> Sent: Thursday, October 31, 2019 00:56
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org
> Subject: Re: [dpdk-dev] [PATCH v2] net/ice: optimize protocol extraction by dynamic mbuf API
> 
> Hi Haiyue,
> 
> Please see some comments below.
> 
> On Tue, Oct 29, 2019 at 03:34:48PM +0800, Haiyue Wang wrote:
> > The original design is to use rte_mbuf::udata64 to save the metadata of
> > protocol extraction which has network protocol data fields and type, an
> 
> nit: an -> a

Fix next version, thanks.

> 
> > private API is used to decode this metadata. It is not so generic.
> >
> > Use the new dynamic mbuf field and flags to handle protocol extraction.
> > Then the metadata is very clean, only network protocol fields extracted
> > by defined offset, and its type will be indicated by related ol_flags.
> >
> > Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> > ---
> 
> Before the patch, the result of a specific hw parsing was stored in
> m->udata on Rx and this information is supposed to used by an
> application or a library, is that correct?
> 

Yes.

> 
> > v2: - disable the protocol extraction if failed to register some ol_flags
> >     - rewrite the commit message
> >
> >  doc/guides/nics/ice.rst       |  22 +++--
> >  drivers/net/ice/Makefile      |   3 -
> >  drivers/net/ice/ice_ethdev.c  |  67 ++++++++++++++-
> >  drivers/net/ice/ice_ethdev.h  |  18 ++++
> >  drivers/net/ice/ice_rxtx.c    |  82 +++++++++++-------
> >  drivers/net/ice/ice_rxtx.h    |   1 -
> >  drivers/net/ice/meson.build   |   2 -
> >  drivers/net/ice/rte_pmd_ice.h | 152 ----------------------------------
> >  8 files changed, 152 insertions(+), 195 deletions(-)
> >  delete mode 100644 drivers/net/ice/rte_pmd_ice.h
> >
> > diff --git a/doc/guides/nics/ice.rst b/doc/guides/nics/ice.rst
> > index 933f63480..f460d1cfd 100644
> > --- a/doc/guides/nics/ice.rst
> > +++ b/doc/guides/nics/ice.rst
> > @@ -82,8 +82,8 @@ Runtime Config Options
> >
> >  - ``Protocol extraction for per queue``
> >
> > -  Configure the RX queues to do protocol extraction into ``rte_mbuf::udata64``
> > -  for protocol handling acceleration, like checking the TCP SYN packets quickly.
> > +  Configure the RX queues to do protocol extraction into mbuf for protocol
> > +  handling acceleration, like checking the TCP SYN packets quickly.
> >
> >    The argument format is::
> >
> > @@ -111,7 +111,9 @@ Runtime Config Options
> >    This setting means queues 1, 2-3, 8-9 are TCP extraction, queues 10-23 are
> >    IPv6 extraction, other queues use the default VLAN extraction.
> >
> > -  The extraction will be copied into the lower 32 bit of ``rte_mbuf::udata64``.
> > +  The extraction metadata will be copied into the registered dynamic mbuf field
> > +  with name: "proto-xtr-metadata", size: 4B, align: __alignof__(uint32_t), and
> > +  the related dynamic mbuf flags in ``rte_mbuf::ol_flags`` will be set.
> >
> >    .. table:: Protocol extraction : ``vlan``
> >
> > @@ -125,6 +127,8 @@ Runtime Config Options
> >
> >    VLAN2 - C-VLAN (second for QinQ).
> >
> > +  Dynamic mbuf flag : "proto-xtr-ol-vlan"
> > +
> >    .. table:: Protocol extraction : ``ipv4``
> >
> >     +----------------------------+----------------------------+
> > @@ -137,6 +141,8 @@ Runtime Config Options
> >
> >    IPHDR2 - IPv4 header word 0, "Ver", "Hdr Len" and "Type of Service" fields.
> >
> > +  Dynamic mbuf flag : "proto-xtr-ol-ipv4"
> > +
> >    .. table:: Protocol extraction : ``ipv6``
> >
> >     +----------------------------+----------------------------+
> > @@ -150,6 +156,8 @@ Runtime Config Options
> >    IPHDR2 - IPv6 header word 0, "Ver", "Traffic class" and high 4 bits of
> >    "Flow Label" fields.
> >
> > +  Dynamic mbuf flag : "proto-xtr-ol-ipv6"
> > +
> >    .. table:: Protocol extraction : ``ipv6_flow``
> >
> >     +----------------------------+----------------------------+
> > @@ -163,6 +171,8 @@ Runtime Config Options
> >    IPHDR2 - IPv6 header word 0, "Ver", "Traffic class" and high 4 bits of
> >    "Flow Label" fields.
> >
> > +  Dynamic mbuf flag : "proto-xtr-ol-ipv6-flow"
> > +
> >    .. table:: Protocol extraction : ``tcp``
> >
> >     +----------------------------+----------------------------+
> > @@ -175,11 +185,7 @@ Runtime Config Options
> >
> >    TCPHDR2 - Reserved
> >
> > -  Use ``get_proto_xtr_flds(struct rte_mbuf *mb)`` to access the protocol
> > -  extraction, do not use ``rte_mbuf::udata64`` directly.
> > -
> > -  The ``dump_proto_xtr_flds(struct rte_mbuf *mb)`` routine shows how to
> > -  access the protocol extraction result in ``struct rte_mbuf``.
> > +  Dynamic mbuf flag : "proto-xtr-ol-tcp"
> >
> >  Driver compilation and testing
> >  ------------------------------
> > diff --git a/drivers/net/ice/Makefile b/drivers/net/ice/Makefile
> > index 7c3b6a7ff..a1817e14c 100644
> > --- a/drivers/net/ice/Makefile
> > +++ b/drivers/net/ice/Makefile
> > @@ -84,7 +84,4 @@ ifeq ($(CC_AVX2_SUPPORT), 1)
> >  endif
> >  SRCS-$(CONFIG_RTE_LIBRTE_ICE_PMD) += ice_generic_flow.c
> >
> > -# install this header file
> > -SYMLINK-$(CONFIG_RTE_LIBRTE_ICE_PMD)-include := rte_pmd_ice.h
> > -
> >  include $(RTE_SDK)/mk/rte.lib.mk
> > diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
> > index d74675842..c68acdef0 100644
> > --- a/drivers/net/ice/ice_ethdev.c
> > +++ b/drivers/net/ice/ice_ethdev.c
> > @@ -30,6 +30,26 @@ static const char * const ice_valid_args[] = {
> >  	NULL
> >  };
> >
> > +static const struct rte_mbuf_dynfield proto_xtr_metadata_param = {
> > +	.name = "proto-xtr-metadata",
> > +	.size = sizeof(uint32_t),
> > +	.align = __alignof__(uint32_t),
> > +	.flags = 0,
> > +};
> > +
> > +struct proto_xtr_ol_flag {
> > +	const struct rte_mbuf_dynflag param;
> > +	uint64_t *flag;
> > +};
> > +
> > +static const struct proto_xtr_ol_flag proto_xtr_ol_flag_params[] = {
> > +	{ { "proto-xtr-ol-vlan", 0 },      &proto_xtr_ol_vlan },
> > +	{ { "proto-xtr-ol-ipv4",  0 },     &proto_xtr_ol_ipv4 },
> > +	{ { "proto-xtr-ol-ipv6", 0 },      &proto_xtr_ol_ipv6 },
> > +	{ { "proto-xtr-ol-ipv6-flow", 0 }, &proto_xtr_ol_ipv6_flow },
> > +	{ { "proto-xtr-ol-tcp", 0 },       &proto_xtr_ol_tcp },
> > +};
> 
> If this fields are going to be used by an application, I think you need
> to provide an api to access to the fields.
> 
> Solution 1, do the same than in Slava's patch (flow metadata):
> 
> - provide a function to do the registration (something like
>   rte_pmd_ice_proto_xtr_register()), that sets global offset and masks.
> - export these global offset and masks
> - provide static inline helpers to access to the dyn fields/flags
> 
> Solution 2, without global variable export and helpers:
> 
> - define the rte_mbuf_dynfield and rte_mbuf_dynflag structures in
>   the .h as static const, so they can be used by the application
> - to access to the dynamic fields/flags, the application calls
>   rte_mbuf_dynfield_register(&rte_pmd_ice_proto_xtr_metadata_param) to get
>   the offset, and store it privately. Then it uses the generic macros
>   to access to the field.
> 
> 

Good practice for reducing the application's effort, I will refer
to Slava's design.

> >  #define ICE_DFLT_OUTER_TAG_TYPE ICE_AQ_VSI_OUTER_TAG_VLAN_9100
> >
> >  /* DDP package search path */
> > @@ -1385,6 +1405,9 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
> >  			ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
> >  	struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
> >  	struct ice_hw *hw = ICE_PF_TO_HW(pf);
> > +	const struct proto_xtr_ol_flag *ol_flag;
> > +	bool proto_xtr_enable = false;
> > +	int offset;
> >  	uint16_t i;
> >
> >  	if (!ice_proto_xtr_support(hw)) {
> > @@ -1398,10 +1421,52 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
> >  		return;
> >  	}
> >
> > -	for (i = 0; i < pf->lan_nb_qps; i++)
> > +	for (i = 0; i < pf->lan_nb_qps; i++) {
> >  		pf->proto_xtr[i] = ad->devargs.proto_xtr[i] != PROTO_XTR_NONE ?
> >  				   ad->devargs.proto_xtr[i] :
> >  				   ad->devargs.proto_xtr_dflt;
> > +
> > +		if (pf->proto_xtr[i] != PROTO_XTR_NONE && !proto_xtr_enable)
> > +			proto_xtr_enable = true;
> > +	}
> > +
> > +	if (likely(!proto_xtr_enable))
> > +		return;
> > +
> > +	offset = rte_mbuf_dynfield_register(&proto_xtr_metadata_param);
> > +	if (unlikely(offset == -1)) {
> > +		PMD_DRV_LOG(ERR,
> > +			    "Protocol extraction metadata is disabled in mbuf with error %d",
> > +			    -rte_errno);
> > +		return;
> > +	}
> 
> I think it is quite dangerous not to return an error in case the
> registration fails. If the application makes use of a helper (in case of
> solution 1), it may crash if there is no verification in the helper.
> 
> Another thing that can be a problem: from what I see, the registration
> of the dynamic field will always be done as soon as the ice port is
> configured. But maybe the user does not want to use this feature, and
> prefers instead to keep space in mbuf for another purpose. I think this
> should be an option.
> 
> A possibility is to let the application do the registration through the
> registration helper (still solution 1). If the application wants to use
> this feature, it will first register the fields/flags, and they will be
> filled by the PMD on rx if they are registered.
> 

In fact, this feature is only enabled explicitly by "-w 18:00.0,proto_xtr=vlan",
by default, it is disabled. In other words, if the application pass the 'proto_xtr'
dev_args, the ice PMD will call 'rte_mbuf_dynXXX_register' instead, the application
jus needs to call 'rte_mbuf_dynXXX_lookup' to get the offset.

So looks like Solution 2 is better ? Since the application just needs to know
the name, size, align.

	for (i = 0; i < pf->lan_nb_qps; i++) {
		pf->proto_xtr[i] = ad->devargs.proto_xtr[i] != PROTO_XTR_NONE ?
				   ad->devargs.proto_xtr[i] :
				   ad->devargs.proto_xtr_dflt;

		if (pf->proto_xtr[i] != PROTO_XTR_NONE && !proto_xtr_enable)
			proto_xtr_enable = true;
	}

	if (likely(!proto_xtr_enable)) <---- Will return gracefully if no 'proto_xtr' dev_args.
		return;

	offset = rte_mbuf_dynfield_register(&proto_xtr_metadata_param);

> > +
> > +	PMD_DRV_LOG(DEBUG,
> > +		    "Protocol extraction metadata offset in mbuf is : %d",
> > +		    offset);
> > +	proto_xtr_metadata = offset;
> > +
> > +	for (i = 0; i < RTE_DIM(proto_xtr_ol_flag_params); i++) {
> > +		ol_flag = &proto_xtr_ol_flag_params[i];
> > +
> > +		offset = rte_mbuf_dynflag_register(&ol_flag->param);
> > +		if (unlikely(offset == -1)) {
> > +			PMD_DRV_LOG(ERR,
> > +				    "Protocol extraction offload '%s' failed to register with error %d",
> > +				    ol_flag->param.name, -rte_errno);
> > +			break;
> > +		}
> > +
> > +		PMD_DRV_LOG(DEBUG,
> > +			    "Protocol extraction offload '%s' offset in mbuf is : %d",
> > +			    ol_flag->param.name, offset);
> > +		*ol_flag->flag = 1ULL << offset;
> > +	}
> > +
> > +	if (i != RTE_DIM(proto_xtr_ol_flag_params)) {
> > +		PMD_DRV_LOG(ERR, "Protocol extraction metadata is disabled");
> > +		proto_xtr_metadata = -1;
> > +	}
> >  }
> >
> >  /*  Initialize SW parameters of PF */
> > diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
> > index de67e5934..1ca00bda4 100644
> > --- a/drivers/net/ice/ice_ethdev.h
> > +++ b/drivers/net/ice/ice_ethdev.h
> > @@ -8,6 +8,7 @@
> >  #include <rte_kvargs.h>
> >
> >  #include <rte_ethdev_driver.h>
> > +#include <rte_mbuf_dyn.h>
> >
> >  #include "base/ice_common.h"
> >  #include "base/ice_adminq_cmd.h"
> > @@ -241,6 +242,23 @@ struct ice_vsi {
> >  	bool offset_loaded;
> >  };
> >
> > +enum proto_xtr_type {
> > +	PROTO_XTR_NONE,
> > +	PROTO_XTR_VLAN,
> > +	PROTO_XTR_IPV4,
> > +	PROTO_XTR_IPV6,
> > +	PROTO_XTR_IPV6_FLOW,
> > +	PROTO_XTR_TCP,
> > +};
> > +
> > +extern int proto_xtr_metadata;
> > +
> > +extern uint64_t proto_xtr_ol_vlan;
> > +extern uint64_t proto_xtr_ol_ipv4;
> > +extern uint64_t proto_xtr_ol_ipv6;
> > +extern uint64_t proto_xtr_ol_ipv6_flow;
> > +extern uint64_t proto_xtr_ol_tcp;
> > +
> >  enum ice_fdir_tunnel_type {
> >  	ICE_FDIR_TUNNEL_TYPE_NONE = 0,
> >  	ICE_FDIR_TUNNEL_TYPE_VXLAN,
> > diff --git a/drivers/net/ice/ice_rxtx.c b/drivers/net/ice/ice_rxtx.c
> > index 8d4820d3c..6493aa700 100644
> > --- a/drivers/net/ice/ice_rxtx.c
> > +++ b/drivers/net/ice/ice_rxtx.c
> > @@ -13,18 +13,31 @@
> >  		PKT_TX_TCP_SEG |		 \
> >  		PKT_TX_OUTER_IP_CKSUM)
> >
> > -static inline uint8_t
> > -ice_rxdid_to_proto_xtr_type(uint8_t rxdid)
> > -{
> > -	static uint8_t xtr_map[] = {
> > -		[ICE_RXDID_COMMS_AUX_VLAN]      = PROTO_XTR_VLAN,
> > -		[ICE_RXDID_COMMS_AUX_IPV4]      = PROTO_XTR_IPV4,
> > -		[ICE_RXDID_COMMS_AUX_IPV6]      = PROTO_XTR_IPV6,
> > -		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = PROTO_XTR_IPV6_FLOW,
> > -		[ICE_RXDID_COMMS_AUX_TCP]       = PROTO_XTR_TCP,
> > +/* Offset of mbuf dynamic field for protocol extraction data */
> > +int proto_xtr_metadata = -1;
> > +
> > +/* Mask of mbuf dynamic flags for protocol extraction type */
> > +uint64_t proto_xtr_ol_vlan;
> > +uint64_t proto_xtr_ol_ipv4;
> > +uint64_t proto_xtr_ol_ipv6;
> > +uint64_t proto_xtr_ol_ipv6_flow;
> > +uint64_t proto_xtr_ol_tcp;
> > +
> > +static inline uint64_t
> > +ice_rxdid_to_proto_xtr_ol_flag(uint8_t rxdid)
> > +{
> > +	static uint64_t *ol_flag_map[] = {
> > +		[ICE_RXDID_COMMS_AUX_VLAN]      = &proto_xtr_ol_vlan,
> > +		[ICE_RXDID_COMMS_AUX_IPV4]      = &proto_xtr_ol_ipv4,
> > +		[ICE_RXDID_COMMS_AUX_IPV6]      = &proto_xtr_ol_ipv6,
> > +		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = &proto_xtr_ol_ipv6_flow,
> > +		[ICE_RXDID_COMMS_AUX_TCP]       = &proto_xtr_ol_tcp,
> >  	};
> > +	uint64_t *ol_flag;
> >
> > -	return rxdid < RTE_DIM(xtr_map) ? xtr_map[rxdid] : PROTO_XTR_NONE;
> > +	ol_flag = rxdid < RTE_DIM(ol_flag_map) ? ol_flag_map[rxdid] : NULL;
> > +
> > +	return ol_flag != NULL ? *ol_flag : 0;
> 
> Just an observation: I wonder if it wouldn't be more efficient to
> register a 3 consecutive flags, and use it as a value: 0=none, 1=vlan,
> 2=ipv4, 3=ipv6, ...
> Unfortunately, there is no 'count' parameter to the
> rte_mbuf_dynflag_register() function, but I think is doable easily.
> 

Yes, then we can save the flags bits if the count is BIGGER, but no
more resources, and for the registered flags can't be freed now, return
error directly will make other less flags bits request fit. And more,
no need to loop like this:

	for (i = 0; i < RTE_DIM(proto_xtr_ol_flag_params); i++) {
		ol_flag = &proto_xtr_ol_flag_params[i];

		offset = rte_mbuf_dynflag_register(&ol_flag->param);
		if (unlikely(offset == -1)) {
			break;
		}

A new API may be born. ;-)

> 
> >  }
> >
> >  static inline uint8_t
> > @@ -1325,10 +1338,38 @@ ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc
> *rxdp)
> >  		   mb->vlan_tci, mb->vlan_tci_outer);
> >  }
> >
> > +#ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
> >  #define ICE_RX_PROTO_XTR_VALID \
> >  	((1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) | \
> >  	 (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
> >
> > +static void
> > +ice_rxd_to_proto_xtr(struct rte_mbuf *mb,
> > +		     volatile struct ice_32b_rx_flex_desc_comms *desc)
> > +{
> > +	uint16_t stat_err = rte_le_to_cpu_16(desc->status_error1);
> > +	uint32_t metadata;
> > +	uint64_t ol_flag;
> > +
> > +	if (unlikely(!(stat_err & ICE_RX_PROTO_XTR_VALID)))
> > +		return;
> > +
> > +	ol_flag = ice_rxdid_to_proto_xtr_ol_flag(desc->rxdid);
> > +	if (unlikely(!ol_flag))
> > +		return;
> > +
> > +	mb->ol_flags |= ol_flag;
> > +
> > +	metadata = stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) ?
> > +				rte_le_to_cpu_16(desc->flex_ts.flex.aux0) : 0;
> > +
> > +	if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S)))
> > +		metadata |= rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
> > +
> > +	*RTE_MBUF_DYNFIELD(mb, proto_xtr_metadata, uint32_t *) = metadata;
> > +}
> > +#endif
> > +
> >  static inline void
> >  ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
> >  		      volatile union ice_rx_flex_desc *rxdp)
> > @@ -1344,28 +1385,13 @@ ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
> >  	}
> >
> >  #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
> > -	init_proto_xtr_flds(mb);
> > -
> > -	stat_err = rte_le_to_cpu_16(desc->status_error1);
> > -	if (stat_err & ICE_RX_PROTO_XTR_VALID) {
> > -		struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
> > -
> > -		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
> > -			xtr->u.raw.data0 =
> > -				rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
> > -
> > -		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
> > -			xtr->u.raw.data1 =
> > -				rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
> > -
> > -		xtr->type = ice_rxdid_to_proto_xtr_type(desc->rxdid);
> > -		xtr->magic = PROTO_XTR_MAGIC_ID;
> > -	}
> > -
> >  	if (desc->flow_id != 0xFFFFFFFF) {
> >  		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
> >  		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
> >  	}
> > +
> > +	if (unlikely(proto_xtr_metadata != -1))
> > +		ice_rxd_to_proto_xtr(mb, desc);
> >  #endif
> >  }
> >
> > diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h
> > index 5de618976..9e3d2cd07 100644
> > --- a/drivers/net/ice/ice_rxtx.h
> > +++ b/drivers/net/ice/ice_rxtx.h
> > @@ -5,7 +5,6 @@
> >  #ifndef _ICE_RXTX_H_
> >  #define _ICE_RXTX_H_
> >
> > -#include "rte_pmd_ice.h"
> >  #include "ice_ethdev.h"
> >
> >  #define ICE_ALIGN_RING_DESC  32
> > diff --git a/drivers/net/ice/meson.build b/drivers/net/ice/meson.build
> > index f9e897bbc..45e760063 100644
> > --- a/drivers/net/ice/meson.build
> > +++ b/drivers/net/ice/meson.build
> > @@ -36,5 +36,3 @@ if arch_subdir == 'x86'
> >  		objs += ice_avx2_lib.extract_objects('ice_rxtx_vec_avx2.c')
> >  	endif
> >  endif
> > -
> > -install_headers('rte_pmd_ice.h')
> > diff --git a/drivers/net/ice/rte_pmd_ice.h b/drivers/net/ice/rte_pmd_ice.h
> > deleted file mode 100644
> > index 719487e1e..000000000
> > --- a/drivers/net/ice/rte_pmd_ice.h
> > +++ /dev/null
> > @@ -1,152 +0,0 @@
> > -/* SPDX-License-Identifier: BSD-3-Clause
> > - * Copyright(c) 2019 Intel Corporation
> > - */
> > -
> > -#ifndef _RTE_PMD_ICE_H_
> > -#define _RTE_PMD_ICE_H_
> > -
> > -#include <stdio.h>
> > -#include <rte_mbuf.h>
> > -#include <rte_ethdev.h>
> > -
> > -#ifdef __cplusplus
> > -extern "C" {
> > -#endif
> > -
> > -enum proto_xtr_type {
> > -	PROTO_XTR_NONE,
> > -	PROTO_XTR_VLAN,
> > -	PROTO_XTR_IPV4,
> > -	PROTO_XTR_IPV6,
> > -	PROTO_XTR_IPV6_FLOW,
> > -	PROTO_XTR_TCP,
> > -};
> > -
> > -struct proto_xtr_flds {
> > -	union {
> > -		struct {
> > -			uint16_t data0;
> > -			uint16_t data1;
> > -		} raw;
> > -		struct {
> > -			uint16_t stag_vid:12,
> > -				 stag_dei:1,
> > -				 stag_pcp:3;
> > -			uint16_t ctag_vid:12,
> > -				 ctag_dei:1,
> > -				 ctag_pcp:3;
> > -		} vlan;
> > -		struct {
> > -			uint16_t protocol:8,
> > -				 ttl:8;
> > -			uint16_t tos:8,
> > -				 ihl:4,
> > -				 version:4;
> > -		} ipv4;
> > -		struct {
> > -			uint16_t hoplimit:8,
> > -				 nexthdr:8;
> > -			uint16_t flowhi4:4,
> > -				 tc:8,
> > -				 version:4;
> > -		} ipv6;
> > -		struct {
> > -			uint16_t flowlo16;
> > -			uint16_t flowhi4:4,
> > -				 tc:8,
> > -				 version:4;
> > -		} ipv6_flow;
> > -		struct {
> > -			uint16_t fin:1,
> > -				 syn:1,
> > -				 rst:1,
> > -				 psh:1,
> > -				 ack:1,
> > -				 urg:1,
> > -				 ece:1,
> > -				 cwr:1,
> > -				 res1:4,
> > -				 doff:4;
> > -			uint16_t rsvd;
> > -		} tcp;
> > -	} u;
> > -
> > -	uint16_t rsvd;
> > -
> > -	uint8_t type;
> > -
> > -#define PROTO_XTR_MAGIC_ID	0xCE
> > -	uint8_t magic;
> > -};
> > -
> > -static inline void
> > -init_proto_xtr_flds(struct rte_mbuf *mb)
> > -{
> > -	mb->udata64 = 0;
> > -}
> > -
> > -static inline struct proto_xtr_flds *
> > -get_proto_xtr_flds(struct rte_mbuf *mb)
> > -{
> > -	RTE_BUILD_BUG_ON(sizeof(struct proto_xtr_flds) > sizeof(mb->udata64));
> > -
> > -	return (struct proto_xtr_flds *)&mb->udata64;
> > -}
> > -
> > -static inline void
> > -dump_proto_xtr_flds(struct rte_mbuf *mb)
> > -{
> > -	struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
> > -
> > -	if (xtr->magic != PROTO_XTR_MAGIC_ID || xtr->type == PROTO_XTR_NONE)
> > -		return;
> > -
> > -	printf(" - Protocol Extraction:[0x%04x:0x%04x],",
> > -	       xtr->u.raw.data0, xtr->u.raw.data1);
> > -
> > -	if (xtr->type == PROTO_XTR_VLAN)
> > -		printf("vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
> > -		       xtr->u.vlan.stag_pcp,
> > -		       xtr->u.vlan.stag_dei,
> > -		       xtr->u.vlan.stag_vid,
> > -		       xtr->u.vlan.ctag_pcp,
> > -		       xtr->u.vlan.ctag_dei,
> > -		       xtr->u.vlan.ctag_vid);
> > -	else if (xtr->type == PROTO_XTR_IPV4)
> > -		printf("ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
> > -		       xtr->u.ipv4.version,
> > -		       xtr->u.ipv4.ihl,
> > -		       xtr->u.ipv4.tos,
> > -		       xtr->u.ipv4.ttl,
> > -		       xtr->u.ipv4.protocol);
> > -	else if (xtr->type == PROTO_XTR_IPV6)
> > -		printf("ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
> > -		       xtr->u.ipv6.version,
> > -		       xtr->u.ipv6.tc,
> > -		       xtr->u.ipv6.flowhi4,
> > -		       xtr->u.ipv6.nexthdr,
> > -		       xtr->u.ipv6.hoplimit);
> > -	else if (xtr->type == PROTO_XTR_IPV6_FLOW)
> > -		printf("ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
> > -		       xtr->u.ipv6_flow.version,
> > -		       xtr->u.ipv6_flow.tc,
> > -		       xtr->u.ipv6_flow.flowhi4,
> > -		       xtr->u.ipv6_flow.flowlo16);
> > -	else if (xtr->type == PROTO_XTR_TCP)
> > -		printf("tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
> > -		       xtr->u.tcp.doff,
> > -		       xtr->u.tcp.cwr ? "C" : "",
> > -		       xtr->u.tcp.ece ? "E" : "",
> > -		       xtr->u.tcp.urg ? "U" : "",
> > -		       xtr->u.tcp.ack ? "A" : "",
> > -		       xtr->u.tcp.psh ? "P" : "",
> > -		       xtr->u.tcp.rst ? "R" : "",
> > -		       xtr->u.tcp.syn ? "S" : "",
> > -		       xtr->u.tcp.fin ? "F" : "");
> > -}
> > -
> > -#ifdef __cplusplus
> > -}
> > -#endif
> > -
> > -#endif /* _RTE_PMD_ICE_H_ */
> > --
> > 2.17.1
> >

[dpdk-dev] [PATCH v3] net/ice: optimize protocol extraction by dynamic mbuf API

[Haiyue Wang]

The original design is to use rte_mbuf::udata64 to save the metadata of
protocol extraction which has network protocol data fields and type, a
private API is used to decode this metadata. It is not so generic.

Use the new dynamic mbuf field and flags to handle protocol extraction.
Then the metadata is very clean, only network protocol fields extracted
by defined offset, and its type will be indicated by related ol_flags.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
v3: - Use the conventions name "rte_net_<pmd>_dynfield_<name>" since it
      is defined in a in PMD.
    - Add helpers API for easiy access.

v2: - disable the protocol extraction if failed to register some ol_flags
    - rewrite the commit message

 doc/api/doxy-api-index.md               |   1 +
 doc/api/doxy-api.conf.in                |   1 +
 doc/guides/nics/ice.rst                 |  14 +-
 drivers/net/ice/ice_ethdev.c            |  70 ++++-
 drivers/net/ice/ice_ethdev.h            |   9 +
 drivers/net/ice/ice_rxtx.c              |  90 +++++--
 drivers/net/ice/rte_pmd_ice.h           | 328 +++++++++++++++---------
 drivers/net/ice/rte_pmd_ice_version.map |  12 +
 8 files changed, 373 insertions(+), 152 deletions(-)

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index 28a5dd37e..dff496be0 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -42,6 +42,7 @@ The public API headers are grouped by topics:
   [KNI]                (@ref rte_kni.h),
   [ixgbe]              (@ref rte_pmd_ixgbe.h),
   [i40e]               (@ref rte_pmd_i40e.h),
+  [ice]                (@ref rte_pmd_ice.h),
   [bnxt]               (@ref rte_pmd_bnxt.h),
   [dpaa]               (@ref rte_pmd_dpaa.h),
   [dpaa2]              (@ref rte_pmd_dpaa2.h),
diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
index d8dafb289..faa3b1593 100644
--- a/doc/api/doxy-api.conf.in
+++ b/doc/api/doxy-api.conf.in
@@ -12,6 +12,7 @@ INPUT                   = @TOPDIR@/doc/api/doxy-api-index.md \
                           @TOPDIR@/drivers/net/dpaa \
                           @TOPDIR@/drivers/net/dpaa2 \
                           @TOPDIR@/drivers/net/i40e \
+                          @TOPDIR@/drivers/net/ice \
                           @TOPDIR@/drivers/net/ixgbe \
                           @TOPDIR@/drivers/net/softnic \
                           @TOPDIR@/drivers/raw/dpaa2_cmdif \
diff --git a/doc/guides/nics/ice.rst b/doc/guides/nics/ice.rst
index 933f63480..e0eba5654 100644
--- a/doc/guides/nics/ice.rst
+++ b/doc/guides/nics/ice.rst
@@ -82,8 +82,8 @@ Runtime Config Options
 
 - ``Protocol extraction for per queue``
 
-  Configure the RX queues to do protocol extraction into ``rte_mbuf::udata64``
-  for protocol handling acceleration, like checking the TCP SYN packets quickly.
+  Configure the RX queues to do protocol extraction into mbuf for protocol
+  handling acceleration, like checking the TCP SYN packets quickly.
 
   The argument format is::
 
@@ -111,7 +111,8 @@ Runtime Config Options
   This setting means queues 1, 2-3, 8-9 are TCP extraction, queues 10-23 are
   IPv6 extraction, other queues use the default VLAN extraction.
 
-  The extraction will be copied into the lower 32 bit of ``rte_mbuf::udata64``.
+  The extraction metadata will be copied into the registered dynamic mbuf field
+  with, and the related dynamic mbuf flags is set.
 
   .. table:: Protocol extraction : ``vlan``
 
@@ -175,10 +176,11 @@ Runtime Config Options
 
   TCPHDR2 - Reserved
 
-  Use ``get_proto_xtr_flds(struct rte_mbuf *mb)`` to access the protocol
-  extraction, do not use ``rte_mbuf::udata64`` directly.
+  Use ``rte_net_ice_dynf_proto_xtr_metadata_get`` to access the protocol
+  extraction metadata, and use ``PKT_RX_DYNF_PROTO_XTR_OL_*`` to get the
+  metadata type of ``struct rte_mbuf::ol_flags``.
 
-  The ``dump_proto_xtr_flds(struct rte_mbuf *mb)`` routine shows how to
+  The ``rte_net_ice_dump_proto_xtr_metadata`` routine shows how to
   access the protocol extraction result in ``struct rte_mbuf``.
 
 Driver compilation and testing
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index d74675842..83d66887d 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -30,6 +30,29 @@ static const char * const ice_valid_args[] = {
 	NULL
 };
 
+static const struct rte_mbuf_dynfield ice_proto_xtr_metadata_param = {
+	.name = RTE_STR(rte_net_ice_dynfield_proto_xtr_metadata),
+	.size = sizeof(uint32_t),
+	.align = __alignof__(uint32_t),
+	.flags = 0,
+};
+
+struct proto_xtr_ol_flag {
+	const struct rte_mbuf_dynflag param;
+	uint64_t *flag;
+};
+
+#define PROTO_XTR_OL_FLAG_DYNF_MAP(ol_flag) \
+	{ { RTE_STR(ol_flag), 0 }, &(ol_flag) }
+
+static const struct proto_xtr_ol_flag ice_proto_xtr_ol_flag_params[] = {
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_ol_vlan),
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_ol_ipv4),
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_ol_ipv6),
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_ol_ipv6_flow),
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_ol_tcp)
+};
+
 #define ICE_DFLT_OUTER_TAG_TYPE ICE_AQ_VSI_OUTER_TAG_VLAN_9100
 
 /* DDP package search path */
@@ -1385,6 +1408,9 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 			ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 	struct ice_hw *hw = ICE_PF_TO_HW(pf);
+	const struct proto_xtr_ol_flag *ol_flag;
+	bool proto_xtr_enable = false;
+	int offset;
 	uint16_t i;
 
 	if (!ice_proto_xtr_support(hw)) {
@@ -1398,10 +1424,52 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 		return;
 	}
 
-	for (i = 0; i < pf->lan_nb_qps; i++)
+	for (i = 0; i < pf->lan_nb_qps; i++) {
 		pf->proto_xtr[i] = ad->devargs.proto_xtr[i] != PROTO_XTR_NONE ?
 				   ad->devargs.proto_xtr[i] :
 				   ad->devargs.proto_xtr_dflt;
+
+		if (pf->proto_xtr[i] != PROTO_XTR_NONE && !proto_xtr_enable)
+			proto_xtr_enable = true;
+	}
+
+	if (likely(!proto_xtr_enable))
+		return;
+
+	offset = rte_mbuf_dynfield_register(&ice_proto_xtr_metadata_param);
+	if (unlikely(offset == -1)) {
+		PMD_DRV_LOG(ERR,
+			    "Protocol extraction metadata is disabled in mbuf with error %d",
+			    -rte_errno);
+		return;
+	}
+
+	PMD_DRV_LOG(DEBUG,
+		    "Protocol extraction metadata offset in mbuf is : %d",
+		    offset);
+	rte_net_ice_dynfield_proto_xtr_metadata = offset;
+
+	for (i = 0; i < RTE_DIM(ice_proto_xtr_ol_flag_params); i++) {
+		ol_flag = &ice_proto_xtr_ol_flag_params[i];
+
+		offset = rte_mbuf_dynflag_register(&ol_flag->param);
+		if (unlikely(offset == -1)) {
+			PMD_DRV_LOG(ERR,
+				    "Protocol extraction offload '%s' failed to register with error %d",
+				    ol_flag->param.name, -rte_errno);
+			break;
+		}
+
+		PMD_DRV_LOG(DEBUG,
+			    "Protocol extraction offload '%s' offset in mbuf is : %d",
+			    ol_flag->param.name, offset);
+		*ol_flag->flag = 1ULL << offset;
+	}
+
+	if (i != RTE_DIM(ice_proto_xtr_ol_flag_params)) {
+		PMD_DRV_LOG(ERR, "Protocol extraction metadata is disabled");
+		rte_net_ice_dynfield_proto_xtr_metadata = -1;
+	}
 }
 
 /*  Initialize SW parameters of PF */
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index de67e5934..c55713cc1 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -241,6 +241,15 @@ struct ice_vsi {
 	bool offset_loaded;
 };
 
+enum proto_xtr_type {
+	PROTO_XTR_NONE,
+	PROTO_XTR_VLAN,
+	PROTO_XTR_IPV4,
+	PROTO_XTR_IPV6,
+	PROTO_XTR_IPV6_FLOW,
+	PROTO_XTR_TCP,
+};
+
 enum ice_fdir_tunnel_type {
 	ICE_FDIR_TUNNEL_TYPE_NONE = 0,
 	ICE_FDIR_TUNNEL_TYPE_VXLAN,
diff --git a/drivers/net/ice/ice_rxtx.c b/drivers/net/ice/ice_rxtx.c
index 47a654a73..13fa08f8a 100644
--- a/drivers/net/ice/ice_rxtx.c
+++ b/drivers/net/ice/ice_rxtx.c
@@ -13,18 +13,39 @@
 		PKT_TX_TCP_SEG |		 \
 		PKT_TX_OUTER_IP_CKSUM)
 
-static inline uint8_t
-ice_rxdid_to_proto_xtr_type(uint8_t rxdid)
-{
-	static uint8_t xtr_map[] = {
-		[ICE_RXDID_COMMS_AUX_VLAN]      = PROTO_XTR_VLAN,
-		[ICE_RXDID_COMMS_AUX_IPV4]      = PROTO_XTR_IPV4,
-		[ICE_RXDID_COMMS_AUX_IPV6]      = PROTO_XTR_IPV6,
-		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = PROTO_XTR_IPV6_FLOW,
-		[ICE_RXDID_COMMS_AUX_TCP]       = PROTO_XTR_TCP,
+/* Offset of mbuf dynamic field for protocol extraction data */
+int rte_net_ice_dynfield_proto_xtr_metadata = -1;
+
+/* Mask of mbuf dynamic flags for protocol extraction type */
+uint64_t rte_net_ice_dynflag_proto_xtr_ol_vlan;
+uint64_t rte_net_ice_dynflag_proto_xtr_ol_ipv4;
+uint64_t rte_net_ice_dynflag_proto_xtr_ol_ipv6;
+uint64_t rte_net_ice_dynflag_proto_xtr_ol_ipv6_flow;
+uint64_t rte_net_ice_dynflag_proto_xtr_ol_tcp;
+
+#define MAP_RXDID_TO_FLAG(rxdid, flag) \
+	[rxdid] = &(flag)
+
+static inline uint64_t
+ice_rxdid_to_proto_xtr_ol_flag(uint8_t rxdid)
+{
+	static uint64_t *ol_flag_map[] = {
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_VLAN,
+				  rte_net_ice_dynflag_proto_xtr_ol_vlan),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV4,
+				  rte_net_ice_dynflag_proto_xtr_ol_ipv4),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV6,
+				  rte_net_ice_dynflag_proto_xtr_ol_ipv6),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV6_FLOW,
+				  rte_net_ice_dynflag_proto_xtr_ol_ipv6_flow),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_TCP,
+				  rte_net_ice_dynflag_proto_xtr_ol_tcp),
 	};
+	uint64_t *ol_flag;
+
+	ol_flag = rxdid < RTE_DIM(ol_flag_map) ? ol_flag_map[rxdid] : NULL;
 
-	return rxdid < RTE_DIM(xtr_map) ? xtr_map[rxdid] : PROTO_XTR_NONE;
+	return ol_flag != NULL ? *ol_flag : 0;
 }
 
 static inline uint8_t
@@ -1325,10 +1346,38 @@ ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc *rxdp)
 		   mb->vlan_tci, mb->vlan_tci_outer);
 }
 
+#ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 #define ICE_RX_PROTO_XTR_VALID \
 	((1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) | \
 	 (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
 
+static void
+ice_rxd_to_proto_xtr(struct rte_mbuf *mb,
+		     volatile struct ice_32b_rx_flex_desc_comms *desc)
+{
+	uint16_t stat_err = rte_le_to_cpu_16(desc->status_error1);
+	uint32_t metadata;
+	uint64_t ol_flag;
+
+	if (unlikely(!(stat_err & ICE_RX_PROTO_XTR_VALID)))
+		return;
+
+	ol_flag = ice_rxdid_to_proto_xtr_ol_flag(desc->rxdid);
+	if (unlikely(!ol_flag))
+		return;
+
+	mb->ol_flags |= ol_flag;
+
+	metadata = stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) ?
+				rte_le_to_cpu_16(desc->flex_ts.flex.aux0) : 0;
+
+	if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S)))
+		metadata |= rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
+
+	*RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(mb) = metadata;
+}
+#endif
+
 static inline void
 ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 		      volatile union ice_rx_flex_desc *rxdp)
@@ -1344,28 +1393,13 @@ ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 	}
 
 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
-	init_proto_xtr_flds(mb);
-
-	stat_err = rte_le_to_cpu_16(desc->status_error1);
-	if (stat_err & ICE_RX_PROTO_XTR_VALID) {
-		struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
-			xtr->u.raw.data0 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
-			xtr->u.raw.data1 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
-
-		xtr->type = ice_rxdid_to_proto_xtr_type(desc->rxdid);
-		xtr->magic = PROTO_XTR_MAGIC_ID;
-	}
-
 	if (desc->flow_id != 0xFFFFFFFF) {
 		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 	}
+
+	if (unlikely(rte_net_ice_dynf_proto_xtr_metadata_avail()))
+		ice_rxd_to_proto_xtr(mb, desc);
 #endif
 }
 
diff --git a/drivers/net/ice/rte_pmd_ice.h b/drivers/net/ice/rte_pmd_ice.h
index 719487e1e..33457a06b 100644
--- a/drivers/net/ice/rte_pmd_ice.h
+++ b/drivers/net/ice/rte_pmd_ice.h
@@ -5,144 +5,238 @@
 #ifndef _RTE_PMD_ICE_H_
 #define _RTE_PMD_ICE_H_
 
+/**
+ * @file rte_pmd_ice.h
+ *
+ * ice PMD specific functions.
+ *
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ */
+
 #include <stdio.h>
 #include <rte_mbuf.h>
-#include <rte_ethdev.h>
+#include <rte_mbuf_dyn.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-enum proto_xtr_type {
-	PROTO_XTR_NONE,
-	PROTO_XTR_VLAN,
-	PROTO_XTR_IPV4,
-	PROTO_XTR_IPV6,
-	PROTO_XTR_IPV6_FLOW,
-	PROTO_XTR_TCP,
+/**
+ * The supported network protocol extraction metadata format definition.
+ */
+union proto_xtr_metadata {
+	uint32_t metadata;
+
+	struct {
+		uint16_t data0;
+		uint16_t data1;
+	} raw;
+
+	struct {
+		uint16_t stag_vid:12,
+			 stag_dei:1,
+			 stag_pcp:3;
+		uint16_t ctag_vid:12,
+			 ctag_dei:1,
+			 ctag_pcp:3;
+	} vlan;
+
+	struct {
+		uint16_t protocol:8,
+			 ttl:8;
+		uint16_t tos:8,
+			 ihl:4,
+			 version:4;
+	} ipv4;
+
+	struct {
+		uint16_t hoplimit:8,
+			 nexthdr:8;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6;
+
+	struct {
+		uint16_t flowlo16;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6_flow;
+
+	struct {
+		uint16_t fin:1,
+			 syn:1,
+			 rst:1,
+			 psh:1,
+			 ack:1,
+			 urg:1,
+			 ece:1,
+			 cwr:1,
+			 res1:4,
+			 doff:4;
+		uint16_t rsvd;
+	} tcp;
 };
 
-struct proto_xtr_flds {
-	union {
-		struct {
-			uint16_t data0;
-			uint16_t data1;
-		} raw;
-		struct {
-			uint16_t stag_vid:12,
-				 stag_dei:1,
-				 stag_pcp:3;
-			uint16_t ctag_vid:12,
-				 ctag_dei:1,
-				 ctag_pcp:3;
-		} vlan;
-		struct {
-			uint16_t protocol:8,
-				 ttl:8;
-			uint16_t tos:8,
-				 ihl:4,
-				 version:4;
-		} ipv4;
-		struct {
-			uint16_t hoplimit:8,
-				 nexthdr:8;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6;
-		struct {
-			uint16_t flowlo16;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6_flow;
-		struct {
-			uint16_t fin:1,
-				 syn:1,
-				 rst:1,
-				 psh:1,
-				 ack:1,
-				 urg:1,
-				 ece:1,
-				 cwr:1,
-				 res1:4,
-				 doff:4;
-			uint16_t rsvd;
-		} tcp;
-	} u;
-
-	uint16_t rsvd;
-
-	uint8_t type;
-
-#define PROTO_XTR_MAGIC_ID	0xCE
-	uint8_t magic;
-};
+/**
+ * The offset of mbuf dynamic field for protocol extraction metadata, it is
+ * registered when dev_args 'proto_xtr' is set.
+ */
+extern int rte_net_ice_dynfield_proto_xtr_metadata;
 
-static inline void
-init_proto_xtr_flds(struct rte_mbuf *mb)
+/**
+ * The mask of mbuf dynamic flags for VLAN protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ol_vlan;
+
+/**
+ * The mask of mbuf dynamic flags for IPv4 protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ol_ipv4;
+
+/**
+ * The mask of mbuf dynamic flags for IPv6 protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ol_ipv6;
+
+/**
+ * The mask of mbuf dynamic flags for IPv6 with flow protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ol_ipv6_flow;
+
+/**
+ * The mask of mbuf dynamic flags for TCP protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ol_tcp;
+
+/**
+ * The mbuf dynamic field pointer for protocol extraction metadata.
+ */
+#define RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m) \
+	RTE_MBUF_DYNFIELD((m), \
+			  rte_net_ice_dynfield_proto_xtr_metadata, uint32_t *)
+
+/**
+ * The mbuf dynamic flag for VLAN protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_OL_VLAN \
+	(rte_net_ice_dynflag_proto_xtr_ol_vlan)
+
+/**
+ * The mbuf dynamic flag for IPv4 protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_OL_IPV4 \
+	(rte_net_ice_dynflag_proto_xtr_ol_ipv4)
+
+/**
+ * The mbuf dynamic flag for IPv6 protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_OL_IPV6 \
+	(rte_net_ice_dynflag_proto_xtr_ol_ipv6)
+
+/**
+ * The mbuf dynamic flag for IPv6 with flow protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_OL_IPV6_FLOW \
+	(rte_net_ice_dynflag_proto_xtr_ol_ipv6_flow)
+
+/**
+ * The mbuf dynamic flag for TCP protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_OL_TCP \
+	(rte_net_ice_dynflag_proto_xtr_ol_tcp)
+
+/**
+ * Check if mbuf dynamic field for protocol extraction metadata is registered.
+ *
+ * @return
+ *   True if registered, false otherwise.
+ */
+__rte_experimental
+static inline int
+rte_net_ice_dynf_proto_xtr_metadata_avail(void)
 {
-	mb->udata64 = 0;
+	return rte_net_ice_dynfield_proto_xtr_metadata != -1;
 }
 
-static inline struct proto_xtr_flds *
-get_proto_xtr_flds(struct rte_mbuf *mb)
+/**
+ * Get the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ * @return
+ *   The saved protocol extraction metadata.
+ */
+__rte_experimental
+static inline uint32_t
+rte_net_ice_dynf_proto_xtr_metadata_get(struct rte_mbuf *m)
 {
-	RTE_BUILD_BUG_ON(sizeof(struct proto_xtr_flds) > sizeof(mb->udata64));
-
-	return (struct proto_xtr_flds *)&mb->udata64;
+	return *RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m);
 }
 
+/**
+ * Dump the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ */
+__rte_experimental
 static inline void
-dump_proto_xtr_flds(struct rte_mbuf *mb)
+rte_net_ice_dump_proto_xtr_metadata(struct rte_mbuf *m)
 {
-	struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
+	union proto_xtr_metadata data;
 
-	if (xtr->magic != PROTO_XTR_MAGIC_ID || xtr->type == PROTO_XTR_NONE)
+	if (!rte_net_ice_dynf_proto_xtr_metadata_avail())
 		return;
 
-	printf(" - Protocol Extraction:[0x%04x:0x%04x],",
-	       xtr->u.raw.data0, xtr->u.raw.data1);
-
-	if (xtr->type == PROTO_XTR_VLAN)
-		printf("vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
-		       xtr->u.vlan.stag_pcp,
-		       xtr->u.vlan.stag_dei,
-		       xtr->u.vlan.stag_vid,
-		       xtr->u.vlan.ctag_pcp,
-		       xtr->u.vlan.ctag_dei,
-		       xtr->u.vlan.ctag_vid);
-	else if (xtr->type == PROTO_XTR_IPV4)
-		printf("ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
-		       xtr->u.ipv4.version,
-		       xtr->u.ipv4.ihl,
-		       xtr->u.ipv4.tos,
-		       xtr->u.ipv4.ttl,
-		       xtr->u.ipv4.protocol);
-	else if (xtr->type == PROTO_XTR_IPV6)
-		printf("ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
-		       xtr->u.ipv6.version,
-		       xtr->u.ipv6.tc,
-		       xtr->u.ipv6.flowhi4,
-		       xtr->u.ipv6.nexthdr,
-		       xtr->u.ipv6.hoplimit);
-	else if (xtr->type == PROTO_XTR_IPV6_FLOW)
-		printf("ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
-		       xtr->u.ipv6_flow.version,
-		       xtr->u.ipv6_flow.tc,
-		       xtr->u.ipv6_flow.flowhi4,
-		       xtr->u.ipv6_flow.flowlo16);
-	else if (xtr->type == PROTO_XTR_TCP)
-		printf("tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
-		       xtr->u.tcp.doff,
-		       xtr->u.tcp.cwr ? "C" : "",
-		       xtr->u.tcp.ece ? "E" : "",
-		       xtr->u.tcp.urg ? "U" : "",
-		       xtr->u.tcp.ack ? "A" : "",
-		       xtr->u.tcp.psh ? "P" : "",
-		       xtr->u.tcp.rst ? "R" : "",
-		       xtr->u.tcp.syn ? "S" : "",
-		       xtr->u.tcp.fin ? "F" : "");
+	data.metadata = rte_net_ice_dynf_proto_xtr_metadata_get(m);
+
+	if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_OL_VLAN)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
+		       data.raw.data0, data.raw.data1,
+		       data.vlan.stag_pcp,
+		       data.vlan.stag_dei,
+		       data.vlan.stag_vid,
+		       data.vlan.ctag_pcp,
+		       data.vlan.ctag_dei,
+		       data.vlan.ctag_vid);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_OL_IPV4)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv4.version,
+		       data.ipv4.ihl,
+		       data.ipv4.tos,
+		       data.ipv4.ttl,
+		       data.ipv4.protocol);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_OL_IPV6)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6.version,
+		       data.ipv6.tc,
+		       data.ipv6.flowhi4,
+		       data.ipv6.nexthdr,
+		       data.ipv6.hoplimit);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_OL_IPV6_FLOW)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6_flow.version,
+		       data.ipv6_flow.tc,
+		       data.ipv6_flow.flowhi4,
+		       data.ipv6_flow.flowlo16);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_OL_TCP)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
+		       data.raw.data0, data.raw.data1,
+		       data.tcp.doff,
+		       data.tcp.cwr ? "C" : "",
+		       data.tcp.ece ? "E" : "",
+		       data.tcp.urg ? "U" : "",
+		       data.tcp.ack ? "A" : "",
+		       data.tcp.psh ? "P" : "",
+		       data.tcp.rst ? "R" : "",
+		       data.tcp.syn ? "S" : "",
+		       data.tcp.fin ? "F" : "");
 }
 
 #ifdef __cplusplus
diff --git a/drivers/net/ice/rte_pmd_ice_version.map b/drivers/net/ice/rte_pmd_ice_version.map
index 7b23b609d..ba59cbcbd 100644
--- a/drivers/net/ice/rte_pmd_ice_version.map
+++ b/drivers/net/ice/rte_pmd_ice_version.map
@@ -2,3 +2,15 @@ DPDK_19.02 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	# added in 19.11
+	rte_net_ice_dynfield_proto_xtr_metadata;
+	rte_net_ice_dynflag_proto_xtr_ol_vlan;
+	rte_net_ice_dynflag_proto_xtr_ol_ipv4;
+	rte_net_ice_dynflag_proto_xtr_ol_ipv6;
+	rte_net_ice_dynflag_proto_xtr_ol_ipv6_flow;
+	rte_net_ice_dynflag_proto_xtr_ol_tcp;
+};
-- 
2.17.1

[dpdk-dev] [PATCH v4] net/ice: optimize protocol extraction by dynamic mbuf API

[Haiyue Wang]

The original design is to use rte_mbuf::udata64 to save the metadata of
protocol extraction which has network protocol data fields and type, a
private API is used to decode this metadata. It is not so generic.

Use the new dynamic mbuf field and flags to handle protocol extraction.
Then the metadata is very clean, only network protocol fields extracted
by defined offset, and its type will be indicated by related ol_flags.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
v4: - Include the 'rte_pmd_ice.h' header in ICE source as needed, reduce
      its compile scope.

v3: - Use the conventions name "rte_net_<pmd>_dynfield_<name>" since it
      is defined in a in PMD.
    - Add helpers API for easiy access.

v2: - disable the protocol extraction if failed to register some ol_flags
    - rewrite the commit message

 doc/api/doxy-api-index.md               |   1 +
 doc/api/doxy-api.conf.in                |   1 +
 doc/guides/nics/ice.rst                 |  14 +-
 drivers/net/ice/ice_ethdev.c            |  72 +++++-
 drivers/net/ice/ice_ethdev.h            |   9 +
 drivers/net/ice/ice_rxtx.c              |  91 +++++--
 drivers/net/ice/ice_rxtx.h              |   1 -
 drivers/net/ice/rte_pmd_ice.h           | 328 +++++++++++++++---------
 drivers/net/ice/rte_pmd_ice_version.map |  12 +
 9 files changed, 376 insertions(+), 153 deletions(-)

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index 28a5dd37e..dff496be0 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -42,6 +42,7 @@ The public API headers are grouped by topics:
   [KNI]                (@ref rte_kni.h),
   [ixgbe]              (@ref rte_pmd_ixgbe.h),
   [i40e]               (@ref rte_pmd_i40e.h),
+  [ice]                (@ref rte_pmd_ice.h),
   [bnxt]               (@ref rte_pmd_bnxt.h),
   [dpaa]               (@ref rte_pmd_dpaa.h),
   [dpaa2]              (@ref rte_pmd_dpaa2.h),
diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
index d8dafb289..faa3b1593 100644
--- a/doc/api/doxy-api.conf.in
+++ b/doc/api/doxy-api.conf.in
@@ -12,6 +12,7 @@ INPUT                   = @TOPDIR@/doc/api/doxy-api-index.md \
                           @TOPDIR@/drivers/net/dpaa \
                           @TOPDIR@/drivers/net/dpaa2 \
                           @TOPDIR@/drivers/net/i40e \
+                          @TOPDIR@/drivers/net/ice \
                           @TOPDIR@/drivers/net/ixgbe \
                           @TOPDIR@/drivers/net/softnic \
                           @TOPDIR@/drivers/raw/dpaa2_cmdif \
diff --git a/doc/guides/nics/ice.rst b/doc/guides/nics/ice.rst
index 933f63480..e0eba5654 100644
--- a/doc/guides/nics/ice.rst
+++ b/doc/guides/nics/ice.rst
@@ -82,8 +82,8 @@ Runtime Config Options
 
 - ``Protocol extraction for per queue``
 
-  Configure the RX queues to do protocol extraction into ``rte_mbuf::udata64``
-  for protocol handling acceleration, like checking the TCP SYN packets quickly.
+  Configure the RX queues to do protocol extraction into mbuf for protocol
+  handling acceleration, like checking the TCP SYN packets quickly.
 
   The argument format is::
 
@@ -111,7 +111,8 @@ Runtime Config Options
   This setting means queues 1, 2-3, 8-9 are TCP extraction, queues 10-23 are
   IPv6 extraction, other queues use the default VLAN extraction.
 
-  The extraction will be copied into the lower 32 bit of ``rte_mbuf::udata64``.
+  The extraction metadata will be copied into the registered dynamic mbuf field
+  with, and the related dynamic mbuf flags is set.
 
   .. table:: Protocol extraction : ``vlan``
 
@@ -175,10 +176,11 @@ Runtime Config Options
 
   TCPHDR2 - Reserved
 
-  Use ``get_proto_xtr_flds(struct rte_mbuf *mb)`` to access the protocol
-  extraction, do not use ``rte_mbuf::udata64`` directly.
+  Use ``rte_net_ice_dynf_proto_xtr_metadata_get`` to access the protocol
+  extraction metadata, and use ``PKT_RX_DYNF_PROTO_XTR_OL_*`` to get the
+  metadata type of ``struct rte_mbuf::ol_flags``.
 
-  The ``dump_proto_xtr_flds(struct rte_mbuf *mb)`` routine shows how to
+  The ``rte_net_ice_dump_proto_xtr_metadata`` routine shows how to
   access the protocol extraction result in ``struct rte_mbuf``.
 
 Driver compilation and testing
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index d74675842..0a2d74107 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -14,6 +14,8 @@
 #include "base/ice_flow.h"
 #include "base/ice_dcb.h"
 #include "base/ice_common.h"
+
+#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 #include "ice_rxtx.h"
 #include "ice_generic_flow.h"
@@ -30,6 +32,29 @@ static const char * const ice_valid_args[] = {
 	NULL
 };
 
+static const struct rte_mbuf_dynfield ice_proto_xtr_metadata_param = {
+	.name = RTE_STR(rte_net_ice_dynfield_proto_xtr_metadata),
+	.size = sizeof(uint32_t),
+	.align = __alignof__(uint32_t),
+	.flags = 0,
+};
+
+struct proto_xtr_ol_flag {
+	const struct rte_mbuf_dynflag param;
+	uint64_t *flag;
+};
+
+#define PROTO_XTR_OL_FLAG_DYNF_MAP(ol_flag) \
+	{ { RTE_STR(ol_flag), 0 }, &(ol_flag) }
+
+static const struct proto_xtr_ol_flag ice_proto_xtr_ol_flag_params[] = {
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_ol_vlan),
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_ol_ipv4),
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_ol_ipv6),
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_ol_ipv6_flow),
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_ol_tcp)
+};
+
 #define ICE_DFLT_OUTER_TAG_TYPE ICE_AQ_VSI_OUTER_TAG_VLAN_9100
 
 /* DDP package search path */
@@ -1385,6 +1410,9 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 			ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 	struct ice_hw *hw = ICE_PF_TO_HW(pf);
+	const struct proto_xtr_ol_flag *ol_flag;
+	bool proto_xtr_enable = false;
+	int offset;
 	uint16_t i;
 
 	if (!ice_proto_xtr_support(hw)) {
@@ -1398,10 +1426,52 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 		return;
 	}
 
-	for (i = 0; i < pf->lan_nb_qps; i++)
+	for (i = 0; i < pf->lan_nb_qps; i++) {
 		pf->proto_xtr[i] = ad->devargs.proto_xtr[i] != PROTO_XTR_NONE ?
 				   ad->devargs.proto_xtr[i] :
 				   ad->devargs.proto_xtr_dflt;
+
+		if (pf->proto_xtr[i] != PROTO_XTR_NONE && !proto_xtr_enable)
+			proto_xtr_enable = true;
+	}
+
+	if (likely(!proto_xtr_enable))
+		return;
+
+	offset = rte_mbuf_dynfield_register(&ice_proto_xtr_metadata_param);
+	if (unlikely(offset == -1)) {
+		PMD_DRV_LOG(ERR,
+			    "Protocol extraction metadata is disabled in mbuf with error %d",
+			    -rte_errno);
+		return;
+	}
+
+	PMD_DRV_LOG(DEBUG,
+		    "Protocol extraction metadata offset in mbuf is : %d",
+		    offset);
+	rte_net_ice_dynfield_proto_xtr_metadata = offset;
+
+	for (i = 0; i < RTE_DIM(ice_proto_xtr_ol_flag_params); i++) {
+		ol_flag = &ice_proto_xtr_ol_flag_params[i];
+
+		offset = rte_mbuf_dynflag_register(&ol_flag->param);
+		if (unlikely(offset == -1)) {
+			PMD_DRV_LOG(ERR,
+				    "Protocol extraction offload '%s' failed to register with error %d",
+				    ol_flag->param.name, -rte_errno);
+			break;
+		}
+
+		PMD_DRV_LOG(DEBUG,
+			    "Protocol extraction offload '%s' offset in mbuf is : %d",
+			    ol_flag->param.name, offset);
+		*ol_flag->flag = 1ULL << offset;
+	}
+
+	if (i != RTE_DIM(ice_proto_xtr_ol_flag_params)) {
+		PMD_DRV_LOG(ERR, "Protocol extraction metadata is disabled");
+		rte_net_ice_dynfield_proto_xtr_metadata = -1;
+	}
 }
 
 /*  Initialize SW parameters of PF */
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index de67e5934..c55713cc1 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -241,6 +241,15 @@ struct ice_vsi {
 	bool offset_loaded;
 };
 
+enum proto_xtr_type {
+	PROTO_XTR_NONE,
+	PROTO_XTR_VLAN,
+	PROTO_XTR_IPV4,
+	PROTO_XTR_IPV6,
+	PROTO_XTR_IPV6_FLOW,
+	PROTO_XTR_TCP,
+};
+
 enum ice_fdir_tunnel_type {
 	ICE_FDIR_TUNNEL_TYPE_NONE = 0,
 	ICE_FDIR_TUNNEL_TYPE_VXLAN,
diff --git a/drivers/net/ice/ice_rxtx.c b/drivers/net/ice/ice_rxtx.c
index 47a654a73..3b14cba3a 100644
--- a/drivers/net/ice/ice_rxtx.c
+++ b/drivers/net/ice/ice_rxtx.c
@@ -5,6 +5,7 @@
 #include <rte_ethdev_driver.h>
 #include <rte_net.h>
 
+#include "rte_pmd_ice.h"
 #include "ice_rxtx.h"
 
 #define ICE_TX_CKSUM_OFFLOAD_MASK (		 \
@@ -13,18 +14,39 @@
 		PKT_TX_TCP_SEG |		 \
 		PKT_TX_OUTER_IP_CKSUM)
 
-static inline uint8_t
-ice_rxdid_to_proto_xtr_type(uint8_t rxdid)
-{
-	static uint8_t xtr_map[] = {
-		[ICE_RXDID_COMMS_AUX_VLAN]      = PROTO_XTR_VLAN,
-		[ICE_RXDID_COMMS_AUX_IPV4]      = PROTO_XTR_IPV4,
-		[ICE_RXDID_COMMS_AUX_IPV6]      = PROTO_XTR_IPV6,
-		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = PROTO_XTR_IPV6_FLOW,
-		[ICE_RXDID_COMMS_AUX_TCP]       = PROTO_XTR_TCP,
+/* Offset of mbuf dynamic field for protocol extraction data */
+int rte_net_ice_dynfield_proto_xtr_metadata = -1;
+
+/* Mask of mbuf dynamic flags for protocol extraction type */
+uint64_t rte_net_ice_dynflag_proto_xtr_ol_vlan;
+uint64_t rte_net_ice_dynflag_proto_xtr_ol_ipv4;
+uint64_t rte_net_ice_dynflag_proto_xtr_ol_ipv6;
+uint64_t rte_net_ice_dynflag_proto_xtr_ol_ipv6_flow;
+uint64_t rte_net_ice_dynflag_proto_xtr_ol_tcp;
+
+#define MAP_RXDID_TO_FLAG(rxdid, flag) \
+	[rxdid] = &(flag)
+
+static inline uint64_t
+ice_rxdid_to_proto_xtr_ol_flag(uint8_t rxdid)
+{
+	static uint64_t *ol_flag_map[] = {
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_VLAN,
+				  rte_net_ice_dynflag_proto_xtr_ol_vlan),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV4,
+				  rte_net_ice_dynflag_proto_xtr_ol_ipv4),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV6,
+				  rte_net_ice_dynflag_proto_xtr_ol_ipv6),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV6_FLOW,
+				  rte_net_ice_dynflag_proto_xtr_ol_ipv6_flow),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_TCP,
+				  rte_net_ice_dynflag_proto_xtr_ol_tcp),
 	};
+	uint64_t *ol_flag;
+
+	ol_flag = rxdid < RTE_DIM(ol_flag_map) ? ol_flag_map[rxdid] : NULL;
 
-	return rxdid < RTE_DIM(xtr_map) ? xtr_map[rxdid] : PROTO_XTR_NONE;
+	return ol_flag != NULL ? *ol_flag : 0;
 }
 
 static inline uint8_t
@@ -1325,10 +1347,38 @@ ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc *rxdp)
 		   mb->vlan_tci, mb->vlan_tci_outer);
 }
 
+#ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 #define ICE_RX_PROTO_XTR_VALID \
 	((1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) | \
 	 (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
 
+static void
+ice_rxd_to_proto_xtr(struct rte_mbuf *mb,
+		     volatile struct ice_32b_rx_flex_desc_comms *desc)
+{
+	uint16_t stat_err = rte_le_to_cpu_16(desc->status_error1);
+	uint32_t metadata;
+	uint64_t ol_flag;
+
+	if (unlikely(!(stat_err & ICE_RX_PROTO_XTR_VALID)))
+		return;
+
+	ol_flag = ice_rxdid_to_proto_xtr_ol_flag(desc->rxdid);
+	if (unlikely(!ol_flag))
+		return;
+
+	mb->ol_flags |= ol_flag;
+
+	metadata = stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) ?
+				rte_le_to_cpu_16(desc->flex_ts.flex.aux0) : 0;
+
+	if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S)))
+		metadata |= rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
+
+	*RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(mb) = metadata;
+}
+#endif
+
 static inline void
 ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 		      volatile union ice_rx_flex_desc *rxdp)
@@ -1344,28 +1394,13 @@ ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 	}
 
 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
-	init_proto_xtr_flds(mb);
-
-	stat_err = rte_le_to_cpu_16(desc->status_error1);
-	if (stat_err & ICE_RX_PROTO_XTR_VALID) {
-		struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
-			xtr->u.raw.data0 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
-			xtr->u.raw.data1 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
-
-		xtr->type = ice_rxdid_to_proto_xtr_type(desc->rxdid);
-		xtr->magic = PROTO_XTR_MAGIC_ID;
-	}
-
 	if (desc->flow_id != 0xFFFFFFFF) {
 		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 	}
+
+	if (unlikely(rte_net_ice_dynf_proto_xtr_metadata_avail()))
+		ice_rxd_to_proto_xtr(mb, desc);
 #endif
 }
 
diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h
index 5de618976..9e3d2cd07 100644
--- a/drivers/net/ice/ice_rxtx.h
+++ b/drivers/net/ice/ice_rxtx.h
@@ -5,7 +5,6 @@
 #ifndef _ICE_RXTX_H_
 #define _ICE_RXTX_H_
 
-#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 
 #define ICE_ALIGN_RING_DESC  32
diff --git a/drivers/net/ice/rte_pmd_ice.h b/drivers/net/ice/rte_pmd_ice.h
index 719487e1e..2ace23c97 100644
--- a/drivers/net/ice/rte_pmd_ice.h
+++ b/drivers/net/ice/rte_pmd_ice.h
@@ -5,144 +5,238 @@
 #ifndef _RTE_PMD_ICE_H_
 #define _RTE_PMD_ICE_H_
 
+/**
+ * @file rte_pmd_ice.h
+ *
+ * ice PMD specific functions.
+ *
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ */
+
 #include <stdio.h>
 #include <rte_mbuf.h>
-#include <rte_ethdev.h>
+#include <rte_mbuf_dyn.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-enum proto_xtr_type {
-	PROTO_XTR_NONE,
-	PROTO_XTR_VLAN,
-	PROTO_XTR_IPV4,
-	PROTO_XTR_IPV6,
-	PROTO_XTR_IPV6_FLOW,
-	PROTO_XTR_TCP,
+/**
+ * The supported network protocol extraction metadata format definition.
+ */
+union proto_xtr_metadata {
+	uint32_t metadata;
+
+	struct {
+		uint16_t data0;
+		uint16_t data1;
+	} raw;
+
+	struct {
+		uint16_t stag_vid:12,
+			 stag_dei:1,
+			 stag_pcp:3;
+		uint16_t ctag_vid:12,
+			 ctag_dei:1,
+			 ctag_pcp:3;
+	} vlan;
+
+	struct {
+		uint16_t protocol:8,
+			 ttl:8;
+		uint16_t tos:8,
+			 ihl:4,
+			 version:4;
+	} ipv4;
+
+	struct {
+		uint16_t hoplimit:8,
+			 nexthdr:8;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6;
+
+	struct {
+		uint16_t flowlo16;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6_flow;
+
+	struct {
+		uint16_t fin:1,
+			 syn:1,
+			 rst:1,
+			 psh:1,
+			 ack:1,
+			 urg:1,
+			 ece:1,
+			 cwr:1,
+			 res1:4,
+			 doff:4;
+		uint16_t rsvd;
+	} tcp;
 };
 
-struct proto_xtr_flds {
-	union {
-		struct {
-			uint16_t data0;
-			uint16_t data1;
-		} raw;
-		struct {
-			uint16_t stag_vid:12,
-				 stag_dei:1,
-				 stag_pcp:3;
-			uint16_t ctag_vid:12,
-				 ctag_dei:1,
-				 ctag_pcp:3;
-		} vlan;
-		struct {
-			uint16_t protocol:8,
-				 ttl:8;
-			uint16_t tos:8,
-				 ihl:4,
-				 version:4;
-		} ipv4;
-		struct {
-			uint16_t hoplimit:8,
-				 nexthdr:8;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6;
-		struct {
-			uint16_t flowlo16;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6_flow;
-		struct {
-			uint16_t fin:1,
-				 syn:1,
-				 rst:1,
-				 psh:1,
-				 ack:1,
-				 urg:1,
-				 ece:1,
-				 cwr:1,
-				 res1:4,
-				 doff:4;
-			uint16_t rsvd;
-		} tcp;
-	} u;
-
-	uint16_t rsvd;
-
-	uint8_t type;
-
-#define PROTO_XTR_MAGIC_ID	0xCE
-	uint8_t magic;
-};
+/**
+ * The offset of mbuf dynamic field for protocol extraction metadata, it is
+ * registered when dev_args 'proto_xtr' is set.
+ */
+extern int rte_net_ice_dynfield_proto_xtr_metadata;
 
-static inline void
-init_proto_xtr_flds(struct rte_mbuf *mb)
+/**
+ * The mask of mbuf dynamic flags for VLAN protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ol_vlan;
+
+/**
+ * The mask of mbuf dynamic flags for IPv4 protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ol_ipv4;
+
+/**
+ * The mask of mbuf dynamic flags for IPv6 protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ol_ipv6;
+
+/**
+ * The mask of mbuf dynamic flags for IPv6 with flow protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ol_ipv6_flow;
+
+/**
+ * The mask of mbuf dynamic flags for TCP protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ol_tcp;
+
+/**
+ * The mbuf dynamic field pointer for protocol extraction metadata.
+ */
+#define RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m) \
+	RTE_MBUF_DYNFIELD((m), \
+			  rte_net_ice_dynfield_proto_xtr_metadata, uint32_t *)
+
+/**
+ * The mbuf dynamic flag for VLAN protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_OL_VLAN \
+	(rte_net_ice_dynflag_proto_xtr_ol_vlan)
+
+/**
+ * The mbuf dynamic flag for IPv4 protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_OL_IPV4 \
+	(rte_net_ice_dynflag_proto_xtr_ol_ipv4)
+
+/**
+ * The mbuf dynamic flag for IPv6 protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_OL_IPV6 \
+	(rte_net_ice_dynflag_proto_xtr_ol_ipv6)
+
+/**
+ * The mbuf dynamic flag for IPv6 with flow protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_OL_IPV6_FLOW \
+	(rte_net_ice_dynflag_proto_xtr_ol_ipv6_flow)
+
+/**
+ * The mbuf dynamic flag for TCP protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_OL_TCP \
+	(rte_net_ice_dynflag_proto_xtr_ol_tcp)
+
+/**
+ * Check if mbuf dynamic field for protocol extraction metadata is registered.
+ *
+ * @return
+ *   True if registered, false otherwise.
+ */
+__rte_experimental
+static __rte_always_inline int
+rte_net_ice_dynf_proto_xtr_metadata_avail(void)
 {
-	mb->udata64 = 0;
+	return rte_net_ice_dynfield_proto_xtr_metadata != -1;
 }
 
-static inline struct proto_xtr_flds *
-get_proto_xtr_flds(struct rte_mbuf *mb)
+/**
+ * Get the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ * @return
+ *   The saved protocol extraction metadata.
+ */
+__rte_experimental
+static __rte_always_inline uint32_t
+rte_net_ice_dynf_proto_xtr_metadata_get(struct rte_mbuf *m)
 {
-	RTE_BUILD_BUG_ON(sizeof(struct proto_xtr_flds) > sizeof(mb->udata64));
-
-	return (struct proto_xtr_flds *)&mb->udata64;
+	return *RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m);
 }
 
+/**
+ * Dump the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ */
+__rte_experimental
 static inline void
-dump_proto_xtr_flds(struct rte_mbuf *mb)
+rte_net_ice_dump_proto_xtr_metadata(struct rte_mbuf *m)
 {
-	struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
+	union proto_xtr_metadata data;
 
-	if (xtr->magic != PROTO_XTR_MAGIC_ID || xtr->type == PROTO_XTR_NONE)
+	if (!rte_net_ice_dynf_proto_xtr_metadata_avail())
 		return;
 
-	printf(" - Protocol Extraction:[0x%04x:0x%04x],",
-	       xtr->u.raw.data0, xtr->u.raw.data1);
-
-	if (xtr->type == PROTO_XTR_VLAN)
-		printf("vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
-		       xtr->u.vlan.stag_pcp,
-		       xtr->u.vlan.stag_dei,
-		       xtr->u.vlan.stag_vid,
-		       xtr->u.vlan.ctag_pcp,
-		       xtr->u.vlan.ctag_dei,
-		       xtr->u.vlan.ctag_vid);
-	else if (xtr->type == PROTO_XTR_IPV4)
-		printf("ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
-		       xtr->u.ipv4.version,
-		       xtr->u.ipv4.ihl,
-		       xtr->u.ipv4.tos,
-		       xtr->u.ipv4.ttl,
-		       xtr->u.ipv4.protocol);
-	else if (xtr->type == PROTO_XTR_IPV6)
-		printf("ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
-		       xtr->u.ipv6.version,
-		       xtr->u.ipv6.tc,
-		       xtr->u.ipv6.flowhi4,
-		       xtr->u.ipv6.nexthdr,
-		       xtr->u.ipv6.hoplimit);
-	else if (xtr->type == PROTO_XTR_IPV6_FLOW)
-		printf("ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
-		       xtr->u.ipv6_flow.version,
-		       xtr->u.ipv6_flow.tc,
-		       xtr->u.ipv6_flow.flowhi4,
-		       xtr->u.ipv6_flow.flowlo16);
-	else if (xtr->type == PROTO_XTR_TCP)
-		printf("tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
-		       xtr->u.tcp.doff,
-		       xtr->u.tcp.cwr ? "C" : "",
-		       xtr->u.tcp.ece ? "E" : "",
-		       xtr->u.tcp.urg ? "U" : "",
-		       xtr->u.tcp.ack ? "A" : "",
-		       xtr->u.tcp.psh ? "P" : "",
-		       xtr->u.tcp.rst ? "R" : "",
-		       xtr->u.tcp.syn ? "S" : "",
-		       xtr->u.tcp.fin ? "F" : "");
+	data.metadata = rte_net_ice_dynf_proto_xtr_metadata_get(m);
+
+	if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_OL_VLAN)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
+		       data.raw.data0, data.raw.data1,
+		       data.vlan.stag_pcp,
+		       data.vlan.stag_dei,
+		       data.vlan.stag_vid,
+		       data.vlan.ctag_pcp,
+		       data.vlan.ctag_dei,
+		       data.vlan.ctag_vid);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_OL_IPV4)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv4.version,
+		       data.ipv4.ihl,
+		       data.ipv4.tos,
+		       data.ipv4.ttl,
+		       data.ipv4.protocol);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_OL_IPV6)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6.version,
+		       data.ipv6.tc,
+		       data.ipv6.flowhi4,
+		       data.ipv6.nexthdr,
+		       data.ipv6.hoplimit);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_OL_IPV6_FLOW)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6_flow.version,
+		       data.ipv6_flow.tc,
+		       data.ipv6_flow.flowhi4,
+		       data.ipv6_flow.flowlo16);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_OL_TCP)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
+		       data.raw.data0, data.raw.data1,
+		       data.tcp.doff,
+		       data.tcp.cwr ? "C" : "",
+		       data.tcp.ece ? "E" : "",
+		       data.tcp.urg ? "U" : "",
+		       data.tcp.ack ? "A" : "",
+		       data.tcp.psh ? "P" : "",
+		       data.tcp.rst ? "R" : "",
+		       data.tcp.syn ? "S" : "",
+		       data.tcp.fin ? "F" : "");
 }
 
 #ifdef __cplusplus
diff --git a/drivers/net/ice/rte_pmd_ice_version.map b/drivers/net/ice/rte_pmd_ice_version.map
index 7b23b609d..ba59cbcbd 100644
--- a/drivers/net/ice/rte_pmd_ice_version.map
+++ b/drivers/net/ice/rte_pmd_ice_version.map
@@ -2,3 +2,15 @@ DPDK_19.02 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	# added in 19.11
+	rte_net_ice_dynfield_proto_xtr_metadata;
+	rte_net_ice_dynflag_proto_xtr_ol_vlan;
+	rte_net_ice_dynflag_proto_xtr_ol_ipv4;
+	rte_net_ice_dynflag_proto_xtr_ol_ipv6;
+	rte_net_ice_dynflag_proto_xtr_ol_ipv6_flow;
+	rte_net_ice_dynflag_proto_xtr_ol_tcp;
+};
-- 
2.17.1

[dpdk-dev] [PATCH v5] net/ice: optimize protocol extraction by dynamic mbuf API

[Haiyue Wang]

The original design is to use rte_mbuf::udata64 to save the metadata of
protocol extraction which has network protocol data fields and type, a
private API is used to decode this metadata. It is not so generic.

Use the new dynamic mbuf field and flags to handle protocol extraction.
Then the metadata is very clean, only network protocol fields extracted
by defined offset, and its type will be indicated by related ol_flags.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
v5: - Remove the '_OL/_ol' in dynamic mbuf flag to make the variable clean.

v4: - Include the 'rte_pmd_ice.h' header in ICE source as needed, reduce
      its compile scope.

v3: - Use the conventions name "rte_net_<pmd>_dynfield_<name>" since it
      is defined in a in PMD.
    - Add helpers API for easiy access.

v2: - disable the protocol extraction if failed to register some ol_flags
    - rewrite the commit message

 doc/api/doxy-api-index.md               |   1 +
 doc/api/doxy-api.conf.in                |   1 +
 doc/guides/nics/ice.rst                 |  14 +-
 drivers/net/ice/ice_ethdev.c            |  72 +++++-
 drivers/net/ice/ice_ethdev.h            |   9 +
 drivers/net/ice/ice_rxtx.c              |  91 +++++--
 drivers/net/ice/ice_rxtx.h              |   1 -
 drivers/net/ice/rte_pmd_ice.h           | 328 +++++++++++++++---------
 drivers/net/ice/rte_pmd_ice_version.map |  12 +
 9 files changed, 376 insertions(+), 153 deletions(-)

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index 28a5dd37e..dff496be0 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -42,6 +42,7 @@ The public API headers are grouped by topics:
   [KNI]                (@ref rte_kni.h),
   [ixgbe]              (@ref rte_pmd_ixgbe.h),
   [i40e]               (@ref rte_pmd_i40e.h),
+  [ice]                (@ref rte_pmd_ice.h),
   [bnxt]               (@ref rte_pmd_bnxt.h),
   [dpaa]               (@ref rte_pmd_dpaa.h),
   [dpaa2]              (@ref rte_pmd_dpaa2.h),
diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
index d8dafb289..faa3b1593 100644
--- a/doc/api/doxy-api.conf.in
+++ b/doc/api/doxy-api.conf.in
@@ -12,6 +12,7 @@ INPUT                   = @TOPDIR@/doc/api/doxy-api-index.md \
                           @TOPDIR@/drivers/net/dpaa \
                           @TOPDIR@/drivers/net/dpaa2 \
                           @TOPDIR@/drivers/net/i40e \
+                          @TOPDIR@/drivers/net/ice \
                           @TOPDIR@/drivers/net/ixgbe \
                           @TOPDIR@/drivers/net/softnic \
                           @TOPDIR@/drivers/raw/dpaa2_cmdif \
diff --git a/doc/guides/nics/ice.rst b/doc/guides/nics/ice.rst
index 933f63480..e0eba5654 100644
--- a/doc/guides/nics/ice.rst
+++ b/doc/guides/nics/ice.rst
@@ -82,8 +82,8 @@ Runtime Config Options
 
 - ``Protocol extraction for per queue``
 
-  Configure the RX queues to do protocol extraction into ``rte_mbuf::udata64``
-  for protocol handling acceleration, like checking the TCP SYN packets quickly.
+  Configure the RX queues to do protocol extraction into mbuf for protocol
+  handling acceleration, like checking the TCP SYN packets quickly.
 
   The argument format is::
 
@@ -111,7 +111,8 @@ Runtime Config Options
   This setting means queues 1, 2-3, 8-9 are TCP extraction, queues 10-23 are
   IPv6 extraction, other queues use the default VLAN extraction.
 
-  The extraction will be copied into the lower 32 bit of ``rte_mbuf::udata64``.
+  The extraction metadata will be copied into the registered dynamic mbuf field
+  with, and the related dynamic mbuf flags is set.
 
   .. table:: Protocol extraction : ``vlan``
 
@@ -175,10 +176,11 @@ Runtime Config Options
 
   TCPHDR2 - Reserved
 
-  Use ``get_proto_xtr_flds(struct rte_mbuf *mb)`` to access the protocol
-  extraction, do not use ``rte_mbuf::udata64`` directly.
+  Use ``rte_net_ice_dynf_proto_xtr_metadata_get`` to access the protocol
+  extraction metadata, and use ``PKT_RX_DYNF_PROTO_XTR_OL_*`` to get the
+  metadata type of ``struct rte_mbuf::ol_flags``.
 
-  The ``dump_proto_xtr_flds(struct rte_mbuf *mb)`` routine shows how to
+  The ``rte_net_ice_dump_proto_xtr_metadata`` routine shows how to
   access the protocol extraction result in ``struct rte_mbuf``.
 
 Driver compilation and testing
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index d74675842..ceed0a89f 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -14,6 +14,8 @@
 #include "base/ice_flow.h"
 #include "base/ice_dcb.h"
 #include "base/ice_common.h"
+
+#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 #include "ice_rxtx.h"
 #include "ice_generic_flow.h"
@@ -30,6 +32,29 @@ static const char * const ice_valid_args[] = {
 	NULL
 };
 
+static const struct rte_mbuf_dynfield ice_proto_xtr_metadata_param = {
+	.name = RTE_STR(rte_net_ice_dynfield_proto_xtr_metadata),
+	.size = sizeof(uint32_t),
+	.align = __alignof__(uint32_t),
+	.flags = 0,
+};
+
+struct proto_xtr_ol_flag {
+	const struct rte_mbuf_dynflag param;
+	uint64_t *flag;
+};
+
+#define PROTO_XTR_OL_FLAG_DYNF_MAP(ol_flag) \
+	{ { RTE_STR(ol_flag), 0 }, &(ol_flag) }
+
+static const struct proto_xtr_ol_flag ice_proto_xtr_ol_flag_params[] = {
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_vlan),
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_ipv4),
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_ipv6),
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_ipv6_flow),
+	PROTO_XTR_OL_FLAG_DYNF_MAP(rte_net_ice_dynflag_proto_xtr_tcp)
+};
+
 #define ICE_DFLT_OUTER_TAG_TYPE ICE_AQ_VSI_OUTER_TAG_VLAN_9100
 
 /* DDP package search path */
@@ -1385,6 +1410,9 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 			ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 	struct ice_hw *hw = ICE_PF_TO_HW(pf);
+	const struct proto_xtr_ol_flag *ol_flag;
+	bool proto_xtr_enable = false;
+	int offset;
 	uint16_t i;
 
 	if (!ice_proto_xtr_support(hw)) {
@@ -1398,10 +1426,52 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 		return;
 	}
 
-	for (i = 0; i < pf->lan_nb_qps; i++)
+	for (i = 0; i < pf->lan_nb_qps; i++) {
 		pf->proto_xtr[i] = ad->devargs.proto_xtr[i] != PROTO_XTR_NONE ?
 				   ad->devargs.proto_xtr[i] :
 				   ad->devargs.proto_xtr_dflt;
+
+		if (pf->proto_xtr[i] != PROTO_XTR_NONE && !proto_xtr_enable)
+			proto_xtr_enable = true;
+	}
+
+	if (likely(!proto_xtr_enable))
+		return;
+
+	offset = rte_mbuf_dynfield_register(&ice_proto_xtr_metadata_param);
+	if (unlikely(offset == -1)) {
+		PMD_DRV_LOG(ERR,
+			    "Protocol extraction metadata is disabled in mbuf with error %d",
+			    -rte_errno);
+		return;
+	}
+
+	PMD_DRV_LOG(DEBUG,
+		    "Protocol extraction metadata offset in mbuf is : %d",
+		    offset);
+	rte_net_ice_dynfield_proto_xtr_metadata = offset;
+
+	for (i = 0; i < RTE_DIM(ice_proto_xtr_ol_flag_params); i++) {
+		ol_flag = &ice_proto_xtr_ol_flag_params[i];
+
+		offset = rte_mbuf_dynflag_register(&ol_flag->param);
+		if (unlikely(offset == -1)) {
+			PMD_DRV_LOG(ERR,
+				    "Protocol extraction offload '%s' failed to register with error %d",
+				    ol_flag->param.name, -rte_errno);
+			break;
+		}
+
+		PMD_DRV_LOG(DEBUG,
+			    "Protocol extraction offload '%s' offset in mbuf is : %d",
+			    ol_flag->param.name, offset);
+		*ol_flag->flag = 1ULL << offset;
+	}
+
+	if (i != RTE_DIM(ice_proto_xtr_ol_flag_params)) {
+		PMD_DRV_LOG(ERR, "Protocol extraction metadata is disabled");
+		rte_net_ice_dynfield_proto_xtr_metadata = -1;
+	}
 }
 
 /*  Initialize SW parameters of PF */
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index de67e5934..c55713cc1 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -241,6 +241,15 @@ struct ice_vsi {
 	bool offset_loaded;
 };
 
+enum proto_xtr_type {
+	PROTO_XTR_NONE,
+	PROTO_XTR_VLAN,
+	PROTO_XTR_IPV4,
+	PROTO_XTR_IPV6,
+	PROTO_XTR_IPV6_FLOW,
+	PROTO_XTR_TCP,
+};
+
 enum ice_fdir_tunnel_type {
 	ICE_FDIR_TUNNEL_TYPE_NONE = 0,
 	ICE_FDIR_TUNNEL_TYPE_VXLAN,
diff --git a/drivers/net/ice/ice_rxtx.c b/drivers/net/ice/ice_rxtx.c
index 47a654a73..581ad1b1e 100644
--- a/drivers/net/ice/ice_rxtx.c
+++ b/drivers/net/ice/ice_rxtx.c
@@ -5,6 +5,7 @@
 #include <rte_ethdev_driver.h>
 #include <rte_net.h>
 
+#include "rte_pmd_ice.h"
 #include "ice_rxtx.h"
 
 #define ICE_TX_CKSUM_OFFLOAD_MASK (		 \
@@ -13,18 +14,39 @@
 		PKT_TX_TCP_SEG |		 \
 		PKT_TX_OUTER_IP_CKSUM)
 
-static inline uint8_t
-ice_rxdid_to_proto_xtr_type(uint8_t rxdid)
-{
-	static uint8_t xtr_map[] = {
-		[ICE_RXDID_COMMS_AUX_VLAN]      = PROTO_XTR_VLAN,
-		[ICE_RXDID_COMMS_AUX_IPV4]      = PROTO_XTR_IPV4,
-		[ICE_RXDID_COMMS_AUX_IPV6]      = PROTO_XTR_IPV6,
-		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = PROTO_XTR_IPV6_FLOW,
-		[ICE_RXDID_COMMS_AUX_TCP]       = PROTO_XTR_TCP,
+/* Offset of mbuf dynamic field for protocol extraction data */
+int rte_net_ice_dynfield_proto_xtr_metadata = -1;
+
+/* Mask of mbuf dynamic flags for protocol extraction type */
+uint64_t rte_net_ice_dynflag_proto_xtr_vlan;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv4;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv6;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow;
+uint64_t rte_net_ice_dynflag_proto_xtr_tcp;
+
+#define MAP_RXDID_TO_FLAG(rxdid, flag) \
+	[rxdid] = &(flag)
+
+static inline uint64_t
+ice_rxdid_to_proto_xtr_ol_flag(uint8_t rxdid)
+{
+	static uint64_t *ol_flag_map[] = {
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_VLAN,
+				  rte_net_ice_dynflag_proto_xtr_vlan),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV4,
+				  rte_net_ice_dynflag_proto_xtr_ipv4),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV6,
+				  rte_net_ice_dynflag_proto_xtr_ipv6),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV6_FLOW,
+				  rte_net_ice_dynflag_proto_xtr_ipv6_flow),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_TCP,
+				  rte_net_ice_dynflag_proto_xtr_tcp),
 	};
+	uint64_t *ol_flag;
+
+	ol_flag = rxdid < RTE_DIM(ol_flag_map) ? ol_flag_map[rxdid] : NULL;
 
-	return rxdid < RTE_DIM(xtr_map) ? xtr_map[rxdid] : PROTO_XTR_NONE;
+	return ol_flag != NULL ? *ol_flag : 0;
 }
 
 static inline uint8_t
@@ -1325,10 +1347,38 @@ ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc *rxdp)
 		   mb->vlan_tci, mb->vlan_tci_outer);
 }
 
+#ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 #define ICE_RX_PROTO_XTR_VALID \
 	((1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) | \
 	 (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
 
+static void
+ice_rxd_to_proto_xtr(struct rte_mbuf *mb,
+		     volatile struct ice_32b_rx_flex_desc_comms *desc)
+{
+	uint16_t stat_err = rte_le_to_cpu_16(desc->status_error1);
+	uint32_t metadata;
+	uint64_t ol_flag;
+
+	if (unlikely(!(stat_err & ICE_RX_PROTO_XTR_VALID)))
+		return;
+
+	ol_flag = ice_rxdid_to_proto_xtr_ol_flag(desc->rxdid);
+	if (unlikely(!ol_flag))
+		return;
+
+	mb->ol_flags |= ol_flag;
+
+	metadata = stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) ?
+				rte_le_to_cpu_16(desc->flex_ts.flex.aux0) : 0;
+
+	if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S)))
+		metadata |= rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
+
+	*RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(mb) = metadata;
+}
+#endif
+
 static inline void
 ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 		      volatile union ice_rx_flex_desc *rxdp)
@@ -1344,28 +1394,13 @@ ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 	}
 
 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
-	init_proto_xtr_flds(mb);
-
-	stat_err = rte_le_to_cpu_16(desc->status_error1);
-	if (stat_err & ICE_RX_PROTO_XTR_VALID) {
-		struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
-			xtr->u.raw.data0 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
-			xtr->u.raw.data1 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
-
-		xtr->type = ice_rxdid_to_proto_xtr_type(desc->rxdid);
-		xtr->magic = PROTO_XTR_MAGIC_ID;
-	}
-
 	if (desc->flow_id != 0xFFFFFFFF) {
 		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 	}
+
+	if (unlikely(rte_net_ice_dynf_proto_xtr_metadata_avail()))
+		ice_rxd_to_proto_xtr(mb, desc);
 #endif
 }
 
diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h
index 5de618976..9e3d2cd07 100644
--- a/drivers/net/ice/ice_rxtx.h
+++ b/drivers/net/ice/ice_rxtx.h
@@ -5,7 +5,6 @@
 #ifndef _ICE_RXTX_H_
 #define _ICE_RXTX_H_
 
-#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 
 #define ICE_ALIGN_RING_DESC  32
diff --git a/drivers/net/ice/rte_pmd_ice.h b/drivers/net/ice/rte_pmd_ice.h
index 719487e1e..194d4b93d 100644
--- a/drivers/net/ice/rte_pmd_ice.h
+++ b/drivers/net/ice/rte_pmd_ice.h
@@ -5,144 +5,238 @@
 #ifndef _RTE_PMD_ICE_H_
 #define _RTE_PMD_ICE_H_
 
+/**
+ * @file rte_pmd_ice.h
+ *
+ * ice PMD specific functions.
+ *
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ */
+
 #include <stdio.h>
 #include <rte_mbuf.h>
-#include <rte_ethdev.h>
+#include <rte_mbuf_dyn.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-enum proto_xtr_type {
-	PROTO_XTR_NONE,
-	PROTO_XTR_VLAN,
-	PROTO_XTR_IPV4,
-	PROTO_XTR_IPV6,
-	PROTO_XTR_IPV6_FLOW,
-	PROTO_XTR_TCP,
+/**
+ * The supported network protocol extraction metadata format definition.
+ */
+union proto_xtr_metadata {
+	uint32_t metadata;
+
+	struct {
+		uint16_t data0;
+		uint16_t data1;
+	} raw;
+
+	struct {
+		uint16_t stag_vid:12,
+			 stag_dei:1,
+			 stag_pcp:3;
+		uint16_t ctag_vid:12,
+			 ctag_dei:1,
+			 ctag_pcp:3;
+	} vlan;
+
+	struct {
+		uint16_t protocol:8,
+			 ttl:8;
+		uint16_t tos:8,
+			 ihl:4,
+			 version:4;
+	} ipv4;
+
+	struct {
+		uint16_t hoplimit:8,
+			 nexthdr:8;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6;
+
+	struct {
+		uint16_t flowlo16;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6_flow;
+
+	struct {
+		uint16_t fin:1,
+			 syn:1,
+			 rst:1,
+			 psh:1,
+			 ack:1,
+			 urg:1,
+			 ece:1,
+			 cwr:1,
+			 res1:4,
+			 doff:4;
+		uint16_t rsvd;
+	} tcp;
 };
 
-struct proto_xtr_flds {
-	union {
-		struct {
-			uint16_t data0;
-			uint16_t data1;
-		} raw;
-		struct {
-			uint16_t stag_vid:12,
-				 stag_dei:1,
-				 stag_pcp:3;
-			uint16_t ctag_vid:12,
-				 ctag_dei:1,
-				 ctag_pcp:3;
-		} vlan;
-		struct {
-			uint16_t protocol:8,
-				 ttl:8;
-			uint16_t tos:8,
-				 ihl:4,
-				 version:4;
-		} ipv4;
-		struct {
-			uint16_t hoplimit:8,
-				 nexthdr:8;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6;
-		struct {
-			uint16_t flowlo16;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6_flow;
-		struct {
-			uint16_t fin:1,
-				 syn:1,
-				 rst:1,
-				 psh:1,
-				 ack:1,
-				 urg:1,
-				 ece:1,
-				 cwr:1,
-				 res1:4,
-				 doff:4;
-			uint16_t rsvd;
-		} tcp;
-	} u;
-
-	uint16_t rsvd;
-
-	uint8_t type;
-
-#define PROTO_XTR_MAGIC_ID	0xCE
-	uint8_t magic;
-};
+/**
+ * The offset of mbuf dynamic field for protocol extraction metadata, it is
+ * registered when dev_args 'proto_xtr' is set.
+ */
+extern int rte_net_ice_dynfield_proto_xtr_metadata;
 
-static inline void
-init_proto_xtr_flds(struct rte_mbuf *mb)
+/**
+ * The mask of mbuf dynamic flags for VLAN protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_vlan;
+
+/**
+ * The mask of mbuf dynamic flags for IPv4 protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv4;
+
+/**
+ * The mask of mbuf dynamic flags for IPv6 protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv6;
+
+/**
+ * The mask of mbuf dynamic flags for IPv6 with flow protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow;
+
+/**
+ * The mask of mbuf dynamic flags for TCP protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_tcp;
+
+/**
+ * The mbuf dynamic field pointer for protocol extraction metadata.
+ */
+#define RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m) \
+	RTE_MBUF_DYNFIELD((m), \
+			  rte_net_ice_dynfield_proto_xtr_metadata, uint32_t *)
+
+/**
+ * The mbuf dynamic flag for VLAN protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_VLAN \
+	(rte_net_ice_dynflag_proto_xtr_vlan)
+
+/**
+ * The mbuf dynamic flag for IPv4 protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV4 \
+	(rte_net_ice_dynflag_proto_xtr_ipv4)
+
+/**
+ * The mbuf dynamic flag for IPv6 protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV6 \
+	(rte_net_ice_dynflag_proto_xtr_ipv6)
+
+/**
+ * The mbuf dynamic flag for IPv6 with flow protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW \
+	(rte_net_ice_dynflag_proto_xtr_ipv6_flow)
+
+/**
+ * The mbuf dynamic flag for TCP protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_TCP \
+	(rte_net_ice_dynflag_proto_xtr_tcp)
+
+/**
+ * Check if mbuf dynamic field for protocol extraction metadata is registered.
+ *
+ * @return
+ *   True if registered, false otherwise.
+ */
+__rte_experimental
+static __rte_always_inline int
+rte_net_ice_dynf_proto_xtr_metadata_avail(void)
 {
-	mb->udata64 = 0;
+	return rte_net_ice_dynfield_proto_xtr_metadata != -1;
 }
 
-static inline struct proto_xtr_flds *
-get_proto_xtr_flds(struct rte_mbuf *mb)
+/**
+ * Get the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ * @return
+ *   The saved protocol extraction metadata.
+ */
+__rte_experimental
+static __rte_always_inline uint32_t
+rte_net_ice_dynf_proto_xtr_metadata_get(struct rte_mbuf *m)
 {
-	RTE_BUILD_BUG_ON(sizeof(struct proto_xtr_flds) > sizeof(mb->udata64));
-
-	return (struct proto_xtr_flds *)&mb->udata64;
+	return *RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m);
 }
 
+/**
+ * Dump the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ */
+__rte_experimental
 static inline void
-dump_proto_xtr_flds(struct rte_mbuf *mb)
+rte_net_ice_dump_proto_xtr_metadata(struct rte_mbuf *m)
 {
-	struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
+	union proto_xtr_metadata data;
 
-	if (xtr->magic != PROTO_XTR_MAGIC_ID || xtr->type == PROTO_XTR_NONE)
+	if (!rte_net_ice_dynf_proto_xtr_metadata_avail())
 		return;
 
-	printf(" - Protocol Extraction:[0x%04x:0x%04x],",
-	       xtr->u.raw.data0, xtr->u.raw.data1);
-
-	if (xtr->type == PROTO_XTR_VLAN)
-		printf("vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
-		       xtr->u.vlan.stag_pcp,
-		       xtr->u.vlan.stag_dei,
-		       xtr->u.vlan.stag_vid,
-		       xtr->u.vlan.ctag_pcp,
-		       xtr->u.vlan.ctag_dei,
-		       xtr->u.vlan.ctag_vid);
-	else if (xtr->type == PROTO_XTR_IPV4)
-		printf("ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
-		       xtr->u.ipv4.version,
-		       xtr->u.ipv4.ihl,
-		       xtr->u.ipv4.tos,
-		       xtr->u.ipv4.ttl,
-		       xtr->u.ipv4.protocol);
-	else if (xtr->type == PROTO_XTR_IPV6)
-		printf("ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
-		       xtr->u.ipv6.version,
-		       xtr->u.ipv6.tc,
-		       xtr->u.ipv6.flowhi4,
-		       xtr->u.ipv6.nexthdr,
-		       xtr->u.ipv6.hoplimit);
-	else if (xtr->type == PROTO_XTR_IPV6_FLOW)
-		printf("ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
-		       xtr->u.ipv6_flow.version,
-		       xtr->u.ipv6_flow.tc,
-		       xtr->u.ipv6_flow.flowhi4,
-		       xtr->u.ipv6_flow.flowlo16);
-	else if (xtr->type == PROTO_XTR_TCP)
-		printf("tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
-		       xtr->u.tcp.doff,
-		       xtr->u.tcp.cwr ? "C" : "",
-		       xtr->u.tcp.ece ? "E" : "",
-		       xtr->u.tcp.urg ? "U" : "",
-		       xtr->u.tcp.ack ? "A" : "",
-		       xtr->u.tcp.psh ? "P" : "",
-		       xtr->u.tcp.rst ? "R" : "",
-		       xtr->u.tcp.syn ? "S" : "",
-		       xtr->u.tcp.fin ? "F" : "");
+	data.metadata = rte_net_ice_dynf_proto_xtr_metadata_get(m);
+
+	if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_VLAN)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
+		       data.raw.data0, data.raw.data1,
+		       data.vlan.stag_pcp,
+		       data.vlan.stag_dei,
+		       data.vlan.stag_vid,
+		       data.vlan.ctag_pcp,
+		       data.vlan.ctag_dei,
+		       data.vlan.ctag_vid);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV4)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv4.version,
+		       data.ipv4.ihl,
+		       data.ipv4.tos,
+		       data.ipv4.ttl,
+		       data.ipv4.protocol);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV6)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6.version,
+		       data.ipv6.tc,
+		       data.ipv6.flowhi4,
+		       data.ipv6.nexthdr,
+		       data.ipv6.hoplimit);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6_flow.version,
+		       data.ipv6_flow.tc,
+		       data.ipv6_flow.flowhi4,
+		       data.ipv6_flow.flowlo16);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_TCP)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
+		       data.raw.data0, data.raw.data1,
+		       data.tcp.doff,
+		       data.tcp.cwr ? "C" : "",
+		       data.tcp.ece ? "E" : "",
+		       data.tcp.urg ? "U" : "",
+		       data.tcp.ack ? "A" : "",
+		       data.tcp.psh ? "P" : "",
+		       data.tcp.rst ? "R" : "",
+		       data.tcp.syn ? "S" : "",
+		       data.tcp.fin ? "F" : "");
 }
 
 #ifdef __cplusplus
diff --git a/drivers/net/ice/rte_pmd_ice_version.map b/drivers/net/ice/rte_pmd_ice_version.map
index 7b23b609d..7e70974d5 100644
--- a/drivers/net/ice/rte_pmd_ice_version.map
+++ b/drivers/net/ice/rte_pmd_ice_version.map
@@ -2,3 +2,15 @@ DPDK_19.02 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	# added in 19.11
+	rte_net_ice_dynfield_proto_xtr_metadata;
+	rte_net_ice_dynflag_proto_xtr_vlan;
+	rte_net_ice_dynflag_proto_xtr_ipv4;
+	rte_net_ice_dynflag_proto_xtr_ipv6;
+	rte_net_ice_dynflag_proto_xtr_ipv6_flow;
+	rte_net_ice_dynflag_proto_xtr_tcp;
+};
-- 
2.17.1

[dpdk-dev] [PATCH v6] net/ice: optimize protocol extraction by dynamic mbuf API

[Haiyue Wang]

The original design is to use rte_mbuf::udata64 to save the metadata of
protocol extraction which has network protocol data fields and type, a
private API is used to decode this metadata. It is not so generic.

Use the new dynamic mbuf field and flags to handle protocol extraction.
Then the metadata is very clean, only network protocol fields extracted
by defined offset, and its type will be indicated by related ol_flags.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
v6: - Don't need to register all flags when if one dev_args is set, register
      it as required.

v5: - Remove the '_OL/_ol' in dynamic mbuf flag to make the variable clean.

v4: - Include the 'rte_pmd_ice.h' header in ICE source as needed, reduce
      its compile scope.

v3: - Use the conventions name "rte_net_<pmd>_dynfield_<name>" since it
      is defined in a in PMD.
    - Add helpers API for easiy access.

v2: - disable the protocol extraction if failed to register some ol_flags
    - rewrite the commit message

 doc/api/doxy-api-index.md               |   1 +
 doc/api/doxy-api.conf.in                |   1 +
 doc/guides/nics/ice.rst                 |  14 +-
 drivers/net/ice/ice_ethdev.c            |  82 +++++-
 drivers/net/ice/ice_ethdev.h            |   9 +
 drivers/net/ice/ice_rxtx.c              |  91 +++++--
 drivers/net/ice/ice_rxtx.h              |   1 -
 drivers/net/ice/rte_pmd_ice.h           | 328 +++++++++++++++---------
 drivers/net/ice/rte_pmd_ice_version.map |  12 +
 9 files changed, 386 insertions(+), 153 deletions(-)

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index 28a5dd37e..dff496be0 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -42,6 +42,7 @@ The public API headers are grouped by topics:
   [KNI]                (@ref rte_kni.h),
   [ixgbe]              (@ref rte_pmd_ixgbe.h),
   [i40e]               (@ref rte_pmd_i40e.h),
+  [ice]                (@ref rte_pmd_ice.h),
   [bnxt]               (@ref rte_pmd_bnxt.h),
   [dpaa]               (@ref rte_pmd_dpaa.h),
   [dpaa2]              (@ref rte_pmd_dpaa2.h),
diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
index d8dafb289..faa3b1593 100644
--- a/doc/api/doxy-api.conf.in
+++ b/doc/api/doxy-api.conf.in
@@ -12,6 +12,7 @@ INPUT                   = @TOPDIR@/doc/api/doxy-api-index.md \
                           @TOPDIR@/drivers/net/dpaa \
                           @TOPDIR@/drivers/net/dpaa2 \
                           @TOPDIR@/drivers/net/i40e \
+                          @TOPDIR@/drivers/net/ice \
                           @TOPDIR@/drivers/net/ixgbe \
                           @TOPDIR@/drivers/net/softnic \
                           @TOPDIR@/drivers/raw/dpaa2_cmdif \
diff --git a/doc/guides/nics/ice.rst b/doc/guides/nics/ice.rst
index 933f63480..04af80545 100644
--- a/doc/guides/nics/ice.rst
+++ b/doc/guides/nics/ice.rst
@@ -82,8 +82,8 @@ Runtime Config Options
 
 - ``Protocol extraction for per queue``
 
-  Configure the RX queues to do protocol extraction into ``rte_mbuf::udata64``
-  for protocol handling acceleration, like checking the TCP SYN packets quickly.
+  Configure the RX queues to do protocol extraction into mbuf for protocol
+  handling acceleration, like checking the TCP SYN packets quickly.
 
   The argument format is::
 
@@ -111,7 +111,8 @@ Runtime Config Options
   This setting means queues 1, 2-3, 8-9 are TCP extraction, queues 10-23 are
   IPv6 extraction, other queues use the default VLAN extraction.
 
-  The extraction will be copied into the lower 32 bit of ``rte_mbuf::udata64``.
+  The extraction metadata will be copied into the registered dynamic mbuf field
+  with, and the related dynamic mbuf flags is set.
 
   .. table:: Protocol extraction : ``vlan``
 
@@ -175,10 +176,11 @@ Runtime Config Options
 
   TCPHDR2 - Reserved
 
-  Use ``get_proto_xtr_flds(struct rte_mbuf *mb)`` to access the protocol
-  extraction, do not use ``rte_mbuf::udata64`` directly.
+  Use ``rte_net_ice_dynf_proto_xtr_metadata_get`` to access the protocol
+  extraction metadata, and use ``PKT_RX_DYNF_PROTO_XTR_*`` to get the
+  metadata type of ``struct rte_mbuf::ol_flags``.
 
-  The ``dump_proto_xtr_flds(struct rte_mbuf *mb)`` routine shows how to
+  The ``rte_net_ice_dump_proto_xtr_metadata`` routine shows how to
   access the protocol extraction result in ``struct rte_mbuf``.
 
 Driver compilation and testing
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index d74675842..140e81d6b 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -14,6 +14,8 @@
 #include "base/ice_flow.h"
 #include "base/ice_dcb.h"
 #include "base/ice_common.h"
+
+#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 #include "ice_rxtx.h"
 #include "ice_generic_flow.h"
@@ -30,6 +32,35 @@ static const char * const ice_valid_args[] = {
 	NULL
 };
 
+static const struct rte_mbuf_dynfield ice_proto_xtr_metadata_param = {
+	.name = RTE_STR(rte_net_ice_dynfield_proto_xtr_metadata),
+	.size = sizeof(uint32_t),
+	.align = __alignof__(uint32_t),
+	.flags = 0,
+};
+
+struct proto_xtr_ol_flag {
+	const struct rte_mbuf_dynflag param;
+	uint64_t *flag;
+	bool required;
+};
+
+#define MAP_TYPE_TO_DYNF(type, flag) \
+	[type] = { { RTE_STR(flag), 0 }, &(flag), false }
+
+static struct proto_xtr_ol_flag ice_proto_xtr_ol_flag_params[] = {
+	MAP_TYPE_TO_DYNF(PROTO_XTR_VLAN,
+			 rte_net_ice_dynflag_proto_xtr_vlan),
+	MAP_TYPE_TO_DYNF(PROTO_XTR_IPV4,
+			 rte_net_ice_dynflag_proto_xtr_ipv4),
+	MAP_TYPE_TO_DYNF(PROTO_XTR_IPV6,
+			 rte_net_ice_dynflag_proto_xtr_ipv6),
+	MAP_TYPE_TO_DYNF(PROTO_XTR_IPV6_FLOW,
+			 rte_net_ice_dynflag_proto_xtr_ipv6_flow),
+	MAP_TYPE_TO_DYNF(PROTO_XTR_TCP,
+			 rte_net_ice_dynflag_proto_xtr_tcp),
+};
+
 #define ICE_DFLT_OUTER_TAG_TYPE ICE_AQ_VSI_OUTER_TAG_VLAN_9100
 
 /* DDP package search path */
@@ -1385,6 +1416,9 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 			ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 	struct ice_hw *hw = ICE_PF_TO_HW(pf);
+	const struct proto_xtr_ol_flag *ol_flag;
+	bool proto_xtr_enable = false;
+	int offset;
 	uint16_t i;
 
 	if (!ice_proto_xtr_support(hw)) {
@@ -1398,10 +1432,56 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 		return;
 	}
 
-	for (i = 0; i < pf->lan_nb_qps; i++)
+	for (i = 0; i < pf->lan_nb_qps; i++) {
 		pf->proto_xtr[i] = ad->devargs.proto_xtr[i] != PROTO_XTR_NONE ?
 				   ad->devargs.proto_xtr[i] :
 				   ad->devargs.proto_xtr_dflt;
+
+		if (pf->proto_xtr[i] != PROTO_XTR_NONE) {
+			uint8_t type = pf->proto_xtr[i];
+
+			ice_proto_xtr_ol_flag_params[type].required = true;
+			proto_xtr_enable = true;
+		}
+	}
+
+	if (likely(!proto_xtr_enable))
+		return;
+
+	offset = rte_mbuf_dynfield_register(&ice_proto_xtr_metadata_param);
+	if (unlikely(offset == -1)) {
+		PMD_DRV_LOG(ERR,
+			    "Protocol extraction metadata is disabled in mbuf with error %d",
+			    -rte_errno);
+		return;
+	}
+
+	PMD_DRV_LOG(DEBUG,
+		    "Protocol extraction metadata offset in mbuf is : %d",
+		    offset);
+	rte_net_ice_dynfield_proto_xtr_metadata = offset;
+
+	for (i = 0; i < RTE_DIM(ice_proto_xtr_ol_flag_params); i++) {
+		ol_flag = &ice_proto_xtr_ol_flag_params[i];
+
+		if (!ol_flag->required)
+			continue;
+
+		offset = rte_mbuf_dynflag_register(&ol_flag->param);
+		if (unlikely(offset == -1)) {
+			PMD_DRV_LOG(ERR,
+				    "Protocol extraction offload '%s' failed to register with error %d",
+				    ol_flag->param.name, -rte_errno);
+
+			rte_net_ice_dynfield_proto_xtr_metadata = -1;
+			break;
+		}
+
+		PMD_DRV_LOG(ERR,
+			    "Protocol extraction offload '%s' offset in mbuf is : %d",
+			    ol_flag->param.name, offset);
+		*ol_flag->flag = 1ULL << offset;
+	}
 }
 
 /*  Initialize SW parameters of PF */
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index de67e5934..c55713cc1 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -241,6 +241,15 @@ struct ice_vsi {
 	bool offset_loaded;
 };
 
+enum proto_xtr_type {
+	PROTO_XTR_NONE,
+	PROTO_XTR_VLAN,
+	PROTO_XTR_IPV4,
+	PROTO_XTR_IPV6,
+	PROTO_XTR_IPV6_FLOW,
+	PROTO_XTR_TCP,
+};
+
 enum ice_fdir_tunnel_type {
 	ICE_FDIR_TUNNEL_TYPE_NONE = 0,
 	ICE_FDIR_TUNNEL_TYPE_VXLAN,
diff --git a/drivers/net/ice/ice_rxtx.c b/drivers/net/ice/ice_rxtx.c
index 47a654a73..581ad1b1e 100644
--- a/drivers/net/ice/ice_rxtx.c
+++ b/drivers/net/ice/ice_rxtx.c
@@ -5,6 +5,7 @@
 #include <rte_ethdev_driver.h>
 #include <rte_net.h>
 
+#include "rte_pmd_ice.h"
 #include "ice_rxtx.h"
 
 #define ICE_TX_CKSUM_OFFLOAD_MASK (		 \
@@ -13,18 +14,39 @@
 		PKT_TX_TCP_SEG |		 \
 		PKT_TX_OUTER_IP_CKSUM)
 
-static inline uint8_t
-ice_rxdid_to_proto_xtr_type(uint8_t rxdid)
-{
-	static uint8_t xtr_map[] = {
-		[ICE_RXDID_COMMS_AUX_VLAN]      = PROTO_XTR_VLAN,
-		[ICE_RXDID_COMMS_AUX_IPV4]      = PROTO_XTR_IPV4,
-		[ICE_RXDID_COMMS_AUX_IPV6]      = PROTO_XTR_IPV6,
-		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = PROTO_XTR_IPV6_FLOW,
-		[ICE_RXDID_COMMS_AUX_TCP]       = PROTO_XTR_TCP,
+/* Offset of mbuf dynamic field for protocol extraction data */
+int rte_net_ice_dynfield_proto_xtr_metadata = -1;
+
+/* Mask of mbuf dynamic flags for protocol extraction type */
+uint64_t rte_net_ice_dynflag_proto_xtr_vlan;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv4;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv6;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow;
+uint64_t rte_net_ice_dynflag_proto_xtr_tcp;
+
+#define MAP_RXDID_TO_FLAG(rxdid, flag) \
+	[rxdid] = &(flag)
+
+static inline uint64_t
+ice_rxdid_to_proto_xtr_ol_flag(uint8_t rxdid)
+{
+	static uint64_t *ol_flag_map[] = {
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_VLAN,
+				  rte_net_ice_dynflag_proto_xtr_vlan),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV4,
+				  rte_net_ice_dynflag_proto_xtr_ipv4),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV6,
+				  rte_net_ice_dynflag_proto_xtr_ipv6),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV6_FLOW,
+				  rte_net_ice_dynflag_proto_xtr_ipv6_flow),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_TCP,
+				  rte_net_ice_dynflag_proto_xtr_tcp),
 	};
+	uint64_t *ol_flag;
+
+	ol_flag = rxdid < RTE_DIM(ol_flag_map) ? ol_flag_map[rxdid] : NULL;
 
-	return rxdid < RTE_DIM(xtr_map) ? xtr_map[rxdid] : PROTO_XTR_NONE;
+	return ol_flag != NULL ? *ol_flag : 0;
 }
 
 static inline uint8_t
@@ -1325,10 +1347,38 @@ ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc *rxdp)
 		   mb->vlan_tci, mb->vlan_tci_outer);
 }
 
+#ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 #define ICE_RX_PROTO_XTR_VALID \
 	((1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) | \
 	 (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
 
+static void
+ice_rxd_to_proto_xtr(struct rte_mbuf *mb,
+		     volatile struct ice_32b_rx_flex_desc_comms *desc)
+{
+	uint16_t stat_err = rte_le_to_cpu_16(desc->status_error1);
+	uint32_t metadata;
+	uint64_t ol_flag;
+
+	if (unlikely(!(stat_err & ICE_RX_PROTO_XTR_VALID)))
+		return;
+
+	ol_flag = ice_rxdid_to_proto_xtr_ol_flag(desc->rxdid);
+	if (unlikely(!ol_flag))
+		return;
+
+	mb->ol_flags |= ol_flag;
+
+	metadata = stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) ?
+				rte_le_to_cpu_16(desc->flex_ts.flex.aux0) : 0;
+
+	if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S)))
+		metadata |= rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
+
+	*RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(mb) = metadata;
+}
+#endif
+
 static inline void
 ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 		      volatile union ice_rx_flex_desc *rxdp)
@@ -1344,28 +1394,13 @@ ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 	}
 
 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
-	init_proto_xtr_flds(mb);
-
-	stat_err = rte_le_to_cpu_16(desc->status_error1);
-	if (stat_err & ICE_RX_PROTO_XTR_VALID) {
-		struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
-			xtr->u.raw.data0 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
-			xtr->u.raw.data1 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
-
-		xtr->type = ice_rxdid_to_proto_xtr_type(desc->rxdid);
-		xtr->magic = PROTO_XTR_MAGIC_ID;
-	}
-
 	if (desc->flow_id != 0xFFFFFFFF) {
 		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 	}
+
+	if (unlikely(rte_net_ice_dynf_proto_xtr_metadata_avail()))
+		ice_rxd_to_proto_xtr(mb, desc);
 #endif
 }
 
diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h
index 5de618976..9e3d2cd07 100644
--- a/drivers/net/ice/ice_rxtx.h
+++ b/drivers/net/ice/ice_rxtx.h
@@ -5,7 +5,6 @@
 #ifndef _ICE_RXTX_H_
 #define _ICE_RXTX_H_
 
-#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 
 #define ICE_ALIGN_RING_DESC  32
diff --git a/drivers/net/ice/rte_pmd_ice.h b/drivers/net/ice/rte_pmd_ice.h
index 719487e1e..1f1b75e85 100644
--- a/drivers/net/ice/rte_pmd_ice.h
+++ b/drivers/net/ice/rte_pmd_ice.h
@@ -5,144 +5,238 @@
 #ifndef _RTE_PMD_ICE_H_
 #define _RTE_PMD_ICE_H_
 
+/**
+ * @file rte_pmd_ice.h
+ *
+ * ice PMD specific functions.
+ *
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ */
+
 #include <stdio.h>
 #include <rte_mbuf.h>
-#include <rte_ethdev.h>
+#include <rte_mbuf_dyn.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-enum proto_xtr_type {
-	PROTO_XTR_NONE,
-	PROTO_XTR_VLAN,
-	PROTO_XTR_IPV4,
-	PROTO_XTR_IPV6,
-	PROTO_XTR_IPV6_FLOW,
-	PROTO_XTR_TCP,
+/**
+ * The supported network protocol extraction metadata format definition.
+ */
+union proto_xtr_metadata {
+	uint32_t metadata;
+
+	struct {
+		uint16_t data0;
+		uint16_t data1;
+	} raw;
+
+	struct {
+		uint16_t stag_vid:12,
+			 stag_dei:1,
+			 stag_pcp:3;
+		uint16_t ctag_vid:12,
+			 ctag_dei:1,
+			 ctag_pcp:3;
+	} vlan;
+
+	struct {
+		uint16_t protocol:8,
+			 ttl:8;
+		uint16_t tos:8,
+			 ihl:4,
+			 version:4;
+	} ipv4;
+
+	struct {
+		uint16_t hoplimit:8,
+			 nexthdr:8;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6;
+
+	struct {
+		uint16_t flowlo16;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6_flow;
+
+	struct {
+		uint16_t fin:1,
+			 syn:1,
+			 rst:1,
+			 psh:1,
+			 ack:1,
+			 urg:1,
+			 ece:1,
+			 cwr:1,
+			 res1:4,
+			 doff:4;
+		uint16_t rsvd;
+	} tcp;
 };
 
-struct proto_xtr_flds {
-	union {
-		struct {
-			uint16_t data0;
-			uint16_t data1;
-		} raw;
-		struct {
-			uint16_t stag_vid:12,
-				 stag_dei:1,
-				 stag_pcp:3;
-			uint16_t ctag_vid:12,
-				 ctag_dei:1,
-				 ctag_pcp:3;
-		} vlan;
-		struct {
-			uint16_t protocol:8,
-				 ttl:8;
-			uint16_t tos:8,
-				 ihl:4,
-				 version:4;
-		} ipv4;
-		struct {
-			uint16_t hoplimit:8,
-				 nexthdr:8;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6;
-		struct {
-			uint16_t flowlo16;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6_flow;
-		struct {
-			uint16_t fin:1,
-				 syn:1,
-				 rst:1,
-				 psh:1,
-				 ack:1,
-				 urg:1,
-				 ece:1,
-				 cwr:1,
-				 res1:4,
-				 doff:4;
-			uint16_t rsvd;
-		} tcp;
-	} u;
-
-	uint16_t rsvd;
-
-	uint8_t type;
-
-#define PROTO_XTR_MAGIC_ID	0xCE
-	uint8_t magic;
-};
+/**
+ * The offset of mbuf dynamic field for protocol extraction metadata, it is
+ * registered when dev_args 'proto_xtr' is set.
+ */
+extern int rte_net_ice_dynfield_proto_xtr_metadata;
 
-static inline void
-init_proto_xtr_flds(struct rte_mbuf *mb)
+/**
+ * The mask of mbuf dynamic flags for VLAN protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_vlan;
+
+/**
+ * The mask of mbuf dynamic flags for IPv4 protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv4;
+
+/**
+ * The mask of mbuf dynamic flags for IPv6 protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv6;
+
+/**
+ * The mask of mbuf dynamic flags for IPv6 with flow protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow;
+
+/**
+ * The mask of mbuf dynamic flags for TCP protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_tcp;
+
+/**
+ * The mbuf dynamic field pointer for protocol extraction metadata.
+ */
+#define RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m) \
+	RTE_MBUF_DYNFIELD((m), \
+			  rte_net_ice_dynfield_proto_xtr_metadata, uint32_t *)
+
+/**
+ * The mbuf dynamic flag for VLAN protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_VLAN \
+	(rte_net_ice_dynflag_proto_xtr_vlan)
+
+/**
+ * The mbuf dynamic flag for IPv4 protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV4 \
+	(rte_net_ice_dynflag_proto_xtr_ipv4)
+
+/**
+ * The mbuf dynamic flag for IPv6 protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV6 \
+	(rte_net_ice_dynflag_proto_xtr_ipv6)
+
+/**
+ * The mbuf dynamic flag for IPv6 with flow protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW \
+	(rte_net_ice_dynflag_proto_xtr_ipv6_flow)
+
+/**
+ * The mbuf dynamic flag for TCP protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_TCP \
+	(rte_net_ice_dynflag_proto_xtr_tcp)
+
+/**
+ * Check if mbuf dynamic field for protocol extraction metadata is registered.
+ *
+ * @return
+ *   True if registered, false otherwise.
+ */
+__rte_experimental
+static __rte_always_inline int
+rte_net_ice_dynf_proto_xtr_metadata_avail(void)
 {
-	mb->udata64 = 0;
+	return rte_net_ice_dynfield_proto_xtr_metadata != -1;
 }
 
-static inline struct proto_xtr_flds *
-get_proto_xtr_flds(struct rte_mbuf *mb)
+/**
+ * Get the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ * @return
+ *   The saved protocol extraction metadata.
+ */
+__rte_experimental
+static __rte_always_inline uint32_t
+rte_net_ice_dynf_proto_xtr_metadata_get(struct rte_mbuf *m)
 {
-	RTE_BUILD_BUG_ON(sizeof(struct proto_xtr_flds) > sizeof(mb->udata64));
-
-	return (struct proto_xtr_flds *)&mb->udata64;
+	return *RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m);
 }
 
+/**
+ * Dump the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ */
+__rte_experimental
 static inline void
-dump_proto_xtr_flds(struct rte_mbuf *mb)
+rte_net_ice_dump_proto_xtr_metadata(struct rte_mbuf *m)
 {
-	struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
+	union proto_xtr_metadata data;
 
-	if (xtr->magic != PROTO_XTR_MAGIC_ID || xtr->type == PROTO_XTR_NONE)
+	if (!rte_net_ice_dynf_proto_xtr_metadata_avail())
 		return;
 
-	printf(" - Protocol Extraction:[0x%04x:0x%04x],",
-	       xtr->u.raw.data0, xtr->u.raw.data1);
-
-	if (xtr->type == PROTO_XTR_VLAN)
-		printf("vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
-		       xtr->u.vlan.stag_pcp,
-		       xtr->u.vlan.stag_dei,
-		       xtr->u.vlan.stag_vid,
-		       xtr->u.vlan.ctag_pcp,
-		       xtr->u.vlan.ctag_dei,
-		       xtr->u.vlan.ctag_vid);
-	else if (xtr->type == PROTO_XTR_IPV4)
-		printf("ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
-		       xtr->u.ipv4.version,
-		       xtr->u.ipv4.ihl,
-		       xtr->u.ipv4.tos,
-		       xtr->u.ipv4.ttl,
-		       xtr->u.ipv4.protocol);
-	else if (xtr->type == PROTO_XTR_IPV6)
-		printf("ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
-		       xtr->u.ipv6.version,
-		       xtr->u.ipv6.tc,
-		       xtr->u.ipv6.flowhi4,
-		       xtr->u.ipv6.nexthdr,
-		       xtr->u.ipv6.hoplimit);
-	else if (xtr->type == PROTO_XTR_IPV6_FLOW)
-		printf("ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
-		       xtr->u.ipv6_flow.version,
-		       xtr->u.ipv6_flow.tc,
-		       xtr->u.ipv6_flow.flowhi4,
-		       xtr->u.ipv6_flow.flowlo16);
-	else if (xtr->type == PROTO_XTR_TCP)
-		printf("tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
-		       xtr->u.tcp.doff,
-		       xtr->u.tcp.cwr ? "C" : "",
-		       xtr->u.tcp.ece ? "E" : "",
-		       xtr->u.tcp.urg ? "U" : "",
-		       xtr->u.tcp.ack ? "A" : "",
-		       xtr->u.tcp.psh ? "P" : "",
-		       xtr->u.tcp.rst ? "R" : "",
-		       xtr->u.tcp.syn ? "S" : "",
-		       xtr->u.tcp.fin ? "F" : "");
+	data.metadata = rte_net_ice_dynf_proto_xtr_metadata_get(m);
+
+	if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_VLAN)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],vlan,stag=%u:%u:%u,ctag=%u:%u:%u",
+		       data.raw.data0, data.raw.data1,
+		       data.vlan.stag_pcp,
+		       data.vlan.stag_dei,
+		       data.vlan.stag_vid,
+		       data.vlan.ctag_pcp,
+		       data.vlan.ctag_dei,
+		       data.vlan.ctag_vid);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV4)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv4.version,
+		       data.ipv4.ihl,
+		       data.ipv4.tos,
+		       data.ipv4.ttl,
+		       data.ipv4.protocol);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV6)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6.version,
+		       data.ipv6.tc,
+		       data.ipv6.flowhi4,
+		       data.ipv6.nexthdr,
+		       data.ipv6.hoplimit);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6_flow.version,
+		       data.ipv6_flow.tc,
+		       data.ipv6_flow.flowhi4,
+		       data.ipv6_flow.flowlo16);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_TCP)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],tcp,doff=%u,flags=%s%s%s%s%s%s%s%s",
+		       data.raw.data0, data.raw.data1,
+		       data.tcp.doff,
+		       data.tcp.cwr ? "C" : "",
+		       data.tcp.ece ? "E" : "",
+		       data.tcp.urg ? "U" : "",
+		       data.tcp.ack ? "A" : "",
+		       data.tcp.psh ? "P" : "",
+		       data.tcp.rst ? "R" : "",
+		       data.tcp.syn ? "S" : "",
+		       data.tcp.fin ? "F" : "");
 }
 
 #ifdef __cplusplus
diff --git a/drivers/net/ice/rte_pmd_ice_version.map b/drivers/net/ice/rte_pmd_ice_version.map
index 7b23b609d..7e70974d5 100644
--- a/drivers/net/ice/rte_pmd_ice_version.map
+++ b/drivers/net/ice/rte_pmd_ice_version.map
@@ -2,3 +2,15 @@ DPDK_19.02 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	# added in 19.11
+	rte_net_ice_dynfield_proto_xtr_metadata;
+	rte_net_ice_dynflag_proto_xtr_vlan;
+	rte_net_ice_dynflag_proto_xtr_ipv4;
+	rte_net_ice_dynflag_proto_xtr_ipv6;
+	rte_net_ice_dynflag_proto_xtr_ipv6_flow;
+	rte_net_ice_dynflag_proto_xtr_tcp;
+};
-- 
2.17.1

[dpdk-dev] [PATCH v7] net/ice: optimize protocol extraction by dynamic mbuf API

[Haiyue Wang]

The original design is to use rte_mbuf::udata64 to save the metadata of
protocol extraction which has network protocol data fields and type, a
private API is used to decode this metadata. It is not so generic.

Use the new dynamic mbuf field and flags to handle protocol extraction.
Then the metadata is very clean, only network protocol fields extracted
by defined offset, and its type will be indicated by related ol_flags.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
v7: - Change the log level from ERR to DEBUG for a successful registration,
      ERR is used to develop firstly.

v6: - Don't need to register all flags when if one dev_args is set, register
      it as required.

v5: - Remove the '_OL/_ol' in dynamic mbuf flag to make the variable clean.

v4: - Include the 'rte_pmd_ice.h' header in ICE source as needed, reduce
      its compile scope.

v3: - Use the conventions name "rte_net_<pmd>_dynfield_<name>" since it
      is defined in a in PMD.
    - Add helpers API for easiy access.

v2: - disable the protocol extraction if failed to register some ol_flags
    - rewrite the commit message

 doc/api/doxy-api-index.md               |   1 +
 doc/api/doxy-api.conf.in                |   1 +
 doc/guides/nics/ice.rst                 |  14 +-
 drivers/net/ice/ice_ethdev.c            |  82 +++++-
 drivers/net/ice/ice_ethdev.h            |   9 +
 drivers/net/ice/ice_rxtx.c              |  91 +++++--
 drivers/net/ice/ice_rxtx.h              |   1 -
 drivers/net/ice/rte_pmd_ice.h           | 328 +++++++++++++++---------
 drivers/net/ice/rte_pmd_ice_version.map |  12 +
 9 files changed, 386 insertions(+), 153 deletions(-)

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index 28a5dd37e..dff496be0 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -42,6 +42,7 @@ The public API headers are grouped by topics:
   [KNI]                (@ref rte_kni.h),
   [ixgbe]              (@ref rte_pmd_ixgbe.h),
   [i40e]               (@ref rte_pmd_i40e.h),
+  [ice]                (@ref rte_pmd_ice.h),
   [bnxt]               (@ref rte_pmd_bnxt.h),
   [dpaa]               (@ref rte_pmd_dpaa.h),
   [dpaa2]              (@ref rte_pmd_dpaa2.h),
diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
index d8dafb289..faa3b1593 100644
--- a/doc/api/doxy-api.conf.in
+++ b/doc/api/doxy-api.conf.in
@@ -12,6 +12,7 @@ INPUT                   = @TOPDIR@/doc/api/doxy-api-index.md \
                           @TOPDIR@/drivers/net/dpaa \
                           @TOPDIR@/drivers/net/dpaa2 \
                           @TOPDIR@/drivers/net/i40e \
+                          @TOPDIR@/drivers/net/ice \
                           @TOPDIR@/drivers/net/ixgbe \
                           @TOPDIR@/drivers/net/softnic \
                           @TOPDIR@/drivers/raw/dpaa2_cmdif \
diff --git a/doc/guides/nics/ice.rst b/doc/guides/nics/ice.rst
index 933f63480..04af80545 100644
--- a/doc/guides/nics/ice.rst
+++ b/doc/guides/nics/ice.rst
@@ -82,8 +82,8 @@ Runtime Config Options
 
 - ``Protocol extraction for per queue``
 
-  Configure the RX queues to do protocol extraction into ``rte_mbuf::udata64``
-  for protocol handling acceleration, like checking the TCP SYN packets quickly.
+  Configure the RX queues to do protocol extraction into mbuf for protocol
+  handling acceleration, like checking the TCP SYN packets quickly.
 
   The argument format is::
 
@@ -111,7 +111,8 @@ Runtime Config Options
   This setting means queues 1, 2-3, 8-9 are TCP extraction, queues 10-23 are
   IPv6 extraction, other queues use the default VLAN extraction.
 
-  The extraction will be copied into the lower 32 bit of ``rte_mbuf::udata64``.
+  The extraction metadata will be copied into the registered dynamic mbuf field
+  with, and the related dynamic mbuf flags is set.
 
   .. table:: Protocol extraction : ``vlan``
 
@@ -175,10 +176,11 @@ Runtime Config Options
 
   TCPHDR2 - Reserved
 
-  Use ``get_proto_xtr_flds(struct rte_mbuf *mb)`` to access the protocol
-  extraction, do not use ``rte_mbuf::udata64`` directly.
+  Use ``rte_net_ice_dynf_proto_xtr_metadata_get`` to access the protocol
+  extraction metadata, and use ``PKT_RX_DYNF_PROTO_XTR_*`` to get the
+  metadata type of ``struct rte_mbuf::ol_flags``.
 
-  The ``dump_proto_xtr_flds(struct rte_mbuf *mb)`` routine shows how to
+  The ``rte_net_ice_dump_proto_xtr_metadata`` routine shows how to
   access the protocol extraction result in ``struct rte_mbuf``.
 
 Driver compilation and testing
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index d74675842..6b0870c2e 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -14,6 +14,8 @@
 #include "base/ice_flow.h"
 #include "base/ice_dcb.h"
 #include "base/ice_common.h"
+
+#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 #include "ice_rxtx.h"
 #include "ice_generic_flow.h"
@@ -30,6 +32,35 @@ static const char * const ice_valid_args[] = {
 	NULL
 };
 
+static const struct rte_mbuf_dynfield ice_proto_xtr_metadata_param = {
+	.name = RTE_STR(rte_net_ice_dynfield_proto_xtr_metadata),
+	.size = sizeof(uint32_t),
+	.align = __alignof__(uint32_t),
+	.flags = 0,
+};
+
+struct proto_xtr_ol_flag {
+	const struct rte_mbuf_dynflag param;
+	uint64_t *flag;
+	bool required;
+};
+
+#define MAP_TYPE_TO_DYNF(type, flag) \
+	[type] = { { RTE_STR(flag), 0 }, &(flag), false }
+
+static struct proto_xtr_ol_flag ice_proto_xtr_ol_flag_params[] = {
+	MAP_TYPE_TO_DYNF(PROTO_XTR_VLAN,
+			 rte_net_ice_dynflag_proto_xtr_vlan),
+	MAP_TYPE_TO_DYNF(PROTO_XTR_IPV4,
+			 rte_net_ice_dynflag_proto_xtr_ipv4),
+	MAP_TYPE_TO_DYNF(PROTO_XTR_IPV6,
+			 rte_net_ice_dynflag_proto_xtr_ipv6),
+	MAP_TYPE_TO_DYNF(PROTO_XTR_IPV6_FLOW,
+			 rte_net_ice_dynflag_proto_xtr_ipv6_flow),
+	MAP_TYPE_TO_DYNF(PROTO_XTR_TCP,
+			 rte_net_ice_dynflag_proto_xtr_tcp),
+};
+
 #define ICE_DFLT_OUTER_TAG_TYPE ICE_AQ_VSI_OUTER_TAG_VLAN_9100
 
 /* DDP package search path */
@@ -1385,6 +1416,9 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 			ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 	struct ice_hw *hw = ICE_PF_TO_HW(pf);
+	const struct proto_xtr_ol_flag *ol_flag;
+	bool proto_xtr_enable = false;
+	int offset;
 	uint16_t i;
 
 	if (!ice_proto_xtr_support(hw)) {
@@ -1398,10 +1432,56 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 		return;
 	}
 
-	for (i = 0; i < pf->lan_nb_qps; i++)
+	for (i = 0; i < pf->lan_nb_qps; i++) {
 		pf->proto_xtr[i] = ad->devargs.proto_xtr[i] != PROTO_XTR_NONE ?
 				   ad->devargs.proto_xtr[i] :
 				   ad->devargs.proto_xtr_dflt;
+
+		if (pf->proto_xtr[i] != PROTO_XTR_NONE) {
+			uint8_t type = pf->proto_xtr[i];
+
+			ice_proto_xtr_ol_flag_params[type].required = true;
+			proto_xtr_enable = true;
+		}
+	}
+
+	if (likely(!proto_xtr_enable))
+		return;
+
+	offset = rte_mbuf_dynfield_register(&ice_proto_xtr_metadata_param);
+	if (unlikely(offset == -1)) {
+		PMD_DRV_LOG(ERR,
+			    "Protocol extraction metadata is disabled in mbuf with error %d",
+			    -rte_errno);
+		return;
+	}
+
+	PMD_DRV_LOG(DEBUG,
+		    "Protocol extraction metadata offset in mbuf is : %d",
+		    offset);
+	rte_net_ice_dynfield_proto_xtr_metadata = offset;
+
+	for (i = 0; i < RTE_DIM(ice_proto_xtr_ol_flag_params); i++) {
+		ol_flag = &ice_proto_xtr_ol_flag_params[i];
+
+		if (!ol_flag->required)
+			continue;
+
+		offset = rte_mbuf_dynflag_register(&ol_flag->param);
+		if (unlikely(offset == -1)) {
+			PMD_DRV_LOG(ERR,
+				    "Protocol extraction offload '%s' failed to register with error %d",
+				    ol_flag->param.name, -rte_errno);
+
+			rte_net_ice_dynfield_proto_xtr_metadata = -1;
+			break;
+		}
+
+		PMD_DRV_LOG(DEBUG,
+			    "Protocol extraction offload '%s' offset in mbuf is : %d",
+			    ol_flag->param.name, offset);
+		*ol_flag->flag = 1ULL << offset;
+	}
 }
 
 /*  Initialize SW parameters of PF */
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index de67e5934..c55713cc1 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -241,6 +241,15 @@ struct ice_vsi {
 	bool offset_loaded;
 };
 
+enum proto_xtr_type {
+	PROTO_XTR_NONE,
+	PROTO_XTR_VLAN,
+	PROTO_XTR_IPV4,
+	PROTO_XTR_IPV6,
+	PROTO_XTR_IPV6_FLOW,
+	PROTO_XTR_TCP,
+};
+
 enum ice_fdir_tunnel_type {
 	ICE_FDIR_TUNNEL_TYPE_NONE = 0,
 	ICE_FDIR_TUNNEL_TYPE_VXLAN,
diff --git a/drivers/net/ice/ice_rxtx.c b/drivers/net/ice/ice_rxtx.c
index 47a654a73..581ad1b1e 100644
--- a/drivers/net/ice/ice_rxtx.c
+++ b/drivers/net/ice/ice_rxtx.c
@@ -5,6 +5,7 @@
 #include <rte_ethdev_driver.h>
 #include <rte_net.h>
 
+#include "rte_pmd_ice.h"
 #include "ice_rxtx.h"
 
 #define ICE_TX_CKSUM_OFFLOAD_MASK (		 \
@@ -13,18 +14,39 @@
 		PKT_TX_TCP_SEG |		 \
 		PKT_TX_OUTER_IP_CKSUM)
 
-static inline uint8_t
-ice_rxdid_to_proto_xtr_type(uint8_t rxdid)
-{
-	static uint8_t xtr_map[] = {
-		[ICE_RXDID_COMMS_AUX_VLAN]      = PROTO_XTR_VLAN,
-		[ICE_RXDID_COMMS_AUX_IPV4]      = PROTO_XTR_IPV4,
-		[ICE_RXDID_COMMS_AUX_IPV6]      = PROTO_XTR_IPV6,
-		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = PROTO_XTR_IPV6_FLOW,
-		[ICE_RXDID_COMMS_AUX_TCP]       = PROTO_XTR_TCP,
+/* Offset of mbuf dynamic field for protocol extraction data */
+int rte_net_ice_dynfield_proto_xtr_metadata = -1;
+
+/* Mask of mbuf dynamic flags for protocol extraction type */
+uint64_t rte_net_ice_dynflag_proto_xtr_vlan;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv4;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv6;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow;
+uint64_t rte_net_ice_dynflag_proto_xtr_tcp;
+
+#define MAP_RXDID_TO_FLAG(rxdid, flag) \
+	[rxdid] = &(flag)
+
+static inline uint64_t
+ice_rxdid_to_proto_xtr_ol_flag(uint8_t rxdid)
+{
+	static uint64_t *ol_flag_map[] = {
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_VLAN,
+				  rte_net_ice_dynflag_proto_xtr_vlan),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV4,
+				  rte_net_ice_dynflag_proto_xtr_ipv4),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV6,
+				  rte_net_ice_dynflag_proto_xtr_ipv6),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_IPV6_FLOW,
+				  rte_net_ice_dynflag_proto_xtr_ipv6_flow),
+		MAP_RXDID_TO_FLAG(ICE_RXDID_COMMS_AUX_TCP,
+				  rte_net_ice_dynflag_proto_xtr_tcp),
 	};
+	uint64_t *ol_flag;
+
+	ol_flag = rxdid < RTE_DIM(ol_flag_map) ? ol_flag_map[rxdid] : NULL;
 
-	return rxdid < RTE_DIM(xtr_map) ? xtr_map[rxdid] : PROTO_XTR_NONE;
+	return ol_flag != NULL ? *ol_flag : 0;
 }
 
 static inline uint8_t
@@ -1325,10 +1347,38 @@ ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc *rxdp)
 		   mb->vlan_tci, mb->vlan_tci_outer);
 }
 
+#ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 #define ICE_RX_PROTO_XTR_VALID \
 	((1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) | \
 	 (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
 
+static void
+ice_rxd_to_proto_xtr(struct rte_mbuf *mb,
+		     volatile struct ice_32b_rx_flex_desc_comms *desc)
+{
+	uint16_t stat_err = rte_le_to_cpu_16(desc->status_error1);
+	uint32_t metadata;
+	uint64_t ol_flag;
+
+	if (unlikely(!(stat_err & ICE_RX_PROTO_XTR_VALID)))
+		return;
+
+	ol_flag = ice_rxdid_to_proto_xtr_ol_flag(desc->rxdid);
+	if (unlikely(!ol_flag))
+		return;
+
+	mb->ol_flags |= ol_flag;
+
+	metadata = stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) ?
+				rte_le_to_cpu_16(desc->flex_ts.flex.aux0) : 0;
+
+	if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S)))
+		metadata |= rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
+
+	*RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(mb) = metadata;
+}
+#endif
+
 static inline void
 ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 		      volatile union ice_rx_flex_desc *rxdp)
@@ -1344,28 +1394,13 @@ ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 	}
 
 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
-	init_proto_xtr_flds(mb);
-
-	stat_err = rte_le_to_cpu_16(desc->status_error1);
-	if (stat_err & ICE_RX_PROTO_XTR_VALID) {
-		struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
-			xtr->u.raw.data0 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
-			xtr->u.raw.data1 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
-
-		xtr->type = ice_rxdid_to_proto_xtr_type(desc->rxdid);
-		xtr->magic = PROTO_XTR_MAGIC_ID;
-	}
-
 	if (desc->flow_id != 0xFFFFFFFF) {
 		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 	}
+
+	if (unlikely(rte_net_ice_dynf_proto_xtr_metadata_avail()))
+		ice_rxd_to_proto_xtr(mb, desc);
 #endif
 }
 
diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h
index 5de618976..9e3d2cd07 100644
--- a/drivers/net/ice/ice_rxtx.h
+++ b/drivers/net/ice/ice_rxtx.h
@@ -5,7 +5,6 @@
 #ifndef _ICE_RXTX_H_
 #define _ICE_RXTX_H_
 
-#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 
 #define ICE_ALIGN_RING_DESC  32
diff --git a/drivers/net/ice/rte_pmd_ice.h b/drivers/net/ice/rte_pmd_ice.h
index 719487e1e..1f1b75e85 100644
--- a/drivers/net/ice/rte_pmd_ice.h
+++ b/drivers/net/ice/rte_pmd_ice.h
@@ -5,144 +5,238 @@
 #ifndef _RTE_PMD_ICE_H_
 #define _RTE_PMD_ICE_H_
 
+/**
+ * @file rte_pmd_ice.h
+ *
+ * ice PMD specific functions.
+ *
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ */
+
 #include <stdio.h>
 #include <rte_mbuf.h>
-#include <rte_ethdev.h>
+#include <rte_mbuf_dyn.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-enum proto_xtr_type {
-	PROTO_XTR_NONE,
-	PROTO_XTR_VLAN,
-	PROTO_XTR_IPV4,
-	PROTO_XTR_IPV6,
-	PROTO_XTR_IPV6_FLOW,
-	PROTO_XTR_TCP,
+/**
+ * The supported network protocol extraction metadata format definition.
+ */
+union proto_xtr_metadata {
+	uint32_t metadata;
+
+	struct {
+		uint16_t data0;
+		uint16_t data1;
+	} raw;
+
+	struct {
+		uint16_t stag_vid:12,
+			 stag_dei:1,
+			 stag_pcp:3;
+		uint16_t ctag_vid:12,
+			 ctag_dei:1,
+			 ctag_pcp:3;
+	} vlan;
+
+	struct {
+		uint16_t protocol:8,
+			 ttl:8;
+		uint16_t tos:8,
+			 ihl:4,
+			 version:4;
+	} ipv4;
+
+	struct {
+		uint16_t hoplimit:8,
+			 nexthdr:8;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6;
+
+	struct {
+		uint16_t flowlo16;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6_flow;
+
+	struct {
+		uint16_t fin:1,
+			 syn:1,
+			 rst:1,
+			 psh:1,
+			 ack:1,
+			 urg:1,
+			 ece:1,
+			 cwr:1,
+			 res1:4,
+			 doff:4;
+		uint16_t rsvd;
+	} tcp;
 };
 
-struct proto_xtr_flds {
-	union {
-		struct {
-			uint16_t data0;
-			uint16_t data1;
-		} raw;
-		struct {
-			uint16_t stag_vid:12,
-				 stag_dei:1,
-				 stag_pcp:3;
-			uint16_t ctag_vid:12,
-				 ctag_dei:1,
-				 ctag_pcp:3;
-		} vlan;
-		struct {
-			uint16_t protocol:8,
-				 ttl:8;
-			uint16_t tos:8,
-				 ihl:4,
-				 version:4;
-		} ipv4;
-		struct {
-			uint16_t hoplimit:8,
-				 nexthdr:8;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6;
-		struct {
-			uint16_t flowlo16;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6_flow;
-		struct {
-			uint16_t fin:1,
-				 syn:1,
-				 rst:1,
-				 psh:1,
-				 ack:1,
-				 urg:1,
-				 ece:1,
-				 cwr:1,
-				 res1:4,
-				 doff:4;
-			uint16_t rsvd;
-		} tcp;
-	} u;
-
-	uint16_t rsvd;
-
-	uint8_t type;
-
-#define PROTO_XTR_MAGIC_ID	0xCE
-	uint8_t magic;
-};
+/**
+ * The offset of mbuf dynamic field for protocol extraction metadata, it is
+ * registered when dev_args 'proto_xtr' is set.
+ */
+extern int rte_net_ice_dynfield_proto_xtr_metadata;
 
-static inline void
-init_proto_xtr_flds(struct rte_mbuf *mb)
+/**
+ * The mask of mbuf dynamic flags for VLAN protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_vlan;
+
+/**
+ * The mask of mbuf dynamic flags for IPv4 protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv4;
+
+/**
+ * The mask of mbuf dynamic flags for IPv6 protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv6;
+
+/**
+ * The mask of mbuf dynamic flags for IPv6 with flow protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow;
+
+/**
+ * The mask of mbuf dynamic flags for TCP protocol extraction type.
+ */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_tcp;
+
+/**
+ * The mbuf dynamic field pointer for protocol extraction metadata.
+ */
+#define RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m) \
+	RTE_MBUF_DYNFIELD((m), \
+			  rte_net_ice_dynfield_proto_xtr_metadata, uint32_t *)
+
+/**
+ * The mbuf dynamic flag for VLAN protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_VLAN \
+	(rte_net_ice_dynflag_proto_xtr_vlan)
+
+/**
+ * The mbuf dynamic flag for IPv4 protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV4 \
+	(rte_net_ice_dynflag_proto_xtr_ipv4)
+
+/**
+ * The mbuf dynamic flag for IPv6 protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV6 \
+	(rte_net_ice_dynflag_proto_xtr_ipv6)
+
+/**
+ * The mbuf dynamic flag for IPv6 with flow protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW \
+	(rte_net_ice_dynflag_proto_xtr_ipv6_flow)
+
+/**
+ * The mbuf dynamic flag for TCP protocol extraction metadata.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_TCP \
+	(rte_net_ice_dynflag_proto_xtr_tcp)
+
+/**
+ * Check if mbuf dynamic field for protocol extraction metadata is registered.
+ *
+ * @return
+ *   True if registered, false otherwise.
+ */
+__rte_experimental
+static __rte_always_inline int
+rte_net_ice_dynf_proto_xtr_metadata_avail(void)
 {
-	mb->udata64 = 0;
+	return rte_net_ice_dynfield_proto_xtr_metadata != -1;
 }
 
-static inline struct proto_xtr_flds *
-get_proto_xtr_flds(struct rte_mbuf *mb)
+/**
+ * Get the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ * @return
+ *   The saved protocol extraction metadata.
+ */
+__rte_experimental
+static __rte_always_inline uint32_t
+rte_net_ice_dynf_proto_xtr_metadata_get(struct rte_mbuf *m)
 {
-	RTE_BUILD_BUG_ON(sizeof(struct proto_xtr_flds) > sizeof(mb->udata64));
-
-	return (struct proto_xtr_flds *)&mb->udata64;
+	return *RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m);
 }
 
+/**
+ * Dump the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ */
+__rte_experimental
 static inline void
-dump_proto_xtr_flds(struct rte_mbuf *mb)
+rte_net_ice_dump_proto_xtr_metadata(struct rte_mbuf *m)
 {
-	struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
+	union proto_xtr_metadata data;
 
-	if (xtr->magic != PROTO_XTR_MAGIC_ID || xtr->type == PROTO_XTR_NONE)
+	if (!rte_net_ice_dynf_proto_xtr_metadata_avail())
 		return;
 
-	printf(" - Protocol Extraction:[0x%04x:0x%04x],",
-	       xtr->u.raw.data0, xtr->u.raw.data1);
-
-	if (xtr->type == PROTO_XTR_VLAN)
-		printf("vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
-		       xtr->u.vlan.stag_pcp,
-		       xtr->u.vlan.stag_dei,
-		       xtr->u.vlan.stag_vid,
-		       xtr->u.vlan.ctag_pcp,
-		       xtr->u.vlan.ctag_dei,
-		       xtr->u.vlan.ctag_vid);
-	else if (xtr->type == PROTO_XTR_IPV4)
-		printf("ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
-		       xtr->u.ipv4.version,
-		       xtr->u.ipv4.ihl,
-		       xtr->u.ipv4.tos,
-		       xtr->u.ipv4.ttl,
-		       xtr->u.ipv4.protocol);
-	else if (xtr->type == PROTO_XTR_IPV6)
-		printf("ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
-		       xtr->u.ipv6.version,
-		       xtr->u.ipv6.tc,
-		       xtr->u.ipv6.flowhi4,
-		       xtr->u.ipv6.nexthdr,
-		       xtr->u.ipv6.hoplimit);
-	else if (xtr->type == PROTO_XTR_IPV6_FLOW)
-		printf("ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
-		       xtr->u.ipv6_flow.version,
-		       xtr->u.ipv6_flow.tc,
-		       xtr->u.ipv6_flow.flowhi4,
-		       xtr->u.ipv6_flow.flowlo16);
-	else if (xtr->type == PROTO_XTR_TCP)
-		printf("tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
-		       xtr->u.tcp.doff,
-		       xtr->u.tcp.cwr ? "C" : "",
-		       xtr->u.tcp.ece ? "E" : "",
-		       xtr->u.tcp.urg ? "U" : "",
-		       xtr->u.tcp.ack ? "A" : "",
-		       xtr->u.tcp.psh ? "P" : "",
-		       xtr->u.tcp.rst ? "R" : "",
-		       xtr->u.tcp.syn ? "S" : "",
-		       xtr->u.tcp.fin ? "F" : "");
+	data.metadata = rte_net_ice_dynf_proto_xtr_metadata_get(m);
+
+	if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_VLAN)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],vlan,stag=%u:%u:%u,ctag=%u:%u:%u",
+		       data.raw.data0, data.raw.data1,
+		       data.vlan.stag_pcp,
+		       data.vlan.stag_dei,
+		       data.vlan.stag_vid,
+		       data.vlan.ctag_pcp,
+		       data.vlan.ctag_dei,
+		       data.vlan.ctag_vid);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV4)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv4.version,
+		       data.ipv4.ihl,
+		       data.ipv4.tos,
+		       data.ipv4.ttl,
+		       data.ipv4.protocol);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV6)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6.version,
+		       data.ipv6.tc,
+		       data.ipv6.flowhi4,
+		       data.ipv6.nexthdr,
+		       data.ipv6.hoplimit);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6_flow.version,
+		       data.ipv6_flow.tc,
+		       data.ipv6_flow.flowhi4,
+		       data.ipv6_flow.flowlo16);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_TCP)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],tcp,doff=%u,flags=%s%s%s%s%s%s%s%s",
+		       data.raw.data0, data.raw.data1,
+		       data.tcp.doff,
+		       data.tcp.cwr ? "C" : "",
+		       data.tcp.ece ? "E" : "",
+		       data.tcp.urg ? "U" : "",
+		       data.tcp.ack ? "A" : "",
+		       data.tcp.psh ? "P" : "",
+		       data.tcp.rst ? "R" : "",
+		       data.tcp.syn ? "S" : "",
+		       data.tcp.fin ? "F" : "");
 }
 
 #ifdef __cplusplus
diff --git a/drivers/net/ice/rte_pmd_ice_version.map b/drivers/net/ice/rte_pmd_ice_version.map
index 7b23b609d..7e70974d5 100644
--- a/drivers/net/ice/rte_pmd_ice_version.map
+++ b/drivers/net/ice/rte_pmd_ice_version.map
@@ -2,3 +2,15 @@ DPDK_19.02 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	# added in 19.11
+	rte_net_ice_dynfield_proto_xtr_metadata;
+	rte_net_ice_dynflag_proto_xtr_vlan;
+	rte_net_ice_dynflag_proto_xtr_ipv4;
+	rte_net_ice_dynflag_proto_xtr_ipv6;
+	rte_net_ice_dynflag_proto_xtr_ipv6_flow;
+	rte_net_ice_dynflag_proto_xtr_tcp;
+};
-- 
2.17.1

[dpdk-dev] [PATCH v8] net/ice: optimize protocol extraction by dynamic mbuf API

[Haiyue Wang]

The original design is to use rte_mbuf::udata64 to save the metadata of
protocol extraction which has network protocol data fields and type, a
private API is used to decode this metadata.

Use the dynamic mbuf field and flags to register the needed fields in
mbuf, to avoid overwriting 'rte_mbuf::udata64' if the application uses
it. It only needs 4B size to save the protocol extraction data, and its
type and validity is indicated by related bit in 'rte_mbuf::ol_flags'.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
v8: - Cleanup the doxgen, rename the dynamic filed and flags with ice prefix.

v7: - Change the log level from ERR to DEBUG for a successful registration,
      ERR is used to develop firstly.

v6: - Don't need to register all flags when if one dev_args is set, register
      it as required.

v5: - Remove the '_OL/_ol' in dynamic mbuf flag to make the variable clean.

v4: - Include the 'rte_pmd_ice.h' header in ICE source as needed, reduce
      its compile scope.

v3: - Use the conventions name "rte_net_<pmd>_dynfield_<name>" since it
      is defined in a in PMD.
    - Add helpers API for easiy access.

v2: - disable the protocol extraction if failed to register some ol_flags
    - rewrite the commit message

 doc/api/doxy-api-index.md               |   1 +
 doc/api/doxy-api.conf.in                |   1 +
 doc/guides/nics/ice.rst                 |  14 +-
 drivers/net/ice/ice_ethdev.c            |  84 ++++++-
 drivers/net/ice/ice_ethdev.h            |   9 +
 drivers/net/ice/ice_rxtx.c              |  88 ++++---
 drivers/net/ice/ice_rxtx.h              |   1 -
 drivers/net/ice/rte_pmd_ice.h           | 312 +++++++++++++++---------
 drivers/net/ice/rte_pmd_ice_version.map |  12 +
 9 files changed, 369 insertions(+), 153 deletions(-)

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index 28a5dd37e..dff496be0 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -42,6 +42,7 @@ The public API headers are grouped by topics:
   [KNI]                (@ref rte_kni.h),
   [ixgbe]              (@ref rte_pmd_ixgbe.h),
   [i40e]               (@ref rte_pmd_i40e.h),
+  [ice]                (@ref rte_pmd_ice.h),
   [bnxt]               (@ref rte_pmd_bnxt.h),
   [dpaa]               (@ref rte_pmd_dpaa.h),
   [dpaa2]              (@ref rte_pmd_dpaa2.h),
diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
index cb67eb728..350b442a1 100644
--- a/doc/api/doxy-api.conf.in
+++ b/doc/api/doxy-api.conf.in
@@ -12,6 +12,7 @@ INPUT                   = @TOPDIR@/doc/api/doxy-api-index.md \
                           @TOPDIR@/drivers/net/dpaa \
                           @TOPDIR@/drivers/net/dpaa2 \
                           @TOPDIR@/drivers/net/i40e \
+                          @TOPDIR@/drivers/net/ice \
                           @TOPDIR@/drivers/net/ixgbe \
                           @TOPDIR@/drivers/net/softnic \
                           @TOPDIR@/drivers/raw/dpaa2_cmdif \
diff --git a/doc/guides/nics/ice.rst b/doc/guides/nics/ice.rst
index 933f63480..04af80545 100644
--- a/doc/guides/nics/ice.rst
+++ b/doc/guides/nics/ice.rst
@@ -82,8 +82,8 @@ Runtime Config Options
 
 - ``Protocol extraction for per queue``
 
-  Configure the RX queues to do protocol extraction into ``rte_mbuf::udata64``
-  for protocol handling acceleration, like checking the TCP SYN packets quickly.
+  Configure the RX queues to do protocol extraction into mbuf for protocol
+  handling acceleration, like checking the TCP SYN packets quickly.
 
   The argument format is::
 
@@ -111,7 +111,8 @@ Runtime Config Options
   This setting means queues 1, 2-3, 8-9 are TCP extraction, queues 10-23 are
   IPv6 extraction, other queues use the default VLAN extraction.
 
-  The extraction will be copied into the lower 32 bit of ``rte_mbuf::udata64``.
+  The extraction metadata will be copied into the registered dynamic mbuf field
+  with, and the related dynamic mbuf flags is set.
 
   .. table:: Protocol extraction : ``vlan``
 
@@ -175,10 +176,11 @@ Runtime Config Options
 
   TCPHDR2 - Reserved
 
-  Use ``get_proto_xtr_flds(struct rte_mbuf *mb)`` to access the protocol
-  extraction, do not use ``rte_mbuf::udata64`` directly.
+  Use ``rte_net_ice_dynf_proto_xtr_metadata_get`` to access the protocol
+  extraction metadata, and use ``PKT_RX_DYNF_PROTO_XTR_*`` to get the
+  metadata type of ``struct rte_mbuf::ol_flags``.
 
-  The ``dump_proto_xtr_flds(struct rte_mbuf *mb)`` routine shows how to
+  The ``rte_net_ice_dump_proto_xtr_metadata`` routine shows how to
   access the protocol extraction result in ``struct rte_mbuf``.
 
 Driver compilation and testing
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index 9d2210a45..3596ce5a0 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -14,6 +14,8 @@
 #include "base/ice_flow.h"
 #include "base/ice_dcb.h"
 #include "base/ice_common.h"
+
+#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 #include "ice_rxtx.h"
 #include "ice_generic_flow.h"
@@ -30,6 +32,37 @@ static const char * const ice_valid_args[] = {
 	NULL
 };
 
+static const struct rte_mbuf_dynfield ice_proto_xtr_metadata_param = {
+	.name = "ice_dynfield_proto_xtr_metadata",
+	.size = sizeof(uint32_t),
+	.align = __alignof__(uint32_t),
+	.flags = 0,
+};
+
+struct proto_xtr_ol_flag {
+	const struct rte_mbuf_dynflag param;
+	uint64_t *ol_flag;
+	bool required;
+};
+
+static struct proto_xtr_ol_flag ice_proto_xtr_ol_flag_params[] = {
+	[PROTO_XTR_VLAN] = {
+		.param = { .name = "ice_dynflag_proto_xtr_vlan" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_vlan },
+	[PROTO_XTR_IPV4] = {
+		.param = { .name = "ice_dynflag_proto_xtr_ipv4" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_ipv4 },
+	[PROTO_XTR_IPV6] = {
+		.param = { .name = "ice_dynflag_proto_xtr_ipv6" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_ipv6 },
+	[PROTO_XTR_IPV6_FLOW] = {
+		.param = { .name = "ice_dynflag_proto_xtr_ipv6_flow" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_ipv6_flow },
+	[PROTO_XTR_TCP] = {
+		.param = { .name = "ice_dynflag_proto_xtr_tcp" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_tcp },
+};
+
 #define ICE_DFLT_OUTER_TAG_TYPE ICE_AQ_VSI_OUTER_TAG_VLAN_9100
 
 /* DDP package search path */
@@ -1385,6 +1418,9 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 			ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 	struct ice_hw *hw = ICE_PF_TO_HW(pf);
+	const struct proto_xtr_ol_flag *ol_flag;
+	bool proto_xtr_enable = false;
+	int offset;
 	uint16_t i;
 
 	if (!ice_proto_xtr_support(hw)) {
@@ -1398,10 +1434,56 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 		return;
 	}
 
-	for (i = 0; i < pf->lan_nb_qps; i++)
+	for (i = 0; i < pf->lan_nb_qps; i++) {
 		pf->proto_xtr[i] = ad->devargs.proto_xtr[i] != PROTO_XTR_NONE ?
 				   ad->devargs.proto_xtr[i] :
 				   ad->devargs.proto_xtr_dflt;
+
+		if (pf->proto_xtr[i] != PROTO_XTR_NONE) {
+			uint8_t type = pf->proto_xtr[i];
+
+			ice_proto_xtr_ol_flag_params[type].required = true;
+			proto_xtr_enable = true;
+		}
+	}
+
+	if (likely(!proto_xtr_enable))
+		return;
+
+	offset = rte_mbuf_dynfield_register(&ice_proto_xtr_metadata_param);
+	if (unlikely(offset == -1)) {
+		PMD_DRV_LOG(ERR,
+			    "Protocol extraction metadata is disabled in mbuf with error %d",
+			    -rte_errno);
+		return;
+	}
+
+	PMD_DRV_LOG(DEBUG,
+		    "Protocol extraction metadata offset in mbuf is : %d",
+		    offset);
+	rte_net_ice_dynfield_proto_xtr_metadata = offset;
+
+	for (i = 0; i < RTE_DIM(ice_proto_xtr_ol_flag_params); i++) {
+		ol_flag = &ice_proto_xtr_ol_flag_params[i];
+
+		if (!ol_flag->required)
+			continue;
+
+		offset = rte_mbuf_dynflag_register(&ol_flag->param);
+		if (unlikely(offset == -1)) {
+			PMD_DRV_LOG(ERR,
+				    "Protocol extraction offload '%s' failed to register with error %d",
+				    ol_flag->param.name, -rte_errno);
+
+			rte_net_ice_dynfield_proto_xtr_metadata = -1;
+			break;
+		}
+
+		PMD_DRV_LOG(DEBUG,
+			    "Protocol extraction offload '%s' offset in mbuf is : %d",
+			    ol_flag->param.name, offset);
+		*ol_flag->ol_flag = 1ULL << offset;
+	}
 }
 
 /*  Initialize SW parameters of PF */
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index de67e5934..c55713cc1 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -241,6 +241,15 @@ struct ice_vsi {
 	bool offset_loaded;
 };
 
+enum proto_xtr_type {
+	PROTO_XTR_NONE,
+	PROTO_XTR_VLAN,
+	PROTO_XTR_IPV4,
+	PROTO_XTR_IPV6,
+	PROTO_XTR_IPV6_FLOW,
+	PROTO_XTR_TCP,
+};
+
 enum ice_fdir_tunnel_type {
 	ICE_FDIR_TUNNEL_TYPE_NONE = 0,
 	ICE_FDIR_TUNNEL_TYPE_VXLAN,
diff --git a/drivers/net/ice/ice_rxtx.c b/drivers/net/ice/ice_rxtx.c
index 53e4eea7b..0fb5628a9 100644
--- a/drivers/net/ice/ice_rxtx.c
+++ b/drivers/net/ice/ice_rxtx.c
@@ -5,6 +5,7 @@
 #include <rte_ethdev_driver.h>
 #include <rte_net.h>
 
+#include "rte_pmd_ice.h"
 #include "ice_rxtx.h"
 
 #define ICE_TX_CKSUM_OFFLOAD_MASK (		 \
@@ -13,18 +14,36 @@
 		PKT_TX_TCP_SEG |		 \
 		PKT_TX_OUTER_IP_CKSUM)
 
-static inline uint8_t
-ice_rxdid_to_proto_xtr_type(uint8_t rxdid)
-{
-	static uint8_t xtr_map[] = {
-		[ICE_RXDID_COMMS_AUX_VLAN]      = PROTO_XTR_VLAN,
-		[ICE_RXDID_COMMS_AUX_IPV4]      = PROTO_XTR_IPV4,
-		[ICE_RXDID_COMMS_AUX_IPV6]      = PROTO_XTR_IPV6,
-		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = PROTO_XTR_IPV6_FLOW,
-		[ICE_RXDID_COMMS_AUX_TCP]       = PROTO_XTR_TCP,
+/* Offset of mbuf dynamic field for protocol extraction data */
+int rte_net_ice_dynfield_proto_xtr_metadata = -1;
+
+/* Mask of mbuf dynamic flags for protocol extraction type */
+uint64_t rte_net_ice_dynflag_proto_xtr_vlan;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv4;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv6;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow;
+uint64_t rte_net_ice_dynflag_proto_xtr_tcp;
+
+static inline uint64_t
+ice_rxdid_to_proto_xtr_ol_flag(uint8_t rxdid)
+{
+	static uint64_t *ol_flag_map[] = {
+		[ICE_RXDID_COMMS_AUX_VLAN] =
+				&rte_net_ice_dynflag_proto_xtr_vlan,
+		[ICE_RXDID_COMMS_AUX_IPV4] =
+				&rte_net_ice_dynflag_proto_xtr_ipv4,
+		[ICE_RXDID_COMMS_AUX_IPV6] =
+				&rte_net_ice_dynflag_proto_xtr_ipv6,
+		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] =
+				&rte_net_ice_dynflag_proto_xtr_ipv6_flow,
+		[ICE_RXDID_COMMS_AUX_TCP] =
+				&rte_net_ice_dynflag_proto_xtr_tcp,
 	};
+	uint64_t *ol_flag;
 
-	return rxdid < RTE_DIM(xtr_map) ? xtr_map[rxdid] : PROTO_XTR_NONE;
+	ol_flag = rxdid < RTE_DIM(ol_flag_map) ? ol_flag_map[rxdid] : NULL;
+
+	return ol_flag != NULL ? *ol_flag : 0ULL;
 }
 
 static inline uint8_t
@@ -1325,10 +1344,38 @@ ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc *rxdp)
 		   mb->vlan_tci, mb->vlan_tci_outer);
 }
 
+#ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 #define ICE_RX_PROTO_XTR_VALID \
 	((1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) | \
 	 (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
 
+static void
+ice_rxd_to_proto_xtr(struct rte_mbuf *mb,
+		     volatile struct ice_32b_rx_flex_desc_comms *desc)
+{
+	uint16_t stat_err = rte_le_to_cpu_16(desc->status_error1);
+	uint32_t metadata;
+	uint64_t ol_flag;
+
+	if (unlikely(!(stat_err & ICE_RX_PROTO_XTR_VALID)))
+		return;
+
+	ol_flag = ice_rxdid_to_proto_xtr_ol_flag(desc->rxdid);
+	if (unlikely(!ol_flag))
+		return;
+
+	mb->ol_flags |= ol_flag;
+
+	metadata = stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) ?
+				rte_le_to_cpu_16(desc->flex_ts.flex.aux0) : 0;
+
+	if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S)))
+		metadata |= rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
+
+	*RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(mb) = metadata;
+}
+#endif
+
 static inline void
 ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 		      volatile union ice_rx_flex_desc *rxdp)
@@ -1344,28 +1391,13 @@ ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 	}
 
 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
-	init_proto_xtr_flds(mb);
-
-	stat_err = rte_le_to_cpu_16(desc->status_error1);
-	if (stat_err & ICE_RX_PROTO_XTR_VALID) {
-		struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
-			xtr->u.raw.data0 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
-			xtr->u.raw.data1 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
-
-		xtr->type = ice_rxdid_to_proto_xtr_type(desc->rxdid);
-		xtr->magic = PROTO_XTR_MAGIC_ID;
-	}
-
 	if (desc->flow_id != 0xFFFFFFFF) {
 		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 	}
+
+	if (unlikely(rte_net_ice_dynf_proto_xtr_metadata_avail()))
+		ice_rxd_to_proto_xtr(mb, desc);
 #endif
 }
 
diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h
index 5de618976..9e3d2cd07 100644
--- a/drivers/net/ice/ice_rxtx.h
+++ b/drivers/net/ice/ice_rxtx.h
@@ -5,7 +5,6 @@
 #ifndef _ICE_RXTX_H_
 #define _ICE_RXTX_H_
 
-#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 
 #define ICE_ALIGN_RING_DESC  32
diff --git a/drivers/net/ice/rte_pmd_ice.h b/drivers/net/ice/rte_pmd_ice.h
index 719487e1e..8ede6eb02 100644
--- a/drivers/net/ice/rte_pmd_ice.h
+++ b/drivers/net/ice/rte_pmd_ice.h
@@ -5,144 +5,222 @@
 #ifndef _RTE_PMD_ICE_H_
 #define _RTE_PMD_ICE_H_
 
+/**
+ * @file rte_pmd_ice.h
+ *
+ * ice PMD specific functions.
+ *
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ */
+
 #include <stdio.h>
 #include <rte_mbuf.h>
-#include <rte_ethdev.h>
+#include <rte_mbuf_dyn.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-enum proto_xtr_type {
-	PROTO_XTR_NONE,
-	PROTO_XTR_VLAN,
-	PROTO_XTR_IPV4,
-	PROTO_XTR_IPV6,
-	PROTO_XTR_IPV6_FLOW,
-	PROTO_XTR_TCP,
+/**
+ * The supported network protocol extraction metadata format.
+ */
+union proto_xtr_metadata {
+	uint32_t metadata;
+
+	struct {
+		uint16_t data0;
+		uint16_t data1;
+	} raw;
+
+	struct {
+		uint16_t stag_vid:12,
+			 stag_dei:1,
+			 stag_pcp:3;
+		uint16_t ctag_vid:12,
+			 ctag_dei:1,
+			 ctag_pcp:3;
+	} vlan;
+
+	struct {
+		uint16_t protocol:8,
+			 ttl:8;
+		uint16_t tos:8,
+			 ihl:4,
+			 version:4;
+	} ipv4;
+
+	struct {
+		uint16_t hoplimit:8,
+			 nexthdr:8;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6;
+
+	struct {
+		uint16_t flowlo16;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6_flow;
+
+	struct {
+		uint16_t fin:1,
+			 syn:1,
+			 rst:1,
+			 psh:1,
+			 ack:1,
+			 urg:1,
+			 ece:1,
+			 cwr:1,
+			 res1:4,
+			 doff:4;
+		uint16_t rsvd;
+	} tcp;
 };
 
-struct proto_xtr_flds {
-	union {
-		struct {
-			uint16_t data0;
-			uint16_t data1;
-		} raw;
-		struct {
-			uint16_t stag_vid:12,
-				 stag_dei:1,
-				 stag_pcp:3;
-			uint16_t ctag_vid:12,
-				 ctag_dei:1,
-				 ctag_pcp:3;
-		} vlan;
-		struct {
-			uint16_t protocol:8,
-				 ttl:8;
-			uint16_t tos:8,
-				 ihl:4,
-				 version:4;
-		} ipv4;
-		struct {
-			uint16_t hoplimit:8,
-				 nexthdr:8;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6;
-		struct {
-			uint16_t flowlo16;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6_flow;
-		struct {
-			uint16_t fin:1,
-				 syn:1,
-				 rst:1,
-				 psh:1,
-				 ack:1,
-				 urg:1,
-				 ece:1,
-				 cwr:1,
-				 res1:4,
-				 doff:4;
-			uint16_t rsvd;
-		} tcp;
-	} u;
-
-	uint16_t rsvd;
-
-	uint8_t type;
-
-#define PROTO_XTR_MAGIC_ID	0xCE
-	uint8_t magic;
-};
+/* Offset of mbuf dynamic field for protocol extraction data */
+extern int rte_net_ice_dynfield_proto_xtr_metadata;
 
-static inline void
-init_proto_xtr_flds(struct rte_mbuf *mb)
+/* Mask of mbuf dynamic flags for protocol extraction type */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_vlan;
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv4;
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv6;
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow;
+extern uint64_t rte_net_ice_dynflag_proto_xtr_tcp;
+
+/**
+ * The mbuf dynamic field pointer for protocol extraction metadata.
+ */
+#define RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m) \
+	RTE_MBUF_DYNFIELD((m), \
+			  rte_net_ice_dynfield_proto_xtr_metadata, uint32_t *)
+
+/**
+ * The mbuf dynamic flag for VLAN protocol extraction metadata, it is valid
+ * when dev_args 'proto_xtr' has 'vlan' specified.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_VLAN \
+	(rte_net_ice_dynflag_proto_xtr_vlan)
+
+/**
+ * The mbuf dynamic flag for IPv4 protocol extraction metadata, it is valid
+ * when dev_args 'proto_xtr' has 'ipv4' specified.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV4 \
+	(rte_net_ice_dynflag_proto_xtr_ipv4)
+
+/**
+ * The mbuf dynamic flag for IPv6 protocol extraction metadata, it is valid
+ * when dev_args 'proto_xtr' has 'ipv6' specified.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV6 \
+	(rte_net_ice_dynflag_proto_xtr_ipv6)
+
+/**
+ * The mbuf dynamic flag for IPv6 with flow protocol extraction metadata, it is
+ * valid when dev_args 'proto_xtr' has 'ipv6_flow' specified.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW \
+	(rte_net_ice_dynflag_proto_xtr_ipv6_flow)
+
+/**
+ * The mbuf dynamic flag for TCP protocol extraction metadata, it is valid
+ * when dev_args 'proto_xtr' has 'tcp' specified.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_TCP \
+	(rte_net_ice_dynflag_proto_xtr_tcp)
+
+/**
+ * Check if mbuf dynamic field for protocol extraction metadata is registered.
+ *
+ * @return
+ *   True if registered, false otherwise.
+ */
+__rte_experimental
+static __rte_always_inline int
+rte_net_ice_dynf_proto_xtr_metadata_avail(void)
 {
-	mb->udata64 = 0;
+	return rte_net_ice_dynfield_proto_xtr_metadata != -1;
 }
 
-static inline struct proto_xtr_flds *
-get_proto_xtr_flds(struct rte_mbuf *mb)
+/**
+ * Get the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ * @return
+ *   The saved protocol extraction metadata.
+ */
+__rte_experimental
+static __rte_always_inline uint32_t
+rte_net_ice_dynf_proto_xtr_metadata_get(struct rte_mbuf *m)
 {
-	RTE_BUILD_BUG_ON(sizeof(struct proto_xtr_flds) > sizeof(mb->udata64));
-
-	return (struct proto_xtr_flds *)&mb->udata64;
+	return *RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m);
 }
 
+/**
+ * Dump the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ */
+__rte_experimental
 static inline void
-dump_proto_xtr_flds(struct rte_mbuf *mb)
+rte_net_ice_dump_proto_xtr_metadata(struct rte_mbuf *m)
 {
-	struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
+	union proto_xtr_metadata data;
 
-	if (xtr->magic != PROTO_XTR_MAGIC_ID || xtr->type == PROTO_XTR_NONE)
+	if (!rte_net_ice_dynf_proto_xtr_metadata_avail())
 		return;
 
-	printf(" - Protocol Extraction:[0x%04x:0x%04x],",
-	       xtr->u.raw.data0, xtr->u.raw.data1);
-
-	if (xtr->type == PROTO_XTR_VLAN)
-		printf("vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
-		       xtr->u.vlan.stag_pcp,
-		       xtr->u.vlan.stag_dei,
-		       xtr->u.vlan.stag_vid,
-		       xtr->u.vlan.ctag_pcp,
-		       xtr->u.vlan.ctag_dei,
-		       xtr->u.vlan.ctag_vid);
-	else if (xtr->type == PROTO_XTR_IPV4)
-		printf("ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
-		       xtr->u.ipv4.version,
-		       xtr->u.ipv4.ihl,
-		       xtr->u.ipv4.tos,
-		       xtr->u.ipv4.ttl,
-		       xtr->u.ipv4.protocol);
-	else if (xtr->type == PROTO_XTR_IPV6)
-		printf("ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
-		       xtr->u.ipv6.version,
-		       xtr->u.ipv6.tc,
-		       xtr->u.ipv6.flowhi4,
-		       xtr->u.ipv6.nexthdr,
-		       xtr->u.ipv6.hoplimit);
-	else if (xtr->type == PROTO_XTR_IPV6_FLOW)
-		printf("ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
-		       xtr->u.ipv6_flow.version,
-		       xtr->u.ipv6_flow.tc,
-		       xtr->u.ipv6_flow.flowhi4,
-		       xtr->u.ipv6_flow.flowlo16);
-	else if (xtr->type == PROTO_XTR_TCP)
-		printf("tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
-		       xtr->u.tcp.doff,
-		       xtr->u.tcp.cwr ? "C" : "",
-		       xtr->u.tcp.ece ? "E" : "",
-		       xtr->u.tcp.urg ? "U" : "",
-		       xtr->u.tcp.ack ? "A" : "",
-		       xtr->u.tcp.psh ? "P" : "",
-		       xtr->u.tcp.rst ? "R" : "",
-		       xtr->u.tcp.syn ? "S" : "",
-		       xtr->u.tcp.fin ? "F" : "");
+	data.metadata = rte_net_ice_dynf_proto_xtr_metadata_get(m);
+
+	if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_VLAN)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],vlan,stag=%u:%u:%u,ctag=%u:%u:%u",
+		       data.raw.data0, data.raw.data1,
+		       data.vlan.stag_pcp,
+		       data.vlan.stag_dei,
+		       data.vlan.stag_vid,
+		       data.vlan.ctag_pcp,
+		       data.vlan.ctag_dei,
+		       data.vlan.ctag_vid);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV4)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv4.version,
+		       data.ipv4.ihl,
+		       data.ipv4.tos,
+		       data.ipv4.ttl,
+		       data.ipv4.protocol);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV6)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6.version,
+		       data.ipv6.tc,
+		       data.ipv6.flowhi4,
+		       data.ipv6.nexthdr,
+		       data.ipv6.hoplimit);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6_flow.version,
+		       data.ipv6_flow.tc,
+		       data.ipv6_flow.flowhi4,
+		       data.ipv6_flow.flowlo16);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_TCP)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],tcp,doff=%u,flags=%s%s%s%s%s%s%s%s",
+		       data.raw.data0, data.raw.data1,
+		       data.tcp.doff,
+		       data.tcp.cwr ? "C" : "",
+		       data.tcp.ece ? "E" : "",
+		       data.tcp.urg ? "U" : "",
+		       data.tcp.ack ? "A" : "",
+		       data.tcp.psh ? "P" : "",
+		       data.tcp.rst ? "R" : "",
+		       data.tcp.syn ? "S" : "",
+		       data.tcp.fin ? "F" : "");
 }
 
 #ifdef __cplusplus
diff --git a/drivers/net/ice/rte_pmd_ice_version.map b/drivers/net/ice/rte_pmd_ice_version.map
index 7b23b609d..7e70974d5 100644
--- a/drivers/net/ice/rte_pmd_ice_version.map
+++ b/drivers/net/ice/rte_pmd_ice_version.map
@@ -2,3 +2,15 @@ DPDK_19.02 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	# added in 19.11
+	rte_net_ice_dynfield_proto_xtr_metadata;
+	rte_net_ice_dynflag_proto_xtr_vlan;
+	rte_net_ice_dynflag_proto_xtr_ipv4;
+	rte_net_ice_dynflag_proto_xtr_ipv6;
+	rte_net_ice_dynflag_proto_xtr_ipv6_flow;
+	rte_net_ice_dynflag_proto_xtr_tcp;
+};
-- 
2.17.1

Re: [dpdk-dev] [PATCH v8] net/ice: optimize protocol extraction by dynamic mbuf API

[Ye Xiaolong]

Hi, Haiyue

On 11/07, Haiyue Wang wrote:
>The original design is to use rte_mbuf::udata64 to save the metadata of
>protocol extraction which has network protocol data fields and type, a
>private API is used to decode this metadata.
>
>Use the dynamic mbuf field and flags to register the needed fields in
>mbuf, to avoid overwriting 'rte_mbuf::udata64' if the application uses
>it. It only needs 4B size to save the protocol extraction data, and its
>type and validity is indicated by related bit in 'rte_mbuf::ol_flags'.
>
>Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
>---
>v8: - Cleanup the doxgen, rename the dynamic filed and flags with ice prefix.
>
>v7: - Change the log level from ERR to DEBUG for a successful registration,
>      ERR is used to develop firstly.
>
>v6: - Don't need to register all flags when if one dev_args is set, register
>      it as required.
>
>v5: - Remove the '_OL/_ol' in dynamic mbuf flag to make the variable clean.
>
>v4: - Include the 'rte_pmd_ice.h' header in ICE source as needed, reduce
>      its compile scope.
>
>v3: - Use the conventions name "rte_net_<pmd>_dynfield_<name>" since it
>      is defined in a in PMD.
>    - Add helpers API for easiy access.
>
>v2: - disable the protocol extraction if failed to register some ol_flags
>    - rewrite the commit message
>
> doc/api/doxy-api-index.md               |   1 +
> doc/api/doxy-api.conf.in                |   1 +
> doc/guides/nics/ice.rst                 |  14 +-
> drivers/net/ice/ice_ethdev.c            |  84 ++++++-
> drivers/net/ice/ice_ethdev.h            |   9 +
> drivers/net/ice/ice_rxtx.c              |  88 ++++---
> drivers/net/ice/ice_rxtx.h              |   1 -
> drivers/net/ice/rte_pmd_ice.h           | 312 +++++++++++++++---------
> drivers/net/ice/rte_pmd_ice_version.map |  12 +
> 9 files changed, 369 insertions(+), 153 deletions(-)
>

[snip]

> 
>-  The extraction will be copied into the lower 32 bit of ``rte_mbuf::udata64``.
>+  The extraction metadata will be copied into the registered dynamic mbuf field
>+  with, and the related dynamic mbuf flags is set.

Above 'with' is redundant?

> 
>   .. table:: Protocol extraction : ``vlan``
> 
>@@ -175,10 +176,11 @@ Runtime Config Options
> 
>   TCPHDR2 - Reserved
> 
>-  Use ``get_proto_xtr_flds(struct rte_mbuf *mb)`` to access the protocol
>-  extraction, do not use ``rte_mbuf::udata64`` directly.
>+  Use ``rte_net_ice_dynf_proto_xtr_metadata_get`` to access the protocol
>+  extraction metadata, and use ``PKT_RX_DYNF_PROTO_XTR_*`` to get the
>+  metadata type of ``struct rte_mbuf::ol_flags``.
> 
>-  The ``dump_proto_xtr_flds(struct rte_mbuf *mb)`` routine shows how to
>+  The ``rte_net_ice_dump_proto_xtr_metadata`` routine shows how to
>   access the protocol extraction result in ``struct rte_mbuf``.
> 

[snip]

>+
>+	PMD_DRV_LOG(DEBUG,
>+		    "Protocol extraction metadata offset in mbuf is : %d",
>+		    offset);
>+	rte_net_ice_dynfield_proto_xtr_metadata = offset;

Seems rte_net_ice_dump_proto_xtr_metadata_off is a better name judging from
its real meaning.

>+
>+	for (i = 0; i < RTE_DIM(ice_proto_xtr_ol_flag_params); i++) {
>+		ol_flag = &ice_proto_xtr_ol_flag_params[i];
>+
>+		if (!ol_flag->required)

[snip]

>+static void
>+ice_rxd_to_proto_xtr(struct rte_mbuf *mb,
>+		     volatile struct ice_32b_rx_flex_desc_comms *desc)
>+{
>+	uint16_t stat_err = rte_le_to_cpu_16(desc->status_error1);
>+	uint32_t metadata;
>+	uint64_t ol_flag;
>+
>+	if (unlikely(!(stat_err & ICE_RX_PROTO_XTR_VALID)))
>+		return;
>+
>+	ol_flag = ice_rxdid_to_proto_xtr_ol_flag(desc->rxdid);

ol_flag here is obtained through offset which is returned by
rte_mbuf_dynflag_register, will it have any chance to conflict with
existing offload flags such as PKT_RX_VLAN, PKT_RX_VLAN_STRIPPED, ...? 

>+	if (unlikely(!ol_flag))
>+		return;
>+
>+	mb->ol_flags |= ol_flag;
>+

[snip]

>--- a/drivers/net/ice/rte_pmd_ice_version.map
>+++ b/drivers/net/ice/rte_pmd_ice_version.map
>@@ -2,3 +2,15 @@ DPDK_19.02 {
> 
> 	local: *;
> };
>+
>+EXPERIMENTAL {
>+	global:
>+
>+	# added in 19.11
>+	rte_net_ice_dynfield_proto_xtr_metadata;
>+	rte_net_ice_dynflag_proto_xtr_vlan;
>+	rte_net_ice_dynflag_proto_xtr_ipv4;
>+	rte_net_ice_dynflag_proto_xtr_ipv6;
>+	rte_net_ice_dynflag_proto_xtr_ipv6_flow;
>+	rte_net_ice_dynflag_proto_xtr_tcp;

Why put the variable other than the experimental api in the map?

Thanks,
Xiaolong

>+};
>-- 
>2.17.1
>

Re: [dpdk-dev] [PATCH v8] net/ice: optimize protocol extraction by dynamic mbuf API

[Wang, Haiyue]

Hi Xiaolong,

> -----Original Message-----
> From: Ye, Xiaolong <xiaolong.ye@intel.com>
> Sent: Thursday, November 7, 2019 17:09
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; olivier.matz@6wind.com
> Subject: Re: [PATCH v8] net/ice: optimize protocol extraction by dynamic mbuf API
> 
> Hi, Haiyue
> 
> On 11/07, Haiyue Wang wrote:
> >The original design is to use rte_mbuf::udata64 to save the metadata of
> >protocol extraction which has network protocol data fields and type, a
> >private API is used to decode this metadata.
> >
> >Use the dynamic mbuf field and flags to register the needed fields in
> >mbuf, to avoid overwriting 'rte_mbuf::udata64' if the application uses
> >it. It only needs 4B size to save the protocol extraction data, and its
> >type and validity is indicated by related bit in 'rte_mbuf::ol_flags'.
> >
> >Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> >---
> >v8: - Cleanup the doxgen, rename the dynamic filed and flags with ice prefix.
> >
> >v7: - Change the log level from ERR to DEBUG for a successful registration,
> >      ERR is used to develop firstly.
> >
> >v6: - Don't need to register all flags when if one dev_args is set, register
> >      it as required.
> >
> >v5: - Remove the '_OL/_ol' in dynamic mbuf flag to make the variable clean.
> >
> >v4: - Include the 'rte_pmd_ice.h' header in ICE source as needed, reduce
> >      its compile scope.
> >
> >v3: - Use the conventions name "rte_net_<pmd>_dynfield_<name>" since it
> >      is defined in a in PMD.
> >    - Add helpers API for easiy access.
> >
> >v2: - disable the protocol extraction if failed to register some ol_flags
> >    - rewrite the commit message
> >
> > doc/api/doxy-api-index.md               |   1 +
> > doc/api/doxy-api.conf.in                |   1 +
> > doc/guides/nics/ice.rst                 |  14 +-
> > drivers/net/ice/ice_ethdev.c            |  84 ++++++-
> > drivers/net/ice/ice_ethdev.h            |   9 +
> > drivers/net/ice/ice_rxtx.c              |  88 ++++---
> > drivers/net/ice/ice_rxtx.h              |   1 -
> > drivers/net/ice/rte_pmd_ice.h           | 312 +++++++++++++++---------
> > drivers/net/ice/rte_pmd_ice_version.map |  12 +
> > 9 files changed, 369 insertions(+), 153 deletions(-)
> >
> 
> [snip]
> 
> >
> >-  The extraction will be copied into the lower 32 bit of ``rte_mbuf::udata64``.
> >+  The extraction metadata will be copied into the registered dynamic mbuf field
> >+  with, and the related dynamic mbuf flags is set.
> 
> Above 'with' is redundant?

Yes, will fix next version.

> 
> >
> >   .. table:: Protocol extraction : ``vlan``
> >
> >@@ -175,10 +176,11 @@ Runtime Config Options
> >
> >   TCPHDR2 - Reserved
> >
> >-  Use ``get_proto_xtr_flds(struct rte_mbuf *mb)`` to access the protocol
> >-  extraction, do not use ``rte_mbuf::udata64`` directly.
> >+  Use ``rte_net_ice_dynf_proto_xtr_metadata_get`` to access the protocol
> >+  extraction metadata, and use ``PKT_RX_DYNF_PROTO_XTR_*`` to get the
> >+  metadata type of ``struct rte_mbuf::ol_flags``.
> >
> >-  The ``dump_proto_xtr_flds(struct rte_mbuf *mb)`` routine shows how to
> >+  The ``rte_net_ice_dump_proto_xtr_metadata`` routine shows how to
> >   access the protocol extraction result in ``struct rte_mbuf``.
> >
> 
> [snip]
> 
> >+
> >+	PMD_DRV_LOG(DEBUG,
> >+		    "Protocol extraction metadata offset in mbuf is : %d",
> >+		    offset);
> >+	rte_net_ice_dynfield_proto_xtr_metadata = offset;
> 
> Seems rte_net_ice_dump_proto_xtr_metadata_off is a better name judging from
> its real meaning.
> 

Make sense, will change it.

> >+
> >+	for (i = 0; i < RTE_DIM(ice_proto_xtr_ol_flag_params); i++) {
> >+		ol_flag = &ice_proto_xtr_ol_flag_params[i];
> >+
> >+		if (!ol_flag->required)
> 
> [snip]
> 
> >+static void
> >+ice_rxd_to_proto_xtr(struct rte_mbuf *mb,
> >+		     volatile struct ice_32b_rx_flex_desc_comms *desc)
> >+{
> >+	uint16_t stat_err = rte_le_to_cpu_16(desc->status_error1);
> >+	uint32_t metadata;
> >+	uint64_t ol_flag;
> >+
> >+	if (unlikely(!(stat_err & ICE_RX_PROTO_XTR_VALID)))
> >+		return;
> >+
> >+	ol_flag = ice_rxdid_to_proto_xtr_ol_flag(desc->rxdid);
> 
> ol_flag here is obtained through offset which is returned by
> rte_mbuf_dynflag_register, will it have any chance to conflict with
> existing offload flags such as PKT_RX_VLAN, PKT_RX_VLAN_STRIPPED, ...?
> 

This API will choose between "PKT_FIRST_FREE" and "PKT_LAST_FREE",
not conflict. ;-)

> >+	if (unlikely(!ol_flag))
> >+		return;
> >+
> >+	mb->ol_flags |= ol_flag;
> >+
> 
> [snip]
> 
> >--- a/drivers/net/ice/rte_pmd_ice_version.map
> >+++ b/drivers/net/ice/rte_pmd_ice_version.map
> >@@ -2,3 +2,15 @@ DPDK_19.02 {
> >
> > 	local: *;
> > };
> >+
> >+EXPERIMENTAL {
> >+	global:
> >+
> >+	# added in 19.11
> >+	rte_net_ice_dynfield_proto_xtr_metadata;
> >+	rte_net_ice_dynflag_proto_xtr_vlan;
> >+	rte_net_ice_dynflag_proto_xtr_ipv4;
> >+	rte_net_ice_dynflag_proto_xtr_ipv6;
> >+	rte_net_ice_dynflag_proto_xtr_ipv6_flow;
> >+	rte_net_ice_dynflag_proto_xtr_tcp;
> 
> Why put the variable other than the experimental api in the map?
> 

The experimental API is static and inline, not global, so no need
to be put here. But the above variables are global, so need to be
put here for compiling shared.

Please refer to: git log -p lib/librte_ethdev/rte_ethdev_version.map

> Thanks,
> Xiaolong
> 
> >+};
> >--
> >2.17.1
> >

[dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API

[Haiyue Wang]

The original design is to use rte_mbuf::udata64 to save the metadata of
protocol extraction which has network protocol data fields and type, a
private API is used to decode this metadata.

Use the dynamic mbuf field and flags to register the needed fields in
mbuf, to avoid overwriting 'rte_mbuf::udata64' if the application uses
it. It only needs 4B size to save the protocol extraction data, and its
type and validity is indicated by related bit in 'rte_mbuf::ol_flags'.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
v9: - Rename the dynamic filed and flags variables to _offs, _mask, so it
      matches the design.

v8: - Cleanup the doxgen, rename the dynamic filed and flags with ice prefix.

v7: - Change the log level from ERR to DEBUG for a successful registration,
      ERR is used to develop firstly.

v6: - Don't need to register all flags when if one dev_args is set, register
      it as required.

v5: - Remove the '_OL/_ol' in dynamic mbuf flag to make the variable clean.

v4: - Include the 'rte_pmd_ice.h' header in ICE source as needed, reduce
      its compile scope.

v3: - Use the conventions name "rte_net_<pmd>_dynfield_<name>" since it
      is defined in a in PMD.
    - Add helpers API for easiy access.

v2: - disable the protocol extraction if failed to register some ol_flags
    - rewrite the commit message

 doc/api/doxy-api-index.md               |   1 +
 doc/api/doxy-api.conf.in                |   1 +
 doc/guides/nics/ice.rst                 |  14 +-
 drivers/net/ice/ice_ethdev.c            |  84 ++++++-
 drivers/net/ice/ice_ethdev.h            |   9 +
 drivers/net/ice/ice_rxtx.c              |  88 ++++---
 drivers/net/ice/ice_rxtx.h              |   1 -
 drivers/net/ice/rte_pmd_ice.h           | 313 +++++++++++++++---------
 drivers/net/ice/rte_pmd_ice_version.map |  12 +
 9 files changed, 370 insertions(+), 153 deletions(-)

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index 28a5dd37e..dff496be0 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -42,6 +42,7 @@ The public API headers are grouped by topics:
   [KNI]                (@ref rte_kni.h),
   [ixgbe]              (@ref rte_pmd_ixgbe.h),
   [i40e]               (@ref rte_pmd_i40e.h),
+  [ice]                (@ref rte_pmd_ice.h),
   [bnxt]               (@ref rte_pmd_bnxt.h),
   [dpaa]               (@ref rte_pmd_dpaa.h),
   [dpaa2]              (@ref rte_pmd_dpaa2.h),
diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
index cb67eb728..350b442a1 100644
--- a/doc/api/doxy-api.conf.in
+++ b/doc/api/doxy-api.conf.in
@@ -12,6 +12,7 @@ INPUT                   = @TOPDIR@/doc/api/doxy-api-index.md \
                           @TOPDIR@/drivers/net/dpaa \
                           @TOPDIR@/drivers/net/dpaa2 \
                           @TOPDIR@/drivers/net/i40e \
+                          @TOPDIR@/drivers/net/ice \
                           @TOPDIR@/drivers/net/ixgbe \
                           @TOPDIR@/drivers/net/softnic \
                           @TOPDIR@/drivers/raw/dpaa2_cmdif \
diff --git a/doc/guides/nics/ice.rst b/doc/guides/nics/ice.rst
index 933f63480..6c60494c7 100644
--- a/doc/guides/nics/ice.rst
+++ b/doc/guides/nics/ice.rst
@@ -82,8 +82,8 @@ Runtime Config Options
 
 - ``Protocol extraction for per queue``
 
-  Configure the RX queues to do protocol extraction into ``rte_mbuf::udata64``
-  for protocol handling acceleration, like checking the TCP SYN packets quickly.
+  Configure the RX queues to do protocol extraction into mbuf for protocol
+  handling acceleration, like checking the TCP SYN packets quickly.
 
   The argument format is::
 
@@ -111,7 +111,8 @@ Runtime Config Options
   This setting means queues 1, 2-3, 8-9 are TCP extraction, queues 10-23 are
   IPv6 extraction, other queues use the default VLAN extraction.
 
-  The extraction will be copied into the lower 32 bit of ``rte_mbuf::udata64``.
+  The extraction metadata is copied into the registered dynamic mbuf field, and
+  the related dynamic mbuf flags is set.
 
   .. table:: Protocol extraction : ``vlan``
 
@@ -175,10 +176,11 @@ Runtime Config Options
 
   TCPHDR2 - Reserved
 
-  Use ``get_proto_xtr_flds(struct rte_mbuf *mb)`` to access the protocol
-  extraction, do not use ``rte_mbuf::udata64`` directly.
+  Use ``rte_net_ice_dynf_proto_xtr_metadata_get`` to access the protocol
+  extraction metadata, and use ``PKT_RX_DYNF_PROTO_XTR_*`` to get the
+  metadata type of ``struct rte_mbuf::ol_flags``.
 
-  The ``dump_proto_xtr_flds(struct rte_mbuf *mb)`` routine shows how to
+  The ``rte_net_ice_dump_proto_xtr_metadata`` routine shows how to
   access the protocol extraction result in ``struct rte_mbuf``.
 
 Driver compilation and testing
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index 9d2210a45..5c39ea724 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -14,6 +14,8 @@
 #include "base/ice_flow.h"
 #include "base/ice_dcb.h"
 #include "base/ice_common.h"
+
+#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 #include "ice_rxtx.h"
 #include "ice_generic_flow.h"
@@ -30,6 +32,37 @@ static const char * const ice_valid_args[] = {
 	NULL
 };
 
+static const struct rte_mbuf_dynfield ice_proto_xtr_metadata_param = {
+	.name = "ice_dynfield_proto_xtr_metadata",
+	.size = sizeof(uint32_t),
+	.align = __alignof__(uint32_t),
+	.flags = 0,
+};
+
+struct proto_xtr_ol_flag {
+	const struct rte_mbuf_dynflag param;
+	uint64_t *ol_flag;
+	bool required;
+};
+
+static struct proto_xtr_ol_flag ice_proto_xtr_ol_flag_params[] = {
+	[PROTO_XTR_VLAN] = {
+		.param = { .name = "ice_dynflag_proto_xtr_vlan" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_vlan_mask },
+	[PROTO_XTR_IPV4] = {
+		.param = { .name = "ice_dynflag_proto_xtr_ipv4" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_ipv4_mask },
+	[PROTO_XTR_IPV6] = {
+		.param = { .name = "ice_dynflag_proto_xtr_ipv6" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_ipv6_mask },
+	[PROTO_XTR_IPV6_FLOW] = {
+		.param = { .name = "ice_dynflag_proto_xtr_ipv6_flow" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask },
+	[PROTO_XTR_TCP] = {
+		.param = { .name = "ice_dynflag_proto_xtr_tcp" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_tcp_mask },
+};
+
 #define ICE_DFLT_OUTER_TAG_TYPE ICE_AQ_VSI_OUTER_TAG_VLAN_9100
 
 /* DDP package search path */
@@ -1385,6 +1418,9 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 			ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 	struct ice_hw *hw = ICE_PF_TO_HW(pf);
+	const struct proto_xtr_ol_flag *ol_flag;
+	bool proto_xtr_enable = false;
+	int offset;
 	uint16_t i;
 
 	if (!ice_proto_xtr_support(hw)) {
@@ -1398,10 +1434,56 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 		return;
 	}
 
-	for (i = 0; i < pf->lan_nb_qps; i++)
+	for (i = 0; i < pf->lan_nb_qps; i++) {
 		pf->proto_xtr[i] = ad->devargs.proto_xtr[i] != PROTO_XTR_NONE ?
 				   ad->devargs.proto_xtr[i] :
 				   ad->devargs.proto_xtr_dflt;
+
+		if (pf->proto_xtr[i] != PROTO_XTR_NONE) {
+			uint8_t type = pf->proto_xtr[i];
+
+			ice_proto_xtr_ol_flag_params[type].required = true;
+			proto_xtr_enable = true;
+		}
+	}
+
+	if (likely(!proto_xtr_enable))
+		return;
+
+	offset = rte_mbuf_dynfield_register(&ice_proto_xtr_metadata_param);
+	if (unlikely(offset == -1)) {
+		PMD_DRV_LOG(ERR,
+			    "Protocol extraction metadata is disabled in mbuf with error %d",
+			    -rte_errno);
+		return;
+	}
+
+	PMD_DRV_LOG(DEBUG,
+		    "Protocol extraction metadata offset in mbuf is : %d",
+		    offset);
+	rte_net_ice_dynfield_proto_xtr_metadata_offs = offset;
+
+	for (i = 0; i < RTE_DIM(ice_proto_xtr_ol_flag_params); i++) {
+		ol_flag = &ice_proto_xtr_ol_flag_params[i];
+
+		if (!ol_flag->required)
+			continue;
+
+		offset = rte_mbuf_dynflag_register(&ol_flag->param);
+		if (unlikely(offset == -1)) {
+			PMD_DRV_LOG(ERR,
+				    "Protocol extraction offload '%s' failed to register with error %d",
+				    ol_flag->param.name, -rte_errno);
+
+			rte_net_ice_dynfield_proto_xtr_metadata_offs = -1;
+			break;
+		}
+
+		PMD_DRV_LOG(DEBUG,
+			    "Protocol extraction offload '%s' offset in mbuf is : %d",
+			    ol_flag->param.name, offset);
+		*ol_flag->ol_flag = 1ULL << offset;
+	}
 }
 
 /*  Initialize SW parameters of PF */
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index de67e5934..c55713cc1 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -241,6 +241,15 @@ struct ice_vsi {
 	bool offset_loaded;
 };
 
+enum proto_xtr_type {
+	PROTO_XTR_NONE,
+	PROTO_XTR_VLAN,
+	PROTO_XTR_IPV4,
+	PROTO_XTR_IPV6,
+	PROTO_XTR_IPV6_FLOW,
+	PROTO_XTR_TCP,
+};
+
 enum ice_fdir_tunnel_type {
 	ICE_FDIR_TUNNEL_TYPE_NONE = 0,
 	ICE_FDIR_TUNNEL_TYPE_VXLAN,
diff --git a/drivers/net/ice/ice_rxtx.c b/drivers/net/ice/ice_rxtx.c
index 53e4eea7b..18c02979e 100644
--- a/drivers/net/ice/ice_rxtx.c
+++ b/drivers/net/ice/ice_rxtx.c
@@ -5,6 +5,7 @@
 #include <rte_ethdev_driver.h>
 #include <rte_net.h>
 
+#include "rte_pmd_ice.h"
 #include "ice_rxtx.h"
 
 #define ICE_TX_CKSUM_OFFLOAD_MASK (		 \
@@ -13,18 +14,36 @@
 		PKT_TX_TCP_SEG |		 \
 		PKT_TX_OUTER_IP_CKSUM)
 
-static inline uint8_t
-ice_rxdid_to_proto_xtr_type(uint8_t rxdid)
-{
-	static uint8_t xtr_map[] = {
-		[ICE_RXDID_COMMS_AUX_VLAN]      = PROTO_XTR_VLAN,
-		[ICE_RXDID_COMMS_AUX_IPV4]      = PROTO_XTR_IPV4,
-		[ICE_RXDID_COMMS_AUX_IPV6]      = PROTO_XTR_IPV6,
-		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = PROTO_XTR_IPV6_FLOW,
-		[ICE_RXDID_COMMS_AUX_TCP]       = PROTO_XTR_TCP,
+/* Offset of mbuf dynamic field for protocol extraction data */
+int rte_net_ice_dynfield_proto_xtr_metadata_offs = -1;
+
+/* Mask of mbuf dynamic flags for protocol extraction type */
+uint64_t rte_net_ice_dynflag_proto_xtr_vlan_mask;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv4_mask;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_mask;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask;
+uint64_t rte_net_ice_dynflag_proto_xtr_tcp_mask;
+
+static inline uint64_t
+ice_rxdid_to_proto_xtr_ol_flag(uint8_t rxdid)
+{
+	static uint64_t *ol_flag_map[] = {
+		[ICE_RXDID_COMMS_AUX_VLAN] =
+				&rte_net_ice_dynflag_proto_xtr_vlan_mask,
+		[ICE_RXDID_COMMS_AUX_IPV4] =
+				&rte_net_ice_dynflag_proto_xtr_ipv4_mask,
+		[ICE_RXDID_COMMS_AUX_IPV6] =
+				&rte_net_ice_dynflag_proto_xtr_ipv6_mask,
+		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] =
+				&rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask,
+		[ICE_RXDID_COMMS_AUX_TCP] =
+				&rte_net_ice_dynflag_proto_xtr_tcp_mask,
 	};
+	uint64_t *ol_flag;
 
-	return rxdid < RTE_DIM(xtr_map) ? xtr_map[rxdid] : PROTO_XTR_NONE;
+	ol_flag = rxdid < RTE_DIM(ol_flag_map) ? ol_flag_map[rxdid] : NULL;
+
+	return ol_flag != NULL ? *ol_flag : 0ULL;
 }
 
 static inline uint8_t
@@ -1325,10 +1344,38 @@ ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc *rxdp)
 		   mb->vlan_tci, mb->vlan_tci_outer);
 }
 
+#ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 #define ICE_RX_PROTO_XTR_VALID \
 	((1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) | \
 	 (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
 
+static void
+ice_rxd_to_proto_xtr(struct rte_mbuf *mb,
+		     volatile struct ice_32b_rx_flex_desc_comms *desc)
+{
+	uint16_t stat_err = rte_le_to_cpu_16(desc->status_error1);
+	uint32_t metadata;
+	uint64_t ol_flag;
+
+	if (unlikely(!(stat_err & ICE_RX_PROTO_XTR_VALID)))
+		return;
+
+	ol_flag = ice_rxdid_to_proto_xtr_ol_flag(desc->rxdid);
+	if (unlikely(!ol_flag))
+		return;
+
+	mb->ol_flags |= ol_flag;
+
+	metadata = stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) ?
+				rte_le_to_cpu_16(desc->flex_ts.flex.aux0) : 0;
+
+	if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S)))
+		metadata |= rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
+
+	*RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(mb) = metadata;
+}
+#endif
+
 static inline void
 ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 		      volatile union ice_rx_flex_desc *rxdp)
@@ -1344,28 +1391,13 @@ ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 	}
 
 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
-	init_proto_xtr_flds(mb);
-
-	stat_err = rte_le_to_cpu_16(desc->status_error1);
-	if (stat_err & ICE_RX_PROTO_XTR_VALID) {
-		struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
-			xtr->u.raw.data0 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
-			xtr->u.raw.data1 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
-
-		xtr->type = ice_rxdid_to_proto_xtr_type(desc->rxdid);
-		xtr->magic = PROTO_XTR_MAGIC_ID;
-	}
-
 	if (desc->flow_id != 0xFFFFFFFF) {
 		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 	}
+
+	if (unlikely(rte_net_ice_dynf_proto_xtr_metadata_avail()))
+		ice_rxd_to_proto_xtr(mb, desc);
 #endif
 }
 
diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h
index 5de618976..9e3d2cd07 100644
--- a/drivers/net/ice/ice_rxtx.h
+++ b/drivers/net/ice/ice_rxtx.h
@@ -5,7 +5,6 @@
 #ifndef _ICE_RXTX_H_
 #define _ICE_RXTX_H_
 
-#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 
 #define ICE_ALIGN_RING_DESC  32
diff --git a/drivers/net/ice/rte_pmd_ice.h b/drivers/net/ice/rte_pmd_ice.h
index 719487e1e..61201f785 100644
--- a/drivers/net/ice/rte_pmd_ice.h
+++ b/drivers/net/ice/rte_pmd_ice.h
@@ -5,144 +5,223 @@
 #ifndef _RTE_PMD_ICE_H_
 #define _RTE_PMD_ICE_H_
 
+/**
+ * @file rte_pmd_ice.h
+ *
+ * ice PMD specific functions.
+ *
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ */
+
 #include <stdio.h>
 #include <rte_mbuf.h>
-#include <rte_ethdev.h>
+#include <rte_mbuf_dyn.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-enum proto_xtr_type {
-	PROTO_XTR_NONE,
-	PROTO_XTR_VLAN,
-	PROTO_XTR_IPV4,
-	PROTO_XTR_IPV6,
-	PROTO_XTR_IPV6_FLOW,
-	PROTO_XTR_TCP,
+/**
+ * The supported network protocol extraction metadata format.
+ */
+union proto_xtr_metadata {
+	uint32_t metadata;
+
+	struct {
+		uint16_t data0;
+		uint16_t data1;
+	} raw;
+
+	struct {
+		uint16_t stag_vid:12,
+			 stag_dei:1,
+			 stag_pcp:3;
+		uint16_t ctag_vid:12,
+			 ctag_dei:1,
+			 ctag_pcp:3;
+	} vlan;
+
+	struct {
+		uint16_t protocol:8,
+			 ttl:8;
+		uint16_t tos:8,
+			 ihl:4,
+			 version:4;
+	} ipv4;
+
+	struct {
+		uint16_t hoplimit:8,
+			 nexthdr:8;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6;
+
+	struct {
+		uint16_t flowlo16;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6_flow;
+
+	struct {
+		uint16_t fin:1,
+			 syn:1,
+			 rst:1,
+			 psh:1,
+			 ack:1,
+			 urg:1,
+			 ece:1,
+			 cwr:1,
+			 res1:4,
+			 doff:4;
+		uint16_t rsvd;
+	} tcp;
 };
 
-struct proto_xtr_flds {
-	union {
-		struct {
-			uint16_t data0;
-			uint16_t data1;
-		} raw;
-		struct {
-			uint16_t stag_vid:12,
-				 stag_dei:1,
-				 stag_pcp:3;
-			uint16_t ctag_vid:12,
-				 ctag_dei:1,
-				 ctag_pcp:3;
-		} vlan;
-		struct {
-			uint16_t protocol:8,
-				 ttl:8;
-			uint16_t tos:8,
-				 ihl:4,
-				 version:4;
-		} ipv4;
-		struct {
-			uint16_t hoplimit:8,
-				 nexthdr:8;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6;
-		struct {
-			uint16_t flowlo16;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6_flow;
-		struct {
-			uint16_t fin:1,
-				 syn:1,
-				 rst:1,
-				 psh:1,
-				 ack:1,
-				 urg:1,
-				 ece:1,
-				 cwr:1,
-				 res1:4,
-				 doff:4;
-			uint16_t rsvd;
-		} tcp;
-	} u;
-
-	uint16_t rsvd;
-
-	uint8_t type;
-
-#define PROTO_XTR_MAGIC_ID	0xCE
-	uint8_t magic;
-};
+/* Offset of mbuf dynamic field for protocol extraction data */
+extern int rte_net_ice_dynfield_proto_xtr_metadata_offs;
 
-static inline void
-init_proto_xtr_flds(struct rte_mbuf *mb)
+/* Mask of mbuf dynamic flags for protocol extraction type */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_vlan_mask;
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv4_mask;
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_mask;
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask;
+extern uint64_t rte_net_ice_dynflag_proto_xtr_tcp_mask;
+
+/**
+ * The mbuf dynamic field pointer for protocol extraction metadata.
+ */
+#define RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m) \
+	RTE_MBUF_DYNFIELD((m), \
+			  rte_net_ice_dynfield_proto_xtr_metadata_offs, \
+			  uint32_t *)
+
+/**
+ * The mbuf dynamic flag for VLAN protocol extraction metadata, it is valid
+ * when dev_args 'proto_xtr' has 'vlan' specified.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_VLAN \
+	(rte_net_ice_dynflag_proto_xtr_vlan_mask)
+
+/**
+ * The mbuf dynamic flag for IPv4 protocol extraction metadata, it is valid
+ * when dev_args 'proto_xtr' has 'ipv4' specified.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV4 \
+	(rte_net_ice_dynflag_proto_xtr_ipv4_mask)
+
+/**
+ * The mbuf dynamic flag for IPv6 protocol extraction metadata, it is valid
+ * when dev_args 'proto_xtr' has 'ipv6' specified.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV6 \
+	(rte_net_ice_dynflag_proto_xtr_ipv6_mask)
+
+/**
+ * The mbuf dynamic flag for IPv6 with flow protocol extraction metadata, it is
+ * valid when dev_args 'proto_xtr' has 'ipv6_flow' specified.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW \
+	(rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask)
+
+/**
+ * The mbuf dynamic flag for TCP protocol extraction metadata, it is valid
+ * when dev_args 'proto_xtr' has 'tcp' specified.
+ */
+#define PKT_RX_DYNF_PROTO_XTR_TCP \
+	(rte_net_ice_dynflag_proto_xtr_tcp_mask)
+
+/**
+ * Check if mbuf dynamic field for protocol extraction metadata is registered.
+ *
+ * @return
+ *   True if registered, false otherwise.
+ */
+__rte_experimental
+static __rte_always_inline int
+rte_net_ice_dynf_proto_xtr_metadata_avail(void)
 {
-	mb->udata64 = 0;
+	return rte_net_ice_dynfield_proto_xtr_metadata_offs != -1;
 }
 
-static inline struct proto_xtr_flds *
-get_proto_xtr_flds(struct rte_mbuf *mb)
+/**
+ * Get the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ * @return
+ *   The saved protocol extraction metadata.
+ */
+__rte_experimental
+static __rte_always_inline uint32_t
+rte_net_ice_dynf_proto_xtr_metadata_get(struct rte_mbuf *m)
 {
-	RTE_BUILD_BUG_ON(sizeof(struct proto_xtr_flds) > sizeof(mb->udata64));
-
-	return (struct proto_xtr_flds *)&mb->udata64;
+	return *RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m);
 }
 
+/**
+ * Dump the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ */
+__rte_experimental
 static inline void
-dump_proto_xtr_flds(struct rte_mbuf *mb)
+rte_net_ice_dump_proto_xtr_metadata(struct rte_mbuf *m)
 {
-	struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
+	union proto_xtr_metadata data;
 
-	if (xtr->magic != PROTO_XTR_MAGIC_ID || xtr->type == PROTO_XTR_NONE)
+	if (!rte_net_ice_dynf_proto_xtr_metadata_avail())
 		return;
 
-	printf(" - Protocol Extraction:[0x%04x:0x%04x],",
-	       xtr->u.raw.data0, xtr->u.raw.data1);
-
-	if (xtr->type == PROTO_XTR_VLAN)
-		printf("vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
-		       xtr->u.vlan.stag_pcp,
-		       xtr->u.vlan.stag_dei,
-		       xtr->u.vlan.stag_vid,
-		       xtr->u.vlan.ctag_pcp,
-		       xtr->u.vlan.ctag_dei,
-		       xtr->u.vlan.ctag_vid);
-	else if (xtr->type == PROTO_XTR_IPV4)
-		printf("ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
-		       xtr->u.ipv4.version,
-		       xtr->u.ipv4.ihl,
-		       xtr->u.ipv4.tos,
-		       xtr->u.ipv4.ttl,
-		       xtr->u.ipv4.protocol);
-	else if (xtr->type == PROTO_XTR_IPV6)
-		printf("ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
-		       xtr->u.ipv6.version,
-		       xtr->u.ipv6.tc,
-		       xtr->u.ipv6.flowhi4,
-		       xtr->u.ipv6.nexthdr,
-		       xtr->u.ipv6.hoplimit);
-	else if (xtr->type == PROTO_XTR_IPV6_FLOW)
-		printf("ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
-		       xtr->u.ipv6_flow.version,
-		       xtr->u.ipv6_flow.tc,
-		       xtr->u.ipv6_flow.flowhi4,
-		       xtr->u.ipv6_flow.flowlo16);
-	else if (xtr->type == PROTO_XTR_TCP)
-		printf("tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
-		       xtr->u.tcp.doff,
-		       xtr->u.tcp.cwr ? "C" : "",
-		       xtr->u.tcp.ece ? "E" : "",
-		       xtr->u.tcp.urg ? "U" : "",
-		       xtr->u.tcp.ack ? "A" : "",
-		       xtr->u.tcp.psh ? "P" : "",
-		       xtr->u.tcp.rst ? "R" : "",
-		       xtr->u.tcp.syn ? "S" : "",
-		       xtr->u.tcp.fin ? "F" : "");
+	data.metadata = rte_net_ice_dynf_proto_xtr_metadata_get(m);
+
+	if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_VLAN)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],vlan,stag=%u:%u:%u,ctag=%u:%u:%u",
+		       data.raw.data0, data.raw.data1,
+		       data.vlan.stag_pcp,
+		       data.vlan.stag_dei,
+		       data.vlan.stag_vid,
+		       data.vlan.ctag_pcp,
+		       data.vlan.ctag_dei,
+		       data.vlan.ctag_vid);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV4)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv4.version,
+		       data.ipv4.ihl,
+		       data.ipv4.tos,
+		       data.ipv4.ttl,
+		       data.ipv4.protocol);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV6)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6.version,
+		       data.ipv6.tc,
+		       data.ipv6.flowhi4,
+		       data.ipv6.nexthdr,
+		       data.ipv6.hoplimit);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6_flow.version,
+		       data.ipv6_flow.tc,
+		       data.ipv6_flow.flowhi4,
+		       data.ipv6_flow.flowlo16);
+	else if (m->ol_flags & PKT_RX_DYNF_PROTO_XTR_TCP)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],tcp,doff=%u,flags=%s%s%s%s%s%s%s%s",
+		       data.raw.data0, data.raw.data1,
+		       data.tcp.doff,
+		       data.tcp.cwr ? "C" : "",
+		       data.tcp.ece ? "E" : "",
+		       data.tcp.urg ? "U" : "",
+		       data.tcp.ack ? "A" : "",
+		       data.tcp.psh ? "P" : "",
+		       data.tcp.rst ? "R" : "",
+		       data.tcp.syn ? "S" : "",
+		       data.tcp.fin ? "F" : "");
 }
 
 #ifdef __cplusplus
diff --git a/drivers/net/ice/rte_pmd_ice_version.map b/drivers/net/ice/rte_pmd_ice_version.map
index 7b23b609d..0677a1c69 100644
--- a/drivers/net/ice/rte_pmd_ice_version.map
+++ b/drivers/net/ice/rte_pmd_ice_version.map
@@ -2,3 +2,15 @@ DPDK_19.02 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	# added in 19.11
+	rte_net_ice_dynfield_proto_xtr_metadata_offs;
+	rte_net_ice_dynflag_proto_xtr_vlan_mask;
+	rte_net_ice_dynflag_proto_xtr_ipv4_mask;
+	rte_net_ice_dynflag_proto_xtr_ipv6_mask;
+	rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask;
+	rte_net_ice_dynflag_proto_xtr_tcp_mask;
+};
-- 
2.17.1

Re: [dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API

[Ye Xiaolong]

On 11/07, Haiyue Wang wrote:
>The original design is to use rte_mbuf::udata64 to save the metadata of
>protocol extraction which has network protocol data fields and type, a
>private API is used to decode this metadata.
>
>Use the dynamic mbuf field and flags to register the needed fields in
>mbuf, to avoid overwriting 'rte_mbuf::udata64' if the application uses
>it. It only needs 4B size to save the protocol extraction data, and its
>type and validity is indicated by related bit in 'rte_mbuf::ol_flags'.
>
>Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
>---
>v9: - Rename the dynamic filed and flags variables to _offs, _mask, so it
>      matches the design.
>
>v8: - Cleanup the doxgen, rename the dynamic filed and flags with ice prefix.
>
>v7: - Change the log level from ERR to DEBUG for a successful registration,
>      ERR is used to develop firstly.
>
>v6: - Don't need to register all flags when if one dev_args is set, register
>      it as required.
>
>v5: - Remove the '_OL/_ol' in dynamic mbuf flag to make the variable clean.
>
>v4: - Include the 'rte_pmd_ice.h' header in ICE source as needed, reduce
>      its compile scope.
>
>v3: - Use the conventions name "rte_net_<pmd>_dynfield_<name>" since it
>      is defined in a in PMD.
>    - Add helpers API for easiy access.
>
>v2: - disable the protocol extraction if failed to register some ol_flags
>    - rewrite the commit message
>
> doc/api/doxy-api-index.md               |   1 +
> doc/api/doxy-api.conf.in                |   1 +
> doc/guides/nics/ice.rst                 |  14 +-
> drivers/net/ice/ice_ethdev.c            |  84 ++++++-
> drivers/net/ice/ice_ethdev.h            |   9 +
> drivers/net/ice/ice_rxtx.c              |  88 ++++---
> drivers/net/ice/ice_rxtx.h              |   1 -
> drivers/net/ice/rte_pmd_ice.h           | 313 +++++++++++++++---------
> drivers/net/ice/rte_pmd_ice_version.map |  12 +
> 9 files changed, 370 insertions(+), 153 deletions(-)
>

Reviewed-by: Xiaolong Ye <xiaolong.ye@intel.com>

Applied to dpdk-next-net-intel. Thanks.

Re: [dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API

[Thomas Monjalon]

Hi,

I see this patch is already merged in next-net-intel,
but please I would prefer to have below improvements first.

07/11/2019 11:44, Haiyue Wang:
> The original design is to use rte_mbuf::udata64 to save the metadata of
> protocol extraction which has network protocol data fields and type, a
> private API is used to decode this metadata.
> 
> Use the dynamic mbuf field and flags to register the needed fields in
> mbuf, to avoid overwriting 'rte_mbuf::udata64' if the application uses
> it. It only needs 4B size to save the protocol extraction data, and its

Yes using a dynamic field is definitely more correct.

> type and validity is indicated by related bit in 'rte_mbuf::ol_flags'.

Better to say explicitly it uses a dynamic flag.

> --- a/drivers/net/ice/rte_pmd_ice.h
> +++ b/drivers/net/ice/rte_pmd_ice.h
> +/**
> + * @file rte_pmd_ice.h
> + *
> + * ice PMD specific functions.
> + *
> + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
> + *
> + */

Adding the file in doxygen is good.
I think it could be a separate patch for doxygen + cleanups.

> +/**
> + * The supported network protocol extraction metadata format.
> + */
> +union proto_xtr_metadata {
> -struct proto_xtr_flds {

Please add a prefix rte_ice_ or rte_net_ice_ as you wish.

[...]
> +/**
> + * The mbuf dynamic flag for VLAN protocol extraction metadata, it is valid
> + * when dev_args 'proto_xtr' has 'vlan' specified.
> + */
> +#define PKT_RX_DYNF_PROTO_XTR_VLAN \
> +	(rte_net_ice_dynflag_proto_xtr_vlan_mask)
> +
> +/**
> + * The mbuf dynamic flag for IPv4 protocol extraction metadata, it is valid
> + * when dev_args 'proto_xtr' has 'ipv4' specified.
> + */
> +#define PKT_RX_DYNF_PROTO_XTR_IPV4 \
> +	(rte_net_ice_dynflag_proto_xtr_ipv4_mask)
> +
> +/**
> + * The mbuf dynamic flag for IPv6 protocol extraction metadata, it is valid
> + * when dev_args 'proto_xtr' has 'ipv6' specified.
> + */
> +#define PKT_RX_DYNF_PROTO_XTR_IPV6 \
> +	(rte_net_ice_dynflag_proto_xtr_ipv6_mask)
> +
> +/**
> + * The mbuf dynamic flag for IPv6 with flow protocol extraction metadata, it is
> + * valid when dev_args 'proto_xtr' has 'ipv6_flow' specified.
> + */
> +#define PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW \
> +	(rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask)
> +
> +/**
> + * The mbuf dynamic flag for TCP protocol extraction metadata, it is valid
> + * when dev_args 'proto_xtr' has 'tcp' specified.
> + */
> +#define PKT_RX_DYNF_PROTO_XTR_TCP \
> +	(rte_net_ice_dynflag_proto_xtr_tcp_mask)

Those fields and flags are missing a RTE_ prefix.
(Yes I know we are missing such prefix in rte_mbuf.f)

Re: [dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API

[Wang, Haiyue]

Hi Thomas,

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Friday, November 8, 2019 20:34
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; olivier.matz@6wind.com; Ye, Xiaolong <xiaolong.ye@intel.com>; Yigit, Ferruh
> <ferruh.yigit@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API
> 
> Hi,
> 
> I see this patch is already merged in next-net-intel,
> but please I would prefer to have below improvements first.
> 
> 07/11/2019 11:44, Haiyue Wang:
> > The original design is to use rte_mbuf::udata64 to save the metadata of
> > protocol extraction which has network protocol data fields and type, a
> > private API is used to decode this metadata.
> >
> > Use the dynamic mbuf field and flags to register the needed fields in
> > mbuf, to avoid overwriting 'rte_mbuf::udata64' if the application uses
> > it. It only needs 4B size to save the protocol extraction data, and its
> 
> Yes using a dynamic field is definitely more correct.
> 
> > type and validity is indicated by related bit in 'rte_mbuf::ol_flags'.
> 
> Better to say explicitly it uses a dynamic flag.
> 

Will update doc to make the description be better.

> > --- a/drivers/net/ice/rte_pmd_ice.h
> > +++ b/drivers/net/ice/rte_pmd_ice.h
> > +/**
> > + * @file rte_pmd_ice.h
> > + *
> > + * ice PMD specific functions.
> > + *
> > + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
> > + *
> > + */
> 
> Adding the file in doxygen is good.
> I think it could be a separate patch for doxygen + cleanups.
> 
> > +/**
> > + * The supported network protocol extraction metadata format.
> > + */
> > +union proto_xtr_metadata {
> > -struct proto_xtr_flds {
> 
> Please add a prefix rte_ice_ or rte_net_ice_ as you wish.
> 

Missed, will be updated.

> [...]
> > +/**
> > + * The mbuf dynamic flag for VLAN protocol extraction metadata, it is valid
> > + * when dev_args 'proto_xtr' has 'vlan' specified.
> > + */
> > +#define PKT_RX_DYNF_PROTO_XTR_VLAN \
> > +	(rte_net_ice_dynflag_proto_xtr_vlan_mask)
> > +
> > +/**
> > + * The mbuf dynamic flag for IPv4 protocol extraction metadata, it is valid
> > + * when dev_args 'proto_xtr' has 'ipv4' specified.
> > + */
> > +#define PKT_RX_DYNF_PROTO_XTR_IPV4 \
> > +	(rte_net_ice_dynflag_proto_xtr_ipv4_mask)
> > +
> > +/**
> > + * The mbuf dynamic flag for IPv6 protocol extraction metadata, it is valid
> > + * when dev_args 'proto_xtr' has 'ipv6' specified.
> > + */
> > +#define PKT_RX_DYNF_PROTO_XTR_IPV6 \
> > +	(rte_net_ice_dynflag_proto_xtr_ipv6_mask)
> > +
> > +/**
> > + * The mbuf dynamic flag for IPv6 with flow protocol extraction metadata, it is
> > + * valid when dev_args 'proto_xtr' has 'ipv6_flow' specified.
> > + */
> > +#define PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW \
> > +	(rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask)
> > +
> > +/**
> > + * The mbuf dynamic flag for TCP protocol extraction metadata, it is valid
> > + * when dev_args 'proto_xtr' has 'tcp' specified.
> > + */
> > +#define PKT_RX_DYNF_PROTO_XTR_TCP \
> > +	(rte_net_ice_dynflag_proto_xtr_tcp_mask)
> 
> Those fields and flags are missing a RTE_ prefix.
> (Yes I know we are missing such prefix in rte_mbuf.f)
> 

The "PKT_RX_" in ' lib/librte_mbuf/rte_mbuf_core.h' will be changed to
"RTE_PKT_RX_" ?

Or keep the above as it is now, to keep the same ol_flags style, until all
the "PKT_RX_" are changed ?

Re: [dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API

[Thomas Monjalon]

08/11/2019 15:08, Wang, Haiyue:
> Hi Thomas,
> 
> > -----Original Message-----
> > From: Thomas Monjalon <thomas@monjalon.net>
> > Sent: Friday, November 8, 2019 20:34
> > To: Wang, Haiyue <haiyue.wang@intel.com>
> > Cc: dev@dpdk.org; olivier.matz@6wind.com; Ye, Xiaolong <xiaolong.ye@intel.com>; Yigit, Ferruh
> > <ferruh.yigit@intel.com>
> > Subject: Re: [dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API
> > 
> > Hi,
> > 
> > I see this patch is already merged in next-net-intel,
> > but please I would prefer to have below improvements first.
> > 
> > 07/11/2019 11:44, Haiyue Wang:
> > > The original design is to use rte_mbuf::udata64 to save the metadata of
> > > protocol extraction which has network protocol data fields and type, a
> > > private API is used to decode this metadata.
> > >
> > > Use the dynamic mbuf field and flags to register the needed fields in
> > > mbuf, to avoid overwriting 'rte_mbuf::udata64' if the application uses
> > > it. It only needs 4B size to save the protocol extraction data, and its
> > 
> > Yes using a dynamic field is definitely more correct.
> > 
> > > type and validity is indicated by related bit in 'rte_mbuf::ol_flags'.
> > 
> > Better to say explicitly it uses a dynamic flag.
> > 
> 
> Will update doc to make the description be better.
> 
> > > --- a/drivers/net/ice/rte_pmd_ice.h
> > > +++ b/drivers/net/ice/rte_pmd_ice.h
> > > +/**
> > > + * @file rte_pmd_ice.h
> > > + *
> > > + * ice PMD specific functions.
> > > + *
> > > + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
> > > + *
> > > + */
> > 
> > Adding the file in doxygen is good.
> > I think it could be a separate patch for doxygen + cleanups.
> > 
> > > +/**
> > > + * The supported network protocol extraction metadata format.
> > > + */
> > > +union proto_xtr_metadata {
> > > -struct proto_xtr_flds {
> > 
> > Please add a prefix rte_ice_ or rte_net_ice_ as you wish.
> > 
> 
> Missed, will be updated.
> 
> > [...]
> > > +/**
> > > + * The mbuf dynamic flag for VLAN protocol extraction metadata, it is valid
> > > + * when dev_args 'proto_xtr' has 'vlan' specified.
> > > + */
> > > +#define PKT_RX_DYNF_PROTO_XTR_VLAN \
> > > +	(rte_net_ice_dynflag_proto_xtr_vlan_mask)
> > > +
> > > +/**
> > > + * The mbuf dynamic flag for IPv4 protocol extraction metadata, it is valid
> > > + * when dev_args 'proto_xtr' has 'ipv4' specified.
> > > + */
> > > +#define PKT_RX_DYNF_PROTO_XTR_IPV4 \
> > > +	(rte_net_ice_dynflag_proto_xtr_ipv4_mask)
> > > +
> > > +/**
> > > + * The mbuf dynamic flag for IPv6 protocol extraction metadata, it is valid
> > > + * when dev_args 'proto_xtr' has 'ipv6' specified.
> > > + */
> > > +#define PKT_RX_DYNF_PROTO_XTR_IPV6 \
> > > +	(rte_net_ice_dynflag_proto_xtr_ipv6_mask)
> > > +
> > > +/**
> > > + * The mbuf dynamic flag for IPv6 with flow protocol extraction metadata, it is
> > > + * valid when dev_args 'proto_xtr' has 'ipv6_flow' specified.
> > > + */
> > > +#define PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW \
> > > +	(rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask)
> > > +
> > > +/**
> > > + * The mbuf dynamic flag for TCP protocol extraction metadata, it is valid
> > > + * when dev_args 'proto_xtr' has 'tcp' specified.
> > > + */
> > > +#define PKT_RX_DYNF_PROTO_XTR_TCP \
> > > +	(rte_net_ice_dynflag_proto_xtr_tcp_mask)
> > 
> > Those fields and flags are missing a RTE_ prefix.
> > (Yes I know we are missing such prefix in rte_mbuf.f)
> > 
> 
> The "PKT_RX_" in ' lib/librte_mbuf/rte_mbuf_core.h' will be changed to
> "RTE_PKT_RX_" ?

I started to do such change a long time ago and never finished.
Yes it will be changed for 20.11.

> Or keep the above as it is now, to keep the same ol_flags style, until all
> the "PKT_RX_" are changed ?

Please I prefer you change yours, without waiting global change.

Re: [dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API

[Wang, Haiyue]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Friday, November 8, 2019 22:40
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; olivier.matz@6wind.com; Ye, Xiaolong <xiaolong.ye@intel.com>; Yigit, Ferruh
> <ferruh.yigit@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API
> 
> 08/11/2019 15:08, Wang, Haiyue:
> > Hi Thomas,
> >
> > > -----Original Message-----
> > > From: Thomas Monjalon <thomas@monjalon.net>
> > > Sent: Friday, November 8, 2019 20:34
> > > To: Wang, Haiyue <haiyue.wang@intel.com>
> > > Cc: dev@dpdk.org; olivier.matz@6wind.com; Ye, Xiaolong <xiaolong.ye@intel.com>; Yigit, Ferruh
> > > <ferruh.yigit@intel.com>
> > > Subject: Re: [dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API
> > >
> > > Hi,
> > >
> > > I see this patch is already merged in next-net-intel,
> > > but please I would prefer to have below improvements first.
> > >
> > > 07/11/2019 11:44, Haiyue Wang:
> > > > The original design is to use rte_mbuf::udata64 to save the metadata of
> > > > protocol extraction which has network protocol data fields and type, a
> > > > private API is used to decode this metadata.
> > > >
> > > > Use the dynamic mbuf field and flags to register the needed fields in
> > > > mbuf, to avoid overwriting 'rte_mbuf::udata64' if the application uses
> > > > it. It only needs 4B size to save the protocol extraction data, and its
> > >
> > > Yes using a dynamic field is definitely more correct.
> > >
> > > > type and validity is indicated by related bit in 'rte_mbuf::ol_flags'.
> > >
> > > Better to say explicitly it uses a dynamic flag.
> > >
> >
> > Will update doc to make the description be better.
> >
> > > > --- a/drivers/net/ice/rte_pmd_ice.h
> > > > +++ b/drivers/net/ice/rte_pmd_ice.h
> > > > +/**
> > > > + * @file rte_pmd_ice.h
> > > > + *
> > > > + * ice PMD specific functions.
> > > > + *
> > > > + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
> > > > + *
> > > > + */
> > >
> > > Adding the file in doxygen is good.
> > > I think it could be a separate patch for doxygen + cleanups.
> > >
> > > > +/**
> > > > + * The supported network protocol extraction metadata format.
> > > > + */
> > > > +union proto_xtr_metadata {
> > > > -struct proto_xtr_flds {
> > >
> > > Please add a prefix rte_ice_ or rte_net_ice_ as you wish.
> > >
> >
> > Missed, will be updated.
> >
> > > [...]
> > > > +/**
> > > > + * The mbuf dynamic flag for VLAN protocol extraction metadata, it is valid
> > > > + * when dev_args 'proto_xtr' has 'vlan' specified.
> > > > + */
> > > > +#define PKT_RX_DYNF_PROTO_XTR_VLAN \
> > > > +	(rte_net_ice_dynflag_proto_xtr_vlan_mask)
> > > > +
> > > > +/**
> > > > + * The mbuf dynamic flag for IPv4 protocol extraction metadata, it is valid
> > > > + * when dev_args 'proto_xtr' has 'ipv4' specified.
> > > > + */
> > > > +#define PKT_RX_DYNF_PROTO_XTR_IPV4 \
> > > > +	(rte_net_ice_dynflag_proto_xtr_ipv4_mask)
> > > > +
> > > > +/**
> > > > + * The mbuf dynamic flag for IPv6 protocol extraction metadata, it is valid
> > > > + * when dev_args 'proto_xtr' has 'ipv6' specified.
> > > > + */
> > > > +#define PKT_RX_DYNF_PROTO_XTR_IPV6 \
> > > > +	(rte_net_ice_dynflag_proto_xtr_ipv6_mask)
> > > > +
> > > > +/**
> > > > + * The mbuf dynamic flag for IPv6 with flow protocol extraction metadata, it is
> > > > + * valid when dev_args 'proto_xtr' has 'ipv6_flow' specified.
> > > > + */
> > > > +#define PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW \
> > > > +	(rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask)
> > > > +
> > > > +/**
> > > > + * The mbuf dynamic flag for TCP protocol extraction metadata, it is valid
> > > > + * when dev_args 'proto_xtr' has 'tcp' specified.
> > > > + */
> > > > +#define PKT_RX_DYNF_PROTO_XTR_TCP \
> > > > +	(rte_net_ice_dynflag_proto_xtr_tcp_mask)
> > >
> > > Those fields and flags are missing a RTE_ prefix.
> > > (Yes I know we are missing such prefix in rte_mbuf.f)
> > >
> >
> > The "PKT_RX_" in ' lib/librte_mbuf/rte_mbuf_core.h' will be changed to
> > "RTE_PKT_RX_" ?
> 
> I started to do such change a long time ago and never finished.
> Yes it will be changed for 20.11.
> 
> > Or keep the above as it is now, to keep the same ol_flags style, until all
> > the "PKT_RX_" are changed ?
> 
> Please I prefer you change yours, without waiting global change.
> 

OK, will update it.

Re: [dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API

[Ye Xiaolong]

On 11/08, Thomas Monjalon wrote:
>Hi,
>
>I see this patch is already merged in next-net-intel,
>but please I would prefer to have below improvements first.
>

Got it, I'll drop v9 in next-net-intel and wait for the improved new version.

Thanks,
Xiaolong

Re: [dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API

[Thomas Monjalon]

07/11/2019 11:44, Haiyue Wang:
> --- a/drivers/net/ice/rte_pmd_ice_version.map
> +++ b/drivers/net/ice/rte_pmd_ice_version.map
> +EXPERIMENTAL {
> +	global:
> +
> +	# added in 19.11
> +	rte_net_ice_dynfield_proto_xtr_metadata_offs;
> +	rte_net_ice_dynflag_proto_xtr_vlan_mask;
> +	rte_net_ice_dynflag_proto_xtr_ipv4_mask;
> +	rte_net_ice_dynflag_proto_xtr_ipv6_mask;
> +	rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask;
> +	rte_net_ice_dynflag_proto_xtr_tcp_mask;
> +};

Given that you provide some functions to access to the metadata,
why do you need to export these flags and field in the .map?

However, the functions are missing in the .map.
Did you try to compile as a shared library?

Re: [dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API

[Wang, Haiyue]

Hi Thomas,

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Friday, November 8, 2019 20:55
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; olivier.matz@6wind.com; Ye, Xiaolong <xiaolong.ye@intel.com>; Yigit, Ferruh
> <ferruh.yigit@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API
> 
> 07/11/2019 11:44, Haiyue Wang:
> > --- a/drivers/net/ice/rte_pmd_ice_version.map
> > +++ b/drivers/net/ice/rte_pmd_ice_version.map
> > +EXPERIMENTAL {
> > +	global:
> > +
> > +	# added in 19.11
> > +	rte_net_ice_dynfield_proto_xtr_metadata_offs;
> > +	rte_net_ice_dynflag_proto_xtr_vlan_mask;
> > +	rte_net_ice_dynflag_proto_xtr_ipv4_mask;
> > +	rte_net_ice_dynflag_proto_xtr_ipv6_mask;
> > +	rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask;
> > +	rte_net_ice_dynflag_proto_xtr_tcp_mask;
> > +};
> 
> Given that you provide some functions to access to the metadata,
> why do you need to export these flags and field in the .map?
>
> However, the functions are missing in the .map.
> Did you try to compile as a shared library?
> 

These functions are 'static inline', no need to be exported in the
.map. And the macros like 'PKT_RX_DYNF_PROTO_XTR_XXX', in fact, their
real definitions are global values defined in rte_pmd_ice like:
rte_net_ice_dynflag_proto_xtr_xxx_mask.

Since rte_pmd_ice are required to compiled as a shared library, so
it is needed to export these flags and field in the .map.

This design is referred to the below upstream practice about dynamic
mbuf.

commit 7743e81854944ed17df05bfdcba26556cb41ca0c
Author: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
Date:   Tue Nov 5 14:19:30 2019 +0000

    ethdev: extend flow metadata

--- a/lib/librte_ethdev/rte_ethdev_version.map
+++ b/lib/librte_ethdev/rte_ethdev_version.map
@@ -289,4 +289,7 @@ EXPERIMENTAL {
        rte_eth_rx_hairpin_queue_setup;
        rte_eth_tx_burst_mode_get;
        rte_eth_tx_hairpin_queue_setup;
+       rte_flow_dynf_metadata_offs;
+       rte_flow_dynf_metadata_mask;
+       rte_flow_dynf_metadata_register;
 };

Re: [dpdk-dev] [PATCH v9] net/ice: optimize protocol extraction by dynamic mbuf API

[Thomas Monjalon]

08/11/2019 15:01, Wang, Haiyue:
> From: Thomas Monjalon <thomas@monjalon.net>
> > 07/11/2019 11:44, Haiyue Wang:
> > > --- a/drivers/net/ice/rte_pmd_ice_version.map
> > > +++ b/drivers/net/ice/rte_pmd_ice_version.map
> > > +EXPERIMENTAL {
> > > +	global:
> > > +
> > > +	# added in 19.11
> > > +	rte_net_ice_dynfield_proto_xtr_metadata_offs;
> > > +	rte_net_ice_dynflag_proto_xtr_vlan_mask;
> > > +	rte_net_ice_dynflag_proto_xtr_ipv4_mask;
> > > +	rte_net_ice_dynflag_proto_xtr_ipv6_mask;
> > > +	rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask;
> > > +	rte_net_ice_dynflag_proto_xtr_tcp_mask;
> > > +};
> > 
> > Given that you provide some functions to access to the metadata,
> > why do you need to export these flags and field in the .map?
> >
> > However, the functions are missing in the .map.
> > Did you try to compile as a shared library?
> > 
> 
> These functions are 'static inline', no need to be exported in the

Yes I missed they are inline.

> .map. And the macros like 'PKT_RX_DYNF_PROTO_XTR_XXX', in fact, their
> real definitions are global values defined in rte_pmd_ice like:
> rte_net_ice_dynflag_proto_xtr_xxx_mask.
> 
> Since rte_pmd_ice are required to compiled as a shared library, so
> it is needed to export these flags and field in the .map.
> 
> This design is referred to the below upstream practice about dynamic
> mbuf.
> 
> commit 7743e81854944ed17df05bfdcba26556cb41ca0c
> Author: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> Date:   Tue Nov 5 14:19:30 2019 +0000
> 
>     ethdev: extend flow metadata
> 
> --- a/lib/librte_ethdev/rte_ethdev_version.map
> +++ b/lib/librte_ethdev/rte_ethdev_version.map
> @@ -289,4 +289,7 @@ EXPERIMENTAL {
>         rte_eth_rx_hairpin_queue_setup;
>         rte_eth_tx_burst_mode_get;
>         rte_eth_tx_hairpin_queue_setup;
> +       rte_flow_dynf_metadata_offs;
> +       rte_flow_dynf_metadata_mask;
> +       rte_flow_dynf_metadata_register;
>  };

Yes all these internals are exported because we use macros
or inline functions.
Matter of tradeoff between API cleaning and performance...

[dpdk-dev] [PATCH v10 0/2] net/ice: optimize protocol extraction by dynamic mbuf API

[Haiyue Wang]

v10: - Rename the exported variables with 'rte_net_ice' and 'RTE_'. Update
       the commit message, and separate patch for doxygen.

v9: - Rename the dynamic filed and flags variables to _offs, _mask, so it
      matches the design.

v8: - Cleanup the doxgen, rename the dynamic filed and flags with ice prefix.

v7: - Change the log level from ERR to DEBUG for a successful registration,
      ERR is used to develop firstly.

v6: - Don't need to register all flags when if one dev_args is set, register
      it as required.

v5: - Remove the '_OL/_ol' in dynamic mbuf flag to make the variable clean.

v4: - Include the 'rte_pmd_ice.h' header in ICE source as needed, reduce
      its compile scope.

v3: - Use the conventions name "rte_net_<pmd>_dynfield_<name>" since it
      is defined in a in PMD.
    - Add helpers API for easiy access.

v2: - disable the protocol extraction if failed to register some ol_flags
    - rewrite the commit message

Haiyue Wang (2):
  net/ice: optimize protocol extraction by dynamic mbuf API
  doc: add the ice PMD doxygen

 doc/api/doxy-api-index.md               |   1 +
 doc/api/doxy-api.conf.in                |   1 +
 doc/guides/nics/ice.rst                 |  14 +-
 drivers/net/ice/ice_ethdev.c            |  84 ++++++-
 drivers/net/ice/ice_ethdev.h            |   9 +
 drivers/net/ice/ice_rxtx.c              |  88 ++++---
 drivers/net/ice/ice_rxtx.h              |   1 -
 drivers/net/ice/rte_pmd_ice.h           | 313 +++++++++++++++---------
 drivers/net/ice/rte_pmd_ice_version.map |  12 +
 9 files changed, 370 insertions(+), 153 deletions(-)

-- 
2.17.1

[dpdk-dev] [PATCH v10 1/2] net/ice: optimize protocol extraction by dynamic mbuf API

[Haiyue Wang]

The original design is to use rte_mbuf::udata64 to save the metadata of
protocol extraction which has network protocol data fields and type, a
private API is used to decode this metadata.

Use the dynamic mbuf field and flags to register the needed fields in
mbuf, to avoid overwriting 'rte_mbuf::udata64', since the application
may use it. Now the protocol extraction metadate is saved into dynamic
mbuf field with 4B size, and its type and validity is indicated by the
related dynamic mbuf flags in 'rte_mbuf::ol_flags'.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 doc/guides/nics/ice.rst                 |  14 +-
 drivers/net/ice/ice_ethdev.c            |  84 +++++++-
 drivers/net/ice/ice_ethdev.h            |   9 +
 drivers/net/ice/ice_rxtx.c              |  88 +++++---
 drivers/net/ice/ice_rxtx.h              |   1 -
 drivers/net/ice/rte_pmd_ice.h           | 258 +++++++++++++-----------
 drivers/net/ice/rte_pmd_ice_version.map |  12 ++
 7 files changed, 313 insertions(+), 153 deletions(-)

diff --git a/doc/guides/nics/ice.rst b/doc/guides/nics/ice.rst
index 933f63480..1a426438d 100644
--- a/doc/guides/nics/ice.rst
+++ b/doc/guides/nics/ice.rst
@@ -82,8 +82,8 @@ Runtime Config Options
 
 - ``Protocol extraction for per queue``
 
-  Configure the RX queues to do protocol extraction into ``rte_mbuf::udata64``
-  for protocol handling acceleration, like checking the TCP SYN packets quickly.
+  Configure the RX queues to do protocol extraction into mbuf for protocol
+  handling acceleration, like checking the TCP SYN packets quickly.
 
   The argument format is::
 
@@ -111,7 +111,8 @@ Runtime Config Options
   This setting means queues 1, 2-3, 8-9 are TCP extraction, queues 10-23 are
   IPv6 extraction, other queues use the default VLAN extraction.
 
-  The extraction will be copied into the lower 32 bit of ``rte_mbuf::udata64``.
+  The extraction metadata is copied into the registered dynamic mbuf field, and
+  the related dynamic mbuf flags is set.
 
   .. table:: Protocol extraction : ``vlan``
 
@@ -175,10 +176,11 @@ Runtime Config Options
 
   TCPHDR2 - Reserved
 
-  Use ``get_proto_xtr_flds(struct rte_mbuf *mb)`` to access the protocol
-  extraction, do not use ``rte_mbuf::udata64`` directly.
+  Use ``rte_net_ice_dynf_proto_xtr_metadata_get`` to access the protocol
+  extraction metadata, and use ``RTE_PKT_RX_DYNF_PROTO_XTR_*`` to get the
+  metadata type of ``struct rte_mbuf::ol_flags``.
 
-  The ``dump_proto_xtr_flds(struct rte_mbuf *mb)`` routine shows how to
+  The ``rte_net_ice_dump_proto_xtr_metadata`` routine shows how to
   access the protocol extraction result in ``struct rte_mbuf``.
 
 Driver compilation and testing
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index d81eb5eeb..5189974e3 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -14,6 +14,8 @@
 #include "base/ice_flow.h"
 #include "base/ice_dcb.h"
 #include "base/ice_common.h"
+
+#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 #include "ice_rxtx.h"
 #include "ice_generic_flow.h"
@@ -30,6 +32,37 @@ static const char * const ice_valid_args[] = {
 	NULL
 };
 
+static const struct rte_mbuf_dynfield ice_proto_xtr_metadata_param = {
+	.name = "ice_dynfield_proto_xtr_metadata",
+	.size = sizeof(uint32_t),
+	.align = __alignof__(uint32_t),
+	.flags = 0,
+};
+
+struct proto_xtr_ol_flag {
+	const struct rte_mbuf_dynflag param;
+	uint64_t *ol_flag;
+	bool required;
+};
+
+static struct proto_xtr_ol_flag ice_proto_xtr_ol_flag_params[] = {
+	[PROTO_XTR_VLAN] = {
+		.param = { .name = "ice_dynflag_proto_xtr_vlan" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_vlan_mask },
+	[PROTO_XTR_IPV4] = {
+		.param = { .name = "ice_dynflag_proto_xtr_ipv4" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_ipv4_mask },
+	[PROTO_XTR_IPV6] = {
+		.param = { .name = "ice_dynflag_proto_xtr_ipv6" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_ipv6_mask },
+	[PROTO_XTR_IPV6_FLOW] = {
+		.param = { .name = "ice_dynflag_proto_xtr_ipv6_flow" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask },
+	[PROTO_XTR_TCP] = {
+		.param = { .name = "ice_dynflag_proto_xtr_tcp" },
+		.ol_flag = &rte_net_ice_dynflag_proto_xtr_tcp_mask },
+};
+
 #define ICE_DFLT_OUTER_TAG_TYPE ICE_AQ_VSI_OUTER_TAG_VLAN_9100
 
 /* DDP package search path */
@@ -1388,6 +1421,9 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 			ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 	struct ice_hw *hw = ICE_PF_TO_HW(pf);
+	const struct proto_xtr_ol_flag *ol_flag;
+	bool proto_xtr_enable = false;
+	int offset;
 	uint16_t i;
 
 	if (!ice_proto_xtr_support(hw)) {
@@ -1401,10 +1437,56 @@ ice_init_proto_xtr(struct rte_eth_dev *dev)
 		return;
 	}
 
-	for (i = 0; i < pf->lan_nb_qps; i++)
+	for (i = 0; i < pf->lan_nb_qps; i++) {
 		pf->proto_xtr[i] = ad->devargs.proto_xtr[i] != PROTO_XTR_NONE ?
 				   ad->devargs.proto_xtr[i] :
 				   ad->devargs.proto_xtr_dflt;
+
+		if (pf->proto_xtr[i] != PROTO_XTR_NONE) {
+			uint8_t type = pf->proto_xtr[i];
+
+			ice_proto_xtr_ol_flag_params[type].required = true;
+			proto_xtr_enable = true;
+		}
+	}
+
+	if (likely(!proto_xtr_enable))
+		return;
+
+	offset = rte_mbuf_dynfield_register(&ice_proto_xtr_metadata_param);
+	if (unlikely(offset == -1)) {
+		PMD_DRV_LOG(ERR,
+			    "Protocol extraction metadata is disabled in mbuf with error %d",
+			    -rte_errno);
+		return;
+	}
+
+	PMD_DRV_LOG(DEBUG,
+		    "Protocol extraction metadata offset in mbuf is : %d",
+		    offset);
+	rte_net_ice_dynfield_proto_xtr_metadata_offs = offset;
+
+	for (i = 0; i < RTE_DIM(ice_proto_xtr_ol_flag_params); i++) {
+		ol_flag = &ice_proto_xtr_ol_flag_params[i];
+
+		if (!ol_flag->required)
+			continue;
+
+		offset = rte_mbuf_dynflag_register(&ol_flag->param);
+		if (unlikely(offset == -1)) {
+			PMD_DRV_LOG(ERR,
+				    "Protocol extraction offload '%s' failed to register with error %d",
+				    ol_flag->param.name, -rte_errno);
+
+			rte_net_ice_dynfield_proto_xtr_metadata_offs = -1;
+			break;
+		}
+
+		PMD_DRV_LOG(DEBUG,
+			    "Protocol extraction offload '%s' offset in mbuf is : %d",
+			    ol_flag->param.name, offset);
+		*ol_flag->ol_flag = 1ULL << offset;
+	}
 }
 
 /*  Initialize SW parameters of PF */
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index de67e5934..c55713cc1 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -241,6 +241,15 @@ struct ice_vsi {
 	bool offset_loaded;
 };
 
+enum proto_xtr_type {
+	PROTO_XTR_NONE,
+	PROTO_XTR_VLAN,
+	PROTO_XTR_IPV4,
+	PROTO_XTR_IPV6,
+	PROTO_XTR_IPV6_FLOW,
+	PROTO_XTR_TCP,
+};
+
 enum ice_fdir_tunnel_type {
 	ICE_FDIR_TUNNEL_TYPE_NONE = 0,
 	ICE_FDIR_TUNNEL_TYPE_VXLAN,
diff --git a/drivers/net/ice/ice_rxtx.c b/drivers/net/ice/ice_rxtx.c
index 53e4eea7b..18c02979e 100644
--- a/drivers/net/ice/ice_rxtx.c
+++ b/drivers/net/ice/ice_rxtx.c
@@ -5,6 +5,7 @@
 #include <rte_ethdev_driver.h>
 #include <rte_net.h>
 
+#include "rte_pmd_ice.h"
 #include "ice_rxtx.h"
 
 #define ICE_TX_CKSUM_OFFLOAD_MASK (		 \
@@ -13,18 +14,36 @@
 		PKT_TX_TCP_SEG |		 \
 		PKT_TX_OUTER_IP_CKSUM)
 
-static inline uint8_t
-ice_rxdid_to_proto_xtr_type(uint8_t rxdid)
-{
-	static uint8_t xtr_map[] = {
-		[ICE_RXDID_COMMS_AUX_VLAN]      = PROTO_XTR_VLAN,
-		[ICE_RXDID_COMMS_AUX_IPV4]      = PROTO_XTR_IPV4,
-		[ICE_RXDID_COMMS_AUX_IPV6]      = PROTO_XTR_IPV6,
-		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] = PROTO_XTR_IPV6_FLOW,
-		[ICE_RXDID_COMMS_AUX_TCP]       = PROTO_XTR_TCP,
+/* Offset of mbuf dynamic field for protocol extraction data */
+int rte_net_ice_dynfield_proto_xtr_metadata_offs = -1;
+
+/* Mask of mbuf dynamic flags for protocol extraction type */
+uint64_t rte_net_ice_dynflag_proto_xtr_vlan_mask;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv4_mask;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_mask;
+uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask;
+uint64_t rte_net_ice_dynflag_proto_xtr_tcp_mask;
+
+static inline uint64_t
+ice_rxdid_to_proto_xtr_ol_flag(uint8_t rxdid)
+{
+	static uint64_t *ol_flag_map[] = {
+		[ICE_RXDID_COMMS_AUX_VLAN] =
+				&rte_net_ice_dynflag_proto_xtr_vlan_mask,
+		[ICE_RXDID_COMMS_AUX_IPV4] =
+				&rte_net_ice_dynflag_proto_xtr_ipv4_mask,
+		[ICE_RXDID_COMMS_AUX_IPV6] =
+				&rte_net_ice_dynflag_proto_xtr_ipv6_mask,
+		[ICE_RXDID_COMMS_AUX_IPV6_FLOW] =
+				&rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask,
+		[ICE_RXDID_COMMS_AUX_TCP] =
+				&rte_net_ice_dynflag_proto_xtr_tcp_mask,
 	};
+	uint64_t *ol_flag;
 
-	return rxdid < RTE_DIM(xtr_map) ? xtr_map[rxdid] : PROTO_XTR_NONE;
+	ol_flag = rxdid < RTE_DIM(ol_flag_map) ? ol_flag_map[rxdid] : NULL;
+
+	return ol_flag != NULL ? *ol_flag : 0ULL;
 }
 
 static inline uint8_t
@@ -1325,10 +1344,38 @@ ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc *rxdp)
 		   mb->vlan_tci, mb->vlan_tci_outer);
 }
 
+#ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 #define ICE_RX_PROTO_XTR_VALID \
 	((1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) | \
 	 (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
 
+static void
+ice_rxd_to_proto_xtr(struct rte_mbuf *mb,
+		     volatile struct ice_32b_rx_flex_desc_comms *desc)
+{
+	uint16_t stat_err = rte_le_to_cpu_16(desc->status_error1);
+	uint32_t metadata;
+	uint64_t ol_flag;
+
+	if (unlikely(!(stat_err & ICE_RX_PROTO_XTR_VALID)))
+		return;
+
+	ol_flag = ice_rxdid_to_proto_xtr_ol_flag(desc->rxdid);
+	if (unlikely(!ol_flag))
+		return;
+
+	mb->ol_flags |= ol_flag;
+
+	metadata = stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S) ?
+				rte_le_to_cpu_16(desc->flex_ts.flex.aux0) : 0;
+
+	if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S)))
+		metadata |= rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
+
+	*RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(mb) = metadata;
+}
+#endif
+
 static inline void
 ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 		      volatile union ice_rx_flex_desc *rxdp)
@@ -1344,28 +1391,13 @@ ice_rxd_to_pkt_fields(struct rte_mbuf *mb,
 	}
 
 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
-	init_proto_xtr_flds(mb);
-
-	stat_err = rte_le_to_cpu_16(desc->status_error1);
-	if (stat_err & ICE_RX_PROTO_XTR_VALID) {
-		struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
-			xtr->u.raw.data0 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
-
-		if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
-			xtr->u.raw.data1 =
-				rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
-
-		xtr->type = ice_rxdid_to_proto_xtr_type(desc->rxdid);
-		xtr->magic = PROTO_XTR_MAGIC_ID;
-	}
-
 	if (desc->flow_id != 0xFFFFFFFF) {
 		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 	}
+
+	if (unlikely(rte_net_ice_dynf_proto_xtr_metadata_avail()))
+		ice_rxd_to_proto_xtr(mb, desc);
 #endif
 }
 
diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h
index 5de618976..9e3d2cd07 100644
--- a/drivers/net/ice/ice_rxtx.h
+++ b/drivers/net/ice/ice_rxtx.h
@@ -5,7 +5,6 @@
 #ifndef _ICE_RXTX_H_
 #define _ICE_RXTX_H_
 
-#include "rte_pmd_ice.h"
 #include "ice_ethdev.h"
 
 #define ICE_ALIGN_RING_DESC  32
diff --git a/drivers/net/ice/rte_pmd_ice.h b/drivers/net/ice/rte_pmd_ice.h
index 719487e1e..717c2130b 100644
--- a/drivers/net/ice/rte_pmd_ice.h
+++ b/drivers/net/ice/rte_pmd_ice.h
@@ -7,142 +7,166 @@
 
 #include <stdio.h>
 #include <rte_mbuf.h>
-#include <rte_ethdev.h>
+#include <rte_mbuf_dyn.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-enum proto_xtr_type {
-	PROTO_XTR_NONE,
-	PROTO_XTR_VLAN,
-	PROTO_XTR_IPV4,
-	PROTO_XTR_IPV6,
-	PROTO_XTR_IPV6_FLOW,
-	PROTO_XTR_TCP,
+union rte_net_ice_proto_xtr_metadata {
+	uint32_t metadata;
+
+	struct {
+		uint16_t data0;
+		uint16_t data1;
+	} raw;
+
+	struct {
+		uint16_t stag_vid:12,
+			 stag_dei:1,
+			 stag_pcp:3;
+		uint16_t ctag_vid:12,
+			 ctag_dei:1,
+			 ctag_pcp:3;
+	} vlan;
+
+	struct {
+		uint16_t protocol:8,
+			 ttl:8;
+		uint16_t tos:8,
+			 ihl:4,
+			 version:4;
+	} ipv4;
+
+	struct {
+		uint16_t hoplimit:8,
+			 nexthdr:8;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6;
+
+	struct {
+		uint16_t flowlo16;
+		uint16_t flowhi4:4,
+			 tc:8,
+			 version:4;
+	} ipv6_flow;
+
+	struct {
+		uint16_t fin:1,
+			 syn:1,
+			 rst:1,
+			 psh:1,
+			 ack:1,
+			 urg:1,
+			 ece:1,
+			 cwr:1,
+			 res1:4,
+			 doff:4;
+		uint16_t rsvd;
+	} tcp;
 };
 
-struct proto_xtr_flds {
-	union {
-		struct {
-			uint16_t data0;
-			uint16_t data1;
-		} raw;
-		struct {
-			uint16_t stag_vid:12,
-				 stag_dei:1,
-				 stag_pcp:3;
-			uint16_t ctag_vid:12,
-				 ctag_dei:1,
-				 ctag_pcp:3;
-		} vlan;
-		struct {
-			uint16_t protocol:8,
-				 ttl:8;
-			uint16_t tos:8,
-				 ihl:4,
-				 version:4;
-		} ipv4;
-		struct {
-			uint16_t hoplimit:8,
-				 nexthdr:8;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6;
-		struct {
-			uint16_t flowlo16;
-			uint16_t flowhi4:4,
-				 tc:8,
-				 version:4;
-		} ipv6_flow;
-		struct {
-			uint16_t fin:1,
-				 syn:1,
-				 rst:1,
-				 psh:1,
-				 ack:1,
-				 urg:1,
-				 ece:1,
-				 cwr:1,
-				 res1:4,
-				 doff:4;
-			uint16_t rsvd;
-		} tcp;
-	} u;
-
-	uint16_t rsvd;
-
-	uint8_t type;
-
-#define PROTO_XTR_MAGIC_ID	0xCE
-	uint8_t magic;
-};
+/* Offset of mbuf dynamic field for protocol extraction data */
+extern int rte_net_ice_dynfield_proto_xtr_metadata_offs;
 
-static inline void
-init_proto_xtr_flds(struct rte_mbuf *mb)
+/* Mask of mbuf dynamic flags for protocol extraction type */
+extern uint64_t rte_net_ice_dynflag_proto_xtr_vlan_mask;
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv4_mask;
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_mask;
+extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask;
+extern uint64_t rte_net_ice_dynflag_proto_xtr_tcp_mask;
+
+#define RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m) \
+	RTE_MBUF_DYNFIELD((m), \
+			  rte_net_ice_dynfield_proto_xtr_metadata_offs, \
+			  uint32_t *)
+
+#define RTE_PKT_RX_DYNF_PROTO_XTR_VLAN \
+	(rte_net_ice_dynflag_proto_xtr_vlan_mask)
+
+#define RTE_PKT_RX_DYNF_PROTO_XTR_IPV4 \
+	(rte_net_ice_dynflag_proto_xtr_ipv4_mask)
+
+#define RTE_PKT_RX_DYNF_PROTO_XTR_IPV6 \
+	(rte_net_ice_dynflag_proto_xtr_ipv6_mask)
+
+#define RTE_PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW \
+	(rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask)
+
+#define RTE_PKT_RX_DYNF_PROTO_XTR_TCP \
+	(rte_net_ice_dynflag_proto_xtr_tcp_mask)
+
+__rte_experimental
+static __rte_always_inline int
+rte_net_ice_dynf_proto_xtr_metadata_avail(void)
 {
-	mb->udata64 = 0;
+	return rte_net_ice_dynfield_proto_xtr_metadata_offs != -1;
 }
 
-static inline struct proto_xtr_flds *
-get_proto_xtr_flds(struct rte_mbuf *mb)
+__rte_experimental
+static __rte_always_inline uint32_t
+rte_net_ice_dynf_proto_xtr_metadata_get(struct rte_mbuf *m)
 {
-	RTE_BUILD_BUG_ON(sizeof(struct proto_xtr_flds) > sizeof(mb->udata64));
-
-	return (struct proto_xtr_flds *)&mb->udata64;
+	return *RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m);
 }
 
+__rte_experimental
 static inline void
-dump_proto_xtr_flds(struct rte_mbuf *mb)
+rte_net_ice_dump_proto_xtr_metadata(struct rte_mbuf *m)
 {
-	struct proto_xtr_flds *xtr = get_proto_xtr_flds(mb);
+	union rte_net_ice_proto_xtr_metadata data;
 
-	if (xtr->magic != PROTO_XTR_MAGIC_ID || xtr->type == PROTO_XTR_NONE)
+	if (!rte_net_ice_dynf_proto_xtr_metadata_avail())
 		return;
 
-	printf(" - Protocol Extraction:[0x%04x:0x%04x],",
-	       xtr->u.raw.data0, xtr->u.raw.data1);
-
-	if (xtr->type == PROTO_XTR_VLAN)
-		printf("vlan,stag=%u:%u:%u,ctag=%u:%u:%u ",
-		       xtr->u.vlan.stag_pcp,
-		       xtr->u.vlan.stag_dei,
-		       xtr->u.vlan.stag_vid,
-		       xtr->u.vlan.ctag_pcp,
-		       xtr->u.vlan.ctag_dei,
-		       xtr->u.vlan.ctag_vid);
-	else if (xtr->type == PROTO_XTR_IPV4)
-		printf("ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u ",
-		       xtr->u.ipv4.version,
-		       xtr->u.ipv4.ihl,
-		       xtr->u.ipv4.tos,
-		       xtr->u.ipv4.ttl,
-		       xtr->u.ipv4.protocol);
-	else if (xtr->type == PROTO_XTR_IPV6)
-		printf("ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u ",
-		       xtr->u.ipv6.version,
-		       xtr->u.ipv6.tc,
-		       xtr->u.ipv6.flowhi4,
-		       xtr->u.ipv6.nexthdr,
-		       xtr->u.ipv6.hoplimit);
-	else if (xtr->type == PROTO_XTR_IPV6_FLOW)
-		printf("ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x ",
-		       xtr->u.ipv6_flow.version,
-		       xtr->u.ipv6_flow.tc,
-		       xtr->u.ipv6_flow.flowhi4,
-		       xtr->u.ipv6_flow.flowlo16);
-	else if (xtr->type == PROTO_XTR_TCP)
-		printf("tcp,doff=%u,flags=%s%s%s%s%s%s%s%s ",
-		       xtr->u.tcp.doff,
-		       xtr->u.tcp.cwr ? "C" : "",
-		       xtr->u.tcp.ece ? "E" : "",
-		       xtr->u.tcp.urg ? "U" : "",
-		       xtr->u.tcp.ack ? "A" : "",
-		       xtr->u.tcp.psh ? "P" : "",
-		       xtr->u.tcp.rst ? "R" : "",
-		       xtr->u.tcp.syn ? "S" : "",
-		       xtr->u.tcp.fin ? "F" : "");
+	data.metadata = rte_net_ice_dynf_proto_xtr_metadata_get(m);
+
+	if (m->ol_flags & RTE_PKT_RX_DYNF_PROTO_XTR_VLAN)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],vlan,stag=%u:%u:%u,ctag=%u:%u:%u",
+		       data.raw.data0, data.raw.data1,
+		       data.vlan.stag_pcp,
+		       data.vlan.stag_dei,
+		       data.vlan.stag_vid,
+		       data.vlan.ctag_pcp,
+		       data.vlan.ctag_dei,
+		       data.vlan.ctag_vid);
+	else if (m->ol_flags & RTE_PKT_RX_DYNF_PROTO_XTR_IPV4)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv4,ver=%u,hdrlen=%u,tos=%u,ttl=%u,proto=%u",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv4.version,
+		       data.ipv4.ihl,
+		       data.ipv4.tos,
+		       data.ipv4.ttl,
+		       data.ipv4.protocol);
+	else if (m->ol_flags & RTE_PKT_RX_DYNF_PROTO_XTR_IPV6)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6,ver=%u,tc=%u,flow_hi4=0x%x,nexthdr=%u,hoplimit=%u",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6.version,
+		       data.ipv6.tc,
+		       data.ipv6.flowhi4,
+		       data.ipv6.nexthdr,
+		       data.ipv6.hoplimit);
+	else if (m->ol_flags & RTE_PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],ipv6_flow,ver=%u,tc=%u,flow=0x%x%04x",
+		       data.raw.data0, data.raw.data1,
+		       data.ipv6_flow.version,
+		       data.ipv6_flow.tc,
+		       data.ipv6_flow.flowhi4,
+		       data.ipv6_flow.flowlo16);
+	else if (m->ol_flags & RTE_PKT_RX_DYNF_PROTO_XTR_TCP)
+		printf(" - Protocol Extraction:[0x%04x:0x%04x],tcp,doff=%u,flags=%s%s%s%s%s%s%s%s",
+		       data.raw.data0, data.raw.data1,
+		       data.tcp.doff,
+		       data.tcp.cwr ? "C" : "",
+		       data.tcp.ece ? "E" : "",
+		       data.tcp.urg ? "U" : "",
+		       data.tcp.ack ? "A" : "",
+		       data.tcp.psh ? "P" : "",
+		       data.tcp.rst ? "R" : "",
+		       data.tcp.syn ? "S" : "",
+		       data.tcp.fin ? "F" : "");
 }
 
 #ifdef __cplusplus
diff --git a/drivers/net/ice/rte_pmd_ice_version.map b/drivers/net/ice/rte_pmd_ice_version.map
index 7b23b609d..0677a1c69 100644
--- a/drivers/net/ice/rte_pmd_ice_version.map
+++ b/drivers/net/ice/rte_pmd_ice_version.map
@@ -2,3 +2,15 @@ DPDK_19.02 {
 
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	# added in 19.11
+	rte_net_ice_dynfield_proto_xtr_metadata_offs;
+	rte_net_ice_dynflag_proto_xtr_vlan_mask;
+	rte_net_ice_dynflag_proto_xtr_ipv4_mask;
+	rte_net_ice_dynflag_proto_xtr_ipv6_mask;
+	rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask;
+	rte_net_ice_dynflag_proto_xtr_tcp_mask;
+};
-- 
2.17.1

[dpdk-dev] [PATCH v10 2/2] doc: add the ice PMD doxygen

[Haiyue Wang]

Add the doxygen for ice protocol extraction feature APIs.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 doc/api/doxy-api-index.md     |  1 +
 doc/api/doxy-api.conf.in      |  1 +
 drivers/net/ice/rte_pmd_ice.h | 55 +++++++++++++++++++++++++++++++++++
 3 files changed, 57 insertions(+)

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index 28a5dd37e..dff496be0 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -42,6 +42,7 @@ The public API headers are grouped by topics:
   [KNI]                (@ref rte_kni.h),
   [ixgbe]              (@ref rte_pmd_ixgbe.h),
   [i40e]               (@ref rte_pmd_i40e.h),
+  [ice]                (@ref rte_pmd_ice.h),
   [bnxt]               (@ref rte_pmd_bnxt.h),
   [dpaa]               (@ref rte_pmd_dpaa.h),
   [dpaa2]              (@ref rte_pmd_dpaa2.h),
diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
index cb67eb728..350b442a1 100644
--- a/doc/api/doxy-api.conf.in
+++ b/doc/api/doxy-api.conf.in
@@ -12,6 +12,7 @@ INPUT                   = @TOPDIR@/doc/api/doxy-api-index.md \
                           @TOPDIR@/drivers/net/dpaa \
                           @TOPDIR@/drivers/net/dpaa2 \
                           @TOPDIR@/drivers/net/i40e \
+                          @TOPDIR@/drivers/net/ice \
                           @TOPDIR@/drivers/net/ixgbe \
                           @TOPDIR@/drivers/net/softnic \
                           @TOPDIR@/drivers/raw/dpaa2_cmdif \
diff --git a/drivers/net/ice/rte_pmd_ice.h b/drivers/net/ice/rte_pmd_ice.h
index 717c2130b..e254db053 100644
--- a/drivers/net/ice/rte_pmd_ice.h
+++ b/drivers/net/ice/rte_pmd_ice.h
@@ -5,6 +5,15 @@
 #ifndef _RTE_PMD_ICE_H_
 #define _RTE_PMD_ICE_H_
 
+/**
+ * @file rte_pmd_ice.h
+ *
+ * ice PMD specific functions.
+ *
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ */
+
 #include <stdio.h>
 #include <rte_mbuf.h>
 #include <rte_mbuf_dyn.h>
@@ -13,6 +22,9 @@
 extern "C" {
 #endif
 
+/**
+ * The supported network protocol extraction metadata format.
+ */
 union rte_net_ice_proto_xtr_metadata {
 	uint32_t metadata;
 
@@ -78,26 +90,55 @@ extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_mask;
 extern uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask;
 extern uint64_t rte_net_ice_dynflag_proto_xtr_tcp_mask;
 
+/**
+ * The mbuf dynamic field pointer for protocol extraction metadata.
+ */
 #define RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m) \
 	RTE_MBUF_DYNFIELD((m), \
 			  rte_net_ice_dynfield_proto_xtr_metadata_offs, \
 			  uint32_t *)
 
+/**
+ * The mbuf dynamic flag for VLAN protocol extraction metadata, it is valid
+ * when dev_args 'proto_xtr' has 'vlan' specified.
+ */
 #define RTE_PKT_RX_DYNF_PROTO_XTR_VLAN \
 	(rte_net_ice_dynflag_proto_xtr_vlan_mask)
 
+/**
+ * The mbuf dynamic flag for IPv4 protocol extraction metadata, it is valid
+ * when dev_args 'proto_xtr' has 'ipv4' specified.
+ */
 #define RTE_PKT_RX_DYNF_PROTO_XTR_IPV4 \
 	(rte_net_ice_dynflag_proto_xtr_ipv4_mask)
 
+/**
+ * The mbuf dynamic flag for IPv6 protocol extraction metadata, it is valid
+ * when dev_args 'proto_xtr' has 'ipv6' specified.
+ */
 #define RTE_PKT_RX_DYNF_PROTO_XTR_IPV6 \
 	(rte_net_ice_dynflag_proto_xtr_ipv6_mask)
 
+/**
+ * The mbuf dynamic flag for IPv6 with flow protocol extraction metadata, it is
+ * valid when dev_args 'proto_xtr' has 'ipv6_flow' specified.
+ */
 #define RTE_PKT_RX_DYNF_PROTO_XTR_IPV6_FLOW \
 	(rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask)
 
+/**
+ * The mbuf dynamic flag for TCP protocol extraction metadata, it is valid
+ * when dev_args 'proto_xtr' has 'tcp' specified.
+ */
 #define RTE_PKT_RX_DYNF_PROTO_XTR_TCP \
 	(rte_net_ice_dynflag_proto_xtr_tcp_mask)
 
+/**
+ * Check if mbuf dynamic field for protocol extraction metadata is registered.
+ *
+ * @return
+ *   True if registered, false otherwise.
+ */
 __rte_experimental
 static __rte_always_inline int
 rte_net_ice_dynf_proto_xtr_metadata_avail(void)
@@ -105,6 +146,14 @@ rte_net_ice_dynf_proto_xtr_metadata_avail(void)
 	return rte_net_ice_dynfield_proto_xtr_metadata_offs != -1;
 }
 
+/**
+ * Get the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ * @return
+ *   The saved protocol extraction metadata.
+ */
 __rte_experimental
 static __rte_always_inline uint32_t
 rte_net_ice_dynf_proto_xtr_metadata_get(struct rte_mbuf *m)
@@ -112,6 +161,12 @@ rte_net_ice_dynf_proto_xtr_metadata_get(struct rte_mbuf *m)
 	return *RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(m);
 }
 
+/**
+ * Dump the mbuf dynamic field for protocol extraction metadata.
+ *
+ * @param m
+ *    The pointer to the mbuf.
+ */
 __rte_experimental
 static inline void
 rte_net_ice_dump_proto_xtr_metadata(struct rte_mbuf *m)
-- 
2.17.1

Re: [dpdk-dev] [PATCH v10 0/2] net/ice: optimize protocol extraction by dynamic mbuf API

[Thomas Monjalon]

08/11/2019 16:44, Haiyue Wang:
> v10: - Rename the exported variables with 'rte_net_ice' and 'RTE_'. Update
>        the commit message, and separate patch for doxygen.

Thank you
I think this version can replace v9 in next-net

Re: [dpdk-dev] [PATCH v10 0/2] net/ice: optimize protocol extraction by dynamic mbuf API

[Ye Xiaolong]

On 11/08, Thomas Monjalon wrote:
>08/11/2019 16:44, Haiyue Wang:
>> v10: - Rename the exported variables with 'rte_net_ice' and 'RTE_'. Update
>>        the commit message, and separate patch for doxygen.
>
>Thank you
>I think this version can replace v9 in next-net
>
>

Dropped v9 and applied this series to dpdk-next-net-intel. Thanks.