[dpdk-dev] [PATCH v4 5/6] net/ark: generalize meta data between FPGA and PMD

[Ed Czeck <ed.czeck@atomicrules.com>]

In this commit we generalize the movement of user-specified
meta data between mbufs and FPGA AXIS tuser fields using
user-defined hook functions.

- Previous use of PMD dynfields are removed
- Hook function added to ark_user_ext
- Add hook function calls in Rx and Tx paths
- Update guide with example of hook function use
- Add release notes

Signed-off-by: Ed Czeck <ed.czeck@atomicrules.com>
---
v3:
- split function rename to separate commit

v4:
- reorder patches renaming before adding
-
---
 doc/guides/nics/ark.rst                | 139 ++++++++++++++++++++++-
 doc/guides/rel_notes/release_21_05.rst |  11 ++
 drivers/net/ark/ark_ethdev.c           |  58 +---------
 drivers/net/ark/ark_ethdev_rx.c        |  27 +++--
 drivers/net/ark/ark_ethdev_rx.h        |   3 -
 drivers/net/ark/ark_ethdev_tx.c        |  23 ++--
 drivers/net/ark/ark_global.h           |  20 ++++
 drivers/net/ark/ark_udm.h              |   5 +-
 drivers/net/ark/rte_pmd_ark.h          | 151 +++++++------------------
 drivers/net/ark/version.map            |   7 --
 10 files changed, 249 insertions(+), 195 deletions(-)

diff --git a/doc/guides/nics/ark.rst b/doc/guides/nics/ark.rst
index dcccfee26..bcc3babd5 100644
--- a/doc/guides/nics/ark.rst
+++ b/doc/guides/nics/ark.rst
@@ -1,5 +1,5 @@
 .. SPDX-License-Identifier: BSD-3-Clause
-    Copyright (c) 2015-2017 Atomic Rules LLC
+    Copyright (c) 2015-2021 Atomic Rules LLC
     All rights reserved.
 
 ARK Poll Mode Driver
@@ -130,6 +130,141 @@ Configuration Information
      be offloaded or remain in host software.
 
 
+Dynamic PMD Extension
+---------------------
+
+Dynamic PMD extensions allow users to customize net/ark functionality
+using their own code. Arkville RTL and this PMD support high-throughput data
+movement, and these extensions allow PMD support for users' FPGA
+features.
+Dynamic PMD extensions operate by having users supply a shared object
+file which is loaded by Arkville PMD during initialization.  The
+object file contains extension (or hook) functions that are registered
+and then called during PMD operations.
+
+The allowable set of extension functions are defined and documented in
+``rte_pmd_ark.h``, only the initialization function,
+``rte_pmd_ark_dev_init()``, is required; all others are optional. The
+following sections give a small extension example along with
+instructions for compiling and using the extension.
+
+
+Extension Example
+^^^^^^^^^^^^^^^^^
+
+The following example shows an extension which populates mbuf fields
+during RX from user meta data coming from FPGA hardware.
+
+.. code-block:: c
+
+   #include <rte_pmd_ark.h>
+   #include <rte_mbuf.h>
+   #include <rte_ethdev.h>
+   #include <rte_malloc.h>
+
+   /* Global structure passed to extension/hook functions */
+   struct ark_user_extension {
+       int timestamp_dynfield_offset;
+   };
+
+   /* RX tuser field based on user's hardware */
+   struct user_rx_meta {
+      uint64_t timestamp;
+      uint32_t rss;
+   } __rte_packed;
+
+   /* Create ark_user_extension object for use in other hook functions */
+   void *rte_pmd_ark_dev_init(struct rte_eth_dev * dev,
+                              void * abar, int port_id )
+   {
+      RTE_SET_USED(dev);
+      RTE_SET_USED(abar);
+      fprintf(stderr, "Called Arkville user extension for port %u\n",
+              port_id);
+
+      struct ark_user_extension *xdata = rte_zmalloc("macExtS",
+             sizeof(struct ark_user_extension), 64);
+      if (!xdata)
+         return NULL;
+
+      /* register dynfield for rx timestamp */
+      rte_mbuf_dyn_rx_timestamp_register(&xdata->timestamp_dynfield_offset,
+                                         NULL);
+
+      fprintf(stderr, "timestamp fields offset in extension is %d\n",
+              xdata->timestamp_dynfield_offset);
+      return xdata;
+   }
+
+   /* uninitialization */
+   void rte_pmd_ark_dev_uninit(struct rte_eth_dev * dev, void *user_data)
+   {
+      rte_free(user_data);
+   }
+
+   /* Hook function -- called for each RX packet
+    * Extract RX timestamp and RSS from meta and place in mbuf
+    */
+   void rte_pmd_ark_rx_user_meta_hook(struct rte_mbuf *mbuf,
+                                      const uint32_t *meta,
+                                      void *user_data)
+   {
+      struct ark_user_extension *xdata = user_data;
+      struct user_rx_meta *user_rx = (struct user_rx_meta*)meta;
+      *RTE_MBUF_DYNFIELD(mbuf, xdata->timestamp_dynfield_offset, uint64_t*) =
+                         user_rx->timestamp;
+      mbuf->hash.rss = user_rx->rss;
+   }
+
+
+Compiling Extension
+^^^^^^^^^^^^^^^^^^^
+
+It is recommended to the compile the extension code with
+``-Wmissing-prototypes`` flag to insure correct function types. Typical
+DPDK options will also be needed.
+
+
+An example command line is give below
+
+.. code-block:: console
+
+    cc `pkg-config --cflags libdpdk` \
+    -O3 -DALLOW_EXPERIMENTAL_API -fPIC -Wall -Wmissing-prototypes -c \
+    -o pmd_net_ark_ext.o pmd_net_ark_ext.c
+    # Linking
+    cc -o libfx1_100g_ext.so.1 -shared \
+    `pkg-config --libs libdpdk` \
+    -Wl,--unresolved-symbols=ignore-all \
+    -Wl,-soname,libpmd_net_ark_ext.so.1 pmd_net_ark_ext.o
+
+In a ``Makefile`` this would be
+
+.. code-block:: Makefile
+
+   CFLAGS += $(shell pkg-config --cflags libdpdk)
+   CFLAGS += -O3 -DALLOW_EXPERIMENTAL_API -fPIC -Wall -Wmissing-prototypes
+   # Linking
+   LDFLAGS += $(shell pkg-config --libs libdpdk)
+   LDFLAGS += -Wl,--unresolved-symbols=ignore-all -Wl,-soname,libpmd_net_ark_ext.so.1
+
+The application must be linked with the ``-export-dynamic`` flags if any
+DPDK or application specific code will called from the extension.
+
+
+Enabling Extension
+^^^^^^^^^^^^^^^^^^
+
+The extensions are enabled in the application through the use of an
+environment variable ``ARK_EXT_PATH`` This variable points to the lib
+extension file generated above.  For example:
+
+.. code-block:: console
+
+   export ARK_EXT_PATH=$(PWD)/libpmd_net_ark_ext.so.1
+   testpmd ...
+
+
 Building DPDK
 -------------
 
@@ -144,6 +279,8 @@ documentation that comes with DPDK suite <linux_gsg>`.
 To build with a non-zero minimum tx packet length, set the above macro in your
 CFLAGS environment prior to the meson build step. I.e.,
 
+.. code-block:: console
+
     export CFLAGS="-DRTE_LIBRTE_ARK_MIN_TX_PKTLEN=60"
     meson build
 
diff --git a/doc/guides/rel_notes/release_21_05.rst b/doc/guides/rel_notes/release_21_05.rst
index 5aa9ed7db..4789bf782 100644
--- a/doc/guides/rel_notes/release_21_05.rst
+++ b/doc/guides/rel_notes/release_21_05.rst
@@ -55,6 +55,17 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Updated Arkville PMD driver.**
+
+  Updated Arkville net driver with new features and improvements, including:
+
+  * Generalized passing meta data between PMD and FPGA, allowing up to 20
+    bytes of user specified information in RX and TX paths.
+
+  * Updated dynamic PMD extensions API using standardized names.
+
+  * Added support for new Atomic Rules PCI device IDs ``0x100f, 0x1010, 0x1017,
+    0x1018, 0x1019``.
 
 Removed Items
 -------------
diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c
index 5282534d3..bdebfdba4 100644
--- a/drivers/net/ark/ark_ethdev.c
+++ b/drivers/net/ark/ark_ethdev.c
@@ -79,12 +79,6 @@ static int  eth_ark_set_mtu(struct rte_eth_dev *dev, uint16_t size);
 #define ARK_TX_MAX_QUEUE (4096 * 4)
 #define ARK_TX_MIN_QUEUE (256)
 
-uint64_t ark_timestamp_rx_dynflag;
-int ark_timestamp_dynfield_offset = -1;
-
-int rte_pmd_ark_rx_userdata_dynfield_offset = -1;
-int rte_pmd_ark_tx_userdata_dynfield_offset = -1;
-
 static const char * const valid_arguments[] = {
 	ARK_PKTGEN_ARG,
 	ARK_PKTCHKR_ARG,
@@ -245,6 +239,12 @@ check_for_ext(struct ark_adapter *ark)
 		(int (*)(struct rte_eth_dev *, uint16_t,
 			  void *))
 		dlsym(ark->d_handle, "rte_pmd_ark_set_mtu");
+	ark->user_ext.rx_user_meta_hook =
+		(rx_user_meta_hook_fn)dlsym(ark->d_handle,
+					    "rte_pmd_ark_rx_user_meta_hook");
+	ark->user_ext.tx_user_meta_hook =
+		(tx_user_meta_hook_fn)dlsym(ark->d_handle,
+					    "rte_pmd_ark_tx_user_meta_hook");
 
 	return found;
 }
@@ -257,16 +257,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 	int ret;
 	int port_count = 1;
 	int p;
-	static const struct rte_mbuf_dynfield ark_tx_userdata_dynfield_desc = {
-		.name = RTE_PMD_ARK_TX_USERDATA_DYNFIELD_NAME,
-		.size = sizeof(rte_pmd_ark_tx_userdata_t),
-		.align = __alignof__(rte_pmd_ark_tx_userdata_t),
-	};
-	static const struct rte_mbuf_dynfield ark_rx_userdata_dynfield_desc = {
-		.name = RTE_PMD_ARK_RX_USERDATA_DYNFIELD_NAME,
-		.size = sizeof(rte_pmd_ark_rx_userdata_t),
-		.align = __alignof__(rte_pmd_ark_rx_userdata_t),
-	};
 
 	ark->eth_dev = dev;
 
@@ -277,30 +267,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 	if (ret)
 		return ret;
 
-	/* Extra mbuf fields for user data */
-	if (RTE_PMD_ARK_TX_USERDATA_ENABLE) {
-		rte_pmd_ark_tx_userdata_dynfield_offset =
-		    rte_mbuf_dynfield_register(&ark_tx_userdata_dynfield_desc);
-		if (rte_pmd_ark_tx_userdata_dynfield_offset < 0) {
-			ARK_PMD_LOG(ERR,
-				    "Failed to register mbuf field for tx userdata\n");
-			return -rte_errno;
-		}
-		ARK_PMD_LOG(INFO, "Registered TX-meta dynamic field at %d\n",
-			    rte_pmd_ark_tx_userdata_dynfield_offset);
-	}
-	if (RTE_PMD_ARK_RX_USERDATA_ENABLE) {
-		rte_pmd_ark_rx_userdata_dynfield_offset =
-		    rte_mbuf_dynfield_register(&ark_rx_userdata_dynfield_desc);
-		if (rte_pmd_ark_rx_userdata_dynfield_offset < 0) {
-			ARK_PMD_LOG(ERR,
-				    "Failed to register mbuf field for rx userdata\n");
-			return -rte_errno;
-		}
-		ARK_PMD_LOG(INFO, "Registered RX-meta dynamic field at %d\n",
-			    rte_pmd_ark_rx_userdata_dynfield_offset);
-	}
-
 	pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 	rte_eth_copy_pci_info(dev, pci_dev);
 	dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
@@ -561,18 +527,6 @@ static int
 eth_ark_dev_configure(struct rte_eth_dev *dev)
 {
 	struct ark_adapter *ark = dev->data->dev_private;
-	int ret;
-
-	if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_TIMESTAMP) {
-		ret = rte_mbuf_dyn_rx_timestamp_register(
-				&ark_timestamp_dynfield_offset,
-				&ark_timestamp_rx_dynflag);
-		if (ret != 0) {
-			ARK_PMD_LOG(ERR,
-				"Failed to register Rx timestamp field/flag\n");
-			return -rte_errno;
-		}
-	}
 
 	eth_ark_dev_set_link_up(dev);
 	if (ark->user_ext.dev_configure)
diff --git a/drivers/net/ark/ark_ethdev_rx.c b/drivers/net/ark/ark_ethdev_rx.c
index 21a9af41a..48ea48404 100644
--- a/drivers/net/ark/ark_ethdev_rx.c
+++ b/drivers/net/ark/ark_ethdev_rx.c
@@ -39,6 +39,9 @@ struct ark_rx_queue {
 	struct ark_udm_t *udm;
 	struct ark_mpu_t *mpu;
 
+	rx_user_meta_hook_fn rx_user_meta_hook;
+	void *ext_user_data;
+
 	uint32_t queue_size;
 	uint32_t queue_mask;
 
@@ -53,8 +56,7 @@ struct ark_rx_queue {
 
 	uint32_t unused;
 
-	/* separate cache line */
-	/* second cache line - fields only used in slow path */
+	/* next cache line - fields written by device */
 	RTE_MARKER cacheline1 __rte_cache_min_aligned;
 
 	volatile uint32_t prod_index;	/* step 2 filled by FPGA */
@@ -167,6 +169,8 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	queue->queue_index = queue_idx;
 	queue->queue_size = nb_desc;
 	queue->queue_mask = nb_desc - 1;
+	queue->rx_user_meta_hook = ark->user_ext.rx_user_meta_hook;
+	queue->ext_user_data = ark->user_data[dev->data->port_id];
 
 	queue->reserve_q =
 		rte_zmalloc_socket("Ark_rx_queue mbuf",
@@ -243,8 +247,11 @@ eth_ark_recv_pkts(void *rx_queue,
 	struct ark_rx_queue *queue;
 	register uint32_t cons_index, prod_index;
 	uint16_t nb;
+	uint16_t i;
 	struct rte_mbuf *mbuf;
+	struct rte_mbuf **pmbuf;
 	struct ark_rx_meta *meta;
+	rx_user_meta_hook_fn rx_user_meta_hook;
 
 	queue = (struct ark_rx_queue *)rx_queue;
 	if (unlikely(queue == 0))
@@ -253,6 +260,8 @@ eth_ark_recv_pkts(void *rx_queue,
 		return 0;
 	prod_index = queue->prod_index;
 	cons_index = queue->cons_index;
+	if (prod_index == cons_index)
+		return 0;
 	nb = 0;
 
 	while (prod_index != cons_index) {
@@ -266,13 +275,6 @@ eth_ark_recv_pkts(void *rx_queue,
 
 		mbuf->pkt_len = meta->pkt_len;
 		mbuf->data_len = meta->pkt_len;
-		/* set timestamp if enabled at least on one device */
-		if (ark_timestamp_rx_dynflag > 0) {
-			*RTE_MBUF_DYNFIELD(mbuf, ark_timestamp_dynfield_offset,
-				rte_mbuf_timestamp_t *) = meta->timestamp;
-			mbuf->ol_flags |= ark_timestamp_rx_dynflag;
-		}
-		rte_pmd_ark_mbuf_rx_userdata_set(mbuf, meta->user_data);
 
 		if (ARK_DEBUG_CORE) {	/* debug sanity checks */
 			if ((meta->pkt_len > (1024 * 16)) ||
@@ -315,6 +317,13 @@ eth_ark_recv_pkts(void *rx_queue,
 			break;
 	}
 
+	rx_user_meta_hook = queue->rx_user_meta_hook;
+	for (pmbuf = rx_pkts, i = 0; rx_user_meta_hook && i < nb; i++) {
+		mbuf = *pmbuf++;
+		meta = RTE_PTR_ADD(mbuf->buf_addr, ARK_RX_META_OFFSET);
+		rx_user_meta_hook(mbuf, meta->user_meta, queue->ext_user_data);
+	}
+
 	eth_ark_rx_update_cons_index(queue, cons_index);
 
 	return nb;
diff --git a/drivers/net/ark/ark_ethdev_rx.h b/drivers/net/ark/ark_ethdev_rx.h
index 33c1f2c95..c8dc340a8 100644
--- a/drivers/net/ark/ark_ethdev_rx.h
+++ b/drivers/net/ark/ark_ethdev_rx.h
@@ -11,9 +11,6 @@
 #include <rte_mempool.h>
 #include <ethdev_driver.h>
 
-extern uint64_t ark_timestamp_rx_dynflag;
-extern int ark_timestamp_dynfield_offset;
-
 int eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev,
 			       uint16_t queue_idx,
 			       uint16_t nb_desc,
diff --git a/drivers/net/ark/ark_ethdev_tx.c b/drivers/net/ark/ark_ethdev_tx.c
index 00e5dbf7c..d6e8345b2 100644
--- a/drivers/net/ark/ark_ethdev_tx.c
+++ b/drivers/net/ark/ark_ethdev_tx.c
@@ -33,6 +33,9 @@ struct ark_tx_queue {
 	/* Stats HW tracks bytes and packets, need to count send errors */
 	uint64_t tx_errors;
 
+	tx_user_meta_hook_fn tx_user_meta_hook;
+	void *ext_user_data;
+
 	uint32_t queue_size;
 	uint32_t queue_mask;
 
@@ -45,9 +48,7 @@ struct ark_tx_queue {
 	/* The queue Index within the dpdk device structures */
 	uint16_t queue_index;
 
-	uint32_t pad[1];
-
-	/* second cache line - fields written by device */
+	/* next cache line - fields written by device */
 	RTE_MARKER cacheline1 __rte_cache_min_aligned;
 	volatile int32_t cons_index;		/* hw is done, can be freed */
 } __rte_cache_aligned;
@@ -120,15 +121,17 @@ eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 {
 	struct ark_tx_queue *queue;
 	struct rte_mbuf *mbuf;
-	uint32_t user_meta;
+	uint32_t user_meta[5];
 
 	int stat;
 	int32_t prod_index_limit;
 	uint16_t nb;
-	uint8_t user_len = 1;
+	uint8_t user_len = 0;
 	const uint32_t min_pkt_len = ARK_MIN_TX_PKTLEN;
+	tx_user_meta_hook_fn tx_user_meta_hook;
 
 	queue = (struct ark_tx_queue *)vtxq;
+	tx_user_meta_hook = queue->tx_user_meta_hook;
 
 	/* free any packets after the HW is done with them */
 	free_completed_tx(queue);
@@ -163,16 +166,18 @@ eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 			memset(appended, 0, to_add);
 		}
 
-		user_meta = rte_pmd_ark_mbuf_tx_userdata_get(mbuf);
+		if (tx_user_meta_hook)
+			tx_user_meta_hook(mbuf, user_meta, &user_len,
+					  queue->ext_user_data);
 		if (unlikely(mbuf->nb_segs != 1)) {
 			stat = eth_ark_tx_jumbo(queue, mbuf,
-						&user_meta, user_len);
+						user_meta, user_len);
 			if (unlikely(stat != 0))
 				break;		/* Queue is full */
 		} else {
 			eth_ark_tx_desc_fill(queue, mbuf,
 					     ARK_DDM_SOP | ARK_DDM_EOP,
-					     &user_meta, user_len);
+					     user_meta, user_len);
 		}
 	}
 
@@ -271,6 +276,8 @@ eth_ark_tx_queue_setup(struct rte_eth_dev *dev,
 	queue->phys_qid = qidx;
 	queue->queue_index = queue_idx;
 	dev->data->tx_queues[queue_idx] = queue;
+	queue->tx_user_meta_hook = ark->user_ext.tx_user_meta_hook;
+	queue->ext_user_data = ark->user_data[dev->data->port_id];
 
 	queue->meta_q =
 		rte_zmalloc_socket("Ark_txqueue meta",
diff --git a/drivers/net/ark/ark_global.h b/drivers/net/ark/ark_global.h
index 91726ecc2..6f9b3013d 100644
--- a/drivers/net/ark/ark_global.h
+++ b/drivers/net/ark/ark_global.h
@@ -57,6 +57,23 @@
 		void     *v;	   \
 	} name
 
+
+/* Extension hooks for extraction and placement of user meta data
+ * during RX an TX operations. These functions are the bridge
+ * between the mbuf struct and the tuser fields on the AXIS
+ * interfaces in the FPGA
+ */
+/* RX hook populates mbuf fields from user defined *meta up to 20 bytes */
+typedef void (*rx_user_meta_hook_fn)(struct rte_mbuf *mbuf,
+				     const uint32_t *meta,
+				     void *ext_user_data);
+/* TX hook poplulate *meta, with up to 20 bytes.  meta_cnt
+ * returns the number of uint32_t words populated, 0 to 5
+ */
+typedef void (*tx_user_meta_hook_fn)(const struct rte_mbuf *mbuf,
+				     uint32_t *meta, uint8_t *meta_cnt,
+				     void *ext_user_data);
+
 struct ark_user_ext {
 	void *(*dev_init)(struct rte_eth_dev *, void *abar, int port_id);
 	void (*dev_uninit)(struct rte_eth_dev *, void *);
@@ -79,6 +96,9 @@ struct ark_user_ext {
 	void (*mac_addr_set)(struct rte_eth_dev *, struct rte_ether_addr *,
 			void *);
 	int (*set_mtu)(struct rte_eth_dev *, uint16_t, void *);
+	/* user meta, hook functions  */
+	rx_user_meta_hook_fn rx_user_meta_hook;
+	tx_user_meta_hook_fn tx_user_meta_hook;
 };
 
 struct ark_adapter {
diff --git a/drivers/net/ark/ark_udm.h b/drivers/net/ark/ark_udm.h
index ea92d4b6e..4e51a5e82 100644
--- a/drivers/net/ark/ark_udm.h
+++ b/drivers/net/ark/ark_udm.h
@@ -19,9 +19,8 @@
  * -- 32 bytes
  */
 struct ark_rx_meta {
-	uint64_t timestamp;
-	uint64_t user_data;
-	uint8_t  reserved[14];
+	uint32_t user_meta[5];	/* user defined based on fpga code */
+	uint8_t  reserved[10];
 	uint16_t pkt_len;
 } __rte_packed;
 
diff --git a/drivers/net/ark/rte_pmd_ark.h b/drivers/net/ark/rte_pmd_ark.h
index 0f24a347d..7c093711f 100644
--- a/drivers/net/ark/rte_pmd_ark.h
+++ b/drivers/net/ark/rte_pmd_ark.h
@@ -5,9 +5,6 @@
 #ifndef RTE_PMD_ARK_H
 #define RTE_PMD_ARK_H
 
-#include <rte_mbuf.h>
-#include <rte_mbuf_dyn.h>
-
 #include <stdint.h>
 struct rte_eth_dev;
 struct rte_mbuf;
@@ -19,115 +16,6 @@ struct rte_eth_stats;
  * ARK driver-specific API
  */
 
-#ifndef RTE_PMD_ARK_TX_USERDATA_ENABLE
-#define RTE_PMD_ARK_TX_USERDATA_ENABLE 0
-#endif
-
-#ifndef RTE_PMD_ARK_RX_USERDATA_ENABLE
-#define RTE_PMD_ARK_RX_USERDATA_ENABLE 0
-#endif
-
-typedef uint32_t rte_pmd_ark_tx_userdata_t;
-typedef uint64_t rte_pmd_ark_rx_userdata_t;
-
-extern int rte_pmd_ark_tx_userdata_dynfield_offset;
-extern int rte_pmd_ark_rx_userdata_dynfield_offset;
-
-/** mbuf dynamic field for custom Tx ARK data */
-#define RTE_PMD_ARK_TX_USERDATA_DYNFIELD_NAME "rte_net_ark_dynfield_tx_userdata"
-/** mbuf dynamic field for custom Rx ARK data */
-#define RTE_PMD_ARK_RX_USERDATA_DYNFIELD_NAME "rte_net_ark_dynfield_rx_userdata"
-
-/**
- * @warning
- * @b EXPERIMENTAL: this API may change without prior notice
- *
- * Read Tx user data from mbuf.
- *
- * @param mbuf Structure to read from.
- * @return user data
- */
-__rte_experimental
-static inline rte_pmd_ark_tx_userdata_t
-rte_pmd_ark_mbuf_tx_userdata_get(const struct rte_mbuf *mbuf)
-{
-#if RTE_PMD_ARK_TX_USERDATA_ENABLE
-	return *RTE_MBUF_DYNFIELD(mbuf, rte_pmd_ark_tx_userdata_dynfield_offset,
-				  rte_pmd_ark_tx_userdata_t *);
-#else
-	RTE_SET_USED(mbuf);
-	return 0;
-#endif
-}
-
-/**
- * @warning
- * @b EXPERIMENTAL: this API may change without prior notice
- *
- * Write Tx user data to mbuf.
- *
- * @param mbuf Structure to write into.
- * @param data User data.
- */
-__rte_experimental
-static inline void
-rte_pmd_ark_mbuf_tx_userdata_set(struct rte_mbuf *mbuf,
-		rte_pmd_ark_tx_userdata_t data)
-{
-#if RTE_PMD_ARK_TX_USERDATA_ENABLE
-	*RTE_MBUF_DYNFIELD(mbuf, rte_pmd_ark_tx_userdata_dynfield_offset,
-			rte_pmd_ark_tx_userdata_t *) = data;
-#else
-	RTE_SET_USED(mbuf);
-	RTE_SET_USED(data);
-#endif
-}
-
-/**
- * @warning
- * @b EXPERIMENTAL: this API may change without prior notice
- *
- * Read Rx user data from mbuf.
- *
- * @param mbuf Structure to read from.
- * @return user data
- */
-__rte_experimental
-static inline rte_pmd_ark_rx_userdata_t
-rte_pmd_ark_mbuf_rx_userdata_get(const struct rte_mbuf *mbuf)
-{
-#if RTE_PMD_ARK_RX_USERDATA_ENABLE
-	return *RTE_MBUF_DYNFIELD(mbuf, rte_pmd_ark_rx_userdata_dynfield_offset,
-			rte_pmd_ark_rx_userdata_t *);
-#else
-	RTE_SET_USED(mbuf);
-	return 0;
-#endif
-}
-
-/**
- * @warning
- * @b EXPERIMENTAL: this API may change without prior notice
- *
- * Write Rx user data to mbuf.
- *
- * @param mbuf Structure to write into.
- * @param data User data.
- */
-__rte_experimental
-static inline void
-rte_pmd_ark_mbuf_rx_userdata_set(struct rte_mbuf *mbuf,
-		rte_pmd_ark_rx_userdata_t data)
-{
-#if RTE_PMD_ARK_RX_USERDATA_ENABLE
-	*RTE_MBUF_DYNFIELD(mbuf, rte_pmd_ark_rx_userdata_dynfield_offset,
-			rte_pmd_ark_rx_userdata_t *) = data;
-#else
-	RTE_SET_USED(mbuf);
-	RTE_SET_USED(data);
-#endif
-}
-
 /* The following section lists function prototypes for Arkville's
  * dynamic PMD extension. User's who create an extension
  * must include this file and define the necessary and desired
@@ -365,4 +253,43 @@ int rte_pmd_ark_set_mtu(struct rte_eth_dev *dev,
 			uint16_t size,
 			void *user_data);
 
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_rx_burst() for each packet. This extension
+ * function allows the transfer of meta data from the user's FPGA to
+ * mbuf fields.
+ *
+ * @param mbuf
+ *   The newly received mbuf
+ * @param meta
+ *   The meta data from the user, up to 20 bytes. The
+ *   underlying data in the PMD is of type uint32_t meta[5];
+ * @param user_data
+ *   user argument from dev_init() call.
+ */
+void rte_pmd_ark_rx_user_meta_hook(struct rte_mbuf *mbuf,
+				   const uint32_t *meta,
+				   void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_tx_burst() for each packet. This extension
+ * function allows the transfer of data from the mbuf to the user's
+ * FPGA.  Up to 20 bytes (5 32-bit words) are transferable
+ *
+ * @param mbuf
+ *   The mbuf about to be transmitted.
+ * @param meta
+ *   The meta data to be populate by this call. The
+ *   underlying in the PMD is of type uint32_t meta[5];
+ * @param meta_cnt
+ *   The count in 32-bit words of the meta data populated, 0 to 5.
+ * @param user_data
+ *   user argument from dev_init() call.
+ */
+void rte_pmd_ark_tx_user_meta_hook(const struct rte_mbuf *mbuf,
+				   uint32_t *meta,
+				   uint8_t *meta_cnt,
+				   void *user_data);
+
 #endif /* RTE_PMD_ARK_H */
diff --git a/drivers/net/ark/version.map b/drivers/net/ark/version.map
index 954bea679..4a76d1d52 100644
--- a/drivers/net/ark/version.map
+++ b/drivers/net/ark/version.map
@@ -1,10 +1,3 @@
 DPDK_21 {
 	local: *;
 };
-
-EXPERIMENTAL {
-	global:
-
-	rte_pmd_ark_tx_userdata_dynfield_offset;
-	rte_pmd_ark_rx_userdata_dynfield_offset;
-};
-- 
2.17.1