[dpdk-dev] [PATCH 0/6] Add Marvell CNXK mempool driver

[dpdk-dev] [PATCH 0/6] Add Marvell CNXK mempool driver

[Ashwin Sekhar T K]

This patchset adds the mempool/cnxk driver which provides the support for the
integrated mempool device found in Marvell CN10K SoC.

The code includes mempool driver functionality for Marvell CN9K SoC as well,
but right now it is not enabled. The future plan is to deprecate existing
mempool/octeontx2 driver once the 'CNXK' drivers are feature complete for
Marvell CN9K SoC.

Depends-on: series-15508 ("Add Marvell CNXK common driver")

Ashwin Sekhar T K (6):
  mempool/cnxk: add build infra and device probe
  mempool/cnxk: add generic ops
  mempool/cnxk: add cn9k mempool ops
  mempool/cnxk: add base cn10k mempool ops
  mempool/cnxk: add cn10k batch enqueue/dequeue support
  doc: add Marvell CNXK mempool documentation

 MAINTAINERS                              |   6 +
 doc/guides/mempool/cnxk.rst              |  84 +++++++
 doc/guides/mempool/index.rst             |   1 +
 doc/guides/platform/cnxk.rst             |   3 +
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 294 +++++++++++++++++++++++
 drivers/mempool/cnxk/cn9k_mempool_ops.c  |  90 +++++++
 drivers/mempool/cnxk/cnxk_mempool.c      | 201 ++++++++++++++++
 drivers/mempool/cnxk/cnxk_mempool.h      |  29 +++
 drivers/mempool/cnxk/cnxk_mempool_ops.c  | 201 ++++++++++++++++
 drivers/mempool/cnxk/meson.build         |  32 +++
 drivers/mempool/cnxk/version.map         |   3 +
 drivers/mempool/meson.build              |   3 +-
 12 files changed, 946 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/mempool/cnxk.rst
 create mode 100644 drivers/mempool/cnxk/cn10k_mempool_ops.c
 create mode 100644 drivers/mempool/cnxk/cn9k_mempool_ops.c
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.c
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.h
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool_ops.c
 create mode 100644 drivers/mempool/cnxk/meson.build
 create mode 100644 drivers/mempool/cnxk/version.map

-- 
2.29.2

[dpdk-dev] [PATCH 1/6] mempool/cnxk: add build infra and device probe

[Ashwin Sekhar T K]

Add the meson based build infrastructure along
with mempool device probe.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cnxk_mempool.c | 212 ++++++++++++++++++++++++++++
 drivers/mempool/cnxk/cnxk_mempool.h |  12 ++
 drivers/mempool/cnxk/meson.build    |  29 ++++
 drivers/mempool/cnxk/version.map    |   3 +
 drivers/mempool/meson.build         |   3 +-
 5 files changed, 258 insertions(+), 1 deletion(-)
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.c
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.h
 create mode 100644 drivers/mempool/cnxk/meson.build
 create mode 100644 drivers/mempool/cnxk/version.map

diff --git a/drivers/mempool/cnxk/cnxk_mempool.c b/drivers/mempool/cnxk/cnxk_mempool.c
new file mode 100644
index 0000000000..c24497a6e5
--- /dev/null
+++ b/drivers/mempool/cnxk/cnxk_mempool.c
@@ -0,0 +1,212 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_atomic.h>
+#include <rte_bus_pci.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+#include <rte_eal.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_mbuf_pool_ops.h>
+#include <rte_pci.h>
+
+#include "roc_api.h"
+#include "cnxk_mempool.h"
+
+#define CNXK_NPA_DEV_NAME	 RTE_STR(cnxk_npa_dev_)
+#define CNXK_NPA_DEV_NAME_LEN	 (sizeof(CNXK_NPA_DEV_NAME) + PCI_PRI_STR_SIZE)
+#define CNXK_NPA_MAX_POOLS_PARAM "max_pools"
+
+uintptr_t *cnxk_mempool_internal_data;
+
+static inline uint32_t
+npa_aura_size_to_u32(uint8_t val)
+{
+	if (val == NPA_AURA_SZ_0)
+		return 128;
+	if (val >= NPA_AURA_SZ_MAX)
+		return BIT_ULL(20);
+
+	return 1 << (val + 6);
+}
+
+static int
+parse_max_pools(const char *key, const char *value, void *extra_args)
+{
+	RTE_SET_USED(key);
+	uint32_t val;
+
+	val = atoi(value);
+	if (val < npa_aura_size_to_u32(NPA_AURA_SZ_128))
+		val = 128;
+	if (val > npa_aura_size_to_u32(NPA_AURA_SZ_1M))
+		val = BIT_ULL(20);
+
+	*(uint8_t *)extra_args = rte_log2_u32(val) - 6;
+	return 0;
+}
+
+static inline uint8_t
+parse_aura_size(struct rte_devargs *devargs)
+{
+	uint8_t aura_sz = NPA_AURA_SZ_128;
+	struct rte_kvargs *kvlist;
+
+	if (devargs == NULL)
+		goto exit;
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL)
+		goto exit;
+
+	rte_kvargs_process(kvlist, CNXK_NPA_MAX_POOLS_PARAM, &parse_max_pools,
+			   &aura_sz);
+	rte_kvargs_free(kvlist);
+exit:
+	return aura_sz;
+}
+
+static inline char *
+npa_dev_to_name(struct rte_pci_device *pci_dev, char *name)
+{
+	snprintf(name, CNXK_NPA_DEV_NAME_LEN, CNXK_NPA_DEV_NAME PCI_PRI_FMT,
+		 pci_dev->addr.domain, pci_dev->addr.bus, pci_dev->addr.devid,
+		 pci_dev->addr.function);
+
+	return name;
+}
+
+static int
+npa_init(struct rte_pci_device *pci_dev)
+{
+	char name[CNXK_NPA_DEV_NAME_LEN];
+	size_t idata_offset, idata_sz;
+	const struct rte_memzone *mz;
+	struct roc_npa *dev;
+	int rc, maxpools;
+
+	rc = plt_init();
+	if (rc < 0)
+		goto error;
+
+	maxpools = parse_aura_size(pci_dev->device.devargs);
+	/* Add the space for per-pool internal data pointers to memzone len */
+	idata_offset = RTE_ALIGN_CEIL(sizeof(*dev), ROC_ALIGN);
+	idata_sz = maxpools * sizeof(uintptr_t);
+
+	rc = -ENOMEM;
+	mz = rte_memzone_reserve_aligned(npa_dev_to_name(pci_dev, name),
+					 idata_offset + idata_sz, SOCKET_ID_ANY,
+					 0, RTE_CACHE_LINE_SIZE);
+	if (mz == NULL)
+		goto error;
+
+	dev = mz->addr;
+	dev->pci_dev = pci_dev;
+	cnxk_mempool_internal_data = (uintptr_t *)(mz->addr_64 + idata_offset);
+	memset(cnxk_mempool_internal_data, 0, idata_sz);
+
+	roc_idev_npa_maxpools_set(maxpools);
+	rc = roc_npa_dev_init(dev);
+	if (rc)
+		goto mz_free;
+
+	return 0;
+
+mz_free:
+	rte_memzone_free(mz);
+error:
+	plt_err("failed to initialize npa device rc=%d", rc);
+	return rc;
+}
+
+static int
+npa_fini(struct rte_pci_device *pci_dev)
+{
+	char name[CNXK_NPA_DEV_NAME_LEN];
+	const struct rte_memzone *mz;
+	int rc;
+
+	mz = rte_memzone_lookup(npa_dev_to_name(pci_dev, name));
+	if (mz == NULL)
+		return -EINVAL;
+
+	rc = roc_npa_dev_fini(mz->addr);
+	if (rc) {
+		if (rc != -EAGAIN)
+			plt_err("Failed to remove npa dev, rc=%d", rc);
+		return rc;
+	}
+	rte_memzone_free(mz);
+
+	return 0;
+}
+
+static int
+npa_remove(struct rte_pci_device *pci_dev)
+{
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	return npa_fini(pci_dev);
+}
+
+static int
+npa_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
+{
+	RTE_SET_USED(pci_drv);
+
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	return npa_init(pci_dev);
+}
+
+static const struct rte_pci_id npa_pci_map[] = {
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_PF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KA,
+	},
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_PF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KAS,
+	},
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_VF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KA,
+	},
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_VF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KAS,
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static struct rte_pci_driver npa_pci = {
+	.id_table = npa_pci_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_NEED_IOVA_AS_VA,
+	.probe = npa_probe,
+	.remove = npa_remove,
+};
+
+RTE_PMD_REGISTER_PCI(mempool_cnxk, npa_pci);
+RTE_PMD_REGISTER_PCI_TABLE(mempool_cnxk, npa_pci_map);
+RTE_PMD_REGISTER_KMOD_DEP(mempool_cnxk, "vfio-pci");
+RTE_PMD_REGISTER_PARAM_STRING(mempool_cnxk,
+			      CNXK_NPA_MAX_POOLS_PARAM "=<128-1048576>");
diff --git a/drivers/mempool/cnxk/cnxk_mempool.h b/drivers/mempool/cnxk/cnxk_mempool.h
new file mode 100644
index 0000000000..4ee3d236f2
--- /dev/null
+++ b/drivers/mempool/cnxk/cnxk_mempool.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#ifndef _CNXK_MEMPOOL_H_
+#define _CNXK_MEMPOOL_H_
+
+#include <rte_mempool.h>
+
+extern uintptr_t *cnxk_mempool_internal_data;
+
+#endif
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
new file mode 100644
index 0000000000..23a171c143
--- /dev/null
+++ b/drivers/mempool/cnxk/meson.build
@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(C) 2021 Marvell.
+#
+
+if is_windows
+	build = false
+	reason = 'not supported on Windows'
+	subdir_done()
+endif
+if not dpdk_conf.get('RTE_ARCH_64')
+	build = false
+	reason = 'only supported on 64-bit'
+	subdir_done()
+endif
+
+sources = files('cnxk_mempool.c')
+
+deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
+
+cflags_options = [
+	'-Wno-strict-prototypes',
+        '-Werror'
+]
+
+foreach option:cflags_options
+        if cc.has_argument(option)
+                cflags += option
+        endif
+endforeach
diff --git a/drivers/mempool/cnxk/version.map b/drivers/mempool/cnxk/version.map
new file mode 100644
index 0000000000..ee80c51721
--- /dev/null
+++ b/drivers/mempool/cnxk/version.map
@@ -0,0 +1,3 @@
+INTERNAL {
+	local: *;
+};
diff --git a/drivers/mempool/meson.build b/drivers/mempool/meson.build
index 4428813dae..a2814c1dfa 100644
--- a/drivers/mempool/meson.build
+++ b/drivers/mempool/meson.build
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-drivers = ['bucket', 'dpaa', 'dpaa2', 'octeontx', 'octeontx2', 'ring', 'stack']
+drivers = ['bucket', 'cnxk', 'dpaa', 'dpaa2', 'octeontx', 'octeontx2', 'ring',
+	   'stack']
 std_deps = ['mempool']
-- 
2.29.2

Re: [dpdk-dev] [PATCH 1/6] mempool/cnxk: add build infra and device probe

[Jerin Jacob]

On Fri, Mar 5, 2021 at 11:43 PM Ashwin Sekhar T K <asekhar@marvell.com> wrote:
>
> Add the meson based build infrastructure along
> with mempool device probe.
>
> Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
> ---
>  drivers/mempool/cnxk/cnxk_mempool.c | 212 ++++++++++++++++++++++++++++
>  drivers/mempool/cnxk/cnxk_mempool.h |  12 ++
>  drivers/mempool/cnxk/meson.build    |  29 ++++
>  drivers/mempool/cnxk/version.map    |   3 +
>  drivers/mempool/meson.build         |   3 +-
>  5 files changed, 258 insertions(+), 1 deletion(-)
>  create mode 100644 drivers/mempool/cnxk/cnxk_mempool.c
>  create mode 100644 drivers/mempool/cnxk/cnxk_mempool.h
>  create mode 100644 drivers/mempool/cnxk/meson.build
>  create mode 100644 drivers/mempool/cnxk/version.map
>
> diff --git a/drivers/mempool/cnxk/cnxk_mempool.c b/drivers/mempool/cnxk/cnxk_mempool.c
> new file mode 100644
> index 0000000000..c24497a6e5
> --- /dev/null
> +++ b/drivers/mempool/cnxk/cnxk_mempool.c
> @@ -0,0 +1,212 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(C) 2021 Marvell.
> + */
> +
> +#include <rte_atomic.h>
> +#include <rte_bus_pci.h>
> +#include <rte_common.h>
> +#include <rte_devargs.h>
> +#include <rte_eal.h>
> +#include <rte_io.h>
> +#include <rte_kvargs.h>
> +#include <rte_malloc.h>
> +#include <rte_mbuf_pool_ops.h>
> +#include <rte_pci.h>
> +
> +#include "roc_api.h"
> +#include "cnxk_mempool.h"
> +
> +#define CNXK_NPA_DEV_NAME       RTE_STR(cnxk_npa_dev_)
> +#define CNXK_NPA_DEV_NAME_LEN   (sizeof(CNXK_NPA_DEV_NAME) + PCI_PRI_STR_SIZE)
> +#define CNXK_NPA_MAX_POOLS_PARAM "max_pools"
> +
> +uintptr_t *cnxk_mempool_internal_data;

Could you remove this global variable, either move to dev structure or
use memzone lookup.


> +
> +#endif
> diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
> new file mode 100644
> index 0000000000..23a171c143
> --- /dev/null
> +++ b/drivers/mempool/cnxk/meson.build
> @@ -0,0 +1,29 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(C) 2021 Marvell.
> +#
> +
> +if is_windows
> +       build = false
> +       reason = 'not supported on Windows'
> +       subdir_done()
> +endif
> +if not dpdk_conf.get('RTE_ARCH_64')
> +       build = false
> +       reason = 'only supported on 64-bit'
> +       subdir_done()
> +endif

Please make it as  positive logic and limit driver only for "64bit Linux"

> +
> +sources = files('cnxk_mempool.c')
> +
> +deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
> +
> +cflags_options = [
> +       '-Wno-strict-prototypes',
> +        '-Werror'

Please limit the driver to "64bit Linux" and please remove this fixup.

[dpdk-dev] [PATCH 2/6] mempool/cnxk: add generic ops

[Ashwin Sekhar T K]

Add generic cnxk mempool ops.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cnxk_mempool.h     |  16 +++
 drivers/mempool/cnxk/cnxk_mempool_ops.c | 173 ++++++++++++++++++++++++
 drivers/mempool/cnxk/meson.build        |   3 +-
 3 files changed, 191 insertions(+), 1 deletion(-)
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool_ops.c

diff --git a/drivers/mempool/cnxk/cnxk_mempool.h b/drivers/mempool/cnxk/cnxk_mempool.h
index 4ee3d236f2..8f226f861c 100644
--- a/drivers/mempool/cnxk/cnxk_mempool.h
+++ b/drivers/mempool/cnxk/cnxk_mempool.h
@@ -7,6 +7,22 @@
 
 #include <rte_mempool.h>
 
+unsigned int cnxk_mempool_get_count(const struct rte_mempool *mp);
+ssize_t cnxk_mempool_calc_mem_size(const struct rte_mempool *mp,
+				   uint32_t obj_num, uint32_t pg_shift,
+				   size_t *min_chunk_size, size_t *align);
+int cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
+			  void *vaddr, rte_iova_t iova, size_t len,
+			  rte_mempool_populate_obj_cb_t *obj_cb,
+			  void *obj_cb_arg);
+int cnxk_mempool_alloc(struct rte_mempool *mp);
+void cnxk_mempool_free(struct rte_mempool *mp);
+
+int __rte_hot cnxk_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
+			       unsigned int n);
+int __rte_hot cnxk_mempool_deq(struct rte_mempool *mp, void **obj_table,
+			       unsigned int n);
+
 extern uintptr_t *cnxk_mempool_internal_data;
 
 #endif
diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
new file mode 100644
index 0000000000..29a4c12208
--- /dev/null
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -0,0 +1,173 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_mempool.h>
+
+#include "roc_api.h"
+#include "cnxk_mempool.h"
+
+int __rte_hot
+cnxk_mempool_enq(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
+{
+	unsigned int index;
+
+	/* Ensure mbuf init changes are written before the free pointers
+	 * are enqueued to the stack.
+	 */
+	rte_io_wmb();
+	for (index = 0; index < n; index++)
+		roc_npa_aura_op_free(mp->pool_id, 0,
+				     (uint64_t)obj_table[index]);
+
+	return 0;
+}
+
+int __rte_hot
+cnxk_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
+{
+	unsigned int index;
+	uint64_t obj;
+
+	for (index = 0; index < n; index++, obj_table++) {
+		int retry = 4;
+
+		/* Retry few times before failing */
+		do {
+			obj = roc_npa_aura_op_alloc(mp->pool_id, 0);
+		} while (retry-- && (obj == 0));
+
+		if (obj == 0) {
+			cnxk_mempool_enq(mp, obj_table - index, index);
+			return -ENOENT;
+		}
+		*obj_table = (void *)obj;
+	}
+
+	return 0;
+}
+
+unsigned int
+cnxk_mempool_get_count(const struct rte_mempool *mp)
+{
+	return (unsigned int)roc_npa_aura_op_available(mp->pool_id);
+}
+
+ssize_t
+cnxk_mempool_calc_mem_size(const struct rte_mempool *mp, uint32_t obj_num,
+			   uint32_t pg_shift, size_t *min_chunk_size,
+			   size_t *align)
+{
+	size_t total_elt_sz;
+
+	/* Need space for one more obj on each chunk to fulfill
+	 * alignment requirements.
+	 */
+	total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
+	return rte_mempool_op_calc_mem_size_helper(
+		mp, obj_num, pg_shift, total_elt_sz, min_chunk_size, align);
+}
+
+int
+cnxk_mempool_alloc(struct rte_mempool *mp)
+{
+	uint64_t aura_handle = 0;
+	struct npa_aura_s aura;
+	struct npa_pool_s pool;
+	uint32_t block_count;
+	size_t block_size;
+	int rc = -ERANGE;
+
+	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
+	block_count = mp->size;
+	if (mp->header_size % ROC_ALIGN != 0) {
+		plt_err("Header size should be multiple of %dB", ROC_ALIGN);
+		goto error;
+	}
+
+	if (block_size % ROC_ALIGN != 0) {
+		plt_err("Block size should be multiple of %dB", ROC_ALIGN);
+		goto error;
+	}
+
+	memset(&aura, 0, sizeof(struct npa_aura_s));
+	memset(&pool, 0, sizeof(struct npa_pool_s));
+	pool.nat_align = 1;
+	/* TODO: Check whether to allow buf_offset > 1 ?? */
+	pool.buf_offset = mp->header_size / ROC_ALIGN;
+
+	/* Use driver specific mp->pool_config to override aura config */
+	if (mp->pool_config != NULL)
+		memcpy(&aura, mp->pool_config, sizeof(struct npa_aura_s));
+
+	rc = roc_npa_pool_create(&aura_handle, block_size, block_count, &aura,
+				 &pool);
+	if (rc) {
+		plt_err("Failed to alloc pool or aura rc=%d", rc);
+		goto error;
+	}
+
+	/* Store aura_handle for future queue operations */
+	mp->pool_id = aura_handle;
+	plt_npa_dbg("block_sz=%lu block_count=%d aura_handle=0x%" PRIx64,
+		    block_size, block_count, aura_handle);
+
+	return 0;
+error:
+	return rc;
+}
+
+void
+cnxk_mempool_free(struct rte_mempool *mp)
+{
+	int rc = 0;
+
+	plt_npa_dbg("aura_handle=0x%" PRIx64, mp->pool_id);
+	rc = roc_npa_pool_destroy(mp->pool_id);
+	if (rc)
+		plt_err("Failed to free pool or aura rc=%d", rc);
+}
+
+int
+cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
+		      void *vaddr, rte_iova_t iova, size_t len,
+		      rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
+{
+	size_t total_elt_sz, off;
+	int num_elts;
+
+	if (iova == RTE_BAD_IOVA)
+		return -EINVAL;
+
+	total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
+
+	/* Align object start address to a multiple of total_elt_sz */
+	off = total_elt_sz - ((((uintptr_t)vaddr - 1) % total_elt_sz) + 1);
+
+	if (len < off)
+		return -EINVAL;
+
+	vaddr = (char *)vaddr + off;
+	iova += off;
+	len -= off;
+	num_elts = len / total_elt_sz;
+
+	plt_npa_dbg("iova %" PRIx64 ", aligned iova %" PRIx64 "", iova - off,
+		    iova);
+	plt_npa_dbg("length %" PRIu64 ", aligned length %" PRIu64 "",
+		    (uint64_t)(len + off), (uint64_t)len);
+	plt_npa_dbg("element size %" PRIu64 "", (uint64_t)total_elt_sz);
+	plt_npa_dbg("requested objects %" PRIu64 ", possible objects %" PRIu64
+		    "", (uint64_t)max_objs, (uint64_t)num_elts);
+	plt_npa_dbg("L1D set distribution :");
+
+	roc_npa_aura_op_range_set(mp->pool_id, iova,
+				  iova + num_elts * total_elt_sz);
+
+	if (roc_npa_pool_range_update_check(mp->pool_id) < 0)
+		return -EBUSY;
+
+	return rte_mempool_op_populate_helper(
+		mp, RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ, max_objs, vaddr, iova,
+		len, obj_cb, obj_cb_arg);
+}
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
index 23a171c143..b9a810e021 100644
--- a/drivers/mempool/cnxk/meson.build
+++ b/drivers/mempool/cnxk/meson.build
@@ -13,7 +13,8 @@ if not dpdk_conf.get('RTE_ARCH_64')
 	subdir_done()
 endif
 
-sources = files('cnxk_mempool.c')
+sources = files('cnxk_mempool.c',
+		'cnxk_mempool_ops.c')
 
 deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
 
-- 
2.29.2

Re: [dpdk-dev] [PATCH 2/6] mempool/cnxk: add generic ops

[Jerin Jacob]

On Fri, Mar 5, 2021 at 11:43 PM Ashwin Sekhar T K <asekhar@marvell.com> wrote:
>
> Add generic cnxk mempool ops.
>
> Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
> ---

> +int
> +cnxk_mempool_alloc(struct rte_mempool *mp)
> +{
> +       uint64_t aura_handle = 0;
> +       struct npa_aura_s aura;
> +       struct npa_pool_s pool;
> +       uint32_t block_count;
> +       size_t block_size;
> +       int rc = -ERANGE;
> +
> +       block_size = mp->elt_size + mp->header_size + mp->trailer_size;
> +       block_count = mp->size;
> +       if (mp->header_size % ROC_ALIGN != 0) {
> +               plt_err("Header size should be multiple of %dB", ROC_ALIGN);
> +               goto error;
> +       }
> +
> +       if (block_size % ROC_ALIGN != 0) {
> +               plt_err("Block size should be multiple of %dB", ROC_ALIGN);
> +               goto error;
> +       }
> +
> +       memset(&aura, 0, sizeof(struct npa_aura_s));
> +       memset(&pool, 0, sizeof(struct npa_pool_s));
> +       pool.nat_align = 1;
> +       /* TODO: Check whether to allow buf_offset > 1 ?? */

Please fix up this TODO.


> +       pool.buf_offset = mp->header_size / ROC_ALIGN;
> +

[dpdk-dev] [PATCH 3/6] mempool/cnxk: add cn9k mempool ops

[Ashwin Sekhar T K]

Add mempool ops specific to cn9k.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn9k_mempool_ops.c | 90 +++++++++++++++++++++++++
 drivers/mempool/cnxk/meson.build        |  3 +-
 2 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 drivers/mempool/cnxk/cn9k_mempool_ops.c

diff --git a/drivers/mempool/cnxk/cn9k_mempool_ops.c b/drivers/mempool/cnxk/cn9k_mempool_ops.c
new file mode 100644
index 0000000000..3a7de39db2
--- /dev/null
+++ b/drivers/mempool/cnxk/cn9k_mempool_ops.c
@@ -0,0 +1,90 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_mempool.h>
+
+#include "roc_api.h"
+#include "cnxk_mempool.h"
+
+static int __rte_hot
+cn9k_mempool_enq(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
+{
+	/* Ensure mbuf init changes are written before the free pointers
+	 * are enqueued to the stack.
+	 */
+	rte_io_wmb();
+	roc_npa_aura_op_bulk_free(mp->pool_id, (const uint64_t *)obj_table, n,
+				  0);
+
+	return 0;
+}
+
+static inline int __rte_hot
+cn9k_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
+{
+	unsigned int count;
+
+	count = roc_npa_aura_op_bulk_alloc(mp->pool_id, (uint64_t *)obj_table,
+					   n, 0, 1);
+
+	if (unlikely(count != n)) {
+		/* If bulk alloc failed to allocate all pointers, try
+		 * allocating remaining pointers with the default alloc
+		 * with retry scheme.
+		 */
+		if (cnxk_mempool_deq(mp, &obj_table[count], n - count)) {
+			cn9k_mempool_enq(mp, obj_table, count);
+			return -ENOENT;
+		}
+	}
+
+	return 0;
+}
+
+static int
+cn9k_mempool_alloc(struct rte_mempool *mp)
+{
+	size_t block_size, padding;
+
+	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
+	/* Align header size to ROC_ALIGN */
+	if (mp->header_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(mp->header_size, ROC_ALIGN) -
+			  mp->header_size;
+		mp->header_size += padding;
+		block_size += padding;
+	}
+
+	/* Align block size to ROC_ALIGN */
+	if (block_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(block_size, ROC_ALIGN) - block_size;
+		mp->trailer_size += padding;
+		block_size += padding;
+	}
+
+	/*
+	 * OCTEON TX2 has 8 sets, 41 ways L1D cache, VA<9:7> bits dictate
+	 * the set selection.
+	 * Add additional padding to ensure that the element size always
+	 * occupies odd number of cachelines to ensure even distribution
+	 * of elements among L1D cache sets.
+	 */
+	padding = ((block_size / ROC_ALIGN) % 2) ? 0 : ROC_ALIGN;
+	mp->trailer_size += padding;
+
+	return cnxk_mempool_alloc(mp);
+}
+
+static struct rte_mempool_ops cn9k_mempool_ops = {
+	.name = "cn9k_mempool_ops",
+	.alloc = cn9k_mempool_alloc,
+	.free = cnxk_mempool_free,
+	.enqueue = cn9k_mempool_enq,
+	.dequeue = cn9k_mempool_deq,
+	.get_count = cnxk_mempool_get_count,
+	.calc_mem_size = cnxk_mempool_calc_mem_size,
+	.populate = cnxk_mempool_populate,
+};
+
+MEMPOOL_REGISTER_OPS(cn9k_mempool_ops);
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
index b9a810e021..4ce865e18b 100644
--- a/drivers/mempool/cnxk/meson.build
+++ b/drivers/mempool/cnxk/meson.build
@@ -14,7 +14,8 @@ if not dpdk_conf.get('RTE_ARCH_64')
 endif
 
 sources = files('cnxk_mempool.c',
-		'cnxk_mempool_ops.c')
+		'cnxk_mempool_ops.c',
+		'cn9k_mempool_ops.c')
 
 deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
 
-- 
2.29.2

[dpdk-dev] [PATCH 4/6] mempool/cnxk: add base cn10k mempool ops

[Ashwin Sekhar T K]

Add base cn10k mempool ops.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 46 ++++++++++++++++++++++++
 drivers/mempool/cnxk/meson.build         |  3 +-
 2 files changed, 48 insertions(+), 1 deletion(-)
 create mode 100644 drivers/mempool/cnxk/cn10k_mempool_ops.c

diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
new file mode 100644
index 0000000000..fc7592fd94
--- /dev/null
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_mempool.h>
+
+#include "roc_api.h"
+#include "cnxk_mempool.h"
+
+static int
+cn10k_mempool_alloc(struct rte_mempool *mp)
+{
+	uint32_t block_size;
+	size_t padding;
+
+	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
+	/* Align header size to ROC_ALIGN */
+	if (mp->header_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(mp->header_size, ROC_ALIGN) -
+			  mp->header_size;
+		mp->header_size += padding;
+		block_size += padding;
+	}
+
+	/* Align block size to ROC_ALIGN */
+	if (block_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(block_size, ROC_ALIGN) - block_size;
+		mp->trailer_size += padding;
+		block_size += padding;
+	}
+
+	return cnxk_mempool_alloc(mp);
+}
+
+static struct rte_mempool_ops cn10k_mempool_ops = {
+	.name = "cn10k_mempool_ops",
+	.alloc = cn10k_mempool_alloc,
+	.free = cnxk_mempool_free,
+	.enqueue = cnxk_mempool_enq,
+	.dequeue = cnxk_mempool_deq,
+	.get_count = cnxk_mempool_get_count,
+	.calc_mem_size = cnxk_mempool_calc_mem_size,
+	.populate = cnxk_mempool_populate,
+};
+
+MEMPOOL_REGISTER_OPS(cn10k_mempool_ops);
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
index 4ce865e18b..46f502bf3a 100644
--- a/drivers/mempool/cnxk/meson.build
+++ b/drivers/mempool/cnxk/meson.build
@@ -15,7 +15,8 @@ endif
 
 sources = files('cnxk_mempool.c',
 		'cnxk_mempool_ops.c',
-		'cn9k_mempool_ops.c')
+		'cn9k_mempool_ops.c',
+		'cn10k_mempool_ops.c')
 
 deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
 
-- 
2.29.2

Re: [dpdk-dev] [PATCH 4/6] mempool/cnxk: add base cn10k mempool ops

[Jerin Jacob]

On Fri, Mar 5, 2021 at 11:43 PM Ashwin Sekhar T K <asekhar@marvell.com> wrote:
>
> Add base cn10k mempool ops.

Could you add more description about why cn10k_mempool_alloc()
different from cn9k in
git commit.

>
> Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
> ---
>  drivers/mempool/cnxk/cn10k_mempool_ops.c | 46 ++++++++++++++++++++++++
>  drivers/mempool/cnxk/meson.build         |  3 +-
>  2 files changed, 48 insertions(+), 1 deletion(-)
>  create mode 100644 drivers/mempool/cnxk/cn10k_mempool_ops.c
>
> diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
> new file mode 100644
> index 0000000000..fc7592fd94
> --- /dev/null
> +++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
> @@ -0,0 +1,46 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(C) 2021 Marvell.
> + */
> +
> +#include <rte_mempool.h>
> +
> +#include "roc_api.h"
> +#include "cnxk_mempool.h"
> +
> +static int
> +cn10k_mempool_alloc(struct rte_mempool *mp)
> +{
> +       uint32_t block_size;
> +       size_t padding;
> +
> +       block_size = mp->elt_size + mp->header_size + mp->trailer_size;
> +       /* Align header size to ROC_ALIGN */
> +       if (mp->header_size % ROC_ALIGN != 0) {
> +               padding = RTE_ALIGN_CEIL(mp->header_size, ROC_ALIGN) -
> +                         mp->header_size;
> +               mp->header_size += padding;
> +               block_size += padding;
> +       }
> +
> +       /* Align block size to ROC_ALIGN */
> +       if (block_size % ROC_ALIGN != 0) {
> +               padding = RTE_ALIGN_CEIL(block_size, ROC_ALIGN) - block_size;
> +               mp->trailer_size += padding;
> +               block_size += padding;
> +       }
> +
> +       return cnxk_mempool_alloc(mp);
> +}
> +
> +static struct rte_mempool_ops cn10k_mempool_ops = {
> +       .name = "cn10k_mempool_ops",
> +       .alloc = cn10k_mempool_alloc,
> +       .free = cnxk_mempool_free,
> +       .enqueue = cnxk_mempool_enq,
> +       .dequeue = cnxk_mempool_deq,
> +       .get_count = cnxk_mempool_get_count,
> +       .calc_mem_size = cnxk_mempool_calc_mem_size,
> +       .populate = cnxk_mempool_populate,
> +};
> +
> +MEMPOOL_REGISTER_OPS(cn10k_mempool_ops);
> diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
> index 4ce865e18b..46f502bf3a 100644
> --- a/drivers/mempool/cnxk/meson.build
> +++ b/drivers/mempool/cnxk/meson.build
> @@ -15,7 +15,8 @@ endif
>
>  sources = files('cnxk_mempool.c',
>                 'cnxk_mempool_ops.c',
> -               'cn9k_mempool_ops.c')
> +               'cn9k_mempool_ops.c',
> +               'cn10k_mempool_ops.c')
>
>  deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
>
> --
> 2.29.2
>

[dpdk-dev] [PATCH 5/6] mempool/cnxk: add cn10k batch enqueue/dequeue support

[Ashwin Sekhar T K]

Add support for asynchronous batch enqueue/dequeue
of pointers from NPA pool.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 258 ++++++++++++++++++++++-
 drivers/mempool/cnxk/cnxk_mempool.c      |  19 +-
 drivers/mempool/cnxk/cnxk_mempool.h      |   3 +-
 drivers/mempool/cnxk/cnxk_mempool_ops.c  |  28 +++
 4 files changed, 287 insertions(+), 21 deletions(-)

diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
index fc7592fd94..131abc0723 100644
--- a/drivers/mempool/cnxk/cn10k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -7,11 +7,239 @@
 #include "roc_api.h"
 #include "cnxk_mempool.h"
 
+#define BATCH_ALLOC_SZ ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS
+
+enum batch_op_status {
+	BATCH_ALLOC_OP_NOT_ISSUED = 0,
+	BATCH_ALLOC_OP_ISSUED = 1,
+	BATCH_ALLOC_OP_DONE
+};
+
+struct batch_op_mem {
+	unsigned int sz;
+	enum batch_op_status status;
+	uint64_t objs[BATCH_ALLOC_SZ] __rte_aligned(ROC_ALIGN);
+};
+
+struct batch_op_data {
+	uint64_t lmt_addr;
+	struct batch_op_mem mem[RTE_MAX_LCORE] __rte_aligned(ROC_ALIGN);
+};
+
+static struct batch_op_data **batch_op_data;
+
+#define BATCH_OP_DATA_GET(pool_id)                                             \
+	batch_op_data[roc_npa_aura_handle_to_aura(pool_id)]
+
+#define BATCH_OP_DATA_SET(pool_id, op_data)                                    \
+	do {                                                                   \
+		uint64_t aura = roc_npa_aura_handle_to_aura(pool_id);          \
+		batch_op_data[aura] = op_data;                                 \
+	} while (0)
+
+int
+cn10k_mempool_lf_init(void)
+{
+	unsigned int maxpools, sz;
+
+	maxpools = roc_idev_npa_maxpools_get();
+	sz = maxpools * sizeof(uintptr_t);
+
+	batch_op_data = rte_zmalloc(NULL, sz, ROC_ALIGN);
+	if (!batch_op_data)
+		return -1;
+
+	return 0;
+}
+
+void
+cn10k_mempool_lf_fini(void)
+{
+	if (!batch_op_data)
+		return;
+
+	rte_free(batch_op_data);
+	batch_op_data = NULL;
+}
+
+static int
+batch_op_init(struct rte_mempool *mp)
+{
+	struct batch_op_data *op_data;
+	int i;
+
+	RTE_ASSERT(BATCH_OP_DATA_GET(mp->pool_id) == NULL);
+	op_data = rte_zmalloc(NULL, sizeof(struct batch_op_data), ROC_ALIGN);
+	if (op_data == NULL)
+		return -1;
+
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		op_data->mem[i].sz = 0;
+		op_data->mem[i].status = BATCH_ALLOC_OP_NOT_ISSUED;
+	}
+
+	op_data->lmt_addr = roc_idev_lmt_base_addr_get();
+	BATCH_OP_DATA_SET(mp->pool_id, op_data);
+
+	return 0;
+}
+
+static void
+batch_op_fini(struct rte_mempool *mp)
+{
+	struct batch_op_data *op_data;
+	int i;
+
+	op_data = BATCH_OP_DATA_GET(mp->pool_id);
+
+	rte_wmb();
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct batch_op_mem *mem = &op_data->mem[i];
+
+		if (mem->status == BATCH_ALLOC_OP_ISSUED) {
+			mem->sz = roc_npa_aura_batch_alloc_extract(
+				mem->objs, mem->objs, BATCH_ALLOC_SZ);
+			mem->status = BATCH_ALLOC_OP_DONE;
+		}
+		if (mem->status == BATCH_ALLOC_OP_DONE) {
+			roc_npa_aura_op_bulk_free(mp->pool_id, mem->objs,
+						  mem->sz, 1);
+			mem->status = BATCH_ALLOC_OP_NOT_ISSUED;
+		}
+	}
+
+	rte_free(op_data);
+	BATCH_OP_DATA_SET(mp->pool_id, NULL);
+}
+
+static int __rte_hot
+cn10k_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
+		  unsigned int n)
+{
+	const uint64_t *ptr = (const uint64_t *)obj_table;
+	uint64_t lmt_addr = 0, lmt_id = 0;
+	struct batch_op_data *op_data;
+
+	/* Ensure mbuf init changes are written before the free pointers are
+	 * are enqueued to the stack.
+	 */
+	rte_io_wmb();
+
+	if (n == 1) {
+		roc_npa_aura_op_free(mp->pool_id, 1, ptr[0]);
+		return 0;
+	}
+
+	op_data = BATCH_OP_DATA_GET(mp->pool_id);
+	lmt_addr = op_data->lmt_addr;
+	ROC_LMT_BASE_ID_GET(lmt_addr, lmt_id);
+	roc_npa_aura_op_batch_free(mp->pool_id, ptr, n, 1, lmt_addr, lmt_id);
+
+	return 0;
+}
+
+static int __rte_hot
+cn10k_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
+{
+	struct batch_op_data *op_data;
+	struct batch_op_mem *mem;
+	unsigned int count = 0;
+	int tid, rc, retry;
+	bool loop = true;
+
+	op_data = BATCH_OP_DATA_GET(mp->pool_id);
+	tid = rte_lcore_id();
+	mem = &op_data->mem[tid];
+
+	/* Issue batch alloc */
+	if (mem->status == BATCH_ALLOC_OP_NOT_ISSUED) {
+		rc = roc_npa_aura_batch_alloc_issue(mp->pool_id, mem->objs,
+						    BATCH_ALLOC_SZ, 0, 1);
+		/* If issue fails, try falling back to default alloc */
+		if (unlikely(rc))
+			return cn10k_mempool_enq(mp, obj_table, n);
+		mem->status = BATCH_ALLOC_OP_ISSUED;
+	}
+
+	retry = 4;
+	while (loop) {
+		unsigned int cur_sz;
+
+		if (mem->status == BATCH_ALLOC_OP_ISSUED) {
+			mem->sz = roc_npa_aura_batch_alloc_extract(
+				mem->objs, mem->objs, BATCH_ALLOC_SZ);
+
+			/* If partial alloc reduce the retry count */
+			retry -= (mem->sz != BATCH_ALLOC_SZ);
+			/* Break the loop if retry count exhausted */
+			loop = !!retry;
+			mem->status = BATCH_ALLOC_OP_DONE;
+		}
+
+		cur_sz = n - count;
+		if (cur_sz > mem->sz)
+			cur_sz = mem->sz;
+
+		/* Dequeue the pointers */
+		memcpy(&obj_table[count], &mem->objs[mem->sz - cur_sz],
+		       cur_sz * sizeof(uintptr_t));
+		mem->sz -= cur_sz;
+		count += cur_sz;
+
+		/* Break loop if the required pointers has been dequeued */
+		loop &= (count != n);
+
+		/* Issue next batch alloc if pointers are exhausted */
+		if (mem->sz == 0) {
+			rc = roc_npa_aura_batch_alloc_issue(
+				mp->pool_id, mem->objs, BATCH_ALLOC_SZ, 0, 1);
+			/* Break loop if issue failed and set status */
+			loop &= !rc;
+			mem->status = !rc;
+		}
+	}
+
+	if (unlikely(count != n)) {
+		/* No partial alloc allowed. Free up allocated pointers */
+		cn10k_mempool_enq(mp, obj_table, count);
+		return -ENOENT;
+	}
+
+	return 0;
+}
+
+static unsigned int
+cn10k_mempool_get_count(const struct rte_mempool *mp)
+{
+	struct batch_op_data *op_data;
+	unsigned int count = 0;
+	int i;
+
+	op_data = BATCH_OP_DATA_GET(mp->pool_id);
+
+	rte_wmb();
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct batch_op_mem *mem = &op_data->mem[i];
+
+		if (mem->status == BATCH_ALLOC_OP_ISSUED)
+			count += roc_npa_aura_batch_alloc_count(mem->objs,
+								BATCH_ALLOC_SZ);
+
+		if (mem->status == BATCH_ALLOC_OP_DONE)
+			count += mem->sz;
+	}
+
+	count += cnxk_mempool_get_count(mp);
+
+	return count;
+}
+
 static int
 cn10k_mempool_alloc(struct rte_mempool *mp)
 {
 	uint32_t block_size;
 	size_t padding;
+	int rc;
 
 	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
 	/* Align header size to ROC_ALIGN */
@@ -29,16 +257,36 @@ cn10k_mempool_alloc(struct rte_mempool *mp)
 		block_size += padding;
 	}
 
-	return cnxk_mempool_alloc(mp);
+	rc = cnxk_mempool_alloc(mp);
+	if (rc)
+		return rc;
+
+	rc = batch_op_init(mp);
+	if (rc) {
+		plt_err("Failed to init batch alloc mem rc=%d", rc);
+		goto error;
+	}
+
+	return 0;
+error:
+	cnxk_mempool_free(mp);
+	return rc;
+}
+
+static void
+cn10k_mempool_free(struct rte_mempool *mp)
+{
+	batch_op_fini(mp);
+	cnxk_mempool_free(mp);
 }
 
 static struct rte_mempool_ops cn10k_mempool_ops = {
 	.name = "cn10k_mempool_ops",
 	.alloc = cn10k_mempool_alloc,
-	.free = cnxk_mempool_free,
-	.enqueue = cnxk_mempool_enq,
-	.dequeue = cnxk_mempool_deq,
-	.get_count = cnxk_mempool_get_count,
+	.free = cn10k_mempool_free,
+	.enqueue = cn10k_mempool_enq,
+	.dequeue = cn10k_mempool_deq,
+	.get_count = cn10k_mempool_get_count,
 	.calc_mem_size = cnxk_mempool_calc_mem_size,
 	.populate = cnxk_mempool_populate,
 };
diff --git a/drivers/mempool/cnxk/cnxk_mempool.c b/drivers/mempool/cnxk/cnxk_mempool.c
index c24497a6e5..1bbe384fe7 100644
--- a/drivers/mempool/cnxk/cnxk_mempool.c
+++ b/drivers/mempool/cnxk/cnxk_mempool.c
@@ -14,14 +14,11 @@
 #include <rte_pci.h>
 
 #include "roc_api.h"
-#include "cnxk_mempool.h"
 
 #define CNXK_NPA_DEV_NAME	 RTE_STR(cnxk_npa_dev_)
 #define CNXK_NPA_DEV_NAME_LEN	 (sizeof(CNXK_NPA_DEV_NAME) + PCI_PRI_STR_SIZE)
 #define CNXK_NPA_MAX_POOLS_PARAM "max_pools"
 
-uintptr_t *cnxk_mempool_internal_data;
-
 static inline uint32_t
 npa_aura_size_to_u32(uint8_t val)
 {
@@ -82,33 +79,25 @@ static int
 npa_init(struct rte_pci_device *pci_dev)
 {
 	char name[CNXK_NPA_DEV_NAME_LEN];
-	size_t idata_offset, idata_sz;
 	const struct rte_memzone *mz;
 	struct roc_npa *dev;
-	int rc, maxpools;
+	int rc;
 
 	rc = plt_init();
 	if (rc < 0)
 		goto error;
 
-	maxpools = parse_aura_size(pci_dev->device.devargs);
-	/* Add the space for per-pool internal data pointers to memzone len */
-	idata_offset = RTE_ALIGN_CEIL(sizeof(*dev), ROC_ALIGN);
-	idata_sz = maxpools * sizeof(uintptr_t);
-
 	rc = -ENOMEM;
 	mz = rte_memzone_reserve_aligned(npa_dev_to_name(pci_dev, name),
-					 idata_offset + idata_sz, SOCKET_ID_ANY,
-					 0, RTE_CACHE_LINE_SIZE);
+					 sizeof(*dev), SOCKET_ID_ANY, 0,
+					 RTE_CACHE_LINE_SIZE);
 	if (mz == NULL)
 		goto error;
 
 	dev = mz->addr;
 	dev->pci_dev = pci_dev;
-	cnxk_mempool_internal_data = (uintptr_t *)(mz->addr_64 + idata_offset);
-	memset(cnxk_mempool_internal_data, 0, idata_sz);
 
-	roc_idev_npa_maxpools_set(maxpools);
+	roc_idev_npa_maxpools_set(parse_aura_size(pci_dev->device.devargs));
 	rc = roc_npa_dev_init(dev);
 	if (rc)
 		goto mz_free;
diff --git a/drivers/mempool/cnxk/cnxk_mempool.h b/drivers/mempool/cnxk/cnxk_mempool.h
index 8f226f861c..6e54346e6a 100644
--- a/drivers/mempool/cnxk/cnxk_mempool.h
+++ b/drivers/mempool/cnxk/cnxk_mempool.h
@@ -23,6 +23,7 @@ int __rte_hot cnxk_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
 int __rte_hot cnxk_mempool_deq(struct rte_mempool *mp, void **obj_table,
 			       unsigned int n);
 
-extern uintptr_t *cnxk_mempool_internal_data;
+int cn10k_mempool_lf_init(void);
+void cn10k_mempool_lf_fini(void);
 
 #endif
diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
index 29a4c12208..18f125c7ac 100644
--- a/drivers/mempool/cnxk/cnxk_mempool_ops.c
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -2,6 +2,7 @@
  * Copyright(C) 2021 Marvell.
  */
 
+#include <rte_mbuf_pool_ops.h>
 #include <rte_mempool.h>
 
 #include "roc_api.h"
@@ -171,3 +172,30 @@ cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
 		mp, RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ, max_objs, vaddr, iova,
 		len, obj_cb, obj_cb_arg);
 }
+
+static int
+cnxk_mempool_lf_init(void)
+{
+	int rc = 0;
+
+	if (roc_model_is_cn10k()) {
+		rte_mbuf_set_platform_mempool_ops("cn10k_mempool_ops");
+		rc = cn10k_mempool_lf_init();
+	} else {
+		rte_mbuf_set_platform_mempool_ops("cn9k_mempool_ops");
+	}
+	return rc;
+}
+
+static void
+cnxk_mempool_lf_fini(void)
+{
+	if (roc_model_is_cn10k())
+		cn10k_mempool_lf_fini();
+}
+
+RTE_INIT(cnxk_mempool_ops_init)
+{
+	roc_npa_lf_init_cb_register(cnxk_mempool_lf_init);
+	roc_npa_lf_fini_cb_register(cnxk_mempool_lf_fini);
+}
-- 
2.29.2

Re: [dpdk-dev] [PATCH 5/6] mempool/cnxk: add cn10k batch enqueue/dequeue support

[Jerin Jacob]

On Fri, Mar 5, 2021 at 11:44 PM Ashwin Sekhar T K <asekhar@marvell.com> wrote:
>
> Add support for asynchronous batch enqueue/dequeue
> of pointers from NPA pool.

Please spilt the enq and deq as separate patches. The rest looks good to me.


>
> Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
> ---

[dpdk-dev] [PATCH 6/6] doc: add Marvell CNXK mempool documentation

[Ashwin Sekhar T K]

Add Marvell OCTEON CNXK mempool documentation.

Signed-off-by: Jerin Jacob <jerinj@marvell.com>
Signed-off-by: Nithin Dabilpuram <ndabilpuram@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 MAINTAINERS                  |  6 +++
 doc/guides/mempool/cnxk.rst  | 84 ++++++++++++++++++++++++++++++++++++
 doc/guides/mempool/index.rst |  1 +
 doc/guides/platform/cnxk.rst |  3 ++
 4 files changed, 94 insertions(+)
 create mode 100644 doc/guides/mempool/cnxk.rst

diff --git a/MAINTAINERS b/MAINTAINERS
index 45dcd36dbe..67c179f11b 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -501,6 +501,12 @@ M: Artem V. Andreev <artem.andreev@oktetlabs.ru>
 M: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
 F: drivers/mempool/bucket/
 
+Marvell cnxk
+M: Ashwin Sekhar T K <asekhar@marvell.com>
+M: Pavan Nikhilesh <pbhagavatula@marvell.com>
+F: drivers/mempool/cnxk/
+F: doc/guides/mempool/cnxk.rst
+
 Marvell OCTEON TX2
 M: Jerin Jacob <jerinj@marvell.com>
 M: Nithin Dabilpuram <ndabilpuram@marvell.com>
diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
new file mode 100644
index 0000000000..fe099bb11a
--- /dev/null
+++ b/doc/guides/mempool/cnxk.rst
@@ -0,0 +1,84 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(C) 2021 Marvell.
+
+CNXK NPA Mempool Driver
+============================
+
+The CNXK NPA PMD (**librte_mempool_cnxk**) provides mempool
+driver support for the integrated mempool device found in **Marvell OCTEON CN9K/CN10K** SoC family.
+
+More information about CNXK SoC can be found at `Marvell Official Website
+<https://www.marvell.com/embedded-processors/infrastructure-processors/>`_.
+
+Features
+--------
+
+CNXK NPA PMD supports:
+
+- Up to 128 NPA LFs
+- 1M Pools per LF
+- HW mempool manager
+- Asynchronous batch alloc of up to 512 buffer allocations with single instruction.
+- Batch free of up to 15 buffers with single instruction.
+- Ethdev Rx buffer allocation in HW to save CPU cycles in the Rx path.
+- Ethdev Tx buffer recycling in HW to save CPU cycles in the Tx path.
+
+Prerequisites and Compilation procedure
+---------------------------------------
+
+   See :doc:`../platform/cnxk` for setup information.
+
+Pre-Installation Configuration
+------------------------------
+
+
+Runtime Config Options
+~~~~~~~~~~~~~~~~~~~~~~
+
+- ``Maximum number of mempools per application`` (default ``128``)
+
+  The maximum number of mempools per application needs to be configured on
+  HW during mempool driver initialization. HW can support up to 1M mempools,
+  Since each mempool costs set of HW resources, the ``max_pools`` ``devargs``
+  parameter is being introduced to configure the number of mempools required
+  for the application.
+  For example::
+
+    -a 0002:02:00.0,max_pools=512
+
+  With the above configuration, the driver will set up only 512 mempools for
+  the given application to save HW resources.
+
+.. note::
+
+   Since this configuration is per application, the end user needs to
+   provide ``max_pools`` parameter to the first PCIe device probed by the given
+   application.
+
+Debugging Options
+~~~~~~~~~~~~~~~~~
+
+.. _table_cnxk_mempool_debug_options:
+
+.. table:: CNXK mempool debug options
+
+   +---+------------+-------------------------------------------------------+
+   | # | Component  | EAL log command                                       |
+   +===+============+=======================================================+
+   | 1 | NPA        | --log-level='pmd\.mempool.cnxk,8'                     |
+   +---+------------+-------------------------------------------------------+
+
+Standalone mempool device
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+   The ``usertools/dpdk-devbind.py`` script shall enumerate all the mempool devices
+   available in the system. In order to avoid, the end user to bind the mempool
+   device prior to use ethdev and/or eventdev device, the respective driver
+   configures an NPA LF and attach to the first probed ethdev or eventdev device.
+   In case, if end user need to run mempool as a standalone device
+   (without ethdev or eventdev), end user needs to bind a mempool device using
+   ``usertools/dpdk-devbind.py``
+
+   Example command to run ``mempool_autotest`` test with standalone CN10K NPA device::
+
+     echo "mempool_autotest" | <build_dir>/app/test/dpdk-test -c 0xf0 --mbuf-pool-ops-name="cn10k_mempool_ops"
diff --git a/doc/guides/mempool/index.rst b/doc/guides/mempool/index.rst
index a0e55467e6..ce53bc1ac7 100644
--- a/doc/guides/mempool/index.rst
+++ b/doc/guides/mempool/index.rst
@@ -11,6 +11,7 @@ application through the mempool API.
     :maxdepth: 2
     :numbered:
 
+    cnxk
     octeontx
     octeontx2
     ring
diff --git a/doc/guides/platform/cnxk.rst b/doc/guides/platform/cnxk.rst
index 3b072877a1..9bbba65f2e 100644
--- a/doc/guides/platform/cnxk.rst
+++ b/doc/guides/platform/cnxk.rst
@@ -141,6 +141,9 @@ HW Offload Drivers
 
 This section lists dataplane H/W block(s) available in CNXK SoC.
 
+#. **Mempool Driver**
+   See :doc:`../mempool/cnxk` for NPA mempool driver information.
+
 Procedure to Setup Platform
 ---------------------------
 
-- 
2.29.2

Re: [dpdk-dev] [PATCH 6/6] doc: add Marvell CNXK mempool documentation

[Jerin Jacob]

On Fri, Mar 5, 2021 at 11:44 PM Ashwin Sekhar T K <asekhar@marvell.com> wrote:
>
> Add Marvell OCTEON CNXK mempool documentation.
>
> Signed-off-by: Jerin Jacob <jerinj@marvell.com>
> Signed-off-by: Nithin Dabilpuram <ndabilpuram@marvell.com>
> Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
> ---
>  MAINTAINERS                  |  6 +++
>  doc/guides/mempool/cnxk.rst  | 84 ++++++++++++++++++++++++++++++++++++
>  doc/guides/mempool/index.rst |  1 +
>  doc/guides/platform/cnxk.rst |  3 ++
>  4 files changed, 94 insertions(+)
>  create mode 100644 doc/guides/mempool/cnxk.rst
>
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 45dcd36dbe..67c179f11b 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -501,6 +501,12 @@ M: Artem V. Andreev <artem.andreev@oktetlabs.ru>
>  M: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
>  F: drivers/mempool/bucket/
>
> +Marvell cnxk
> +M: Ashwin Sekhar T K <asekhar@marvell.com>
> +M: Pavan Nikhilesh <pbhagavatula@marvell.com>
> +F: drivers/mempool/cnxk/
> +F: doc/guides/mempool/cnxk.rst


Please move this section to the first patch.

> +
>  Marvell OCTEON TX2
>  M: Jerin Jacob <jerinj@marvell.com>
>  M: Nithin Dabilpuram <ndabilpuram@marvell.com>
> diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
> new file mode 100644
> index 0000000000..fe099bb11a
> --- /dev/null
> +++ b/doc/guides/mempool/cnxk.rst
> @@ -0,0 +1,84 @@
> +..  SPDX-License-Identifier: BSD-3-Clause
> +    Copyright(C) 2021 Marvell.
> +
> +CNXK NPA Mempool Driver
> +============================
> +
> +The CNXK NPA PMD (**librte_mempool_cnxk**) provides mempool
> +driver support for the integrated mempool device found in **Marvell OCTEON CN9K/CN10K** SoC family.
> +
> +More information about CNXK SoC can be found at `Marvell Official Website
> +<https://www.marvell.com/embedded-processors/infrastructure-processors/>`_.
> +
> +Features
> +--------
> +
> +CNXK NPA PMD supports:
> +
> +- Up to 128 NPA LFs
> +- 1M Pools per LF
> +- HW mempool manager
> +- Asynchronous batch alloc of up to 512 buffer allocations with single instruction.
> +- Batch free of up to 15 buffers with single instruction.
> +- Ethdev Rx buffer allocation in HW to save CPU cycles in the Rx path.
> +- Ethdev Tx buffer recycling in HW to save CPU cycles in the Tx path.


Please move this section first patch.


> +
> +Prerequisites and Compilation procedure
> +---------------------------------------
> +
> +   See :doc:`../platform/cnxk` for setup information.
> +
> +Pre-Installation Configuration
> +------------------------------
> +
> +
> +Runtime Config Options
> +~~~~~~~~~~~~~~~~~~~~~~
> +
> +- ``Maximum number of mempools per application`` (default ``128``)
> +
> +  The maximum number of mempools per application needs to be configured on
> +  HW during mempool driver initialization. HW can support up to 1M mempools,
> +  Since each mempool costs set of HW resources, the ``max_pools`` ``devargs``
> +  parameter is being introduced to configure the number of mempools required
> +  for the application.
> +  For example::
> +
> +    -a 0002:02:00.0,max_pools=512


Please add this section on the patch where it adds devargs.

> +
> +  With the above configuration, the driver will set up only 512 mempools for
> +  the given application to save HW resources.
> +
> +.. note::
> +
> +   Since this configuration is per application, the end user needs to
> +   provide ``max_pools`` parameter to the first PCIe device probed by the given
> +   application.
> +
> +Debugging Options
> +~~~~~~~~~~~~~~~~~
> +
> +.. _table_cnxk_mempool_debug_options:
> +
> +.. table:: CNXK mempool debug options
> +
> +   +---+------------+-------------------------------------------------------+
> +   | # | Component  | EAL log command                                       |
> +   +===+============+=======================================================+
> +   | 1 | NPA        | --log-level='pmd\.mempool.cnxk,8'                     |
> +   +---+------------+-------------------------------------------------------+
> +
> +Standalone mempool device
> +~~~~~~~~~~~~~~~~~~~~~~~~~
> +
> +   The ``usertools/dpdk-devbind.py`` script shall enumerate all the mempool devices
> +   available in the system. In order to avoid, the end user to bind the mempool
> +   device prior to use ethdev and/or eventdev device, the respective driver
> +   configures an NPA LF and attach to the first probed ethdev or eventdev device.
> +   In case, if end user need to run mempool as a standalone device
> +   (without ethdev or eventdev), end user needs to bind a mempool device using
> +   ``usertools/dpdk-devbind.py``
> +
> +   Example command to run ``mempool_autotest`` test with standalone CN10K NPA device::
> +
> +     echo "mempool_autotest" | <build_dir>/app/test/dpdk-test -c 0xf0 --mbuf-pool-ops-name="cn10k_mempool_ops"
> diff --git a/doc/guides/mempool/index.rst b/doc/guides/mempool/index.rst
> index a0e55467e6..ce53bc1ac7 100644
> --- a/doc/guides/mempool/index.rst
> +++ b/doc/guides/mempool/index.rst
> @@ -11,6 +11,7 @@ application through the mempool API.
>      :maxdepth: 2
>      :numbered:
>
> +    cnxk
>      octeontx
>      octeontx2
>      ring
> diff --git a/doc/guides/platform/cnxk.rst b/doc/guides/platform/cnxk.rst
> index 3b072877a1..9bbba65f2e 100644
> --- a/doc/guides/platform/cnxk.rst
> +++ b/doc/guides/platform/cnxk.rst
> @@ -141,6 +141,9 @@ HW Offload Drivers
>
>  This section lists dataplane H/W block(s) available in CNXK SoC.
>
> +#. **Mempool Driver**
> +   See :doc:`../mempool/cnxk` for NPA mempool driver information.
> +
>  Procedure to Setup Platform
>  ---------------------------
>
> --
> 2.29.2
>

[dpdk-dev] [PATCH v2 00/11] Add Marvell CNXK mempool driver

[Ashwin Sekhar T K]

This patchset adds the mempool/cnxk driver which provides the support for the
integrated mempool device found in Marvell CN10K SoC.

The code includes mempool driver functionality for Marvell CN9K SoC as well,
but right now it is not enabled. The future plan is to deprecate existing
mempool/octeontx2 driver once the 'CNXK' drivers are feature complete for
Marvell CN9K SoC.

Depends-on: series-16059 ("Add Marvell CNXK common driver")

v2:
 - Addressed Jerin's comments in v1.
 - Splitted mempool ops for cn10k/cn9k into multiple commits.
 - Added more description in the commit messages.
 - Moved MAINTAINERS and doc change to first commit.
 - Moved doc changes into respective commits implementing the change.

Ashwin Sekhar T K (11):
  mempool/cnxk: add build infra and doc
  mempool/cnxk: add device probe/remove
  mempool/cnxk: add generic ops
  mempool/cnxk: register lf init/fini callbacks
  mempool/cnxk: add cn9k mempool ops
  mempool/cnxk: add cn9k optimized mempool enqueue/dequeue
  mempool/cnxk: add cn10k mempool ops
  mempool/cnxk: add batch op init
  mempool/cnxk: add cn10k batch enqueue op
  mempool/cnxk: add cn10k get count op
  mempool/cnxk: add cn10k batch dequeue op

 MAINTAINERS                              |   6 +
 doc/guides/mempool/cnxk.rst              |  91 +++++++
 doc/guides/mempool/index.rst             |   1 +
 doc/guides/platform/cnxk.rst             |   3 +
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 294 +++++++++++++++++++++++
 drivers/mempool/cnxk/cn9k_mempool_ops.c  |  89 +++++++
 drivers/mempool/cnxk/cnxk_mempool.c      | 201 ++++++++++++++++
 drivers/mempool/cnxk/cnxk_mempool.h      |  29 +++
 drivers/mempool/cnxk/cnxk_mempool_ops.c  | 199 +++++++++++++++
 drivers/mempool/cnxk/meson.build         |  16 ++
 drivers/mempool/cnxk/version.map         |   3 +
 drivers/mempool/meson.build              |   3 +-
 12 files changed, 934 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/mempool/cnxk.rst
 create mode 100644 drivers/mempool/cnxk/cn10k_mempool_ops.c
 create mode 100644 drivers/mempool/cnxk/cn9k_mempool_ops.c
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.c
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.h
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool_ops.c
 create mode 100644 drivers/mempool/cnxk/meson.build
 create mode 100644 drivers/mempool/cnxk/version.map

-- 
2.31.0

[dpdk-dev] [PATCH v2 01/11] mempool/cnxk: add build infra and doc

[Ashwin Sekhar T K]

Add the meson based build infrastructure for Marvell
CNXK mempool driver along with stub implementations
for mempool device probe.

Also add Marvell CNXK mempool base documentation.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Jerin Jacob <jerinj@marvell.com>
Signed-off-by: Nithin Dabilpuram <ndabilpuram@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 MAINTAINERS                         |  6 +++
 doc/guides/mempool/cnxk.rst         | 55 ++++++++++++++++++++
 doc/guides/mempool/index.rst        |  1 +
 doc/guides/platform/cnxk.rst        |  3 ++
 drivers/mempool/cnxk/cnxk_mempool.c | 78 +++++++++++++++++++++++++++++
 drivers/mempool/cnxk/meson.build    | 13 +++++
 drivers/mempool/cnxk/version.map    |  3 ++
 drivers/mempool/meson.build         |  3 +-
 8 files changed, 161 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/mempool/cnxk.rst
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.c
 create mode 100644 drivers/mempool/cnxk/meson.build
 create mode 100644 drivers/mempool/cnxk/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index c837516d14..bae8b93030 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -501,6 +501,12 @@ M: Artem V. Andreev <artem.andreev@oktetlabs.ru>
 M: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
 F: drivers/mempool/bucket/
 
+Marvell cnxk
+M: Ashwin Sekhar T K <asekhar@marvell.com>
+M: Pavan Nikhilesh <pbhagavatula@marvell.com>
+F: drivers/mempool/cnxk/
+F: doc/guides/mempool/cnxk.rst
+
 Marvell OCTEON TX2
 M: Jerin Jacob <jerinj@marvell.com>
 M: Nithin Dabilpuram <ndabilpuram@marvell.com>
diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
new file mode 100644
index 0000000000..e72a77c361
--- /dev/null
+++ b/doc/guides/mempool/cnxk.rst
@@ -0,0 +1,55 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(C) 2021 Marvell.
+
+CNXK NPA Mempool Driver
+============================
+
+The CNXK NPA PMD (**librte_mempool_cnxk**) provides mempool driver support for
+the integrated mempool device found in **Marvell OCTEON CN9K/CN10K** SoC family.
+
+More information about CNXK SoC can be found at `Marvell Official Website
+<https://www.marvell.com/embedded-processors/infrastructure-processors/>`_.
+
+Features
+--------
+
+CNXK NPA PMD supports:
+
+- Up to 128 NPA LFs
+- 1M Pools per LF
+- HW mempool manager
+- Ethdev Rx buffer allocation in HW to save CPU cycles in the Rx path.
+- Ethdev Tx buffer recycling in HW to save CPU cycles in the Tx path.
+
+Prerequisites and Compilation procedure
+---------------------------------------
+
+   See :doc:`../platform/cnxk` for setup information.
+
+Pre-Installation Configuration
+------------------------------
+
+
+Debugging Options
+~~~~~~~~~~~~~~~~~
+
+.. _table_cnxk_mempool_debug_options:
+
+.. table:: CNXK mempool debug options
+
+   +---+------------+-------------------------------------------------------+
+   | # | Component  | EAL log command                                       |
+   +===+============+=======================================================+
+   | 1 | NPA        | --log-level='pmd\.mempool.cnxk,8'                     |
+   +---+------------+-------------------------------------------------------+
+
+Standalone mempool device
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+   The ``usertools/dpdk-devbind.py`` script shall enumerate all the mempool
+   devices available in the system. In order to avoid, the end user to bind the
+   mempool device prior to use ethdev and/or eventdev device, the respective
+   driver configures an NPA LF and attach to the first probed ethdev or eventdev
+   device. In case, if end user need to run mempool as a standalone device
+   (without ethdev or eventdev), end user needs to bind a mempool device using
+   ``usertools/dpdk-devbind.py``
diff --git a/doc/guides/mempool/index.rst b/doc/guides/mempool/index.rst
index a0e55467e6..ce53bc1ac7 100644
--- a/doc/guides/mempool/index.rst
+++ b/doc/guides/mempool/index.rst
@@ -11,6 +11,7 @@ application through the mempool API.
     :maxdepth: 2
     :numbered:
 
+    cnxk
     octeontx
     octeontx2
     ring
diff --git a/doc/guides/platform/cnxk.rst b/doc/guides/platform/cnxk.rst
index 3b072877a1..9bbba65f2e 100644
--- a/doc/guides/platform/cnxk.rst
+++ b/doc/guides/platform/cnxk.rst
@@ -141,6 +141,9 @@ HW Offload Drivers
 
 This section lists dataplane H/W block(s) available in CNXK SoC.
 
+#. **Mempool Driver**
+   See :doc:`../mempool/cnxk` for NPA mempool driver information.
+
 Procedure to Setup Platform
 ---------------------------
 
diff --git a/drivers/mempool/cnxk/cnxk_mempool.c b/drivers/mempool/cnxk/cnxk_mempool.c
new file mode 100644
index 0000000000..947078c052
--- /dev/null
+++ b/drivers/mempool/cnxk/cnxk_mempool.c
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_atomic.h>
+#include <rte_bus_pci.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+#include <rte_eal.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_mbuf_pool_ops.h>
+#include <rte_pci.h>
+
+#include "roc_api.h"
+
+static int
+npa_remove(struct rte_pci_device *pci_dev)
+{
+	RTE_SET_USED(pci_dev);
+
+	return 0;
+}
+
+static int
+npa_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
+{
+	RTE_SET_USED(pci_drv);
+	RTE_SET_USED(pci_dev);
+
+	return 0;
+}
+
+static const struct rte_pci_id npa_pci_map[] = {
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_PF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KA,
+	},
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_PF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KAS,
+	},
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_VF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KA,
+	},
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_VF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KAS,
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static struct rte_pci_driver npa_pci = {
+	.id_table = npa_pci_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_NEED_IOVA_AS_VA,
+	.probe = npa_probe,
+	.remove = npa_remove,
+};
+
+RTE_PMD_REGISTER_PCI(mempool_cnxk, npa_pci);
+RTE_PMD_REGISTER_PCI_TABLE(mempool_cnxk, npa_pci_map);
+RTE_PMD_REGISTER_KMOD_DEP(mempool_cnxk, "vfio-pci");
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
new file mode 100644
index 0000000000..0be0802373
--- /dev/null
+++ b/drivers/mempool/cnxk/meson.build
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(C) 2021 Marvell.
+#
+
+if not is_linux or not dpdk_conf.get('RTE_ARCH_64')
+	build = false
+	reason = 'only supported on 64-bit Linux'
+	subdir_done()
+endif
+
+sources = files('cnxk_mempool.c')
+
+deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
diff --git a/drivers/mempool/cnxk/version.map b/drivers/mempool/cnxk/version.map
new file mode 100644
index 0000000000..ee80c51721
--- /dev/null
+++ b/drivers/mempool/cnxk/version.map
@@ -0,0 +1,3 @@
+INTERNAL {
+	local: *;
+};
diff --git a/drivers/mempool/meson.build b/drivers/mempool/meson.build
index 4428813dae..a2814c1dfa 100644
--- a/drivers/mempool/meson.build
+++ b/drivers/mempool/meson.build
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-drivers = ['bucket', 'dpaa', 'dpaa2', 'octeontx', 'octeontx2', 'ring', 'stack']
+drivers = ['bucket', 'cnxk', 'dpaa', 'dpaa2', 'octeontx', 'octeontx2', 'ring',
+	   'stack']
 std_deps = ['mempool']
-- 
2.31.0

[dpdk-dev] [PATCH v2 02/11] mempool/cnxk: add device probe/remove

[Ashwin Sekhar T K]

Add the implementation for CNXk mempool device
probe and remove.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 doc/guides/mempool/cnxk.rst         |  23 +++++
 drivers/mempool/cnxk/cnxk_mempool.c | 131 +++++++++++++++++++++++++++-
 2 files changed, 150 insertions(+), 4 deletions(-)

diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
index e72a77c361..907c19c841 100644
--- a/doc/guides/mempool/cnxk.rst
+++ b/doc/guides/mempool/cnxk.rst
@@ -30,6 +30,29 @@ Pre-Installation Configuration
 ------------------------------
 
 
+Runtime Config Options
+~~~~~~~~~~~~~~~~~~~~~~
+
+- ``Maximum number of mempools per application`` (default ``128``)
+
+  The maximum number of mempools per application needs to be configured on
+  HW during mempool driver initialization. HW can support up to 1M mempools,
+  Since each mempool costs set of HW resources, the ``max_pools`` ``devargs``
+  parameter is being introduced to configure the number of mempools required
+  for the application.
+  For example::
+
+    -a 0002:02:00.0,max_pools=512
+
+  With the above configuration, the driver will set up only 512 mempools for
+  the given application to save HW resources.
+
+.. note::
+
+   Since this configuration is per application, the end user needs to
+   provide ``max_pools`` parameter to the first PCIe device probed by the given
+   application.
+
 Debugging Options
 ~~~~~~~~~~~~~~~~~
 
diff --git a/drivers/mempool/cnxk/cnxk_mempool.c b/drivers/mempool/cnxk/cnxk_mempool.c
index 947078c052..703d15be42 100644
--- a/drivers/mempool/cnxk/cnxk_mempool.c
+++ b/drivers/mempool/cnxk/cnxk_mempool.c
@@ -15,21 +15,142 @@
 
 #include "roc_api.h"
 
+#define CNXK_NPA_DEV_NAME	 RTE_STR(cnxk_npa_dev_)
+#define CNXK_NPA_DEV_NAME_LEN	 (sizeof(CNXK_NPA_DEV_NAME) + PCI_PRI_STR_SIZE)
+#define CNXK_NPA_MAX_POOLS_PARAM "max_pools"
+
+static inline uint32_t
+npa_aura_size_to_u32(uint8_t val)
+{
+	if (val == NPA_AURA_SZ_0)
+		return 128;
+	if (val >= NPA_AURA_SZ_MAX)
+		return BIT_ULL(20);
+
+	return 1 << (val + 6);
+}
+
 static int
-npa_remove(struct rte_pci_device *pci_dev)
+parse_max_pools(const char *key, const char *value, void *extra_args)
 {
-	RTE_SET_USED(pci_dev);
+	RTE_SET_USED(key);
+	uint32_t val;
 
+	val = atoi(value);
+	if (val < npa_aura_size_to_u32(NPA_AURA_SZ_128))
+		val = 128;
+	if (val > npa_aura_size_to_u32(NPA_AURA_SZ_1M))
+		val = BIT_ULL(20);
+
+	*(uint8_t *)extra_args = rte_log2_u32(val) - 6;
 	return 0;
 }
 
+static inline uint8_t
+parse_aura_size(struct rte_devargs *devargs)
+{
+	uint8_t aura_sz = NPA_AURA_SZ_128;
+	struct rte_kvargs *kvlist;
+
+	if (devargs == NULL)
+		goto exit;
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL)
+		goto exit;
+
+	rte_kvargs_process(kvlist, CNXK_NPA_MAX_POOLS_PARAM, &parse_max_pools,
+			   &aura_sz);
+	rte_kvargs_free(kvlist);
+exit:
+	return aura_sz;
+}
+
+static inline char *
+npa_dev_to_name(struct rte_pci_device *pci_dev, char *name)
+{
+	snprintf(name, CNXK_NPA_DEV_NAME_LEN, CNXK_NPA_DEV_NAME PCI_PRI_FMT,
+		 pci_dev->addr.domain, pci_dev->addr.bus, pci_dev->addr.devid,
+		 pci_dev->addr.function);
+
+	return name;
+}
+
+static int
+npa_init(struct rte_pci_device *pci_dev)
+{
+	char name[CNXK_NPA_DEV_NAME_LEN];
+	const struct rte_memzone *mz;
+	struct roc_npa *dev;
+	int rc;
+
+	rc = roc_plt_init();
+	if (rc < 0)
+		goto error;
+
+	rc = -ENOMEM;
+	mz = rte_memzone_reserve_aligned(npa_dev_to_name(pci_dev, name),
+					 sizeof(*dev), SOCKET_ID_ANY, 0,
+					 RTE_CACHE_LINE_SIZE);
+	if (mz == NULL)
+		goto error;
+
+	dev = mz->addr;
+	dev->pci_dev = pci_dev;
+
+	roc_idev_npa_maxpools_set(parse_aura_size(pci_dev->device.devargs));
+	rc = roc_npa_dev_init(dev);
+	if (rc)
+		goto mz_free;
+
+	return 0;
+
+mz_free:
+	rte_memzone_free(mz);
+error:
+	plt_err("failed to initialize npa device rc=%d", rc);
+	return rc;
+}
+
+static int
+npa_fini(struct rte_pci_device *pci_dev)
+{
+	char name[CNXK_NPA_DEV_NAME_LEN];
+	const struct rte_memzone *mz;
+	int rc;
+
+	mz = rte_memzone_lookup(npa_dev_to_name(pci_dev, name));
+	if (mz == NULL)
+		return -EINVAL;
+
+	rc = roc_npa_dev_fini(mz->addr);
+	if (rc) {
+		if (rc != -EAGAIN)
+			plt_err("Failed to remove npa dev, rc=%d", rc);
+		return rc;
+	}
+	rte_memzone_free(mz);
+
+	return 0;
+}
+
+static int
+npa_remove(struct rte_pci_device *pci_dev)
+{
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	return npa_fini(pci_dev);
+}
+
 static int
 npa_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 {
 	RTE_SET_USED(pci_drv);
-	RTE_SET_USED(pci_dev);
 
-	return 0;
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	return npa_init(pci_dev);
 }
 
 static const struct rte_pci_id npa_pci_map[] = {
@@ -76,3 +197,5 @@ static struct rte_pci_driver npa_pci = {
 RTE_PMD_REGISTER_PCI(mempool_cnxk, npa_pci);
 RTE_PMD_REGISTER_PCI_TABLE(mempool_cnxk, npa_pci_map);
 RTE_PMD_REGISTER_KMOD_DEP(mempool_cnxk, "vfio-pci");
+RTE_PMD_REGISTER_PARAM_STRING(mempool_cnxk,
+			      CNXK_NPA_MAX_POOLS_PARAM "=<128-1048576>");
-- 
2.31.0

[dpdk-dev] [PATCH v2 03/11] mempool/cnxk: add generic ops

[Ashwin Sekhar T K]

Add generic CNXk mempool ops which will enqueue/dequeue
from pool one element at a time.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cnxk_mempool.h     |  26 ++++
 drivers/mempool/cnxk/cnxk_mempool_ops.c | 171 ++++++++++++++++++++++++
 drivers/mempool/cnxk/meson.build        |   3 +-
 3 files changed, 199 insertions(+), 1 deletion(-)
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.h
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool_ops.c

diff --git a/drivers/mempool/cnxk/cnxk_mempool.h b/drivers/mempool/cnxk/cnxk_mempool.h
new file mode 100644
index 0000000000..099b7f6998
--- /dev/null
+++ b/drivers/mempool/cnxk/cnxk_mempool.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#ifndef _CNXK_MEMPOOL_H_
+#define _CNXK_MEMPOOL_H_
+
+#include <rte_mempool.h>
+
+unsigned int cnxk_mempool_get_count(const struct rte_mempool *mp);
+ssize_t cnxk_mempool_calc_mem_size(const struct rte_mempool *mp,
+				   uint32_t obj_num, uint32_t pg_shift,
+				   size_t *min_chunk_size, size_t *align);
+int cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
+			  void *vaddr, rte_iova_t iova, size_t len,
+			  rte_mempool_populate_obj_cb_t *obj_cb,
+			  void *obj_cb_arg);
+int cnxk_mempool_alloc(struct rte_mempool *mp);
+void cnxk_mempool_free(struct rte_mempool *mp);
+
+int __rte_hot cnxk_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
+			       unsigned int n);
+int __rte_hot cnxk_mempool_deq(struct rte_mempool *mp, void **obj_table,
+			       unsigned int n);
+
+#endif
diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
new file mode 100644
index 0000000000..2ce1816c04
--- /dev/null
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -0,0 +1,171 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_mempool.h>
+
+#include "roc_api.h"
+#include "cnxk_mempool.h"
+
+int __rte_hot
+cnxk_mempool_enq(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
+{
+	unsigned int index;
+
+	/* Ensure mbuf init changes are written before the free pointers
+	 * are enqueued to the stack.
+	 */
+	rte_io_wmb();
+	for (index = 0; index < n; index++)
+		roc_npa_aura_op_free(mp->pool_id, 0,
+				     (uint64_t)obj_table[index]);
+
+	return 0;
+}
+
+int __rte_hot
+cnxk_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
+{
+	unsigned int index;
+	uint64_t obj;
+
+	for (index = 0; index < n; index++, obj_table++) {
+		int retry = 4;
+
+		/* Retry few times before failing */
+		do {
+			obj = roc_npa_aura_op_alloc(mp->pool_id, 0);
+		} while (retry-- && (obj == 0));
+
+		if (obj == 0) {
+			cnxk_mempool_enq(mp, obj_table - index, index);
+			return -ENOENT;
+		}
+		*obj_table = (void *)obj;
+	}
+
+	return 0;
+}
+
+unsigned int
+cnxk_mempool_get_count(const struct rte_mempool *mp)
+{
+	return (unsigned int)roc_npa_aura_op_available(mp->pool_id);
+}
+
+ssize_t
+cnxk_mempool_calc_mem_size(const struct rte_mempool *mp, uint32_t obj_num,
+			   uint32_t pg_shift, size_t *min_chunk_size,
+			   size_t *align)
+{
+	size_t total_elt_sz;
+
+	/* Need space for one more obj on each chunk to fulfill
+	 * alignment requirements.
+	 */
+	total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
+	return rte_mempool_op_calc_mem_size_helper(
+		mp, obj_num, pg_shift, total_elt_sz, min_chunk_size, align);
+}
+
+int
+cnxk_mempool_alloc(struct rte_mempool *mp)
+{
+	uint64_t aura_handle = 0;
+	struct npa_aura_s aura;
+	struct npa_pool_s pool;
+	uint32_t block_count;
+	size_t block_size;
+	int rc = -ERANGE;
+
+	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
+	block_count = mp->size;
+	if (mp->header_size % ROC_ALIGN != 0) {
+		plt_err("Header size should be multiple of %dB", ROC_ALIGN);
+		goto error;
+	}
+
+	if (block_size % ROC_ALIGN != 0) {
+		plt_err("Block size should be multiple of %dB", ROC_ALIGN);
+		goto error;
+	}
+
+	memset(&aura, 0, sizeof(struct npa_aura_s));
+	memset(&pool, 0, sizeof(struct npa_pool_s));
+	pool.nat_align = 1;
+	pool.buf_offset = mp->header_size / ROC_ALIGN;
+
+	/* Use driver specific mp->pool_config to override aura config */
+	if (mp->pool_config != NULL)
+		memcpy(&aura, mp->pool_config, sizeof(struct npa_aura_s));
+
+	rc = roc_npa_pool_create(&aura_handle, block_size, block_count, &aura,
+				 &pool);
+	if (rc) {
+		plt_err("Failed to alloc pool or aura rc=%d", rc);
+		goto error;
+	}
+
+	/* Store aura_handle for future queue operations */
+	mp->pool_id = aura_handle;
+	plt_npa_dbg("block_sz=%lu block_count=%d aura_handle=0x%" PRIx64,
+		    block_size, block_count, aura_handle);
+
+	return 0;
+error:
+	return rc;
+}
+
+void
+cnxk_mempool_free(struct rte_mempool *mp)
+{
+	int rc = 0;
+
+	plt_npa_dbg("aura_handle=0x%" PRIx64, mp->pool_id);
+	rc = roc_npa_pool_destroy(mp->pool_id);
+	if (rc)
+		plt_err("Failed to free pool or aura rc=%d", rc);
+}
+
+int
+cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
+		      void *vaddr, rte_iova_t iova, size_t len,
+		      rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
+{
+	size_t total_elt_sz, off;
+	int num_elts;
+
+	if (iova == RTE_BAD_IOVA)
+		return -EINVAL;
+
+	total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
+
+	/* Align object start address to a multiple of total_elt_sz */
+	off = total_elt_sz - ((((uintptr_t)vaddr - 1) % total_elt_sz) + 1);
+
+	if (len < off)
+		return -EINVAL;
+
+	vaddr = (char *)vaddr + off;
+	iova += off;
+	len -= off;
+	num_elts = len / total_elt_sz;
+
+	plt_npa_dbg("iova %" PRIx64 ", aligned iova %" PRIx64 "", iova - off,
+		    iova);
+	plt_npa_dbg("length %" PRIu64 ", aligned length %" PRIu64 "",
+		    (uint64_t)(len + off), (uint64_t)len);
+	plt_npa_dbg("element size %" PRIu64 "", (uint64_t)total_elt_sz);
+	plt_npa_dbg("requested objects %" PRIu64 ", possible objects %" PRIu64
+		    "", (uint64_t)max_objs, (uint64_t)num_elts);
+
+	roc_npa_aura_op_range_set(mp->pool_id, iova,
+				  iova + num_elts * total_elt_sz);
+
+	if (roc_npa_pool_range_update_check(mp->pool_id) < 0)
+		return -EBUSY;
+
+	return rte_mempool_op_populate_helper(
+		mp, RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ, max_objs, vaddr, iova,
+		len, obj_cb, obj_cb_arg);
+}
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
index 0be0802373..52244e728b 100644
--- a/drivers/mempool/cnxk/meson.build
+++ b/drivers/mempool/cnxk/meson.build
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.get('RTE_ARCH_64')
 	subdir_done()
 endif
 
-sources = files('cnxk_mempool.c')
+sources = files('cnxk_mempool.c',
+		'cnxk_mempool_ops.c')
 
 deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
-- 
2.31.0

[dpdk-dev] [PATCH v2 04/11] mempool/cnxk: register lf init/fini callbacks

[Ashwin Sekhar T K]

Register the CNXk mempool lf init/fini callbacks which
will set the appropriate mempool ops to be used according
to the platform.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cnxk_mempool_ops.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
index 2ce1816c04..18c307288c 100644
--- a/drivers/mempool/cnxk/cnxk_mempool_ops.c
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -2,6 +2,7 @@
  * Copyright(C) 2021 Marvell.
  */
 
+#include <rte_mbuf_pool_ops.h>
 #include <rte_mempool.h>
 
 #include "roc_api.h"
@@ -169,3 +170,23 @@ cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
 		mp, RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ, max_objs, vaddr, iova,
 		len, obj_cb, obj_cb_arg);
 }
+
+static int
+cnxk_mempool_lf_init(void)
+{
+	if (roc_model_is_cn10k() || roc_model_is_cn9k())
+		rte_mbuf_set_platform_mempool_ops("cnxk_mempool_ops");
+
+	return 0;
+}
+
+static void
+cnxk_mempool_lf_fini(void)
+{
+}
+
+RTE_INIT(cnxk_mempool_ops_init)
+{
+	roc_npa_lf_init_cb_register(cnxk_mempool_lf_init);
+	roc_npa_lf_fini_cb_register(cnxk_mempool_lf_fini);
+}
-- 
2.31.0

[dpdk-dev] [PATCH v2 05/11] mempool/cnxk: add cn9k mempool ops

[Ashwin Sekhar T K]

Add Marvell CN9k mempool ops and implement CN9k mempool
alloc which makes sure that the element size always occupy
odd number of cachelines to ensure even distribution among
of elements among L1D cache sets.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn9k_mempool_ops.c | 54 +++++++++++++++++++++++++
 drivers/mempool/cnxk/cnxk_mempool_ops.c |  4 +-
 drivers/mempool/cnxk/meson.build        |  3 +-
 3 files changed, 59 insertions(+), 2 deletions(-)
 create mode 100644 drivers/mempool/cnxk/cn9k_mempool_ops.c

diff --git a/drivers/mempool/cnxk/cn9k_mempool_ops.c b/drivers/mempool/cnxk/cn9k_mempool_ops.c
new file mode 100644
index 0000000000..f5ac163af9
--- /dev/null
+++ b/drivers/mempool/cnxk/cn9k_mempool_ops.c
@@ -0,0 +1,54 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_mempool.h>
+
+#include "roc_api.h"
+#include "cnxk_mempool.h"
+
+static int
+cn9k_mempool_alloc(struct rte_mempool *mp)
+{
+	size_t block_size, padding;
+
+	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
+	/* Align header size to ROC_ALIGN */
+	if (mp->header_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(mp->header_size, ROC_ALIGN) -
+			  mp->header_size;
+		mp->header_size += padding;
+		block_size += padding;
+	}
+
+	/* Align block size to ROC_ALIGN */
+	if (block_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(block_size, ROC_ALIGN) - block_size;
+		mp->trailer_size += padding;
+		block_size += padding;
+	}
+
+	/*
+	 * Marvell CN9k has 8 sets, 41 ways L1D cache, VA<9:7> bits dictate the
+	 * set selection. Add additional padding to ensure that the element size
+	 * always occupies odd number of cachelines to ensure even distribution
+	 * of elements among L1D cache sets.
+	 */
+	padding = ((block_size / ROC_ALIGN) % 2) ? 0 : ROC_ALIGN;
+	mp->trailer_size += padding;
+
+	return cnxk_mempool_alloc(mp);
+}
+
+static struct rte_mempool_ops cn9k_mempool_ops = {
+	.name = "cn9k_mempool_ops",
+	.alloc = cn9k_mempool_alloc,
+	.free = cnxk_mempool_free,
+	.enqueue = cnxk_mempool_enq,
+	.dequeue = cnxk_mempool_deq,
+	.get_count = cnxk_mempool_get_count,
+	.calc_mem_size = cnxk_mempool_calc_mem_size,
+	.populate = cnxk_mempool_populate,
+};
+
+MEMPOOL_REGISTER_OPS(cn9k_mempool_ops);
diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
index 18c307288c..45c45e9943 100644
--- a/drivers/mempool/cnxk/cnxk_mempool_ops.c
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -174,7 +174,9 @@ cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
 static int
 cnxk_mempool_lf_init(void)
 {
-	if (roc_model_is_cn10k() || roc_model_is_cn9k())
+	if (roc_model_is_cn9k())
+		rte_mbuf_set_platform_mempool_ops("cn9k_mempool_ops");
+	else if (roc_model_is_cn10k())
 		rte_mbuf_set_platform_mempool_ops("cnxk_mempool_ops");
 
 	return 0;
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
index 52244e728b..ff31893ff4 100644
--- a/drivers/mempool/cnxk/meson.build
+++ b/drivers/mempool/cnxk/meson.build
@@ -9,6 +9,7 @@ if not is_linux or not dpdk_conf.get('RTE_ARCH_64')
 endif
 
 sources = files('cnxk_mempool.c',
-		'cnxk_mempool_ops.c')
+		'cnxk_mempool_ops.c',
+		'cn9k_mempool_ops.c')
 
 deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
-- 
2.31.0

[dpdk-dev] [PATCH v2 06/11] mempool/cnxk: add cn9k optimized mempool enqueue/dequeue

[Ashwin Sekhar T K]

Add Marvell CN9k mempool enqueue/dequeue. Marvell CN9k
supports burst dequeue which allows to dequeue up to 32
pointers using pipelined casp instructions.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 doc/guides/mempool/cnxk.rst             |  4 +++
 drivers/mempool/cnxk/cn9k_mempool_ops.c | 39 +++++++++++++++++++++++--
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
index 907c19c841..f51532b101 100644
--- a/doc/guides/mempool/cnxk.rst
+++ b/doc/guides/mempool/cnxk.rst
@@ -21,6 +21,10 @@ CNXK NPA PMD supports:
 - Ethdev Rx buffer allocation in HW to save CPU cycles in the Rx path.
 - Ethdev Tx buffer recycling in HW to save CPU cycles in the Tx path.
 
+CN9k NPA supports:
+
+- Burst alloc of up to 32 pointers.
+
 Prerequisites and Compilation procedure
 ---------------------------------------
 
diff --git a/drivers/mempool/cnxk/cn9k_mempool_ops.c b/drivers/mempool/cnxk/cn9k_mempool_ops.c
index f5ac163af9..c0cdba640b 100644
--- a/drivers/mempool/cnxk/cn9k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn9k_mempool_ops.c
@@ -7,6 +7,41 @@
 #include "roc_api.h"
 #include "cnxk_mempool.h"
 
+static int __rte_hot
+cn9k_mempool_enq(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
+{
+	/* Ensure mbuf init changes are written before the free pointers
+	 * are enqueued to the stack.
+	 */
+	rte_io_wmb();
+	roc_npa_aura_op_bulk_free(mp->pool_id, (const uint64_t *)obj_table, n,
+				  0);
+
+	return 0;
+}
+
+static inline int __rte_hot
+cn9k_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
+{
+	unsigned int count;
+
+	count = roc_npa_aura_op_bulk_alloc(mp->pool_id, (uint64_t *)obj_table,
+					   n, 0, 1);
+
+	if (unlikely(count != n)) {
+		/* If bulk alloc failed to allocate all pointers, try
+		 * allocating remaining pointers with the default alloc
+		 * with retry scheme.
+		 */
+		if (cnxk_mempool_deq(mp, &obj_table[count], n - count)) {
+			cn9k_mempool_enq(mp, obj_table, count);
+			return -ENOENT;
+		}
+	}
+
+	return 0;
+}
+
 static int
 cn9k_mempool_alloc(struct rte_mempool *mp)
 {
@@ -44,8 +79,8 @@ static struct rte_mempool_ops cn9k_mempool_ops = {
 	.name = "cn9k_mempool_ops",
 	.alloc = cn9k_mempool_alloc,
 	.free = cnxk_mempool_free,
-	.enqueue = cnxk_mempool_enq,
-	.dequeue = cnxk_mempool_deq,
+	.enqueue = cn9k_mempool_enq,
+	.dequeue = cn9k_mempool_deq,
 	.get_count = cnxk_mempool_get_count,
 	.calc_mem_size = cnxk_mempool_calc_mem_size,
 	.populate = cnxk_mempool_populate,
-- 
2.31.0

[dpdk-dev] [PATCH v2 07/11] mempool/cnxk: add cn10k mempool ops

[Ashwin Sekhar T K]

Add Marvell CN10k mempool ops and implement CN10k mempool alloc.

CN10k has 64 bytes L1D cache line size. Hence the CN10k mempool
alloc does not make the element size an odd multiple L1D cache
line size as NPA requires the element sizes to be multiples of
128 bytes.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 doc/guides/mempool/cnxk.rst              |  4 ++
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 52 ++++++++++++++++++++++++
 drivers/mempool/cnxk/cnxk_mempool_ops.c  |  2 +-
 drivers/mempool/cnxk/meson.build         |  3 +-
 4 files changed, 59 insertions(+), 2 deletions(-)
 create mode 100644 drivers/mempool/cnxk/cn10k_mempool_ops.c

diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
index f51532b101..783368e690 100644
--- a/doc/guides/mempool/cnxk.rst
+++ b/doc/guides/mempool/cnxk.rst
@@ -80,3 +80,7 @@ Standalone mempool device
    device. In case, if end user need to run mempool as a standalone device
    (without ethdev or eventdev), end user needs to bind a mempool device using
    ``usertools/dpdk-devbind.py``
+
+   Example command to run ``mempool_autotest`` test with standalone CN10K NPA device::
+
+     echo "mempool_autotest" | <build_dir>/app/test/dpdk-test -c 0xf0 --mbuf-pool-ops-name="cn10k_mempool_ops"
diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
new file mode 100644
index 0000000000..9b63789006
--- /dev/null
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_mempool.h>
+
+#include "roc_api.h"
+#include "cnxk_mempool.h"
+
+static int
+cn10k_mempool_alloc(struct rte_mempool *mp)
+{
+	uint32_t block_size;
+	size_t padding;
+
+	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
+	/* Align header size to ROC_ALIGN */
+	if (mp->header_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(mp->header_size, ROC_ALIGN) -
+			  mp->header_size;
+		mp->header_size += padding;
+		block_size += padding;
+	}
+
+	/* Align block size to ROC_ALIGN */
+	if (block_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(block_size, ROC_ALIGN) - block_size;
+		mp->trailer_size += padding;
+		block_size += padding;
+	}
+
+	return cnxk_mempool_alloc(mp);
+}
+
+static void
+cn10k_mempool_free(struct rte_mempool *mp)
+{
+	cnxk_mempool_free(mp);
+}
+
+static struct rte_mempool_ops cn10k_mempool_ops = {
+	.name = "cn10k_mempool_ops",
+	.alloc = cn10k_mempool_alloc,
+	.free = cn10k_mempool_free,
+	.enqueue = cnxk_mempool_enq,
+	.dequeue = cnxk_mempool_deq,
+	.get_count = cnxk_mempool_get_count,
+	.calc_mem_size = cnxk_mempool_calc_mem_size,
+	.populate = cnxk_mempool_populate,
+};
+
+MEMPOOL_REGISTER_OPS(cn10k_mempool_ops);
diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
index 45c45e9943..0ec131a475 100644
--- a/drivers/mempool/cnxk/cnxk_mempool_ops.c
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -177,7 +177,7 @@ cnxk_mempool_lf_init(void)
 	if (roc_model_is_cn9k())
 		rte_mbuf_set_platform_mempool_ops("cn9k_mempool_ops");
 	else if (roc_model_is_cn10k())
-		rte_mbuf_set_platform_mempool_ops("cnxk_mempool_ops");
+		rte_mbuf_set_platform_mempool_ops("cn10k_mempool_ops");
 
 	return 0;
 }
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
index ff31893ff4..3282b5e5a6 100644
--- a/drivers/mempool/cnxk/meson.build
+++ b/drivers/mempool/cnxk/meson.build
@@ -10,6 +10,7 @@ endif
 
 sources = files('cnxk_mempool.c',
 		'cnxk_mempool_ops.c',
-		'cn9k_mempool_ops.c')
+		'cn9k_mempool_ops.c',
+		'cn10k_mempool_ops.c')
 
 deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
-- 
2.31.0

[dpdk-dev] [PATCH v2 08/11] mempool/cnxk: add batch op init

[Ashwin Sekhar T K]

Marvell CN10k mempool supports batch enqueue/dequeue which can
dequeue up to 512 pointers and enqueue up to 15 pointers using
a single instruction.

These batch operations require a DMA memory to enqueue/dequeue
pointers. This patch adds the initialization of this DMA memory.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 doc/guides/mempool/cnxk.rst              |   5 +
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 122 ++++++++++++++++++++++-
 drivers/mempool/cnxk/cnxk_mempool.h      |   3 +
 drivers/mempool/cnxk/cnxk_mempool_ops.c  |  13 ++-
 4 files changed, 138 insertions(+), 5 deletions(-)

diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
index 783368e690..286ee29003 100644
--- a/doc/guides/mempool/cnxk.rst
+++ b/doc/guides/mempool/cnxk.rst
@@ -25,6 +25,11 @@ CN9k NPA supports:
 
 - Burst alloc of up to 32 pointers.
 
+CN10k NPA supports:
+
+- Batch dequeue of up to 512 pointers with single instruction.
+- Batch enqueue of up to 15 pointers with single instruction.
+
 Prerequisites and Compilation procedure
 ---------------------------------------
 
diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
index 9b63789006..d34041528a 100644
--- a/drivers/mempool/cnxk/cn10k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -7,11 +7,117 @@
 #include "roc_api.h"
 #include "cnxk_mempool.h"
 
+#define BATCH_ALLOC_SZ ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS
+
+enum batch_op_status {
+	BATCH_ALLOC_OP_NOT_ISSUED = 0,
+	BATCH_ALLOC_OP_ISSUED = 1,
+	BATCH_ALLOC_OP_DONE
+};
+
+struct batch_op_mem {
+	unsigned int sz;
+	enum batch_op_status status;
+	uint64_t objs[BATCH_ALLOC_SZ] __rte_aligned(ROC_ALIGN);
+};
+
+struct batch_op_data {
+	uint64_t lmt_addr;
+	struct batch_op_mem mem[RTE_MAX_LCORE] __rte_aligned(ROC_ALIGN);
+};
+
+static struct batch_op_data **batch_op_data;
+
+#define BATCH_OP_DATA_GET(pool_id)                                             \
+	batch_op_data[roc_npa_aura_handle_to_aura(pool_id)]
+
+#define BATCH_OP_DATA_SET(pool_id, op_data)                                    \
+	do {                                                                   \
+		uint64_t aura = roc_npa_aura_handle_to_aura(pool_id);          \
+		batch_op_data[aura] = op_data;                                 \
+	} while (0)
+
+int
+cn10k_mempool_lf_init(void)
+{
+	unsigned int maxpools, sz;
+
+	maxpools = roc_idev_npa_maxpools_get();
+	sz = maxpools * sizeof(struct batch_op_data *);
+
+	batch_op_data = rte_zmalloc(NULL, sz, ROC_ALIGN);
+	if (!batch_op_data)
+		return -1;
+
+	return 0;
+}
+
+void
+cn10k_mempool_lf_fini(void)
+{
+	if (!batch_op_data)
+		return;
+
+	rte_free(batch_op_data);
+	batch_op_data = NULL;
+}
+
+static int
+batch_op_init(struct rte_mempool *mp)
+{
+	struct batch_op_data *op_data;
+	int i;
+
+	RTE_ASSERT(BATCH_OP_DATA_GET(mp->pool_id) == NULL);
+	op_data = rte_zmalloc(NULL, sizeof(struct batch_op_data), ROC_ALIGN);
+	if (op_data == NULL)
+		return -1;
+
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		op_data->mem[i].sz = 0;
+		op_data->mem[i].status = BATCH_ALLOC_OP_NOT_ISSUED;
+	}
+
+	op_data->lmt_addr = roc_idev_lmt_base_addr_get();
+	BATCH_OP_DATA_SET(mp->pool_id, op_data);
+
+	return 0;
+}
+
+static void
+batch_op_fini(struct rte_mempool *mp)
+{
+	struct batch_op_data *op_data;
+	int i;
+
+	op_data = BATCH_OP_DATA_GET(mp->pool_id);
+
+	rte_wmb();
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct batch_op_mem *mem = &op_data->mem[i];
+
+		if (mem->status == BATCH_ALLOC_OP_ISSUED) {
+			mem->sz = roc_npa_aura_batch_alloc_extract(
+				mem->objs, mem->objs, BATCH_ALLOC_SZ);
+			mem->status = BATCH_ALLOC_OP_DONE;
+		}
+		if (mem->status == BATCH_ALLOC_OP_DONE) {
+			roc_npa_aura_op_bulk_free(mp->pool_id, mem->objs,
+						  mem->sz, 1);
+			mem->status = BATCH_ALLOC_OP_NOT_ISSUED;
+		}
+	}
+
+	rte_free(op_data);
+	BATCH_OP_DATA_SET(mp->pool_id, NULL);
+}
+
 static int
 cn10k_mempool_alloc(struct rte_mempool *mp)
 {
 	uint32_t block_size;
 	size_t padding;
+	int rc;
 
 	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
 	/* Align header size to ROC_ALIGN */
@@ -29,12 +135,26 @@ cn10k_mempool_alloc(struct rte_mempool *mp)
 		block_size += padding;
 	}
 
-	return cnxk_mempool_alloc(mp);
+	rc = cnxk_mempool_alloc(mp);
+	if (rc)
+		return rc;
+
+	rc = batch_op_init(mp);
+	if (rc) {
+		plt_err("Failed to init batch alloc mem rc=%d", rc);
+		goto error;
+	}
+
+	return 0;
+error:
+	cnxk_mempool_free(mp);
+	return rc;
 }
 
 static void
 cn10k_mempool_free(struct rte_mempool *mp)
 {
+	batch_op_fini(mp);
 	cnxk_mempool_free(mp);
 }
 
diff --git a/drivers/mempool/cnxk/cnxk_mempool.h b/drivers/mempool/cnxk/cnxk_mempool.h
index 099b7f6998..6e54346e6a 100644
--- a/drivers/mempool/cnxk/cnxk_mempool.h
+++ b/drivers/mempool/cnxk/cnxk_mempool.h
@@ -23,4 +23,7 @@ int __rte_hot cnxk_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
 int __rte_hot cnxk_mempool_deq(struct rte_mempool *mp, void **obj_table,
 			       unsigned int n);
 
+int cn10k_mempool_lf_init(void);
+void cn10k_mempool_lf_fini(void);
+
 #endif
diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
index 0ec131a475..389c3622fd 100644
--- a/drivers/mempool/cnxk/cnxk_mempool_ops.c
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -174,17 +174,22 @@ cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
 static int
 cnxk_mempool_lf_init(void)
 {
-	if (roc_model_is_cn9k())
+	int rc = 0;
+
+	if (roc_model_is_cn9k()) {
 		rte_mbuf_set_platform_mempool_ops("cn9k_mempool_ops");
-	else if (roc_model_is_cn10k())
+	} else if (roc_model_is_cn10k()) {
 		rte_mbuf_set_platform_mempool_ops("cn10k_mempool_ops");
-
-	return 0;
+		rc = cn10k_mempool_lf_init();
+	}
+	return rc;
 }
 
 static void
 cnxk_mempool_lf_fini(void)
 {
+	if (roc_model_is_cn10k())
+		cn10k_mempool_lf_fini();
 }
 
 RTE_INIT(cnxk_mempool_ops_init)
-- 
2.31.0

Re: [dpdk-dev] [PATCH v2 08/11] mempool/cnxk: add batch op init

[Jerin Jacob]

On Sat, Apr 3, 2021 at 7:49 PM Ashwin Sekhar T K <asekhar@marvell.com> wrote:
>
> Marvell CN10k mempool supports batch enqueue/dequeue which can
> dequeue up to 512 pointers and enqueue up to 15 pointers using
> a single instruction.
>
> These batch operations require a DMA memory to enqueue/dequeue
> pointers. This patch adds the initialization of this DMA memory.
>
> Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
> ---
>  doc/guides/mempool/cnxk.rst              |   5 +
>  drivers/mempool/cnxk/cn10k_mempool_ops.c | 122 ++++++++++++++++++++++-
>  drivers/mempool/cnxk/cnxk_mempool.h      |   3 +
>  drivers/mempool/cnxk/cnxk_mempool_ops.c  |  13 ++-
>  4 files changed, 138 insertions(+), 5 deletions(-)
>
> +
> +static struct batch_op_data **batch_op_data;

Please remove the global variable as it will break the multi-process.

> +
> +#define BATCH_OP_DATA_GET(pool_id)                                             \
> +       batch_op_data[roc_npa_aura_handle_to_aura(pool_id)]
> +
> +#define BATCH_OP_DATA_SET(pool_id, op_data)                                    \
> +       do {                                                                   \
> +               uint64_t aura = roc_npa_aura_handle_to_aura(pool_id);          \
> +               batch_op_data[aura] = op_data;                                 \
> +       } while (0)
> +

Please check this can be made as static inline if there is NO performance cost.

[dpdk-dev] [PATCH v2 09/11] mempool/cnxk: add cn10k batch enqueue op

[Ashwin Sekhar T K]

Add the implementation for Marvell CN10k mempool batch enqueue op.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 28 +++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
index d34041528a..2e3ec414da 100644
--- a/drivers/mempool/cnxk/cn10k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -112,6 +112,32 @@ batch_op_fini(struct rte_mempool *mp)
 	BATCH_OP_DATA_SET(mp->pool_id, NULL);
 }
 
+static int __rte_hot
+cn10k_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
+		  unsigned int n)
+{
+	const uint64_t *ptr = (const uint64_t *)obj_table;
+	uint64_t lmt_addr = 0, lmt_id = 0;
+	struct batch_op_data *op_data;
+
+	/* Ensure mbuf init changes are written before the free pointers are
+	 * enqueued to the stack.
+	 */
+	rte_io_wmb();
+
+	if (n == 1) {
+		roc_npa_aura_op_free(mp->pool_id, 1, ptr[0]);
+		return 0;
+	}
+
+	op_data = BATCH_OP_DATA_GET(mp->pool_id);
+	lmt_addr = op_data->lmt_addr;
+	ROC_LMT_BASE_ID_GET(lmt_addr, lmt_id);
+	roc_npa_aura_op_batch_free(mp->pool_id, ptr, n, 1, lmt_addr, lmt_id);
+
+	return 0;
+}
+
 static int
 cn10k_mempool_alloc(struct rte_mempool *mp)
 {
@@ -162,7 +188,7 @@ static struct rte_mempool_ops cn10k_mempool_ops = {
 	.name = "cn10k_mempool_ops",
 	.alloc = cn10k_mempool_alloc,
 	.free = cn10k_mempool_free,
-	.enqueue = cnxk_mempool_enq,
+	.enqueue = cn10k_mempool_enq,
 	.dequeue = cnxk_mempool_deq,
 	.get_count = cnxk_mempool_get_count,
 	.calc_mem_size = cnxk_mempool_calc_mem_size,
-- 
2.31.0

Re: [dpdk-dev] [PATCH v2 09/11] mempool/cnxk: add cn10k batch enqueue op

[Jerin Jacob]

On Sat, Apr 3, 2021 at 7:49 PM Ashwin Sekhar T K <asekhar@marvell.com> wrote:
>
> Add the implementation for Marvell CN10k mempool batch enqueue op.
>
> Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
> ---
>  drivers/mempool/cnxk/cn10k_mempool_ops.c | 28 +++++++++++++++++++++++-
>  1 file changed, 27 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
> index d34041528a..2e3ec414da 100644
> --- a/drivers/mempool/cnxk/cn10k_mempool_ops.c
> +++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
> @@ -112,6 +112,32 @@ batch_op_fini(struct rte_mempool *mp)
>         BATCH_OP_DATA_SET(mp->pool_id, NULL);
>  }
>
> +static int __rte_hot
> +cn10k_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
> +                 unsigned int n)
> +{
> +       const uint64_t *ptr = (const uint64_t *)obj_table;
> +       uint64_t lmt_addr = 0, lmt_id = 0;

Please check the initialization to zero is required or not.

> +       struct batch_op_data *op_data;
> +
> +       /* Ensure mbuf init changes are written before the free pointers are
> +        * enqueued to the stack.
> +        */
> +       rte_io_wmb();
> +
> +       if (n == 1) {
> +               roc_npa_aura_op_free(mp->pool_id, 1, ptr[0]);
> +               return 0;
> +       }
> +
> +       op_data = BATCH_OP_DATA_GET(mp->pool_id);
> +       lmt_addr = op_data->lmt_addr;
> +       ROC_LMT_BASE_ID_GET(lmt_addr, lmt_id);
> +       roc_npa_aura_op_batch_free(mp->pool_id, ptr, n, 1, lmt_addr, lmt_id);
> +
> +       return 0;
> +}
> +
>  static int
>  cn10k_mempool_alloc(struct rte_mempool *mp)
>  {
> @@ -162,7 +188,7 @@ static struct rte_mempool_ops cn10k_mempool_ops = {
>         .name = "cn10k_mempool_ops",
>         .alloc = cn10k_mempool_alloc,
>         .free = cn10k_mempool_free,
> -       .enqueue = cnxk_mempool_enq,
> +       .enqueue = cn10k_mempool_enq,
>         .dequeue = cnxk_mempool_deq,
>         .get_count = cnxk_mempool_get_count,
>         .calc_mem_size = cnxk_mempool_calc_mem_size,
> --
> 2.31.0
>

[dpdk-dev] [PATCH v2 10/11] mempool/cnxk: add cn10k get count op

[Ashwin Sekhar T K]

Add the implementation for Marvell CN10k get count op.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 28 +++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
index 2e3ec414da..16b2f6697f 100644
--- a/drivers/mempool/cnxk/cn10k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -138,6 +138,32 @@ cn10k_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
 	return 0;
 }
 
+static unsigned int
+cn10k_mempool_get_count(const struct rte_mempool *mp)
+{
+	struct batch_op_data *op_data;
+	unsigned int count = 0;
+	int i;
+
+	op_data = BATCH_OP_DATA_GET(mp->pool_id);
+
+	rte_wmb();
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct batch_op_mem *mem = &op_data->mem[i];
+
+		if (mem->status == BATCH_ALLOC_OP_ISSUED)
+			count += roc_npa_aura_batch_alloc_count(mem->objs,
+								BATCH_ALLOC_SZ);
+
+		if (mem->status == BATCH_ALLOC_OP_DONE)
+			count += mem->sz;
+	}
+
+	count += cnxk_mempool_get_count(mp);
+
+	return count;
+}
+
 static int
 cn10k_mempool_alloc(struct rte_mempool *mp)
 {
@@ -190,7 +216,7 @@ static struct rte_mempool_ops cn10k_mempool_ops = {
 	.free = cn10k_mempool_free,
 	.enqueue = cn10k_mempool_enq,
 	.dequeue = cnxk_mempool_deq,
-	.get_count = cnxk_mempool_get_count,
+	.get_count = cn10k_mempool_get_count,
 	.calc_mem_size = cnxk_mempool_calc_mem_size,
 	.populate = cnxk_mempool_populate,
 };
-- 
2.31.0

[dpdk-dev] [PATCH v2 11/11] mempool/cnxk: add cn10k batch dequeue op

[Ashwin Sekhar T K]

Add the implementation for Marvell CN10k mempool batch dequeue op.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 72 +++++++++++++++++++++++-
 1 file changed, 71 insertions(+), 1 deletion(-)

diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
index 16b2f6697f..05f36ff263 100644
--- a/drivers/mempool/cnxk/cn10k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -164,6 +164,76 @@ cn10k_mempool_get_count(const struct rte_mempool *mp)
 	return count;
 }
 
+static int __rte_hot
+cn10k_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
+{
+	struct batch_op_data *op_data;
+	struct batch_op_mem *mem;
+	unsigned int count = 0;
+	int tid, rc, retry;
+	bool loop = true;
+
+	op_data = BATCH_OP_DATA_GET(mp->pool_id);
+	tid = rte_lcore_id();
+	mem = &op_data->mem[tid];
+
+	/* Issue batch alloc */
+	if (mem->status == BATCH_ALLOC_OP_NOT_ISSUED) {
+		rc = roc_npa_aura_batch_alloc_issue(mp->pool_id, mem->objs,
+						    BATCH_ALLOC_SZ, 0, 1);
+		/* If issue fails, try falling back to default alloc */
+		if (unlikely(rc))
+			return cn10k_mempool_enq(mp, obj_table, n);
+		mem->status = BATCH_ALLOC_OP_ISSUED;
+	}
+
+	retry = 4;
+	while (loop) {
+		unsigned int cur_sz;
+
+		if (mem->status == BATCH_ALLOC_OP_ISSUED) {
+			mem->sz = roc_npa_aura_batch_alloc_extract(
+				mem->objs, mem->objs, BATCH_ALLOC_SZ);
+
+			/* If partial alloc reduce the retry count */
+			retry -= (mem->sz != BATCH_ALLOC_SZ);
+			/* Break the loop if retry count exhausted */
+			loop = !!retry;
+			mem->status = BATCH_ALLOC_OP_DONE;
+		}
+
+		cur_sz = n - count;
+		if (cur_sz > mem->sz)
+			cur_sz = mem->sz;
+
+		/* Dequeue the pointers */
+		memcpy(&obj_table[count], &mem->objs[mem->sz - cur_sz],
+		       cur_sz * sizeof(uintptr_t));
+		mem->sz -= cur_sz;
+		count += cur_sz;
+
+		/* Break loop if the required pointers has been dequeued */
+		loop &= (count != n);
+
+		/* Issue next batch alloc if pointers are exhausted */
+		if (mem->sz == 0) {
+			rc = roc_npa_aura_batch_alloc_issue(
+				mp->pool_id, mem->objs, BATCH_ALLOC_SZ, 0, 1);
+			/* Break loop if issue failed and set status */
+			loop &= !rc;
+			mem->status = !rc;
+		}
+	}
+
+	if (unlikely(count != n)) {
+		/* No partial alloc allowed. Free up allocated pointers */
+		cn10k_mempool_enq(mp, obj_table, count);
+		return -ENOENT;
+	}
+
+	return 0;
+}
+
 static int
 cn10k_mempool_alloc(struct rte_mempool *mp)
 {
@@ -215,7 +285,7 @@ static struct rte_mempool_ops cn10k_mempool_ops = {
 	.alloc = cn10k_mempool_alloc,
 	.free = cn10k_mempool_free,
 	.enqueue = cn10k_mempool_enq,
-	.dequeue = cnxk_mempool_deq,
+	.dequeue = cn10k_mempool_deq,
 	.get_count = cn10k_mempool_get_count,
 	.calc_mem_size = cnxk_mempool_calc_mem_size,
 	.populate = cnxk_mempool_populate,
-- 
2.31.0

[dpdk-dev] [PATCH v3 00/11] Add Marvell CNXK mempool driver

[Ashwin Sekhar T K]

This patchset adds the mempool/cnxk driver which provides the support for the
integrated mempool device found in Marvell CN10K SoC.

The code includes mempool driver functionality for Marvell CN9K SoC as well,
but right now it is not enabled. The future plan is to deprecate existing
mempool/octeontx2 driver once the 'CNXK' drivers are feature complete for
Marvell CN9K SoC.

Depends-on: series-16131 ("Add Marvell CNXK common driver")

v3:
 - Change batch op data initialization to plt init callback.
 - Reserve a memzone for batch op data.
 - Handle batch op data initialization in secondary process.

Ashwin Sekhar T K (11):
  mempool/cnxk: add build infra and doc
  mempool/cnxk: add device probe/remove
  mempool/cnxk: add generic ops
  mempool/cnxk: register plt init callback
  mempool/cnxk: add cn9k mempool ops
  mempool/cnxk: add cn9k optimized mempool enqueue/dequeue
  mempool/cnxk: add cn10k mempool ops
  mempool/cnxk: add batch op init
  mempool/cnxk: add cn10k batch enqueue op
  mempool/cnxk: add cn10k get count op
  mempool/cnxk: add cn10k batch dequeue op

 MAINTAINERS                              |   6 +
 doc/guides/mempool/cnxk.rst              |  91 +++++++
 doc/guides/mempool/index.rst             |   1 +
 doc/guides/platform/cnxk.rst             |   3 +
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 319 +++++++++++++++++++++++
 drivers/mempool/cnxk/cn9k_mempool_ops.c  |  89 +++++++
 drivers/mempool/cnxk/cnxk_mempool.c      | 202 ++++++++++++++
 drivers/mempool/cnxk/cnxk_mempool.h      |  28 ++
 drivers/mempool/cnxk/cnxk_mempool_ops.c  | 191 ++++++++++++++
 drivers/mempool/cnxk/meson.build         |  16 ++
 drivers/mempool/cnxk/version.map         |   3 +
 drivers/mempool/meson.build              |   3 +-
 12 files changed, 951 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/mempool/cnxk.rst
 create mode 100644 drivers/mempool/cnxk/cn10k_mempool_ops.c
 create mode 100644 drivers/mempool/cnxk/cn9k_mempool_ops.c
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.c
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.h
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool_ops.c
 create mode 100644 drivers/mempool/cnxk/meson.build
 create mode 100644 drivers/mempool/cnxk/version.map

-- 
2.31.0

[dpdk-dev] [PATCH v3 01/11] mempool/cnxk: add build infra and doc

[Ashwin Sekhar T K]

Add the meson based build infrastructure for Marvell
CNXK mempool driver along with stub implementations
for mempool device probe.

Also add Marvell CNXK mempool base documentation.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Jerin Jacob <jerinj@marvell.com>
Signed-off-by: Nithin Dabilpuram <ndabilpuram@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 MAINTAINERS                         |  6 +++
 doc/guides/mempool/cnxk.rst         | 55 ++++++++++++++++++++
 doc/guides/mempool/index.rst        |  1 +
 doc/guides/platform/cnxk.rst        |  3 ++
 drivers/mempool/cnxk/cnxk_mempool.c | 78 +++++++++++++++++++++++++++++
 drivers/mempool/cnxk/meson.build    | 13 +++++
 drivers/mempool/cnxk/version.map    |  3 ++
 drivers/mempool/meson.build         |  3 +-
 8 files changed, 161 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/mempool/cnxk.rst
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.c
 create mode 100644 drivers/mempool/cnxk/meson.build
 create mode 100644 drivers/mempool/cnxk/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index c837516d14..bae8b93030 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -501,6 +501,12 @@ M: Artem V. Andreev <artem.andreev@oktetlabs.ru>
 M: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
 F: drivers/mempool/bucket/
 
+Marvell cnxk
+M: Ashwin Sekhar T K <asekhar@marvell.com>
+M: Pavan Nikhilesh <pbhagavatula@marvell.com>
+F: drivers/mempool/cnxk/
+F: doc/guides/mempool/cnxk.rst
+
 Marvell OCTEON TX2
 M: Jerin Jacob <jerinj@marvell.com>
 M: Nithin Dabilpuram <ndabilpuram@marvell.com>
diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
new file mode 100644
index 0000000000..e72a77c361
--- /dev/null
+++ b/doc/guides/mempool/cnxk.rst
@@ -0,0 +1,55 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(C) 2021 Marvell.
+
+CNXK NPA Mempool Driver
+============================
+
+The CNXK NPA PMD (**librte_mempool_cnxk**) provides mempool driver support for
+the integrated mempool device found in **Marvell OCTEON CN9K/CN10K** SoC family.
+
+More information about CNXK SoC can be found at `Marvell Official Website
+<https://www.marvell.com/embedded-processors/infrastructure-processors/>`_.
+
+Features
+--------
+
+CNXK NPA PMD supports:
+
+- Up to 128 NPA LFs
+- 1M Pools per LF
+- HW mempool manager
+- Ethdev Rx buffer allocation in HW to save CPU cycles in the Rx path.
+- Ethdev Tx buffer recycling in HW to save CPU cycles in the Tx path.
+
+Prerequisites and Compilation procedure
+---------------------------------------
+
+   See :doc:`../platform/cnxk` for setup information.
+
+Pre-Installation Configuration
+------------------------------
+
+
+Debugging Options
+~~~~~~~~~~~~~~~~~
+
+.. _table_cnxk_mempool_debug_options:
+
+.. table:: CNXK mempool debug options
+
+   +---+------------+-------------------------------------------------------+
+   | # | Component  | EAL log command                                       |
+   +===+============+=======================================================+
+   | 1 | NPA        | --log-level='pmd\.mempool.cnxk,8'                     |
+   +---+------------+-------------------------------------------------------+
+
+Standalone mempool device
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+   The ``usertools/dpdk-devbind.py`` script shall enumerate all the mempool
+   devices available in the system. In order to avoid, the end user to bind the
+   mempool device prior to use ethdev and/or eventdev device, the respective
+   driver configures an NPA LF and attach to the first probed ethdev or eventdev
+   device. In case, if end user need to run mempool as a standalone device
+   (without ethdev or eventdev), end user needs to bind a mempool device using
+   ``usertools/dpdk-devbind.py``
diff --git a/doc/guides/mempool/index.rst b/doc/guides/mempool/index.rst
index a0e55467e6..ce53bc1ac7 100644
--- a/doc/guides/mempool/index.rst
+++ b/doc/guides/mempool/index.rst
@@ -11,6 +11,7 @@ application through the mempool API.
     :maxdepth: 2
     :numbered:
 
+    cnxk
     octeontx
     octeontx2
     ring
diff --git a/doc/guides/platform/cnxk.rst b/doc/guides/platform/cnxk.rst
index d8fc00df40..489569ef4e 100644
--- a/doc/guides/platform/cnxk.rst
+++ b/doc/guides/platform/cnxk.rst
@@ -142,6 +142,9 @@ HW Offload Drivers
 
 This section lists dataplane H/W block(s) available in CNXK SoC.
 
+#. **Mempool Driver**
+   See :doc:`../mempool/cnxk` for NPA mempool driver information.
+
 Procedure to Setup Platform
 ---------------------------
 
diff --git a/drivers/mempool/cnxk/cnxk_mempool.c b/drivers/mempool/cnxk/cnxk_mempool.c
new file mode 100644
index 0000000000..947078c052
--- /dev/null
+++ b/drivers/mempool/cnxk/cnxk_mempool.c
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_atomic.h>
+#include <rte_bus_pci.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+#include <rte_eal.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_mbuf_pool_ops.h>
+#include <rte_pci.h>
+
+#include "roc_api.h"
+
+static int
+npa_remove(struct rte_pci_device *pci_dev)
+{
+	RTE_SET_USED(pci_dev);
+
+	return 0;
+}
+
+static int
+npa_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
+{
+	RTE_SET_USED(pci_drv);
+	RTE_SET_USED(pci_dev);
+
+	return 0;
+}
+
+static const struct rte_pci_id npa_pci_map[] = {
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_PF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KA,
+	},
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_PF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KAS,
+	},
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_VF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KA,
+	},
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_VF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KAS,
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static struct rte_pci_driver npa_pci = {
+	.id_table = npa_pci_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_NEED_IOVA_AS_VA,
+	.probe = npa_probe,
+	.remove = npa_remove,
+};
+
+RTE_PMD_REGISTER_PCI(mempool_cnxk, npa_pci);
+RTE_PMD_REGISTER_PCI_TABLE(mempool_cnxk, npa_pci_map);
+RTE_PMD_REGISTER_KMOD_DEP(mempool_cnxk, "vfio-pci");
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
new file mode 100644
index 0000000000..0be0802373
--- /dev/null
+++ b/drivers/mempool/cnxk/meson.build
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(C) 2021 Marvell.
+#
+
+if not is_linux or not dpdk_conf.get('RTE_ARCH_64')
+	build = false
+	reason = 'only supported on 64-bit Linux'
+	subdir_done()
+endif
+
+sources = files('cnxk_mempool.c')
+
+deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
diff --git a/drivers/mempool/cnxk/version.map b/drivers/mempool/cnxk/version.map
new file mode 100644
index 0000000000..ee80c51721
--- /dev/null
+++ b/drivers/mempool/cnxk/version.map
@@ -0,0 +1,3 @@
+INTERNAL {
+	local: *;
+};
diff --git a/drivers/mempool/meson.build b/drivers/mempool/meson.build
index 4428813dae..a2814c1dfa 100644
--- a/drivers/mempool/meson.build
+++ b/drivers/mempool/meson.build
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-drivers = ['bucket', 'dpaa', 'dpaa2', 'octeontx', 'octeontx2', 'ring', 'stack']
+drivers = ['bucket', 'cnxk', 'dpaa', 'dpaa2', 'octeontx', 'octeontx2', 'ring',
+	   'stack']
 std_deps = ['mempool']
-- 
2.31.0

[dpdk-dev] [PATCH v3 02/11] mempool/cnxk: add device probe/remove

[Ashwin Sekhar T K]

Add the implementation for CNXk mempool device
probe and remove.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 doc/guides/mempool/cnxk.rst         |  23 +++++
 drivers/mempool/cnxk/cnxk_mempool.c | 132 +++++++++++++++++++++++++++-
 2 files changed, 151 insertions(+), 4 deletions(-)

diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
index e72a77c361..907c19c841 100644
--- a/doc/guides/mempool/cnxk.rst
+++ b/doc/guides/mempool/cnxk.rst
@@ -30,6 +30,29 @@ Pre-Installation Configuration
 ------------------------------
 
 
+Runtime Config Options
+~~~~~~~~~~~~~~~~~~~~~~
+
+- ``Maximum number of mempools per application`` (default ``128``)
+
+  The maximum number of mempools per application needs to be configured on
+  HW during mempool driver initialization. HW can support up to 1M mempools,
+  Since each mempool costs set of HW resources, the ``max_pools`` ``devargs``
+  parameter is being introduced to configure the number of mempools required
+  for the application.
+  For example::
+
+    -a 0002:02:00.0,max_pools=512
+
+  With the above configuration, the driver will set up only 512 mempools for
+  the given application to save HW resources.
+
+.. note::
+
+   Since this configuration is per application, the end user needs to
+   provide ``max_pools`` parameter to the first PCIe device probed by the given
+   application.
+
 Debugging Options
 ~~~~~~~~~~~~~~~~~
 
diff --git a/drivers/mempool/cnxk/cnxk_mempool.c b/drivers/mempool/cnxk/cnxk_mempool.c
index 947078c052..dd4d74ca05 100644
--- a/drivers/mempool/cnxk/cnxk_mempool.c
+++ b/drivers/mempool/cnxk/cnxk_mempool.c
@@ -15,21 +15,143 @@
 
 #include "roc_api.h"
 
+#define CNXK_NPA_DEV_NAME	 RTE_STR(cnxk_npa_dev_)
+#define CNXK_NPA_DEV_NAME_LEN	 (sizeof(CNXK_NPA_DEV_NAME) + PCI_PRI_STR_SIZE)
+#define CNXK_NPA_MAX_POOLS_PARAM "max_pools"
+
+static inline uint32_t
+npa_aura_size_to_u32(uint8_t val)
+{
+	if (val == NPA_AURA_SZ_0)
+		return 128;
+	if (val >= NPA_AURA_SZ_MAX)
+		return BIT_ULL(20);
+
+	return 1 << (val + 6);
+}
+
 static int
-npa_remove(struct rte_pci_device *pci_dev)
+parse_max_pools(const char *key, const char *value, void *extra_args)
+{
+	RTE_SET_USED(key);
+	uint32_t val;
+
+	val = atoi(value);
+	if (val < npa_aura_size_to_u32(NPA_AURA_SZ_128))
+		val = 128;
+	if (val > npa_aura_size_to_u32(NPA_AURA_SZ_1M))
+		val = BIT_ULL(20);
+
+	*(uint8_t *)extra_args = rte_log2_u32(val) - 6;
+	return 0;
+}
+
+static inline uint8_t
+parse_aura_size(struct rte_devargs *devargs)
+{
+	uint8_t aura_sz = NPA_AURA_SZ_128;
+	struct rte_kvargs *kvlist;
+
+	if (devargs == NULL)
+		goto exit;
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL)
+		goto exit;
+
+	rte_kvargs_process(kvlist, CNXK_NPA_MAX_POOLS_PARAM, &parse_max_pools,
+			   &aura_sz);
+	rte_kvargs_free(kvlist);
+exit:
+	return aura_sz;
+}
+
+static inline char *
+npa_dev_to_name(struct rte_pci_device *pci_dev, char *name)
+{
+	snprintf(name, CNXK_NPA_DEV_NAME_LEN, CNXK_NPA_DEV_NAME PCI_PRI_FMT,
+		 pci_dev->addr.domain, pci_dev->addr.bus, pci_dev->addr.devid,
+		 pci_dev->addr.function);
+
+	return name;
+}
+
+static int
+npa_init(struct rte_pci_device *pci_dev)
 {
-	RTE_SET_USED(pci_dev);
+	char name[CNXK_NPA_DEV_NAME_LEN];
+	const struct rte_memzone *mz;
+	struct roc_npa *dev;
+	int rc = -ENOMEM;
+
+	mz = rte_memzone_reserve_aligned(npa_dev_to_name(pci_dev, name),
+					 sizeof(*dev), SOCKET_ID_ANY, 0,
+					 RTE_CACHE_LINE_SIZE);
+	if (mz == NULL)
+		goto error;
+
+	dev = mz->addr;
+	dev->pci_dev = pci_dev;
+
+	roc_idev_npa_maxpools_set(parse_aura_size(pci_dev->device.devargs));
+	rc = roc_npa_dev_init(dev);
+	if (rc)
+		goto mz_free;
+
+	return 0;
+
+mz_free:
+	rte_memzone_free(mz);
+error:
+	plt_err("failed to initialize npa device rc=%d", rc);
+	return rc;
+}
+
+static int
+npa_fini(struct rte_pci_device *pci_dev)
+{
+	char name[CNXK_NPA_DEV_NAME_LEN];
+	const struct rte_memzone *mz;
+	int rc;
+
+	mz = rte_memzone_lookup(npa_dev_to_name(pci_dev, name));
+	if (mz == NULL)
+		return -EINVAL;
+
+	rc = roc_npa_dev_fini(mz->addr);
+	if (rc) {
+		if (rc != -EAGAIN)
+			plt_err("Failed to remove npa dev, rc=%d", rc);
+		return rc;
+	}
+	rte_memzone_free(mz);
 
 	return 0;
 }
 
+static int
+npa_remove(struct rte_pci_device *pci_dev)
+{
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	return npa_fini(pci_dev);
+}
+
 static int
 npa_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 {
+	int rc;
+
 	RTE_SET_USED(pci_drv);
-	RTE_SET_USED(pci_dev);
 
-	return 0;
+	rc = roc_plt_init();
+	if (rc < 0)
+		return rc;
+
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	return npa_init(pci_dev);
 }
 
 static const struct rte_pci_id npa_pci_map[] = {
@@ -76,3 +198,5 @@ static struct rte_pci_driver npa_pci = {
 RTE_PMD_REGISTER_PCI(mempool_cnxk, npa_pci);
 RTE_PMD_REGISTER_PCI_TABLE(mempool_cnxk, npa_pci_map);
 RTE_PMD_REGISTER_KMOD_DEP(mempool_cnxk, "vfio-pci");
+RTE_PMD_REGISTER_PARAM_STRING(mempool_cnxk,
+			      CNXK_NPA_MAX_POOLS_PARAM "=<128-1048576>");
-- 
2.31.0

[dpdk-dev] [PATCH v3 03/11] mempool/cnxk: add generic ops

[Ashwin Sekhar T K]

Add generic CNXk mempool ops which will enqueue/dequeue
from pool one element at a time.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cnxk_mempool.h     |  26 ++++
 drivers/mempool/cnxk/cnxk_mempool_ops.c | 171 ++++++++++++++++++++++++
 drivers/mempool/cnxk/meson.build        |   3 +-
 3 files changed, 199 insertions(+), 1 deletion(-)
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.h
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool_ops.c

diff --git a/drivers/mempool/cnxk/cnxk_mempool.h b/drivers/mempool/cnxk/cnxk_mempool.h
new file mode 100644
index 0000000000..099b7f6998
--- /dev/null
+++ b/drivers/mempool/cnxk/cnxk_mempool.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#ifndef _CNXK_MEMPOOL_H_
+#define _CNXK_MEMPOOL_H_
+
+#include <rte_mempool.h>
+
+unsigned int cnxk_mempool_get_count(const struct rte_mempool *mp);
+ssize_t cnxk_mempool_calc_mem_size(const struct rte_mempool *mp,
+				   uint32_t obj_num, uint32_t pg_shift,
+				   size_t *min_chunk_size, size_t *align);
+int cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
+			  void *vaddr, rte_iova_t iova, size_t len,
+			  rte_mempool_populate_obj_cb_t *obj_cb,
+			  void *obj_cb_arg);
+int cnxk_mempool_alloc(struct rte_mempool *mp);
+void cnxk_mempool_free(struct rte_mempool *mp);
+
+int __rte_hot cnxk_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
+			       unsigned int n);
+int __rte_hot cnxk_mempool_deq(struct rte_mempool *mp, void **obj_table,
+			       unsigned int n);
+
+#endif
diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
new file mode 100644
index 0000000000..2ce1816c04
--- /dev/null
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -0,0 +1,171 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_mempool.h>
+
+#include "roc_api.h"
+#include "cnxk_mempool.h"
+
+int __rte_hot
+cnxk_mempool_enq(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
+{
+	unsigned int index;
+
+	/* Ensure mbuf init changes are written before the free pointers
+	 * are enqueued to the stack.
+	 */
+	rte_io_wmb();
+	for (index = 0; index < n; index++)
+		roc_npa_aura_op_free(mp->pool_id, 0,
+				     (uint64_t)obj_table[index]);
+
+	return 0;
+}
+
+int __rte_hot
+cnxk_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
+{
+	unsigned int index;
+	uint64_t obj;
+
+	for (index = 0; index < n; index++, obj_table++) {
+		int retry = 4;
+
+		/* Retry few times before failing */
+		do {
+			obj = roc_npa_aura_op_alloc(mp->pool_id, 0);
+		} while (retry-- && (obj == 0));
+
+		if (obj == 0) {
+			cnxk_mempool_enq(mp, obj_table - index, index);
+			return -ENOENT;
+		}
+		*obj_table = (void *)obj;
+	}
+
+	return 0;
+}
+
+unsigned int
+cnxk_mempool_get_count(const struct rte_mempool *mp)
+{
+	return (unsigned int)roc_npa_aura_op_available(mp->pool_id);
+}
+
+ssize_t
+cnxk_mempool_calc_mem_size(const struct rte_mempool *mp, uint32_t obj_num,
+			   uint32_t pg_shift, size_t *min_chunk_size,
+			   size_t *align)
+{
+	size_t total_elt_sz;
+
+	/* Need space for one more obj on each chunk to fulfill
+	 * alignment requirements.
+	 */
+	total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
+	return rte_mempool_op_calc_mem_size_helper(
+		mp, obj_num, pg_shift, total_elt_sz, min_chunk_size, align);
+}
+
+int
+cnxk_mempool_alloc(struct rte_mempool *mp)
+{
+	uint64_t aura_handle = 0;
+	struct npa_aura_s aura;
+	struct npa_pool_s pool;
+	uint32_t block_count;
+	size_t block_size;
+	int rc = -ERANGE;
+
+	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
+	block_count = mp->size;
+	if (mp->header_size % ROC_ALIGN != 0) {
+		plt_err("Header size should be multiple of %dB", ROC_ALIGN);
+		goto error;
+	}
+
+	if (block_size % ROC_ALIGN != 0) {
+		plt_err("Block size should be multiple of %dB", ROC_ALIGN);
+		goto error;
+	}
+
+	memset(&aura, 0, sizeof(struct npa_aura_s));
+	memset(&pool, 0, sizeof(struct npa_pool_s));
+	pool.nat_align = 1;
+	pool.buf_offset = mp->header_size / ROC_ALIGN;
+
+	/* Use driver specific mp->pool_config to override aura config */
+	if (mp->pool_config != NULL)
+		memcpy(&aura, mp->pool_config, sizeof(struct npa_aura_s));
+
+	rc = roc_npa_pool_create(&aura_handle, block_size, block_count, &aura,
+				 &pool);
+	if (rc) {
+		plt_err("Failed to alloc pool or aura rc=%d", rc);
+		goto error;
+	}
+
+	/* Store aura_handle for future queue operations */
+	mp->pool_id = aura_handle;
+	plt_npa_dbg("block_sz=%lu block_count=%d aura_handle=0x%" PRIx64,
+		    block_size, block_count, aura_handle);
+
+	return 0;
+error:
+	return rc;
+}
+
+void
+cnxk_mempool_free(struct rte_mempool *mp)
+{
+	int rc = 0;
+
+	plt_npa_dbg("aura_handle=0x%" PRIx64, mp->pool_id);
+	rc = roc_npa_pool_destroy(mp->pool_id);
+	if (rc)
+		plt_err("Failed to free pool or aura rc=%d", rc);
+}
+
+int
+cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
+		      void *vaddr, rte_iova_t iova, size_t len,
+		      rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
+{
+	size_t total_elt_sz, off;
+	int num_elts;
+
+	if (iova == RTE_BAD_IOVA)
+		return -EINVAL;
+
+	total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
+
+	/* Align object start address to a multiple of total_elt_sz */
+	off = total_elt_sz - ((((uintptr_t)vaddr - 1) % total_elt_sz) + 1);
+
+	if (len < off)
+		return -EINVAL;
+
+	vaddr = (char *)vaddr + off;
+	iova += off;
+	len -= off;
+	num_elts = len / total_elt_sz;
+
+	plt_npa_dbg("iova %" PRIx64 ", aligned iova %" PRIx64 "", iova - off,
+		    iova);
+	plt_npa_dbg("length %" PRIu64 ", aligned length %" PRIu64 "",
+		    (uint64_t)(len + off), (uint64_t)len);
+	plt_npa_dbg("element size %" PRIu64 "", (uint64_t)total_elt_sz);
+	plt_npa_dbg("requested objects %" PRIu64 ", possible objects %" PRIu64
+		    "", (uint64_t)max_objs, (uint64_t)num_elts);
+
+	roc_npa_aura_op_range_set(mp->pool_id, iova,
+				  iova + num_elts * total_elt_sz);
+
+	if (roc_npa_pool_range_update_check(mp->pool_id) < 0)
+		return -EBUSY;
+
+	return rte_mempool_op_populate_helper(
+		mp, RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ, max_objs, vaddr, iova,
+		len, obj_cb, obj_cb_arg);
+}
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
index 0be0802373..52244e728b 100644
--- a/drivers/mempool/cnxk/meson.build
+++ b/drivers/mempool/cnxk/meson.build
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.get('RTE_ARCH_64')
 	subdir_done()
 endif
 
-sources = files('cnxk_mempool.c')
+sources = files('cnxk_mempool.c',
+		'cnxk_mempool_ops.c')
 
 deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
-- 
2.31.0

[dpdk-dev] [PATCH v3 04/11] mempool/cnxk: register plt init callback

[Ashwin Sekhar T K]

Register the CNXk mempool plt init callback which will set the
appropriate mempool ops to be used for the platform.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cnxk_mempool_ops.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
index 2ce1816c04..e8f64be76b 100644
--- a/drivers/mempool/cnxk/cnxk_mempool_ops.c
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -2,6 +2,7 @@
  * Copyright(C) 2021 Marvell.
  */
 
+#include <rte_mbuf_pool_ops.h>
 #include <rte_mempool.h>
 
 #include "roc_api.h"
@@ -169,3 +170,17 @@ cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
 		mp, RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ, max_objs, vaddr, iova,
 		len, obj_cb, obj_cb_arg);
 }
+
+static int
+cnxk_mempool_plt_init(void)
+{
+	if (roc_model_is_cn10k() || roc_model_is_cn9k())
+		rte_mbuf_set_platform_mempool_ops("cnxk_mempool_ops");
+
+	return 0;
+}
+
+RTE_INIT(cnxk_mempool_ops_init)
+{
+	roc_plt_init_cb_register(cnxk_mempool_plt_init);
+}
-- 
2.31.0

[dpdk-dev] [PATCH v3 05/11] mempool/cnxk: add cn9k mempool ops

[Ashwin Sekhar T K]

Add Marvell CN9k mempool ops and implement CN9k mempool
alloc which makes sure that the element size always occupy
odd number of cachelines to ensure even distribution among
of elements among L1D cache sets.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn9k_mempool_ops.c | 54 +++++++++++++++++++++++++
 drivers/mempool/cnxk/cnxk_mempool_ops.c |  4 +-
 drivers/mempool/cnxk/meson.build        |  3 +-
 3 files changed, 59 insertions(+), 2 deletions(-)
 create mode 100644 drivers/mempool/cnxk/cn9k_mempool_ops.c

diff --git a/drivers/mempool/cnxk/cn9k_mempool_ops.c b/drivers/mempool/cnxk/cn9k_mempool_ops.c
new file mode 100644
index 0000000000..f5ac163af9
--- /dev/null
+++ b/drivers/mempool/cnxk/cn9k_mempool_ops.c
@@ -0,0 +1,54 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_mempool.h>
+
+#include "roc_api.h"
+#include "cnxk_mempool.h"
+
+static int
+cn9k_mempool_alloc(struct rte_mempool *mp)
+{
+	size_t block_size, padding;
+
+	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
+	/* Align header size to ROC_ALIGN */
+	if (mp->header_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(mp->header_size, ROC_ALIGN) -
+			  mp->header_size;
+		mp->header_size += padding;
+		block_size += padding;
+	}
+
+	/* Align block size to ROC_ALIGN */
+	if (block_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(block_size, ROC_ALIGN) - block_size;
+		mp->trailer_size += padding;
+		block_size += padding;
+	}
+
+	/*
+	 * Marvell CN9k has 8 sets, 41 ways L1D cache, VA<9:7> bits dictate the
+	 * set selection. Add additional padding to ensure that the element size
+	 * always occupies odd number of cachelines to ensure even distribution
+	 * of elements among L1D cache sets.
+	 */
+	padding = ((block_size / ROC_ALIGN) % 2) ? 0 : ROC_ALIGN;
+	mp->trailer_size += padding;
+
+	return cnxk_mempool_alloc(mp);
+}
+
+static struct rte_mempool_ops cn9k_mempool_ops = {
+	.name = "cn9k_mempool_ops",
+	.alloc = cn9k_mempool_alloc,
+	.free = cnxk_mempool_free,
+	.enqueue = cnxk_mempool_enq,
+	.dequeue = cnxk_mempool_deq,
+	.get_count = cnxk_mempool_get_count,
+	.calc_mem_size = cnxk_mempool_calc_mem_size,
+	.populate = cnxk_mempool_populate,
+};
+
+MEMPOOL_REGISTER_OPS(cn9k_mempool_ops);
diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
index e8f64be76b..d8ed37ec1a 100644
--- a/drivers/mempool/cnxk/cnxk_mempool_ops.c
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -174,7 +174,9 @@ cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
 static int
 cnxk_mempool_plt_init(void)
 {
-	if (roc_model_is_cn10k() || roc_model_is_cn9k())
+	if (roc_model_is_cn9k())
+		rte_mbuf_set_platform_mempool_ops("cn9k_mempool_ops");
+	else if (roc_model_is_cn10k())
 		rte_mbuf_set_platform_mempool_ops("cnxk_mempool_ops");
 
 	return 0;
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
index 52244e728b..ff31893ff4 100644
--- a/drivers/mempool/cnxk/meson.build
+++ b/drivers/mempool/cnxk/meson.build
@@ -9,6 +9,7 @@ if not is_linux or not dpdk_conf.get('RTE_ARCH_64')
 endif
 
 sources = files('cnxk_mempool.c',
-		'cnxk_mempool_ops.c')
+		'cnxk_mempool_ops.c',
+		'cn9k_mempool_ops.c')
 
 deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
-- 
2.31.0

[dpdk-dev] [PATCH v3 06/11] mempool/cnxk: add cn9k optimized mempool enqueue/dequeue

[Ashwin Sekhar T K]

Add Marvell CN9k mempool enqueue/dequeue. Marvell CN9k
supports burst dequeue which allows to dequeue up to 32
pointers using pipelined casp instructions.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 doc/guides/mempool/cnxk.rst             |  4 +++
 drivers/mempool/cnxk/cn9k_mempool_ops.c | 39 +++++++++++++++++++++++--
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
index 907c19c841..f51532b101 100644
--- a/doc/guides/mempool/cnxk.rst
+++ b/doc/guides/mempool/cnxk.rst
@@ -21,6 +21,10 @@ CNXK NPA PMD supports:
 - Ethdev Rx buffer allocation in HW to save CPU cycles in the Rx path.
 - Ethdev Tx buffer recycling in HW to save CPU cycles in the Tx path.
 
+CN9k NPA supports:
+
+- Burst alloc of up to 32 pointers.
+
 Prerequisites and Compilation procedure
 ---------------------------------------
 
diff --git a/drivers/mempool/cnxk/cn9k_mempool_ops.c b/drivers/mempool/cnxk/cn9k_mempool_ops.c
index f5ac163af9..c0cdba640b 100644
--- a/drivers/mempool/cnxk/cn9k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn9k_mempool_ops.c
@@ -7,6 +7,41 @@
 #include "roc_api.h"
 #include "cnxk_mempool.h"
 
+static int __rte_hot
+cn9k_mempool_enq(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
+{
+	/* Ensure mbuf init changes are written before the free pointers
+	 * are enqueued to the stack.
+	 */
+	rte_io_wmb();
+	roc_npa_aura_op_bulk_free(mp->pool_id, (const uint64_t *)obj_table, n,
+				  0);
+
+	return 0;
+}
+
+static inline int __rte_hot
+cn9k_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
+{
+	unsigned int count;
+
+	count = roc_npa_aura_op_bulk_alloc(mp->pool_id, (uint64_t *)obj_table,
+					   n, 0, 1);
+
+	if (unlikely(count != n)) {
+		/* If bulk alloc failed to allocate all pointers, try
+		 * allocating remaining pointers with the default alloc
+		 * with retry scheme.
+		 */
+		if (cnxk_mempool_deq(mp, &obj_table[count], n - count)) {
+			cn9k_mempool_enq(mp, obj_table, count);
+			return -ENOENT;
+		}
+	}
+
+	return 0;
+}
+
 static int
 cn9k_mempool_alloc(struct rte_mempool *mp)
 {
@@ -44,8 +79,8 @@ static struct rte_mempool_ops cn9k_mempool_ops = {
 	.name = "cn9k_mempool_ops",
 	.alloc = cn9k_mempool_alloc,
 	.free = cnxk_mempool_free,
-	.enqueue = cnxk_mempool_enq,
-	.dequeue = cnxk_mempool_deq,
+	.enqueue = cn9k_mempool_enq,
+	.dequeue = cn9k_mempool_deq,
 	.get_count = cnxk_mempool_get_count,
 	.calc_mem_size = cnxk_mempool_calc_mem_size,
 	.populate = cnxk_mempool_populate,
-- 
2.31.0

[dpdk-dev] [PATCH v3 07/11] mempool/cnxk: add cn10k mempool ops

[Ashwin Sekhar T K]

Add Marvell CN10k mempool ops and implement CN10k mempool alloc.

CN10k has 64 bytes L1D cache line size. Hence the CN10k mempool
alloc does not make the element size an odd multiple L1D cache
line size as NPA requires the element sizes to be multiples of
128 bytes.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 doc/guides/mempool/cnxk.rst              |  4 ++
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 52 ++++++++++++++++++++++++
 drivers/mempool/cnxk/cnxk_mempool_ops.c  |  2 +-
 drivers/mempool/cnxk/meson.build         |  3 +-
 4 files changed, 59 insertions(+), 2 deletions(-)
 create mode 100644 drivers/mempool/cnxk/cn10k_mempool_ops.c

diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
index f51532b101..783368e690 100644
--- a/doc/guides/mempool/cnxk.rst
+++ b/doc/guides/mempool/cnxk.rst
@@ -80,3 +80,7 @@ Standalone mempool device
    device. In case, if end user need to run mempool as a standalone device
    (without ethdev or eventdev), end user needs to bind a mempool device using
    ``usertools/dpdk-devbind.py``
+
+   Example command to run ``mempool_autotest`` test with standalone CN10K NPA device::
+
+     echo "mempool_autotest" | <build_dir>/app/test/dpdk-test -c 0xf0 --mbuf-pool-ops-name="cn10k_mempool_ops"
diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
new file mode 100644
index 0000000000..9b63789006
--- /dev/null
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_mempool.h>
+
+#include "roc_api.h"
+#include "cnxk_mempool.h"
+
+static int
+cn10k_mempool_alloc(struct rte_mempool *mp)
+{
+	uint32_t block_size;
+	size_t padding;
+
+	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
+	/* Align header size to ROC_ALIGN */
+	if (mp->header_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(mp->header_size, ROC_ALIGN) -
+			  mp->header_size;
+		mp->header_size += padding;
+		block_size += padding;
+	}
+
+	/* Align block size to ROC_ALIGN */
+	if (block_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(block_size, ROC_ALIGN) - block_size;
+		mp->trailer_size += padding;
+		block_size += padding;
+	}
+
+	return cnxk_mempool_alloc(mp);
+}
+
+static void
+cn10k_mempool_free(struct rte_mempool *mp)
+{
+	cnxk_mempool_free(mp);
+}
+
+static struct rte_mempool_ops cn10k_mempool_ops = {
+	.name = "cn10k_mempool_ops",
+	.alloc = cn10k_mempool_alloc,
+	.free = cn10k_mempool_free,
+	.enqueue = cnxk_mempool_enq,
+	.dequeue = cnxk_mempool_deq,
+	.get_count = cnxk_mempool_get_count,
+	.calc_mem_size = cnxk_mempool_calc_mem_size,
+	.populate = cnxk_mempool_populate,
+};
+
+MEMPOOL_REGISTER_OPS(cn10k_mempool_ops);
diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
index d8ed37ec1a..42c02bf14e 100644
--- a/drivers/mempool/cnxk/cnxk_mempool_ops.c
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -177,7 +177,7 @@ cnxk_mempool_plt_init(void)
 	if (roc_model_is_cn9k())
 		rte_mbuf_set_platform_mempool_ops("cn9k_mempool_ops");
 	else if (roc_model_is_cn10k())
-		rte_mbuf_set_platform_mempool_ops("cnxk_mempool_ops");
+		rte_mbuf_set_platform_mempool_ops("cn10k_mempool_ops");
 
 	return 0;
 }
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
index ff31893ff4..3282b5e5a6 100644
--- a/drivers/mempool/cnxk/meson.build
+++ b/drivers/mempool/cnxk/meson.build
@@ -10,6 +10,7 @@ endif
 
 sources = files('cnxk_mempool.c',
 		'cnxk_mempool_ops.c',
-		'cn9k_mempool_ops.c')
+		'cn9k_mempool_ops.c',
+		'cn10k_mempool_ops.c')
 
 deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
-- 
2.31.0

[dpdk-dev] [PATCH v3 08/11] mempool/cnxk: add batch op init

[Ashwin Sekhar T K]

Marvell CN10k mempool supports batch enqueue/dequeue which can
dequeue up to 512 pointers and enqueue up to 15 pointers using
a single instruction.

These batch operations require a DMA memory to enqueue/dequeue
pointers. This patch adds the initialization of this DMA memory.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 doc/guides/mempool/cnxk.rst              |   5 +
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 147 ++++++++++++++++++++++-
 drivers/mempool/cnxk/cnxk_mempool.h      |   2 +
 drivers/mempool/cnxk/cnxk_mempool_ops.c  |  11 +-
 4 files changed, 160 insertions(+), 5 deletions(-)

diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
index 783368e690..286ee29003 100644
--- a/doc/guides/mempool/cnxk.rst
+++ b/doc/guides/mempool/cnxk.rst
@@ -25,6 +25,11 @@ CN9k NPA supports:
 
 - Burst alloc of up to 32 pointers.
 
+CN10k NPA supports:
+
+- Batch dequeue of up to 512 pointers with single instruction.
+- Batch enqueue of up to 15 pointers with single instruction.
+
 Prerequisites and Compilation procedure
 ---------------------------------------
 
diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
index 9b63789006..a3aef0ddb2 100644
--- a/drivers/mempool/cnxk/cn10k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -7,11 +7,136 @@
 #include "roc_api.h"
 #include "cnxk_mempool.h"
 
+#define BATCH_ALLOC_SZ              ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS
+#define BATCH_OP_DATA_TABLE_MZ_NAME "batch_op_data_table_mz"
+
+enum batch_op_status {
+	BATCH_ALLOC_OP_NOT_ISSUED = 0,
+	BATCH_ALLOC_OP_ISSUED = 1,
+	BATCH_ALLOC_OP_DONE
+};
+
+struct batch_op_mem {
+	unsigned int sz;
+	enum batch_op_status status;
+	uint64_t objs[BATCH_ALLOC_SZ] __rte_aligned(ROC_ALIGN);
+};
+
+struct batch_op_data {
+	uint64_t lmt_addr;
+	struct batch_op_mem mem[RTE_MAX_LCORE] __rte_aligned(ROC_ALIGN);
+};
+
+static struct batch_op_data **batch_op_data_tbl;
+
+static int
+batch_op_data_table_create(void)
+{
+	const struct rte_memzone *mz;
+
+	/* If table is already set, nothing to do */
+	if (batch_op_data_tbl)
+		return 0;
+
+	mz = rte_memzone_lookup(BATCH_OP_DATA_TABLE_MZ_NAME);
+	if (mz == NULL) {
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+			unsigned int maxpools, sz;
+
+			maxpools = roc_idev_npa_maxpools_get();
+			sz = maxpools * sizeof(struct batch_op_data *);
+
+			mz = rte_memzone_reserve_aligned(
+				BATCH_OP_DATA_TABLE_MZ_NAME, sz, SOCKET_ID_ANY,
+				0, ROC_ALIGN);
+		}
+		if (mz == NULL) {
+			plt_err("Failed to reserve batch op data table");
+			return -ENOMEM;
+		}
+	}
+	batch_op_data_tbl = mz->addr;
+	rte_wmb();
+	return 0;
+}
+
+static inline struct batch_op_data *
+batch_op_data_get(uint64_t pool_id)
+{
+	uint64_t aura = roc_npa_aura_handle_to_aura(pool_id);
+
+	return batch_op_data_tbl[aura];
+}
+
+static inline void
+batch_op_data_set(uint64_t pool_id, struct batch_op_data *op_data)
+{
+	uint64_t aura = roc_npa_aura_handle_to_aura(pool_id);
+
+	batch_op_data_tbl[aura] = op_data;
+}
+
+static int
+batch_op_init(struct rte_mempool *mp)
+{
+	struct batch_op_data *op_data;
+	int i;
+
+	op_data = batch_op_data_get(mp->pool_id);
+	/* The data should not have been allocated previously */
+	RTE_ASSERT(op_data == NULL);
+
+	op_data = rte_zmalloc(NULL, sizeof(struct batch_op_data), ROC_ALIGN);
+	if (op_data == NULL)
+		return -ENOMEM;
+
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		op_data->mem[i].sz = 0;
+		op_data->mem[i].status = BATCH_ALLOC_OP_NOT_ISSUED;
+	}
+
+	op_data->lmt_addr = roc_idev_lmt_base_addr_get();
+	batch_op_data_set(mp->pool_id, op_data);
+	rte_wmb();
+
+	return 0;
+}
+
+static void
+batch_op_fini(struct rte_mempool *mp)
+{
+	struct batch_op_data *op_data;
+	int i;
+
+	op_data = batch_op_data_get(mp->pool_id);
+
+	rte_wmb();
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct batch_op_mem *mem = &op_data->mem[i];
+
+		if (mem->status == BATCH_ALLOC_OP_ISSUED) {
+			mem->sz = roc_npa_aura_batch_alloc_extract(
+				mem->objs, mem->objs, BATCH_ALLOC_SZ);
+			mem->status = BATCH_ALLOC_OP_DONE;
+		}
+		if (mem->status == BATCH_ALLOC_OP_DONE) {
+			roc_npa_aura_op_bulk_free(mp->pool_id, mem->objs,
+						  mem->sz, 1);
+			mem->status = BATCH_ALLOC_OP_NOT_ISSUED;
+		}
+	}
+
+	rte_free(op_data);
+	batch_op_data_set(mp->pool_id, NULL);
+	rte_wmb();
+}
+
 static int
 cn10k_mempool_alloc(struct rte_mempool *mp)
 {
 	uint32_t block_size;
 	size_t padding;
+	int rc;
 
 	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
 	/* Align header size to ROC_ALIGN */
@@ -29,15 +154,35 @@ cn10k_mempool_alloc(struct rte_mempool *mp)
 		block_size += padding;
 	}
 
-	return cnxk_mempool_alloc(mp);
+	rc = cnxk_mempool_alloc(mp);
+	if (rc)
+		return rc;
+
+	rc = batch_op_init(mp);
+	if (rc) {
+		plt_err("Failed to init batch alloc mem rc=%d", rc);
+		goto error;
+	}
+
+	return 0;
+error:
+	cnxk_mempool_free(mp);
+	return rc;
 }
 
 static void
 cn10k_mempool_free(struct rte_mempool *mp)
 {
+	batch_op_fini(mp);
 	cnxk_mempool_free(mp);
 }
 
+int
+cn10k_mempool_plt_init(void)
+{
+	return batch_op_data_table_create();
+}
+
 static struct rte_mempool_ops cn10k_mempool_ops = {
 	.name = "cn10k_mempool_ops",
 	.alloc = cn10k_mempool_alloc,
diff --git a/drivers/mempool/cnxk/cnxk_mempool.h b/drivers/mempool/cnxk/cnxk_mempool.h
index 099b7f6998..3405aa7663 100644
--- a/drivers/mempool/cnxk/cnxk_mempool.h
+++ b/drivers/mempool/cnxk/cnxk_mempool.h
@@ -23,4 +23,6 @@ int __rte_hot cnxk_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
 int __rte_hot cnxk_mempool_deq(struct rte_mempool *mp, void **obj_table,
 			       unsigned int n);
 
+int cn10k_mempool_plt_init(void);
+
 #endif
diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
index 42c02bf14e..c7b75f026d 100644
--- a/drivers/mempool/cnxk/cnxk_mempool_ops.c
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -174,12 +174,15 @@ cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
 static int
 cnxk_mempool_plt_init(void)
 {
-	if (roc_model_is_cn9k())
+	int rc = 0;
+
+	if (roc_model_is_cn9k()) {
 		rte_mbuf_set_platform_mempool_ops("cn9k_mempool_ops");
-	else if (roc_model_is_cn10k())
+	} else if (roc_model_is_cn10k()) {
 		rte_mbuf_set_platform_mempool_ops("cn10k_mempool_ops");
-
-	return 0;
+		rc = cn10k_mempool_plt_init();
+	}
+	return rc;
 }
 
 RTE_INIT(cnxk_mempool_ops_init)
-- 
2.31.0

[dpdk-dev] [PATCH v3 09/11] mempool/cnxk: add cn10k batch enqueue op

[Ashwin Sekhar T K]

Add the implementation for Marvell CN10k mempool batch enqueue op.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 28 +++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
index a3aef0ddb2..c225c227df 100644
--- a/drivers/mempool/cnxk/cn10k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -131,6 +131,32 @@ batch_op_fini(struct rte_mempool *mp)
 	rte_wmb();
 }
 
+static int __rte_hot
+cn10k_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
+		  unsigned int n)
+{
+	const uint64_t *ptr = (const uint64_t *)obj_table;
+	uint64_t lmt_addr = 0, lmt_id = 0;
+	struct batch_op_data *op_data;
+
+	/* Ensure mbuf init changes are written before the free pointers are
+	 * enqueued to the stack.
+	 */
+	rte_io_wmb();
+
+	if (n == 1) {
+		roc_npa_aura_op_free(mp->pool_id, 1, ptr[0]);
+		return 0;
+	}
+
+	op_data = batch_op_data_get(mp->pool_id);
+	lmt_addr = op_data->lmt_addr;
+	ROC_LMT_BASE_ID_GET(lmt_addr, lmt_id);
+	roc_npa_aura_op_batch_free(mp->pool_id, ptr, n, 1, lmt_addr, lmt_id);
+
+	return 0;
+}
+
 static int
 cn10k_mempool_alloc(struct rte_mempool *mp)
 {
@@ -187,7 +213,7 @@ static struct rte_mempool_ops cn10k_mempool_ops = {
 	.name = "cn10k_mempool_ops",
 	.alloc = cn10k_mempool_alloc,
 	.free = cn10k_mempool_free,
-	.enqueue = cnxk_mempool_enq,
+	.enqueue = cn10k_mempool_enq,
 	.dequeue = cnxk_mempool_deq,
 	.get_count = cnxk_mempool_get_count,
 	.calc_mem_size = cnxk_mempool_calc_mem_size,
-- 
2.31.0

[dpdk-dev] [PATCH v3 10/11] mempool/cnxk: add cn10k get count op

[Ashwin Sekhar T K]

Add the implementation for Marvell CN10k get count op.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 28 +++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
index c225c227df..d244a5e90f 100644
--- a/drivers/mempool/cnxk/cn10k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -157,6 +157,32 @@ cn10k_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
 	return 0;
 }
 
+static unsigned int
+cn10k_mempool_get_count(const struct rte_mempool *mp)
+{
+	struct batch_op_data *op_data;
+	unsigned int count = 0;
+	int i;
+
+	op_data = batch_op_data_get(mp->pool_id);
+
+	rte_wmb();
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct batch_op_mem *mem = &op_data->mem[i];
+
+		if (mem->status == BATCH_ALLOC_OP_ISSUED)
+			count += roc_npa_aura_batch_alloc_count(mem->objs,
+								BATCH_ALLOC_SZ);
+
+		if (mem->status == BATCH_ALLOC_OP_DONE)
+			count += mem->sz;
+	}
+
+	count += cnxk_mempool_get_count(mp);
+
+	return count;
+}
+
 static int
 cn10k_mempool_alloc(struct rte_mempool *mp)
 {
@@ -215,7 +241,7 @@ static struct rte_mempool_ops cn10k_mempool_ops = {
 	.free = cn10k_mempool_free,
 	.enqueue = cn10k_mempool_enq,
 	.dequeue = cnxk_mempool_deq,
-	.get_count = cnxk_mempool_get_count,
+	.get_count = cn10k_mempool_get_count,
 	.calc_mem_size = cnxk_mempool_calc_mem_size,
 	.populate = cnxk_mempool_populate,
 };
-- 
2.31.0

[dpdk-dev] [PATCH v3 11/11] mempool/cnxk: add cn10k batch dequeue op

[Ashwin Sekhar T K]

Add the implementation for Marvell CN10k mempool batch dequeue op.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 72 +++++++++++++++++++++++-
 1 file changed, 71 insertions(+), 1 deletion(-)

diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
index d244a5e90f..95458b34b7 100644
--- a/drivers/mempool/cnxk/cn10k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -183,6 +183,76 @@ cn10k_mempool_get_count(const struct rte_mempool *mp)
 	return count;
 }
 
+static int __rte_hot
+cn10k_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
+{
+	struct batch_op_data *op_data;
+	struct batch_op_mem *mem;
+	unsigned int count = 0;
+	int tid, rc, retry;
+	bool loop = true;
+
+	op_data = batch_op_data_get(mp->pool_id);
+	tid = rte_lcore_id();
+	mem = &op_data->mem[tid];
+
+	/* Issue batch alloc */
+	if (mem->status == BATCH_ALLOC_OP_NOT_ISSUED) {
+		rc = roc_npa_aura_batch_alloc_issue(mp->pool_id, mem->objs,
+						    BATCH_ALLOC_SZ, 0, 1);
+		/* If issue fails, try falling back to default alloc */
+		if (unlikely(rc))
+			return cn10k_mempool_enq(mp, obj_table, n);
+		mem->status = BATCH_ALLOC_OP_ISSUED;
+	}
+
+	retry = 4;
+	while (loop) {
+		unsigned int cur_sz;
+
+		if (mem->status == BATCH_ALLOC_OP_ISSUED) {
+			mem->sz = roc_npa_aura_batch_alloc_extract(
+				mem->objs, mem->objs, BATCH_ALLOC_SZ);
+
+			/* If partial alloc reduce the retry count */
+			retry -= (mem->sz != BATCH_ALLOC_SZ);
+			/* Break the loop if retry count exhausted */
+			loop = !!retry;
+			mem->status = BATCH_ALLOC_OP_DONE;
+		}
+
+		cur_sz = n - count;
+		if (cur_sz > mem->sz)
+			cur_sz = mem->sz;
+
+		/* Dequeue the pointers */
+		memcpy(&obj_table[count], &mem->objs[mem->sz - cur_sz],
+		       cur_sz * sizeof(uintptr_t));
+		mem->sz -= cur_sz;
+		count += cur_sz;
+
+		/* Break loop if the required pointers has been dequeued */
+		loop &= (count != n);
+
+		/* Issue next batch alloc if pointers are exhausted */
+		if (mem->sz == 0) {
+			rc = roc_npa_aura_batch_alloc_issue(
+				mp->pool_id, mem->objs, BATCH_ALLOC_SZ, 0, 1);
+			/* Break loop if issue failed and set status */
+			loop &= !rc;
+			mem->status = !rc;
+		}
+	}
+
+	if (unlikely(count != n)) {
+		/* No partial alloc allowed. Free up allocated pointers */
+		cn10k_mempool_enq(mp, obj_table, count);
+		return -ENOENT;
+	}
+
+	return 0;
+}
+
 static int
 cn10k_mempool_alloc(struct rte_mempool *mp)
 {
@@ -240,7 +310,7 @@ static struct rte_mempool_ops cn10k_mempool_ops = {
 	.alloc = cn10k_mempool_alloc,
 	.free = cn10k_mempool_free,
 	.enqueue = cn10k_mempool_enq,
-	.dequeue = cnxk_mempool_deq,
+	.dequeue = cn10k_mempool_deq,
 	.get_count = cn10k_mempool_get_count,
 	.calc_mem_size = cnxk_mempool_calc_mem_size,
 	.populate = cnxk_mempool_populate,
-- 
2.31.0

Re: [dpdk-dev] [PATCH v3 00/11] Add Marvell CNXK mempool driver

[Jerin Jacob]

On Tue, Apr 6, 2021 at 8:41 PM Ashwin Sekhar T K <asekhar@marvell.com> wrote:
>
> This patchset adds the mempool/cnxk driver which provides the support for the
> integrated mempool device found in Marvell CN10K SoC.
>
> The code includes mempool driver functionality for Marvell CN9K SoC as well,
> but right now it is not enabled. The future plan is to deprecate existing
> mempool/octeontx2 driver once the 'CNXK' drivers are feature complete for
> Marvell CN9K SoC.
>
> Depends-on: series-16131 ("Add Marvell CNXK common driver")
>
> v3:
>  - Change batch op data initialization to plt init callback.
>  - Reserve a memzone for batch op data.
>  - Handle batch op data initialization in secondary process.

1) http://patches.dpdk.org/project/dpdk/patch/20210406151115.1889455-12-asekhar@marvell.com/
shows some CI issues, Could you check, Is this valid or not?
2) Common code series has added the following [1] section, Could you
add mempool driver update as the second bullet.

Rest looks good to me.

[1]
diff --git a/doc/guides/rel_notes/release_21_05.rst
b/doc/guides/rel_notes/release_21_05.rst
index 2ffeb92..cc6e53e 100644
--- a/doc/guides/rel_notes/release_21_05.rst
+++ b/doc/guides/rel_notes/release_21_05.rst
@@ -65,6 +65,15 @@  New Features
       representor=[[c#]pf#]sf# sf[0,2-1023] /* 1023 SFs.                     */
       representor=[c#]pf#      c2pf[0,1]    /* 2 PFs on controller 2.        */

+* **Added support for Marvell CN10K SoC drivers.**
+
+  Added Marvell CN10K SoC support. Marvell CN10K SoC are based on Octeon 10
+  family of ARM64 processors with ARM Neoverse N2 core with accelerators for
+  packet processing, timers, cryptography, etc.
+
+  * Added common/cnxk driver consisting of common API to be used by
+    net, crypto and event PMD's.


>
> Ashwin Sekhar T K (11):
>   mempool/cnxk: add build infra and doc
>   mempool/cnxk: add device probe/remove
>   mempool/cnxk: add generic ops
>   mempool/cnxk: register plt init callback
>   mempool/cnxk: add cn9k mempool ops
>   mempool/cnxk: add cn9k optimized mempool enqueue/dequeue
>   mempool/cnxk: add cn10k mempool ops
>   mempool/cnxk: add batch op init
>   mempool/cnxk: add cn10k batch enqueue op
>   mempool/cnxk: add cn10k get count op
>   mempool/cnxk: add cn10k batch dequeue op
>
>  MAINTAINERS                              |   6 +
>  doc/guides/mempool/cnxk.rst              |  91 +++++++
>  doc/guides/mempool/index.rst             |   1 +
>  doc/guides/platform/cnxk.rst             |   3 +
>  drivers/mempool/cnxk/cn10k_mempool_ops.c | 319 +++++++++++++++++++++++
>  drivers/mempool/cnxk/cn9k_mempool_ops.c  |  89 +++++++
>  drivers/mempool/cnxk/cnxk_mempool.c      | 202 ++++++++++++++
>  drivers/mempool/cnxk/cnxk_mempool.h      |  28 ++
>  drivers/mempool/cnxk/cnxk_mempool_ops.c  | 191 ++++++++++++++
>  drivers/mempool/cnxk/meson.build         |  16 ++
>  drivers/mempool/cnxk/version.map         |   3 +
>  drivers/mempool/meson.build              |   3 +-
>  12 files changed, 951 insertions(+), 1 deletion(-)
>  create mode 100644 doc/guides/mempool/cnxk.rst
>  create mode 100644 drivers/mempool/cnxk/cn10k_mempool_ops.c
>  create mode 100644 drivers/mempool/cnxk/cn9k_mempool_ops.c
>  create mode 100644 drivers/mempool/cnxk/cnxk_mempool.c
>  create mode 100644 drivers/mempool/cnxk/cnxk_mempool.h
>  create mode 100644 drivers/mempool/cnxk/cnxk_mempool_ops.c
>  create mode 100644 drivers/mempool/cnxk/meson.build
>  create mode 100644 drivers/mempool/cnxk/version.map
>
> --
> 2.31.0
>

[dpdk-dev] [PATCH v4 00/11] Add Marvell CNXK mempool driver

[Ashwin Sekhar T K]

This patchset adds the mempool/cnxk driver which provides the support for the
integrated mempool device found in Marvell CN10K SoC.

The code includes mempool driver functionality for Marvell CN9K SoC as well,
but right now it is not enabled. The future plan is to deprecate existing
mempool/octeontx2 driver once the 'CNXK' drivers are feature complete for
Marvell CN9K SoC.

Depends-on: series-16131 ("Add Marvell CNXK common driver")

v4:
 - Added a bullet point for cnxk mempool driver in release_21_05.rst.

Ashwin Sekhar T K (11):
  mempool/cnxk: add build infra and doc
  mempool/cnxk: add device probe/remove
  mempool/cnxk: add generic ops
  mempool/cnxk: register plt init callback
  mempool/cnxk: add cn9k mempool ops
  mempool/cnxk: add cn9k optimized mempool enqueue/dequeue
  mempool/cnxk: add cn10k mempool ops
  mempool/cnxk: add batch op init
  mempool/cnxk: add cn10k batch enqueue op
  mempool/cnxk: add cn10k get count op
  mempool/cnxk: add cn10k batch dequeue op

 MAINTAINERS                              |   6 +
 doc/guides/mempool/cnxk.rst              |  91 +++++++
 doc/guides/mempool/index.rst             |   1 +
 doc/guides/platform/cnxk.rst             |   3 +
 doc/guides/rel_notes/release_21_05.rst   |   2 +
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 319 +++++++++++++++++++++++
 drivers/mempool/cnxk/cn9k_mempool_ops.c  |  89 +++++++
 drivers/mempool/cnxk/cnxk_mempool.c      | 202 ++++++++++++++
 drivers/mempool/cnxk/cnxk_mempool.h      |  28 ++
 drivers/mempool/cnxk/cnxk_mempool_ops.c  | 191 ++++++++++++++
 drivers/mempool/cnxk/meson.build         |  16 ++
 drivers/mempool/cnxk/version.map         |   3 +
 drivers/mempool/meson.build              |   3 +-
 13 files changed, 953 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/mempool/cnxk.rst
 create mode 100644 drivers/mempool/cnxk/cn10k_mempool_ops.c
 create mode 100644 drivers/mempool/cnxk/cn9k_mempool_ops.c
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.c
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.h
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool_ops.c
 create mode 100644 drivers/mempool/cnxk/meson.build
 create mode 100644 drivers/mempool/cnxk/version.map

-- 
2.31.0

[dpdk-dev] [PATCH v4 01/11] mempool/cnxk: add build infra and doc

[Ashwin Sekhar T K]

Add the meson based build infrastructure for Marvell
CNXK mempool driver along with stub implementations
for mempool device probe.

Also add Marvell CNXK mempool base documentation.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Jerin Jacob <jerinj@marvell.com>
Signed-off-by: Nithin Dabilpuram <ndabilpuram@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 MAINTAINERS                            |  6 ++
 doc/guides/mempool/cnxk.rst            | 55 ++++++++++++++++++
 doc/guides/mempool/index.rst           |  1 +
 doc/guides/platform/cnxk.rst           |  3 +
 doc/guides/rel_notes/release_21_05.rst |  2 +
 drivers/mempool/cnxk/cnxk_mempool.c    | 78 ++++++++++++++++++++++++++
 drivers/mempool/cnxk/meson.build       | 13 +++++
 drivers/mempool/cnxk/version.map       |  3 +
 drivers/mempool/meson.build            |  3 +-
 9 files changed, 163 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/mempool/cnxk.rst
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.c
 create mode 100644 drivers/mempool/cnxk/meson.build
 create mode 100644 drivers/mempool/cnxk/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index f72ce479e2..aa819bcd16 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -501,6 +501,12 @@ M: Artem V. Andreev <artem.andreev@oktetlabs.ru>
 M: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
 F: drivers/mempool/bucket/
 
+Marvell cnxk
+M: Ashwin Sekhar T K <asekhar@marvell.com>
+M: Pavan Nikhilesh <pbhagavatula@marvell.com>
+F: drivers/mempool/cnxk/
+F: doc/guides/mempool/cnxk.rst
+
 Marvell OCTEON TX2
 M: Jerin Jacob <jerinj@marvell.com>
 M: Nithin Dabilpuram <ndabilpuram@marvell.com>
diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
new file mode 100644
index 0000000000..e72a77c361
--- /dev/null
+++ b/doc/guides/mempool/cnxk.rst
@@ -0,0 +1,55 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(C) 2021 Marvell.
+
+CNXK NPA Mempool Driver
+============================
+
+The CNXK NPA PMD (**librte_mempool_cnxk**) provides mempool driver support for
+the integrated mempool device found in **Marvell OCTEON CN9K/CN10K** SoC family.
+
+More information about CNXK SoC can be found at `Marvell Official Website
+<https://www.marvell.com/embedded-processors/infrastructure-processors/>`_.
+
+Features
+--------
+
+CNXK NPA PMD supports:
+
+- Up to 128 NPA LFs
+- 1M Pools per LF
+- HW mempool manager
+- Ethdev Rx buffer allocation in HW to save CPU cycles in the Rx path.
+- Ethdev Tx buffer recycling in HW to save CPU cycles in the Tx path.
+
+Prerequisites and Compilation procedure
+---------------------------------------
+
+   See :doc:`../platform/cnxk` for setup information.
+
+Pre-Installation Configuration
+------------------------------
+
+
+Debugging Options
+~~~~~~~~~~~~~~~~~
+
+.. _table_cnxk_mempool_debug_options:
+
+.. table:: CNXK mempool debug options
+
+   +---+------------+-------------------------------------------------------+
+   | # | Component  | EAL log command                                       |
+   +===+============+=======================================================+
+   | 1 | NPA        | --log-level='pmd\.mempool.cnxk,8'                     |
+   +---+------------+-------------------------------------------------------+
+
+Standalone mempool device
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+   The ``usertools/dpdk-devbind.py`` script shall enumerate all the mempool
+   devices available in the system. In order to avoid, the end user to bind the
+   mempool device prior to use ethdev and/or eventdev device, the respective
+   driver configures an NPA LF and attach to the first probed ethdev or eventdev
+   device. In case, if end user need to run mempool as a standalone device
+   (without ethdev or eventdev), end user needs to bind a mempool device using
+   ``usertools/dpdk-devbind.py``
diff --git a/doc/guides/mempool/index.rst b/doc/guides/mempool/index.rst
index a0e55467e6..ce53bc1ac7 100644
--- a/doc/guides/mempool/index.rst
+++ b/doc/guides/mempool/index.rst
@@ -11,6 +11,7 @@ application through the mempool API.
     :maxdepth: 2
     :numbered:
 
+    cnxk
     octeontx
     octeontx2
     ring
diff --git a/doc/guides/platform/cnxk.rst b/doc/guides/platform/cnxk.rst
index d8fc00df40..489569ef4e 100644
--- a/doc/guides/platform/cnxk.rst
+++ b/doc/guides/platform/cnxk.rst
@@ -142,6 +142,9 @@ HW Offload Drivers
 
 This section lists dataplane H/W block(s) available in CNXK SoC.
 
+#. **Mempool Driver**
+   See :doc:`../mempool/cnxk` for NPA mempool driver information.
+
 Procedure to Setup Platform
 ---------------------------
 
diff --git a/doc/guides/rel_notes/release_21_05.rst b/doc/guides/rel_notes/release_21_05.rst
index 9ca1cb244c..4255ab3372 100644
--- a/doc/guides/rel_notes/release_21_05.rst
+++ b/doc/guides/rel_notes/release_21_05.rst
@@ -73,6 +73,8 @@ New Features
 
   * Added common/cnxk driver consisting of common API to be used by
     net, crypto and event PMD's.
+  * Added mempool/cnxk driver which provides the support for the integrated
+    mempool device.
 
 * **Updated Arkville PMD driver.**
 
diff --git a/drivers/mempool/cnxk/cnxk_mempool.c b/drivers/mempool/cnxk/cnxk_mempool.c
new file mode 100644
index 0000000000..947078c052
--- /dev/null
+++ b/drivers/mempool/cnxk/cnxk_mempool.c
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_atomic.h>
+#include <rte_bus_pci.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+#include <rte_eal.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_mbuf_pool_ops.h>
+#include <rte_pci.h>
+
+#include "roc_api.h"
+
+static int
+npa_remove(struct rte_pci_device *pci_dev)
+{
+	RTE_SET_USED(pci_dev);
+
+	return 0;
+}
+
+static int
+npa_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
+{
+	RTE_SET_USED(pci_drv);
+	RTE_SET_USED(pci_dev);
+
+	return 0;
+}
+
+static const struct rte_pci_id npa_pci_map[] = {
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_PF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KA,
+	},
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_PF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KAS,
+	},
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_VF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KA,
+	},
+	{
+		.class_id = RTE_CLASS_ANY_ID,
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVID_CNXK_RVU_NPA_VF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUBSYSTEM_DEVID_CN10KAS,
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static struct rte_pci_driver npa_pci = {
+	.id_table = npa_pci_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_NEED_IOVA_AS_VA,
+	.probe = npa_probe,
+	.remove = npa_remove,
+};
+
+RTE_PMD_REGISTER_PCI(mempool_cnxk, npa_pci);
+RTE_PMD_REGISTER_PCI_TABLE(mempool_cnxk, npa_pci_map);
+RTE_PMD_REGISTER_KMOD_DEP(mempool_cnxk, "vfio-pci");
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
new file mode 100644
index 0000000000..0be0802373
--- /dev/null
+++ b/drivers/mempool/cnxk/meson.build
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(C) 2021 Marvell.
+#
+
+if not is_linux or not dpdk_conf.get('RTE_ARCH_64')
+	build = false
+	reason = 'only supported on 64-bit Linux'
+	subdir_done()
+endif
+
+sources = files('cnxk_mempool.c')
+
+deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
diff --git a/drivers/mempool/cnxk/version.map b/drivers/mempool/cnxk/version.map
new file mode 100644
index 0000000000..ee80c51721
--- /dev/null
+++ b/drivers/mempool/cnxk/version.map
@@ -0,0 +1,3 @@
+INTERNAL {
+	local: *;
+};
diff --git a/drivers/mempool/meson.build b/drivers/mempool/meson.build
index 4428813dae..a2814c1dfa 100644
--- a/drivers/mempool/meson.build
+++ b/drivers/mempool/meson.build
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-drivers = ['bucket', 'dpaa', 'dpaa2', 'octeontx', 'octeontx2', 'ring', 'stack']
+drivers = ['bucket', 'cnxk', 'dpaa', 'dpaa2', 'octeontx', 'octeontx2', 'ring',
+	   'stack']
 std_deps = ['mempool']
-- 
2.31.0

[dpdk-dev] [PATCH v4 02/11] mempool/cnxk: add device probe/remove

[Ashwin Sekhar T K]

Add the implementation for CNXk mempool device
probe and remove.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 doc/guides/mempool/cnxk.rst         |  23 +++++
 drivers/mempool/cnxk/cnxk_mempool.c | 132 +++++++++++++++++++++++++++-
 2 files changed, 151 insertions(+), 4 deletions(-)

diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
index e72a77c361..907c19c841 100644
--- a/doc/guides/mempool/cnxk.rst
+++ b/doc/guides/mempool/cnxk.rst
@@ -30,6 +30,29 @@ Pre-Installation Configuration
 ------------------------------
 
 
+Runtime Config Options
+~~~~~~~~~~~~~~~~~~~~~~
+
+- ``Maximum number of mempools per application`` (default ``128``)
+
+  The maximum number of mempools per application needs to be configured on
+  HW during mempool driver initialization. HW can support up to 1M mempools,
+  Since each mempool costs set of HW resources, the ``max_pools`` ``devargs``
+  parameter is being introduced to configure the number of mempools required
+  for the application.
+  For example::
+
+    -a 0002:02:00.0,max_pools=512
+
+  With the above configuration, the driver will set up only 512 mempools for
+  the given application to save HW resources.
+
+.. note::
+
+   Since this configuration is per application, the end user needs to
+   provide ``max_pools`` parameter to the first PCIe device probed by the given
+   application.
+
 Debugging Options
 ~~~~~~~~~~~~~~~~~
 
diff --git a/drivers/mempool/cnxk/cnxk_mempool.c b/drivers/mempool/cnxk/cnxk_mempool.c
index 947078c052..dd4d74ca05 100644
--- a/drivers/mempool/cnxk/cnxk_mempool.c
+++ b/drivers/mempool/cnxk/cnxk_mempool.c
@@ -15,21 +15,143 @@
 
 #include "roc_api.h"
 
+#define CNXK_NPA_DEV_NAME	 RTE_STR(cnxk_npa_dev_)
+#define CNXK_NPA_DEV_NAME_LEN	 (sizeof(CNXK_NPA_DEV_NAME) + PCI_PRI_STR_SIZE)
+#define CNXK_NPA_MAX_POOLS_PARAM "max_pools"
+
+static inline uint32_t
+npa_aura_size_to_u32(uint8_t val)
+{
+	if (val == NPA_AURA_SZ_0)
+		return 128;
+	if (val >= NPA_AURA_SZ_MAX)
+		return BIT_ULL(20);
+
+	return 1 << (val + 6);
+}
+
 static int
-npa_remove(struct rte_pci_device *pci_dev)
+parse_max_pools(const char *key, const char *value, void *extra_args)
+{
+	RTE_SET_USED(key);
+	uint32_t val;
+
+	val = atoi(value);
+	if (val < npa_aura_size_to_u32(NPA_AURA_SZ_128))
+		val = 128;
+	if (val > npa_aura_size_to_u32(NPA_AURA_SZ_1M))
+		val = BIT_ULL(20);
+
+	*(uint8_t *)extra_args = rte_log2_u32(val) - 6;
+	return 0;
+}
+
+static inline uint8_t
+parse_aura_size(struct rte_devargs *devargs)
+{
+	uint8_t aura_sz = NPA_AURA_SZ_128;
+	struct rte_kvargs *kvlist;
+
+	if (devargs == NULL)
+		goto exit;
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL)
+		goto exit;
+
+	rte_kvargs_process(kvlist, CNXK_NPA_MAX_POOLS_PARAM, &parse_max_pools,
+			   &aura_sz);
+	rte_kvargs_free(kvlist);
+exit:
+	return aura_sz;
+}
+
+static inline char *
+npa_dev_to_name(struct rte_pci_device *pci_dev, char *name)
+{
+	snprintf(name, CNXK_NPA_DEV_NAME_LEN, CNXK_NPA_DEV_NAME PCI_PRI_FMT,
+		 pci_dev->addr.domain, pci_dev->addr.bus, pci_dev->addr.devid,
+		 pci_dev->addr.function);
+
+	return name;
+}
+
+static int
+npa_init(struct rte_pci_device *pci_dev)
 {
-	RTE_SET_USED(pci_dev);
+	char name[CNXK_NPA_DEV_NAME_LEN];
+	const struct rte_memzone *mz;
+	struct roc_npa *dev;
+	int rc = -ENOMEM;
+
+	mz = rte_memzone_reserve_aligned(npa_dev_to_name(pci_dev, name),
+					 sizeof(*dev), SOCKET_ID_ANY, 0,
+					 RTE_CACHE_LINE_SIZE);
+	if (mz == NULL)
+		goto error;
+
+	dev = mz->addr;
+	dev->pci_dev = pci_dev;
+
+	roc_idev_npa_maxpools_set(parse_aura_size(pci_dev->device.devargs));
+	rc = roc_npa_dev_init(dev);
+	if (rc)
+		goto mz_free;
+
+	return 0;
+
+mz_free:
+	rte_memzone_free(mz);
+error:
+	plt_err("failed to initialize npa device rc=%d", rc);
+	return rc;
+}
+
+static int
+npa_fini(struct rte_pci_device *pci_dev)
+{
+	char name[CNXK_NPA_DEV_NAME_LEN];
+	const struct rte_memzone *mz;
+	int rc;
+
+	mz = rte_memzone_lookup(npa_dev_to_name(pci_dev, name));
+	if (mz == NULL)
+		return -EINVAL;
+
+	rc = roc_npa_dev_fini(mz->addr);
+	if (rc) {
+		if (rc != -EAGAIN)
+			plt_err("Failed to remove npa dev, rc=%d", rc);
+		return rc;
+	}
+	rte_memzone_free(mz);
 
 	return 0;
 }
 
+static int
+npa_remove(struct rte_pci_device *pci_dev)
+{
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	return npa_fini(pci_dev);
+}
+
 static int
 npa_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 {
+	int rc;
+
 	RTE_SET_USED(pci_drv);
-	RTE_SET_USED(pci_dev);
 
-	return 0;
+	rc = roc_plt_init();
+	if (rc < 0)
+		return rc;
+
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	return npa_init(pci_dev);
 }
 
 static const struct rte_pci_id npa_pci_map[] = {
@@ -76,3 +198,5 @@ static struct rte_pci_driver npa_pci = {
 RTE_PMD_REGISTER_PCI(mempool_cnxk, npa_pci);
 RTE_PMD_REGISTER_PCI_TABLE(mempool_cnxk, npa_pci_map);
 RTE_PMD_REGISTER_KMOD_DEP(mempool_cnxk, "vfio-pci");
+RTE_PMD_REGISTER_PARAM_STRING(mempool_cnxk,
+			      CNXK_NPA_MAX_POOLS_PARAM "=<128-1048576>");
-- 
2.31.0

[dpdk-dev] [PATCH v4 03/11] mempool/cnxk: add generic ops

[Ashwin Sekhar T K]

Add generic CNXk mempool ops which will enqueue/dequeue
from pool one element at a time.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cnxk_mempool.h     |  26 ++++
 drivers/mempool/cnxk/cnxk_mempool_ops.c | 171 ++++++++++++++++++++++++
 drivers/mempool/cnxk/meson.build        |   3 +-
 3 files changed, 199 insertions(+), 1 deletion(-)
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool.h
 create mode 100644 drivers/mempool/cnxk/cnxk_mempool_ops.c

diff --git a/drivers/mempool/cnxk/cnxk_mempool.h b/drivers/mempool/cnxk/cnxk_mempool.h
new file mode 100644
index 0000000000..099b7f6998
--- /dev/null
+++ b/drivers/mempool/cnxk/cnxk_mempool.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#ifndef _CNXK_MEMPOOL_H_
+#define _CNXK_MEMPOOL_H_
+
+#include <rte_mempool.h>
+
+unsigned int cnxk_mempool_get_count(const struct rte_mempool *mp);
+ssize_t cnxk_mempool_calc_mem_size(const struct rte_mempool *mp,
+				   uint32_t obj_num, uint32_t pg_shift,
+				   size_t *min_chunk_size, size_t *align);
+int cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
+			  void *vaddr, rte_iova_t iova, size_t len,
+			  rte_mempool_populate_obj_cb_t *obj_cb,
+			  void *obj_cb_arg);
+int cnxk_mempool_alloc(struct rte_mempool *mp);
+void cnxk_mempool_free(struct rte_mempool *mp);
+
+int __rte_hot cnxk_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
+			       unsigned int n);
+int __rte_hot cnxk_mempool_deq(struct rte_mempool *mp, void **obj_table,
+			       unsigned int n);
+
+#endif
diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
new file mode 100644
index 0000000000..2ce1816c04
--- /dev/null
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -0,0 +1,171 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_mempool.h>
+
+#include "roc_api.h"
+#include "cnxk_mempool.h"
+
+int __rte_hot
+cnxk_mempool_enq(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
+{
+	unsigned int index;
+
+	/* Ensure mbuf init changes are written before the free pointers
+	 * are enqueued to the stack.
+	 */
+	rte_io_wmb();
+	for (index = 0; index < n; index++)
+		roc_npa_aura_op_free(mp->pool_id, 0,
+				     (uint64_t)obj_table[index]);
+
+	return 0;
+}
+
+int __rte_hot
+cnxk_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
+{
+	unsigned int index;
+	uint64_t obj;
+
+	for (index = 0; index < n; index++, obj_table++) {
+		int retry = 4;
+
+		/* Retry few times before failing */
+		do {
+			obj = roc_npa_aura_op_alloc(mp->pool_id, 0);
+		} while (retry-- && (obj == 0));
+
+		if (obj == 0) {
+			cnxk_mempool_enq(mp, obj_table - index, index);
+			return -ENOENT;
+		}
+		*obj_table = (void *)obj;
+	}
+
+	return 0;
+}
+
+unsigned int
+cnxk_mempool_get_count(const struct rte_mempool *mp)
+{
+	return (unsigned int)roc_npa_aura_op_available(mp->pool_id);
+}
+
+ssize_t
+cnxk_mempool_calc_mem_size(const struct rte_mempool *mp, uint32_t obj_num,
+			   uint32_t pg_shift, size_t *min_chunk_size,
+			   size_t *align)
+{
+	size_t total_elt_sz;
+
+	/* Need space for one more obj on each chunk to fulfill
+	 * alignment requirements.
+	 */
+	total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
+	return rte_mempool_op_calc_mem_size_helper(
+		mp, obj_num, pg_shift, total_elt_sz, min_chunk_size, align);
+}
+
+int
+cnxk_mempool_alloc(struct rte_mempool *mp)
+{
+	uint64_t aura_handle = 0;
+	struct npa_aura_s aura;
+	struct npa_pool_s pool;
+	uint32_t block_count;
+	size_t block_size;
+	int rc = -ERANGE;
+
+	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
+	block_count = mp->size;
+	if (mp->header_size % ROC_ALIGN != 0) {
+		plt_err("Header size should be multiple of %dB", ROC_ALIGN);
+		goto error;
+	}
+
+	if (block_size % ROC_ALIGN != 0) {
+		plt_err("Block size should be multiple of %dB", ROC_ALIGN);
+		goto error;
+	}
+
+	memset(&aura, 0, sizeof(struct npa_aura_s));
+	memset(&pool, 0, sizeof(struct npa_pool_s));
+	pool.nat_align = 1;
+	pool.buf_offset = mp->header_size / ROC_ALIGN;
+
+	/* Use driver specific mp->pool_config to override aura config */
+	if (mp->pool_config != NULL)
+		memcpy(&aura, mp->pool_config, sizeof(struct npa_aura_s));
+
+	rc = roc_npa_pool_create(&aura_handle, block_size, block_count, &aura,
+				 &pool);
+	if (rc) {
+		plt_err("Failed to alloc pool or aura rc=%d", rc);
+		goto error;
+	}
+
+	/* Store aura_handle for future queue operations */
+	mp->pool_id = aura_handle;
+	plt_npa_dbg("block_sz=%lu block_count=%d aura_handle=0x%" PRIx64,
+		    block_size, block_count, aura_handle);
+
+	return 0;
+error:
+	return rc;
+}
+
+void
+cnxk_mempool_free(struct rte_mempool *mp)
+{
+	int rc = 0;
+
+	plt_npa_dbg("aura_handle=0x%" PRIx64, mp->pool_id);
+	rc = roc_npa_pool_destroy(mp->pool_id);
+	if (rc)
+		plt_err("Failed to free pool or aura rc=%d", rc);
+}
+
+int
+cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
+		      void *vaddr, rte_iova_t iova, size_t len,
+		      rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
+{
+	size_t total_elt_sz, off;
+	int num_elts;
+
+	if (iova == RTE_BAD_IOVA)
+		return -EINVAL;
+
+	total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
+
+	/* Align object start address to a multiple of total_elt_sz */
+	off = total_elt_sz - ((((uintptr_t)vaddr - 1) % total_elt_sz) + 1);
+
+	if (len < off)
+		return -EINVAL;
+
+	vaddr = (char *)vaddr + off;
+	iova += off;
+	len -= off;
+	num_elts = len / total_elt_sz;
+
+	plt_npa_dbg("iova %" PRIx64 ", aligned iova %" PRIx64 "", iova - off,
+		    iova);
+	plt_npa_dbg("length %" PRIu64 ", aligned length %" PRIu64 "",
+		    (uint64_t)(len + off), (uint64_t)len);
+	plt_npa_dbg("element size %" PRIu64 "", (uint64_t)total_elt_sz);
+	plt_npa_dbg("requested objects %" PRIu64 ", possible objects %" PRIu64
+		    "", (uint64_t)max_objs, (uint64_t)num_elts);
+
+	roc_npa_aura_op_range_set(mp->pool_id, iova,
+				  iova + num_elts * total_elt_sz);
+
+	if (roc_npa_pool_range_update_check(mp->pool_id) < 0)
+		return -EBUSY;
+
+	return rte_mempool_op_populate_helper(
+		mp, RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ, max_objs, vaddr, iova,
+		len, obj_cb, obj_cb_arg);
+}
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
index 0be0802373..52244e728b 100644
--- a/drivers/mempool/cnxk/meson.build
+++ b/drivers/mempool/cnxk/meson.build
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.get('RTE_ARCH_64')
 	subdir_done()
 endif
 
-sources = files('cnxk_mempool.c')
+sources = files('cnxk_mempool.c',
+		'cnxk_mempool_ops.c')
 
 deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
-- 
2.31.0

[dpdk-dev] [PATCH v4 04/11] mempool/cnxk: register plt init callback

[Ashwin Sekhar T K]

Register the CNXk mempool plt init callback which will set the
appropriate mempool ops to be used for the platform.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cnxk_mempool_ops.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
index 2ce1816c04..e8f64be76b 100644
--- a/drivers/mempool/cnxk/cnxk_mempool_ops.c
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -2,6 +2,7 @@
  * Copyright(C) 2021 Marvell.
  */
 
+#include <rte_mbuf_pool_ops.h>
 #include <rte_mempool.h>
 
 #include "roc_api.h"
@@ -169,3 +170,17 @@ cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
 		mp, RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ, max_objs, vaddr, iova,
 		len, obj_cb, obj_cb_arg);
 }
+
+static int
+cnxk_mempool_plt_init(void)
+{
+	if (roc_model_is_cn10k() || roc_model_is_cn9k())
+		rte_mbuf_set_platform_mempool_ops("cnxk_mempool_ops");
+
+	return 0;
+}
+
+RTE_INIT(cnxk_mempool_ops_init)
+{
+	roc_plt_init_cb_register(cnxk_mempool_plt_init);
+}
-- 
2.31.0

[dpdk-dev] [PATCH v4 05/11] mempool/cnxk: add cn9k mempool ops

[Ashwin Sekhar T K]

Add Marvell CN9k mempool ops and implement CN9k mempool
alloc which makes sure that the element size always occupy
odd number of cachelines to ensure even distribution among
of elements among L1D cache sets.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn9k_mempool_ops.c | 54 +++++++++++++++++++++++++
 drivers/mempool/cnxk/cnxk_mempool_ops.c |  4 +-
 drivers/mempool/cnxk/meson.build        |  3 +-
 3 files changed, 59 insertions(+), 2 deletions(-)
 create mode 100644 drivers/mempool/cnxk/cn9k_mempool_ops.c

diff --git a/drivers/mempool/cnxk/cn9k_mempool_ops.c b/drivers/mempool/cnxk/cn9k_mempool_ops.c
new file mode 100644
index 0000000000..f5ac163af9
--- /dev/null
+++ b/drivers/mempool/cnxk/cn9k_mempool_ops.c
@@ -0,0 +1,54 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_mempool.h>
+
+#include "roc_api.h"
+#include "cnxk_mempool.h"
+
+static int
+cn9k_mempool_alloc(struct rte_mempool *mp)
+{
+	size_t block_size, padding;
+
+	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
+	/* Align header size to ROC_ALIGN */
+	if (mp->header_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(mp->header_size, ROC_ALIGN) -
+			  mp->header_size;
+		mp->header_size += padding;
+		block_size += padding;
+	}
+
+	/* Align block size to ROC_ALIGN */
+	if (block_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(block_size, ROC_ALIGN) - block_size;
+		mp->trailer_size += padding;
+		block_size += padding;
+	}
+
+	/*
+	 * Marvell CN9k has 8 sets, 41 ways L1D cache, VA<9:7> bits dictate the
+	 * set selection. Add additional padding to ensure that the element size
+	 * always occupies odd number of cachelines to ensure even distribution
+	 * of elements among L1D cache sets.
+	 */
+	padding = ((block_size / ROC_ALIGN) % 2) ? 0 : ROC_ALIGN;
+	mp->trailer_size += padding;
+
+	return cnxk_mempool_alloc(mp);
+}
+
+static struct rte_mempool_ops cn9k_mempool_ops = {
+	.name = "cn9k_mempool_ops",
+	.alloc = cn9k_mempool_alloc,
+	.free = cnxk_mempool_free,
+	.enqueue = cnxk_mempool_enq,
+	.dequeue = cnxk_mempool_deq,
+	.get_count = cnxk_mempool_get_count,
+	.calc_mem_size = cnxk_mempool_calc_mem_size,
+	.populate = cnxk_mempool_populate,
+};
+
+MEMPOOL_REGISTER_OPS(cn9k_mempool_ops);
diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
index e8f64be76b..d8ed37ec1a 100644
--- a/drivers/mempool/cnxk/cnxk_mempool_ops.c
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -174,7 +174,9 @@ cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
 static int
 cnxk_mempool_plt_init(void)
 {
-	if (roc_model_is_cn10k() || roc_model_is_cn9k())
+	if (roc_model_is_cn9k())
+		rte_mbuf_set_platform_mempool_ops("cn9k_mempool_ops");
+	else if (roc_model_is_cn10k())
 		rte_mbuf_set_platform_mempool_ops("cnxk_mempool_ops");
 
 	return 0;
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
index 52244e728b..ff31893ff4 100644
--- a/drivers/mempool/cnxk/meson.build
+++ b/drivers/mempool/cnxk/meson.build
@@ -9,6 +9,7 @@ if not is_linux or not dpdk_conf.get('RTE_ARCH_64')
 endif
 
 sources = files('cnxk_mempool.c',
-		'cnxk_mempool_ops.c')
+		'cnxk_mempool_ops.c',
+		'cn9k_mempool_ops.c')
 
 deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
-- 
2.31.0

[dpdk-dev] [PATCH v4 06/11] mempool/cnxk: add cn9k optimized mempool enqueue/dequeue

[Ashwin Sekhar T K]

Add Marvell CN9k mempool enqueue/dequeue. Marvell CN9k
supports burst dequeue which allows to dequeue up to 32
pointers using pipelined casp instructions.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 doc/guides/mempool/cnxk.rst             |  4 +++
 drivers/mempool/cnxk/cn9k_mempool_ops.c | 39 +++++++++++++++++++++++--
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
index 907c19c841..f51532b101 100644
--- a/doc/guides/mempool/cnxk.rst
+++ b/doc/guides/mempool/cnxk.rst
@@ -21,6 +21,10 @@ CNXK NPA PMD supports:
 - Ethdev Rx buffer allocation in HW to save CPU cycles in the Rx path.
 - Ethdev Tx buffer recycling in HW to save CPU cycles in the Tx path.
 
+CN9k NPA supports:
+
+- Burst alloc of up to 32 pointers.
+
 Prerequisites and Compilation procedure
 ---------------------------------------
 
diff --git a/drivers/mempool/cnxk/cn9k_mempool_ops.c b/drivers/mempool/cnxk/cn9k_mempool_ops.c
index f5ac163af9..c0cdba640b 100644
--- a/drivers/mempool/cnxk/cn9k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn9k_mempool_ops.c
@@ -7,6 +7,41 @@
 #include "roc_api.h"
 #include "cnxk_mempool.h"
 
+static int __rte_hot
+cn9k_mempool_enq(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
+{
+	/* Ensure mbuf init changes are written before the free pointers
+	 * are enqueued to the stack.
+	 */
+	rte_io_wmb();
+	roc_npa_aura_op_bulk_free(mp->pool_id, (const uint64_t *)obj_table, n,
+				  0);
+
+	return 0;
+}
+
+static inline int __rte_hot
+cn9k_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
+{
+	unsigned int count;
+
+	count = roc_npa_aura_op_bulk_alloc(mp->pool_id, (uint64_t *)obj_table,
+					   n, 0, 1);
+
+	if (unlikely(count != n)) {
+		/* If bulk alloc failed to allocate all pointers, try
+		 * allocating remaining pointers with the default alloc
+		 * with retry scheme.
+		 */
+		if (cnxk_mempool_deq(mp, &obj_table[count], n - count)) {
+			cn9k_mempool_enq(mp, obj_table, count);
+			return -ENOENT;
+		}
+	}
+
+	return 0;
+}
+
 static int
 cn9k_mempool_alloc(struct rte_mempool *mp)
 {
@@ -44,8 +79,8 @@ static struct rte_mempool_ops cn9k_mempool_ops = {
 	.name = "cn9k_mempool_ops",
 	.alloc = cn9k_mempool_alloc,
 	.free = cnxk_mempool_free,
-	.enqueue = cnxk_mempool_enq,
-	.dequeue = cnxk_mempool_deq,
+	.enqueue = cn9k_mempool_enq,
+	.dequeue = cn9k_mempool_deq,
 	.get_count = cnxk_mempool_get_count,
 	.calc_mem_size = cnxk_mempool_calc_mem_size,
 	.populate = cnxk_mempool_populate,
-- 
2.31.0

[dpdk-dev] [PATCH v4 07/11] mempool/cnxk: add cn10k mempool ops

[Ashwin Sekhar T K]

Add Marvell CN10k mempool ops and implement CN10k mempool alloc.

CN10k has 64 bytes L1D cache line size. Hence the CN10k mempool
alloc does not make the element size an odd multiple L1D cache
line size as NPA requires the element sizes to be multiples of
128 bytes.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 doc/guides/mempool/cnxk.rst              |  4 ++
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 52 ++++++++++++++++++++++++
 drivers/mempool/cnxk/cnxk_mempool_ops.c  |  2 +-
 drivers/mempool/cnxk/meson.build         |  3 +-
 4 files changed, 59 insertions(+), 2 deletions(-)
 create mode 100644 drivers/mempool/cnxk/cn10k_mempool_ops.c

diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
index f51532b101..783368e690 100644
--- a/doc/guides/mempool/cnxk.rst
+++ b/doc/guides/mempool/cnxk.rst
@@ -80,3 +80,7 @@ Standalone mempool device
    device. In case, if end user need to run mempool as a standalone device
    (without ethdev or eventdev), end user needs to bind a mempool device using
    ``usertools/dpdk-devbind.py``
+
+   Example command to run ``mempool_autotest`` test with standalone CN10K NPA device::
+
+     echo "mempool_autotest" | <build_dir>/app/test/dpdk-test -c 0xf0 --mbuf-pool-ops-name="cn10k_mempool_ops"
diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
new file mode 100644
index 0000000000..9b63789006
--- /dev/null
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include <rte_mempool.h>
+
+#include "roc_api.h"
+#include "cnxk_mempool.h"
+
+static int
+cn10k_mempool_alloc(struct rte_mempool *mp)
+{
+	uint32_t block_size;
+	size_t padding;
+
+	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
+	/* Align header size to ROC_ALIGN */
+	if (mp->header_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(mp->header_size, ROC_ALIGN) -
+			  mp->header_size;
+		mp->header_size += padding;
+		block_size += padding;
+	}
+
+	/* Align block size to ROC_ALIGN */
+	if (block_size % ROC_ALIGN != 0) {
+		padding = RTE_ALIGN_CEIL(block_size, ROC_ALIGN) - block_size;
+		mp->trailer_size += padding;
+		block_size += padding;
+	}
+
+	return cnxk_mempool_alloc(mp);
+}
+
+static void
+cn10k_mempool_free(struct rte_mempool *mp)
+{
+	cnxk_mempool_free(mp);
+}
+
+static struct rte_mempool_ops cn10k_mempool_ops = {
+	.name = "cn10k_mempool_ops",
+	.alloc = cn10k_mempool_alloc,
+	.free = cn10k_mempool_free,
+	.enqueue = cnxk_mempool_enq,
+	.dequeue = cnxk_mempool_deq,
+	.get_count = cnxk_mempool_get_count,
+	.calc_mem_size = cnxk_mempool_calc_mem_size,
+	.populate = cnxk_mempool_populate,
+};
+
+MEMPOOL_REGISTER_OPS(cn10k_mempool_ops);
diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
index d8ed37ec1a..42c02bf14e 100644
--- a/drivers/mempool/cnxk/cnxk_mempool_ops.c
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -177,7 +177,7 @@ cnxk_mempool_plt_init(void)
 	if (roc_model_is_cn9k())
 		rte_mbuf_set_platform_mempool_ops("cn9k_mempool_ops");
 	else if (roc_model_is_cn10k())
-		rte_mbuf_set_platform_mempool_ops("cnxk_mempool_ops");
+		rte_mbuf_set_platform_mempool_ops("cn10k_mempool_ops");
 
 	return 0;
 }
diff --git a/drivers/mempool/cnxk/meson.build b/drivers/mempool/cnxk/meson.build
index ff31893ff4..3282b5e5a6 100644
--- a/drivers/mempool/cnxk/meson.build
+++ b/drivers/mempool/cnxk/meson.build
@@ -10,6 +10,7 @@ endif
 
 sources = files('cnxk_mempool.c',
 		'cnxk_mempool_ops.c',
-		'cn9k_mempool_ops.c')
+		'cn9k_mempool_ops.c',
+		'cn10k_mempool_ops.c')
 
 deps += ['eal', 'mbuf', 'kvargs', 'bus_pci', 'common_cnxk', 'mempool']
-- 
2.31.0

[dpdk-dev] [PATCH v4 08/11] mempool/cnxk: add batch op init

[Ashwin Sekhar T K]

Marvell CN10k mempool supports batch enqueue/dequeue which can
dequeue up to 512 pointers and enqueue up to 15 pointers using
a single instruction.

These batch operations require a DMA memory to enqueue/dequeue
pointers. This patch adds the initialization of this DMA memory.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 doc/guides/mempool/cnxk.rst              |   5 +
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 147 ++++++++++++++++++++++-
 drivers/mempool/cnxk/cnxk_mempool.h      |   2 +
 drivers/mempool/cnxk/cnxk_mempool_ops.c  |  11 +-
 4 files changed, 160 insertions(+), 5 deletions(-)

diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst
index 783368e690..286ee29003 100644
--- a/doc/guides/mempool/cnxk.rst
+++ b/doc/guides/mempool/cnxk.rst
@@ -25,6 +25,11 @@ CN9k NPA supports:
 
 - Burst alloc of up to 32 pointers.
 
+CN10k NPA supports:
+
+- Batch dequeue of up to 512 pointers with single instruction.
+- Batch enqueue of up to 15 pointers with single instruction.
+
 Prerequisites and Compilation procedure
 ---------------------------------------
 
diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
index 9b63789006..a3aef0ddb2 100644
--- a/drivers/mempool/cnxk/cn10k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -7,11 +7,136 @@
 #include "roc_api.h"
 #include "cnxk_mempool.h"
 
+#define BATCH_ALLOC_SZ              ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS
+#define BATCH_OP_DATA_TABLE_MZ_NAME "batch_op_data_table_mz"
+
+enum batch_op_status {
+	BATCH_ALLOC_OP_NOT_ISSUED = 0,
+	BATCH_ALLOC_OP_ISSUED = 1,
+	BATCH_ALLOC_OP_DONE
+};
+
+struct batch_op_mem {
+	unsigned int sz;
+	enum batch_op_status status;
+	uint64_t objs[BATCH_ALLOC_SZ] __rte_aligned(ROC_ALIGN);
+};
+
+struct batch_op_data {
+	uint64_t lmt_addr;
+	struct batch_op_mem mem[RTE_MAX_LCORE] __rte_aligned(ROC_ALIGN);
+};
+
+static struct batch_op_data **batch_op_data_tbl;
+
+static int
+batch_op_data_table_create(void)
+{
+	const struct rte_memzone *mz;
+
+	/* If table is already set, nothing to do */
+	if (batch_op_data_tbl)
+		return 0;
+
+	mz = rte_memzone_lookup(BATCH_OP_DATA_TABLE_MZ_NAME);
+	if (mz == NULL) {
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+			unsigned int maxpools, sz;
+
+			maxpools = roc_idev_npa_maxpools_get();
+			sz = maxpools * sizeof(struct batch_op_data *);
+
+			mz = rte_memzone_reserve_aligned(
+				BATCH_OP_DATA_TABLE_MZ_NAME, sz, SOCKET_ID_ANY,
+				0, ROC_ALIGN);
+		}
+		if (mz == NULL) {
+			plt_err("Failed to reserve batch op data table");
+			return -ENOMEM;
+		}
+	}
+	batch_op_data_tbl = mz->addr;
+	rte_wmb();
+	return 0;
+}
+
+static inline struct batch_op_data *
+batch_op_data_get(uint64_t pool_id)
+{
+	uint64_t aura = roc_npa_aura_handle_to_aura(pool_id);
+
+	return batch_op_data_tbl[aura];
+}
+
+static inline void
+batch_op_data_set(uint64_t pool_id, struct batch_op_data *op_data)
+{
+	uint64_t aura = roc_npa_aura_handle_to_aura(pool_id);
+
+	batch_op_data_tbl[aura] = op_data;
+}
+
+static int
+batch_op_init(struct rte_mempool *mp)
+{
+	struct batch_op_data *op_data;
+	int i;
+
+	op_data = batch_op_data_get(mp->pool_id);
+	/* The data should not have been allocated previously */
+	RTE_ASSERT(op_data == NULL);
+
+	op_data = rte_zmalloc(NULL, sizeof(struct batch_op_data), ROC_ALIGN);
+	if (op_data == NULL)
+		return -ENOMEM;
+
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		op_data->mem[i].sz = 0;
+		op_data->mem[i].status = BATCH_ALLOC_OP_NOT_ISSUED;
+	}
+
+	op_data->lmt_addr = roc_idev_lmt_base_addr_get();
+	batch_op_data_set(mp->pool_id, op_data);
+	rte_wmb();
+
+	return 0;
+}
+
+static void
+batch_op_fini(struct rte_mempool *mp)
+{
+	struct batch_op_data *op_data;
+	int i;
+
+	op_data = batch_op_data_get(mp->pool_id);
+
+	rte_wmb();
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct batch_op_mem *mem = &op_data->mem[i];
+
+		if (mem->status == BATCH_ALLOC_OP_ISSUED) {
+			mem->sz = roc_npa_aura_batch_alloc_extract(
+				mem->objs, mem->objs, BATCH_ALLOC_SZ);
+			mem->status = BATCH_ALLOC_OP_DONE;
+		}
+		if (mem->status == BATCH_ALLOC_OP_DONE) {
+			roc_npa_aura_op_bulk_free(mp->pool_id, mem->objs,
+						  mem->sz, 1);
+			mem->status = BATCH_ALLOC_OP_NOT_ISSUED;
+		}
+	}
+
+	rte_free(op_data);
+	batch_op_data_set(mp->pool_id, NULL);
+	rte_wmb();
+}
+
 static int
 cn10k_mempool_alloc(struct rte_mempool *mp)
 {
 	uint32_t block_size;
 	size_t padding;
+	int rc;
 
 	block_size = mp->elt_size + mp->header_size + mp->trailer_size;
 	/* Align header size to ROC_ALIGN */
@@ -29,15 +154,35 @@ cn10k_mempool_alloc(struct rte_mempool *mp)
 		block_size += padding;
 	}
 
-	return cnxk_mempool_alloc(mp);
+	rc = cnxk_mempool_alloc(mp);
+	if (rc)
+		return rc;
+
+	rc = batch_op_init(mp);
+	if (rc) {
+		plt_err("Failed to init batch alloc mem rc=%d", rc);
+		goto error;
+	}
+
+	return 0;
+error:
+	cnxk_mempool_free(mp);
+	return rc;
 }
 
 static void
 cn10k_mempool_free(struct rte_mempool *mp)
 {
+	batch_op_fini(mp);
 	cnxk_mempool_free(mp);
 }
 
+int
+cn10k_mempool_plt_init(void)
+{
+	return batch_op_data_table_create();
+}
+
 static struct rte_mempool_ops cn10k_mempool_ops = {
 	.name = "cn10k_mempool_ops",
 	.alloc = cn10k_mempool_alloc,
diff --git a/drivers/mempool/cnxk/cnxk_mempool.h b/drivers/mempool/cnxk/cnxk_mempool.h
index 099b7f6998..3405aa7663 100644
--- a/drivers/mempool/cnxk/cnxk_mempool.h
+++ b/drivers/mempool/cnxk/cnxk_mempool.h
@@ -23,4 +23,6 @@ int __rte_hot cnxk_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
 int __rte_hot cnxk_mempool_deq(struct rte_mempool *mp, void **obj_table,
 			       unsigned int n);
 
+int cn10k_mempool_plt_init(void);
+
 #endif
diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c
index 42c02bf14e..c7b75f026d 100644
--- a/drivers/mempool/cnxk/cnxk_mempool_ops.c
+++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c
@@ -174,12 +174,15 @@ cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs,
 static int
 cnxk_mempool_plt_init(void)
 {
-	if (roc_model_is_cn9k())
+	int rc = 0;
+
+	if (roc_model_is_cn9k()) {
 		rte_mbuf_set_platform_mempool_ops("cn9k_mempool_ops");
-	else if (roc_model_is_cn10k())
+	} else if (roc_model_is_cn10k()) {
 		rte_mbuf_set_platform_mempool_ops("cn10k_mempool_ops");
-
-	return 0;
+		rc = cn10k_mempool_plt_init();
+	}
+	return rc;
 }
 
 RTE_INIT(cnxk_mempool_ops_init)
-- 
2.31.0

[dpdk-dev] [PATCH v4 09/11] mempool/cnxk: add cn10k batch enqueue op

[Ashwin Sekhar T K]

Add the implementation for Marvell CN10k mempool batch enqueue op.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 28 +++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
index a3aef0ddb2..c225c227df 100644
--- a/drivers/mempool/cnxk/cn10k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -131,6 +131,32 @@ batch_op_fini(struct rte_mempool *mp)
 	rte_wmb();
 }
 
+static int __rte_hot
+cn10k_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
+		  unsigned int n)
+{
+	const uint64_t *ptr = (const uint64_t *)obj_table;
+	uint64_t lmt_addr = 0, lmt_id = 0;
+	struct batch_op_data *op_data;
+
+	/* Ensure mbuf init changes are written before the free pointers are
+	 * enqueued to the stack.
+	 */
+	rte_io_wmb();
+
+	if (n == 1) {
+		roc_npa_aura_op_free(mp->pool_id, 1, ptr[0]);
+		return 0;
+	}
+
+	op_data = batch_op_data_get(mp->pool_id);
+	lmt_addr = op_data->lmt_addr;
+	ROC_LMT_BASE_ID_GET(lmt_addr, lmt_id);
+	roc_npa_aura_op_batch_free(mp->pool_id, ptr, n, 1, lmt_addr, lmt_id);
+
+	return 0;
+}
+
 static int
 cn10k_mempool_alloc(struct rte_mempool *mp)
 {
@@ -187,7 +213,7 @@ static struct rte_mempool_ops cn10k_mempool_ops = {
 	.name = "cn10k_mempool_ops",
 	.alloc = cn10k_mempool_alloc,
 	.free = cn10k_mempool_free,
-	.enqueue = cnxk_mempool_enq,
+	.enqueue = cn10k_mempool_enq,
 	.dequeue = cnxk_mempool_deq,
 	.get_count = cnxk_mempool_get_count,
 	.calc_mem_size = cnxk_mempool_calc_mem_size,
-- 
2.31.0

[dpdk-dev] [PATCH v4 10/11] mempool/cnxk: add cn10k get count op

[Ashwin Sekhar T K]

Add the implementation for Marvell CN10k get count op.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 28 +++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
index c225c227df..d244a5e90f 100644
--- a/drivers/mempool/cnxk/cn10k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -157,6 +157,32 @@ cn10k_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
 	return 0;
 }
 
+static unsigned int
+cn10k_mempool_get_count(const struct rte_mempool *mp)
+{
+	struct batch_op_data *op_data;
+	unsigned int count = 0;
+	int i;
+
+	op_data = batch_op_data_get(mp->pool_id);
+
+	rte_wmb();
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct batch_op_mem *mem = &op_data->mem[i];
+
+		if (mem->status == BATCH_ALLOC_OP_ISSUED)
+			count += roc_npa_aura_batch_alloc_count(mem->objs,
+								BATCH_ALLOC_SZ);
+
+		if (mem->status == BATCH_ALLOC_OP_DONE)
+			count += mem->sz;
+	}
+
+	count += cnxk_mempool_get_count(mp);
+
+	return count;
+}
+
 static int
 cn10k_mempool_alloc(struct rte_mempool *mp)
 {
@@ -215,7 +241,7 @@ static struct rte_mempool_ops cn10k_mempool_ops = {
 	.free = cn10k_mempool_free,
 	.enqueue = cn10k_mempool_enq,
 	.dequeue = cnxk_mempool_deq,
-	.get_count = cnxk_mempool_get_count,
+	.get_count = cn10k_mempool_get_count,
 	.calc_mem_size = cnxk_mempool_calc_mem_size,
 	.populate = cnxk_mempool_populate,
 };
-- 
2.31.0

[dpdk-dev] [PATCH v4 11/11] mempool/cnxk: add cn10k batch dequeue op

[Ashwin Sekhar T K]

Add the implementation for Marvell CN10k mempool batch dequeue op.

Signed-off-by: Ashwin Sekhar T K <asekhar@marvell.com>
---
 drivers/mempool/cnxk/cn10k_mempool_ops.c | 72 +++++++++++++++++++++++-
 1 file changed, 71 insertions(+), 1 deletion(-)

diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c
index d244a5e90f..95458b34b7 100644
--- a/drivers/mempool/cnxk/cn10k_mempool_ops.c
+++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c
@@ -183,6 +183,76 @@ cn10k_mempool_get_count(const struct rte_mempool *mp)
 	return count;
 }
 
+static int __rte_hot
+cn10k_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
+{
+	struct batch_op_data *op_data;
+	struct batch_op_mem *mem;
+	unsigned int count = 0;
+	int tid, rc, retry;
+	bool loop = true;
+
+	op_data = batch_op_data_get(mp->pool_id);
+	tid = rte_lcore_id();
+	mem = &op_data->mem[tid];
+
+	/* Issue batch alloc */
+	if (mem->status == BATCH_ALLOC_OP_NOT_ISSUED) {
+		rc = roc_npa_aura_batch_alloc_issue(mp->pool_id, mem->objs,
+						    BATCH_ALLOC_SZ, 0, 1);
+		/* If issue fails, try falling back to default alloc */
+		if (unlikely(rc))
+			return cn10k_mempool_enq(mp, obj_table, n);
+		mem->status = BATCH_ALLOC_OP_ISSUED;
+	}
+
+	retry = 4;
+	while (loop) {
+		unsigned int cur_sz;
+
+		if (mem->status == BATCH_ALLOC_OP_ISSUED) {
+			mem->sz = roc_npa_aura_batch_alloc_extract(
+				mem->objs, mem->objs, BATCH_ALLOC_SZ);
+
+			/* If partial alloc reduce the retry count */
+			retry -= (mem->sz != BATCH_ALLOC_SZ);
+			/* Break the loop if retry count exhausted */
+			loop = !!retry;
+			mem->status = BATCH_ALLOC_OP_DONE;
+		}
+
+		cur_sz = n - count;
+		if (cur_sz > mem->sz)
+			cur_sz = mem->sz;
+
+		/* Dequeue the pointers */
+		memcpy(&obj_table[count], &mem->objs[mem->sz - cur_sz],
+		       cur_sz * sizeof(uintptr_t));
+		mem->sz -= cur_sz;
+		count += cur_sz;
+
+		/* Break loop if the required pointers has been dequeued */
+		loop &= (count != n);
+
+		/* Issue next batch alloc if pointers are exhausted */
+		if (mem->sz == 0) {
+			rc = roc_npa_aura_batch_alloc_issue(
+				mp->pool_id, mem->objs, BATCH_ALLOC_SZ, 0, 1);
+			/* Break loop if issue failed and set status */
+			loop &= !rc;
+			mem->status = !rc;
+		}
+	}
+
+	if (unlikely(count != n)) {
+		/* No partial alloc allowed. Free up allocated pointers */
+		cn10k_mempool_enq(mp, obj_table, count);
+		return -ENOENT;
+	}
+
+	return 0;
+}
+
 static int
 cn10k_mempool_alloc(struct rte_mempool *mp)
 {
@@ -240,7 +310,7 @@ static struct rte_mempool_ops cn10k_mempool_ops = {
 	.alloc = cn10k_mempool_alloc,
 	.free = cn10k_mempool_free,
 	.enqueue = cn10k_mempool_enq,
-	.dequeue = cnxk_mempool_deq,
+	.dequeue = cn10k_mempool_deq,
 	.get_count = cn10k_mempool_get_count,
 	.calc_mem_size = cnxk_mempool_calc_mem_size,
 	.populate = cnxk_mempool_populate,
-- 
2.31.0

Re: [dpdk-dev] [PATCH v4 00/11] Add Marvell CNXK mempool driver

[Jerin Jacob]

On Thu, Apr 8, 2021 at 3:21 PM Ashwin Sekhar T K <asekhar@marvell.com> wrote:
>
> This patchset adds the mempool/cnxk driver which provides the support for the
> integrated mempool device found in Marvell CN10K SoC.
>
> The code includes mempool driver functionality for Marvell CN9K SoC as well,
> but right now it is not enabled. The future plan is to deprecate existing
> mempool/octeontx2 driver once the 'CNXK' drivers are feature complete for
> Marvell CN9K SoC.
>
> Depends-on: series-16131 ("Add Marvell CNXK common driver")

Series applied to dpdk-next-net-mrvl/for-dpdk-main. Thanks.


>
> v4:
>  - Added a bullet point for cnxk mempool driver in release_21_05.rst.
>
> Ashwin Sekhar T K (11):
>   mempool/cnxk: add build infra and doc
>   mempool/cnxk: add device probe/remove
>   mempool/cnxk: add generic ops
>   mempool/cnxk: register plt init callback
>   mempool/cnxk: add cn9k mempool ops
>   mempool/cnxk: add cn9k optimized mempool enqueue/dequeue
>   mempool/cnxk: add cn10k mempool ops
>   mempool/cnxk: add batch op init
>   mempool/cnxk: add cn10k batch enqueue op
>   mempool/cnxk: add cn10k get count op
>   mempool/cnxk: add cn10k batch dequeue op
>
>  MAINTAINERS                              |   6 +
>  doc/guides/mempool/cnxk.rst              |  91 +++++++
>  doc/guides/mempool/index.rst             |   1 +
>  doc/guides/platform/cnxk.rst             |   3 +
>  doc/guides/rel_notes/release_21_05.rst   |   2 +
>  drivers/mempool/cnxk/cn10k_mempool_ops.c | 319 +++++++++++++++++++++++
>  drivers/mempool/cnxk/cn9k_mempool_ops.c  |  89 +++++++
>  drivers/mempool/cnxk/cnxk_mempool.c      | 202 ++++++++++++++
>  drivers/mempool/cnxk/cnxk_mempool.h      |  28 ++
>  drivers/mempool/cnxk/cnxk_mempool_ops.c  | 191 ++++++++++++++
>  drivers/mempool/cnxk/meson.build         |  16 ++
>  drivers/mempool/cnxk/version.map         |   3 +
>  drivers/mempool/meson.build              |   3 +-
>  13 files changed, 953 insertions(+), 1 deletion(-)
>  create mode 100644 doc/guides/mempool/cnxk.rst
>  create mode 100644 drivers/mempool/cnxk/cn10k_mempool_ops.c
>  create mode 100644 drivers/mempool/cnxk/cn9k_mempool_ops.c
>  create mode 100644 drivers/mempool/cnxk/cnxk_mempool.c
>  create mode 100644 drivers/mempool/cnxk/cnxk_mempool.h
>  create mode 100644 drivers/mempool/cnxk/cnxk_mempool_ops.c
>  create mode 100644 drivers/mempool/cnxk/meson.build
>  create mode 100644 drivers/mempool/cnxk/version.map
>
> --
> 2.31.0
>