Re: [dpdk-dev] [PATCH] gpudev: introduce memory API

[Honnappa Nagarahalli <Honnappa.Nagarahalli@arm.com>]

<snip>

> 
> From: Elena Agostini <eagostini@nvidia.com>
> 
> The new library gpudev is for dealing with GPU from a DPDK application in a
> vendor-agnostic way.
It would be good to explain how the application using GPU+DPDK would look like.

Which parts of the workload need DPDK's support?

Any requirements on co-existence of GPU with other accelerators?

> 
> As a first step, the features are focused on memory management.
> A function allows to allocate memory inside the GPU, while another one
> allows to use main (CPU) memory from the GPU.
Is this memory for packet buffers or something else?

> 
> The infrastructure is prepared to welcome drivers in drivers/gpu/ as the
> upcoming NVIDIA one, implementing the gpudev API.
> Other additions planned for next revisions:
>   - C implementation file
>   - guide documentation
>   - unit tests
>   - integration in testpmd to enable Rx/Tx to/from GPU memory.
> 
> The next step should focus on GPU processing task control.
> 
> Signed-off-by: Elena Agostini <eagostini@nvidia.com>
> Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
> ---
>  .gitignore                           |   1 +
>  MAINTAINERS                          |   6 +
>  doc/api/doxy-api-index.md            |   1 +
>  doc/api/doxy-api.conf.in             |   1 +
>  doc/guides/conf.py                   |   8 ++
>  doc/guides/gpus/features/default.ini |  13 ++
>  doc/guides/gpus/index.rst            |  11 ++
>  doc/guides/gpus/overview.rst         |   7 +
>  doc/guides/index.rst                 |   1 +
>  doc/guides/prog_guide/gpu.rst        |   5 +
>  doc/guides/prog_guide/index.rst      |   1 +
>  drivers/gpu/meson.build              |   4 +
>  drivers/meson.build                  |   1 +
>  lib/gpudev/gpu_driver.h              |  44 +++++++
>  lib/gpudev/meson.build               |   9 ++
>  lib/gpudev/rte_gpudev.h              | 183 +++++++++++++++++++++++++++
>  lib/gpudev/version.map               |  11 ++
>  lib/meson.build                      |   1 +
>  18 files changed, 308 insertions(+)
>  create mode 100644 doc/guides/gpus/features/default.ini
>  create mode 100644 doc/guides/gpus/index.rst  create mode 100644
> doc/guides/gpus/overview.rst  create mode 100644
> doc/guides/prog_guide/gpu.rst  create mode 100644
> drivers/gpu/meson.build  create mode 100644 lib/gpudev/gpu_driver.h
> create mode 100644 lib/gpudev/meson.build  create mode 100644
> lib/gpudev/rte_gpudev.h  create mode 100644 lib/gpudev/version.map
> 
> diff --git a/.gitignore b/.gitignore
> index b19c0717e6..49494e0c6c 100644
> --- a/.gitignore
> +++ b/.gitignore
> @@ -14,6 +14,7 @@ doc/guides/compressdevs/overview_feature_table.txt
>  doc/guides/regexdevs/overview_feature_table.txt
>  doc/guides/vdpadevs/overview_feature_table.txt
>  doc/guides/bbdevs/overview_feature_table.txt
> +doc/guides/gpus/overview_feature_table.txt
> 
>  # ignore generated ctags/cscope files
>  cscope.out.po
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 5877a16971..c4755dfe9a 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -452,6 +452,12 @@ F: app/test-regex/
>  F: doc/guides/prog_guide/regexdev.rst
>  F: doc/guides/regexdevs/features/default.ini
> 
> +GPU API - EXPERIMENTAL
> +M: Elena Agostini <eagostini@nvidia.com>
> +F: lib/gpudev/
> +F: doc/guides/prog_guide/gpu.rst
> +F: doc/guides/gpus/features/default.ini
> +
>  Eventdev API
>  M: Jerin Jacob <jerinj@marvell.com>
>  T: git://dpdk.org/next/dpdk-next-eventdev
> diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md index
> 1992107a03..bd10342ca2 100644
> --- a/doc/api/doxy-api-index.md
> +++ b/doc/api/doxy-api-index.md
> @@ -21,6 +21,7 @@ The public API headers are grouped by topics:
>    [compressdev]        (@ref rte_compressdev.h),
>    [compress]           (@ref rte_comp.h),
>    [regexdev]           (@ref rte_regexdev.h),
> +  [gpudev]             (@ref rte_gpudev.h),
>    [eventdev]           (@ref rte_eventdev.h),
>    [event_eth_rx_adapter]   (@ref rte_event_eth_rx_adapter.h),
>    [event_eth_tx_adapter]   (@ref rte_event_eth_tx_adapter.h),
> diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in index
> 325a0195c6..831b9a6b33 100644
> --- a/doc/api/doxy-api.conf.in
> +++ b/doc/api/doxy-api.conf.in
> @@ -40,6 +40,7 @@ INPUT                   = @TOPDIR@/doc/api/doxy-api-
> index.md \
>                            @TOPDIR@/lib/eventdev \
>                            @TOPDIR@/lib/fib \
>                            @TOPDIR@/lib/flow_classify \
> +                          @TOPDIR@/lib/gpudev \
>                            @TOPDIR@/lib/graph \
>                            @TOPDIR@/lib/gro \
>                            @TOPDIR@/lib/gso \ diff --git a/doc/guides/conf.py
> b/doc/guides/conf.py index 67d2dd62c7..7930da9ceb 100644
> --- a/doc/guides/conf.py
> +++ b/doc/guides/conf.py
> @@ -152,6 +152,9 @@ def generate_overview_table(output_filename,
> table_id, section, table_name, titl
>          name = ini_filename[:-4]
>          name = name.replace('_vf', 'vf')
>          pmd_names.append(name)
> +    if not pmd_names:
> +        # Add an empty column if table is empty (required by RST syntax)
> +        pmd_names.append(' ')
> 
>      # Pad the table header names.
>      max_header_len = len(max(pmd_names, key=len)) @@ -388,6 +391,11
> @@ def setup(app):
>                              'Features',
>                              'Features availability in bbdev drivers',
>                              'Feature')
> +    table_file = dirname(__file__) + '/gpus/overview_feature_table.txt'
> +    generate_overview_table(table_file, 1,
> +                            'Features',
> +                            'Features availability in GPU drivers',
> +                            'Feature')
> 
>      if LooseVersion(sphinx_version) < LooseVersion('1.3.1'):
>          print('Upgrade sphinx to version >= 1.3.1 for '
> diff --git a/doc/guides/gpus/features/default.ini
> b/doc/guides/gpus/features/default.ini
> new file mode 100644
> index 0000000000..c363447b0d
> --- /dev/null
> +++ b/doc/guides/gpus/features/default.ini
> @@ -0,0 +1,13 @@
> +;
> +; Features of a GPU driver.
> +;
> +; This file defines the features that are valid for inclusion in ; the
> +other driver files and also the order that they appear in ; the
> +features table in the documentation. The feature description ; string
> +should not exceed feature_str_len defined in conf.py.
> +;
> +[Features]
> +Get device info                =
> +Share CPU memory with GPU      =
> +Allocate GPU memory            =
> +Free memory                    =
> diff --git a/doc/guides/gpus/index.rst b/doc/guides/gpus/index.rst new file
> mode 100644 index 0000000000..f9c62aeb36
> --- /dev/null
> +++ b/doc/guides/gpus/index.rst
> @@ -0,0 +1,11 @@
> +.. SPDX-License-Identifier: BSD-3-Clause
> +   Copyright 2021 NVIDIA Corporation & Affiliates
> +
> +GPU Drivers
> +===========
> +
> +.. toctree::
> +   :maxdepth: 2
> +   :numbered:
> +
> +   overview
> diff --git a/doc/guides/gpus/overview.rst b/doc/guides/gpus/overview.rst
> new file mode 100644 index 0000000000..e7f985e98b
> --- /dev/null
> +++ b/doc/guides/gpus/overview.rst
> @@ -0,0 +1,7 @@
> +.. SPDX-License-Identifier: BSD-3-Clause
> +   Copyright 2021 NVIDIA Corporation & Affiliates
> +
> +Overview of GPU Drivers
> +=======================
> +
> +.. include:: overview_feature_table.txt
> diff --git a/doc/guides/index.rst b/doc/guides/index.rst index
> 857f0363d3..ee4d79a4eb 100644
> --- a/doc/guides/index.rst
> +++ b/doc/guides/index.rst
> @@ -21,6 +21,7 @@ DPDK documentation
>     compressdevs/index
>     vdpadevs/index
>     regexdevs/index
> +   gpus/index
>     eventdevs/index
>     rawdevs/index
>     mempool/index
> diff --git a/doc/guides/prog_guide/gpu.rst b/doc/guides/prog_guide/gpu.rst
> new file mode 100644 index 0000000000..54f9fa8300
> --- /dev/null
> +++ b/doc/guides/prog_guide/gpu.rst
> @@ -0,0 +1,5 @@
> +.. SPDX-License-Identifier: BSD-3-Clause
> +   Copyright 2021 NVIDIA Corporation & Affiliates
> +
> +GPU Library
> +===========
> diff --git a/doc/guides/prog_guide/index.rst
> b/doc/guides/prog_guide/index.rst index 2dce507f46..dfddf90b51 100644
> --- a/doc/guides/prog_guide/index.rst
> +++ b/doc/guides/prog_guide/index.rst
> @@ -27,6 +27,7 @@ Programmer's Guide
>      cryptodev_lib
>      compressdev
>      regexdev
> +    gpu
>      rte_security
>      rawdev
>      link_bonding_poll_mode_drv_lib
> diff --git a/drivers/gpu/meson.build b/drivers/gpu/meson.build new file mode
> 100644 index 0000000000..5189950616
> --- /dev/null
> +++ b/drivers/gpu/meson.build
> @@ -0,0 +1,4 @@
> +# SPDX-License-Identifier: BSD-3-Clause # Copyright 2021 NVIDIA
> +Corporation & Affiliates
> +
> +drivers = []
> diff --git a/drivers/meson.build b/drivers/meson.build index
> bc6f4f567f..f607040d79 100644
> --- a/drivers/meson.build
> +++ b/drivers/meson.build
> @@ -18,6 +18,7 @@ subdirs = [
>          'vdpa',           # depends on common, bus and mempool.
>          'event',          # depends on common, bus, mempool and net.
>          'baseband',       # depends on common and bus.
> +        'gpu',            # depends on common and bus.
>  ]
> 
>  if meson.is_cross_build()
> diff --git a/lib/gpudev/gpu_driver.h b/lib/gpudev/gpu_driver.h new file mode
> 100644 index 0000000000..5ff609e49d
> --- /dev/null
> +++ b/lib/gpudev/gpu_driver.h
> @@ -0,0 +1,44 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 NVIDIA Corporation & Affiliates  */
> +
> +#ifndef GPU_DRIVER_H
> +#define GPU_DRIVER_H
> +
> +#include <stdint.h>
> +
> +#include <rte_common.h>
> +
> +#include "rte_gpudev.h"
> +
> +struct rte_gpu_dev;
> +
> +typedef int (*gpu_malloc_t)(struct rte_gpu_dev *dev, size_t size, void
> +**ptr); typedef int (*gpu_free_t)(struct rte_gpu_dev *dev, void *ptr);
> +
> +struct rte_gpu_dev {
> +	/* Backing device. */
> +	struct rte_device *device;
> +	/* GPU info structure. */
> +	struct rte_gpu_info info;
> +	/* Counter of processes using the device. */
> +	uint16_t process_cnt;
> +	/* If device is currently used or not. */
> +	enum rte_gpu_state state;
> +	/* FUNCTION: Allocate memory on the GPU. */
> +	gpu_malloc_t gpu_malloc;
> +	/* FUNCTION: Allocate memory on the CPU visible from the GPU. */
> +	gpu_malloc_t gpu_malloc_visible;
> +	/* FUNCTION: Free allocated memory on the GPU. */
> +	gpu_free_t gpu_free;
> +	/* Device interrupt handle. */
> +	struct rte_intr_handle *intr_handle;
> +	/* Driver-specific private data. */
> +	void *dev_private;
> +} __rte_cache_aligned;
> +
> +struct rte_gpu_dev *rte_gpu_dev_allocate(const char *name); struct
> +rte_gpu_dev *rte_gpu_dev_get_by_name(const char *name); int
> +rte_gpu_dev_release(struct rte_gpu_dev *gpudev);
> +
> +#endif /* GPU_DRIVER_H */
> diff --git a/lib/gpudev/meson.build b/lib/gpudev/meson.build new file mode
> 100644 index 0000000000..f05459e18d
> --- /dev/null
> +++ b/lib/gpudev/meson.build
> @@ -0,0 +1,9 @@
> +# SPDX-License-Identifier: BSD-3-Clause # Copyright 2021 NVIDIA
> +Corporation & Affiliates
> +
> +headers = files(
> +        'rte_gpudev.h',
> +)
> +
> +sources = files(
> +)
> diff --git a/lib/gpudev/rte_gpudev.h b/lib/gpudev/rte_gpudev.h new file mode
> 100644 index 0000000000..b12f35c17e
> --- /dev/null
> +++ b/lib/gpudev/rte_gpudev.h
> @@ -0,0 +1,183 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 NVIDIA Corporation & Affiliates  */
> +
> +#ifndef RTE_GPUDEV_H
> +#define RTE_GPUDEV_H
> +
> +#include <stdint.h>
> +#include <stdbool.h>
> +
> +#include <rte_common.h>
> +
> +/**
> + * @file
> + * Generic library to interact with a GPU.
> + *
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + */
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +/** Maximum number of GPU engines. */
> +#define RTE_GPU_MAX_DEVS UINT16_C(32)
> +/** Maximum length of device name. */
> +#define RTE_GPU_NAME_MAX_LEN 128
> +
> +/** Flags indicate current state of GPU device. */ enum rte_gpu_state {
> +	RTE_GPU_STATE_UNUSED,        /**< not initialized */
> +	RTE_GPU_STATE_INITIALIZED,   /**< initialized */
> +};
> +
> +/** Store a list of info for a given GPU. */ struct rte_gpu_info {
> +	/** GPU device ID. */
> +	uint16_t gpu_id;
> +	/** Unique identifier name. */
> +	char name[RTE_GPU_NAME_MAX_LEN];
> +	/** Total memory available on device. */
> +	size_t total_memory;
> +	/** Total processors available on device. */
> +	int processor_count;
> +};
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Returns the number of GPUs detected and associated to DPDK.
> + *
> + * @return
> + *   The number of available GPUs.
> + */
> +__rte_experimental
> +uint16_t rte_gpu_dev_count_avail(void);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Check if the device is valid and initialized in DPDK.
> + *
> + * @param gpu_id
> + *   The input GPU ID.
> + *
> + * @return
> + *   - True if gpu_id is a valid and initialized GPU.
> + *   - False otherwise.
> + */
> +__rte_experimental
> +bool rte_gpu_dev_is_valid(uint16_t gpu_id);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Get the GPU ID of the next valid GPU initialized in DPDK.
> + *
> + * @param gpu_id
> + *   The initial GPU ID to start the research.
> + *
> + * @return
> + *   Next GPU ID corresponding to a valid and initialized GPU device.
> + */
> +__rte_experimental
> +uint16_t rte_gpu_dev_find_next(uint16_t gpu_id);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Macro to iterate over all valid GPUs.
> + *
> + * @param gpu_id
> + *   The ID of the next possible valid GPU.
> + * @return
> + *   Next valid GPU ID, RTE_GPU_MAX_DEVS if there is none.
> + */
> +#define RTE_GPU_FOREACH_DEV(gpu_id) \
> +	for (gpu_id = rte_gpu_find_next(0); \
> +	     gpu_id < RTE_GPU_MAX_DEVS; \
> +	     gpu_id = rte_gpu_find_next(gpu_id + 1))
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Return GPU specific info.
> + *
> + * @param gpu_id
> + *   GPU ID to get info.
> + * @param info
> + *   Memory structure to fill with the info.
> + *
> + * @return
> + *   0 on success, -1 otherwise.
> + */
> +__rte_experimental
> +int rte_gpu_dev_info_get(uint16_t gpu_id, struct rte_gpu_info **info);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Allocate a chunk of memory on the GPU.
> + *
> + * @param gpu_id
> + *   GPU ID to allocate memory.
> + * @param size
> + *   Number of bytes to allocate.
> + * @param ptr
> + *   Pointer to store the address of the allocated memory.
> + *
> + * @return
> + *   0 on success, -1 otherwise.
> + */
> +__rte_experimental
> +int rte_gpu_malloc(uint16_t gpu_id, size_t size, void **ptr);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Allocate a chunk of memory on the CPU that is visible from the GPU.
> + *
> + * @param gpu_id
> + *   Reference GPU ID.
> + * @param size
> + *   Number of bytes to allocate.
> + * @param ptr
> + *   Pointer to store the address of the allocated memory.
> + *
> + * @return
> + *   0 on success, -1 otherwise.
> + */
> +__rte_experimental
> +int rte_gpu_malloc_visible(uint16_t gpu_id, size_t size, void **ptr);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Deallocate a chunk of memory allocated with rte_gpu_malloc*.
> + *
> + * @param gpu_id
> + *   Reference GPU ID.
> + * @param ptr
> + *   Pointer to the memory area to be deallocated.
> + *
> + * @return
> + *   0 on success, -1 otherwise.
> + */
> +__rte_experimental
> +int rte_gpu_free(uint16_t gpu_id, void *ptr);
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif /* RTE_GPUDEV_H */
> diff --git a/lib/gpudev/version.map b/lib/gpudev/version.map new file mode
> 100644 index 0000000000..9e0f218e8b
> --- /dev/null
> +++ b/lib/gpudev/version.map
> @@ -0,0 +1,11 @@
> +EXPERIMENTAL {
> +	global:
> +
> +	rte_gpu_dev_count_avail;
> +	rte_gpu_dev_find_next;
> +	rte_gpu_dev_info_get;
> +	rte_gpu_dev_is_valid;
> +	rte_gpu_free;
> +	rte_gpu_malloc;
> +	rte_gpu_malloc_visible;
> +};
> diff --git a/lib/meson.build b/lib/meson.build index 4a64756a68..ffefc64c69
> 100644
> --- a/lib/meson.build
> +++ b/lib/meson.build
> @@ -33,6 +33,7 @@ libraries = [
>          'distributor',
>          'efd',
>          'eventdev',
> +        'gpudev',
>          'gro',
>          'gso',
>          'ip_frag',
> --
> 2.31.1