Re: [dpdk-dev] [PATCH v3 1/8] raw/ioat: add initial support for ioat rawdev driver

["Hu, Jiayu" <jiayu.hu@intel.com>]

Sorry for the incorrect format in the last mail.
The comment is repeated below.

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Bruce Richardson
> Sent: Thursday, June 27, 2019 6:41 PM
> To: dev@dpdk.org
> Cc: thomas@monjalon.net; jerinj@marvell.com; Richardson, Bruce
> <bruce.richardson@intel.com>
> Subject: [dpdk-dev] [PATCH v3 1/8] raw/ioat: add initial support for ioat
> rawdev driver
> 
> Add stubs for ioat rawdev driver support in DPDK, specifically:
> 
>   * makefile and meson build hooks
>   * initial public header file
>   * rawdev main C file, with probe and release functions
>   * release note update announcing the driver
>   * initial documentation for the new section in the rawdev doc
>   * unit test stubs for device unit tests
> 
> Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
> 
> ---
> V2: don't create a new file for unit testing, add to existing rawdev test
>     file, and place test cases in the driver selftest routine (added later
>     in set)
>     Add new section in document about identifying hardware using lspci
> ---
>  MAINTAINERS                                 |  6 +-
>  app/test/test_rawdev.c                      |  8 ++
>  config/common_armv8a_linux                  |  1 +
>  config/common_base                          |  5 ++
>  config/defconfig_arm-armv7a-linuxapp-gcc    |  1 +
>  config/defconfig_ppc_64-power8-linuxapp-gcc |  1 +
>  doc/guides/rawdevs/index.rst                |  1 +
>  doc/guides/rawdevs/ioat_rawdev.rst          | 63 ++++++++++++++
>  doc/guides/rel_notes/release_19_08.rst      | 11 +++
>  drivers/raw/Makefile                        |  1 +
>  drivers/raw/ioat/Makefile                   | 28 +++++++
>  drivers/raw/ioat/ioat_rawdev.c              | 93 +++++++++++++++++++++
>  drivers/raw/ioat/meson.build                |  8 ++
>  drivers/raw/ioat/rte_ioat_rawdev.h          | 24 ++++++
>  drivers/raw/ioat/rte_pmd_ioat_version.map   |  4 +
>  drivers/raw/meson.build                     |  4 +-
>  mk/rte.app.mk                               |  1 +
>  17 files changed, 258 insertions(+), 2 deletions(-)
>  create mode 100644 doc/guides/rawdevs/ioat_rawdev.rst
>  create mode 100644 drivers/raw/ioat/Makefile
>  create mode 100644 drivers/raw/ioat/ioat_rawdev.c
>  create mode 100644 drivers/raw/ioat/meson.build
>  create mode 100644 drivers/raw/ioat/rte_ioat_rawdev.h
>  create mode 100644 drivers/raw/ioat/rte_pmd_ioat_version.map
> 
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 0c3b48920..f28c526bc 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -1068,6 +1068,11 @@ M: Tianfei zhang <tianfei.zhang@intel.com>
>  F: drivers/raw/ifpga_rawdev/
>  F: doc/guides/rawdevs/ifpga_rawdev.rst
> 
> +IOAT Rawdev
> +M: Bruce Richardson <bruce.richardson@intel.com>
> +F: drivers/raw/ioat/
> +F: doc/guides/rawdevs/ioat_rawdev.rst
> +
>  NXP DPAA2 QDMA
>  M: Nipun Gupta <nipun.gupta@nxp.com>
>  F: drivers/raw/dpaa2_qdma/
> @@ -1078,7 +1083,6 @@ M: Nipun Gupta <nipun.gupta@nxp.com>
>  F: drivers/raw/dpaa2_cmdif/
>  F: doc/guides/rawdevs/dpaa2_cmdif.rst
> 
> -
>  Packet processing
>  -----------------
> 
> diff --git a/app/test/test_rawdev.c b/app/test/test_rawdev.c
> index 043a38a13..88549fb61 100644
> --- a/app/test/test_rawdev.c
> +++ b/app/test/test_rawdev.c
> @@ -25,3 +25,11 @@ test_rawdev_selftest_skeleton(void)
>  }
> 
>  REGISTER_TEST_COMMAND(rawdev_autotest,
> test_rawdev_selftest_skeleton);
> +
> +static int
> +test_rawdev_selftest_ioat(void)
> +{
> +	return TEST_SKIPPED;
> +}
> +
> +REGISTER_TEST_COMMAND(ioat_rawdev_autotest,
> test_rawdev_selftest_ioat);
> diff --git a/config/common_armv8a_linux b/config/common_armv8a_linux
> index 72091de1c..481712ebc 100644
> --- a/config/common_armv8a_linux
> +++ b/config/common_armv8a_linux
> @@ -34,5 +34,6 @@ CONFIG_RTE_ARCH_ARM64_MEMCPY=n
>  CONFIG_RTE_LIBRTE_FM10K_PMD=n
>  CONFIG_RTE_LIBRTE_SFC_EFX_PMD=n
>  CONFIG_RTE_LIBRTE_AVP_PMD=n
> +CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV=n
> 
>  CONFIG_RTE_SCHED_VECTOR=n
> diff --git a/config/common_base b/config/common_base
> index fa1ae249a..e6b830923 100644
> --- a/config/common_base
> +++ b/config/common_base
> @@ -747,6 +747,11 @@
> CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
>  #
>  CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
> 
> +#
> +# Compile PMD for Intel IOAT raw device
> +#
> +CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV=y
> +
>  #
>  # Compile librte_ring
>  #
> diff --git a/config/defconfig_arm-armv7a-linuxapp-gcc
> b/config/defconfig_arm-armv7a-linuxapp-gcc
> index c9509b274..ee158ef9d 100644
> --- a/config/defconfig_arm-armv7a-linuxapp-gcc
> +++ b/config/defconfig_arm-armv7a-linuxapp-gcc
> @@ -54,3 +54,4 @@ CONFIG_RTE_LIBRTE_QEDE_PMD=n
>  CONFIG_RTE_LIBRTE_SFC_EFX_PMD=n
>  CONFIG_RTE_LIBRTE_AVP_PMD=n
>  CONFIG_RTE_LIBRTE_NFP_PMD=n
> +CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV=n
> diff --git a/config/defconfig_ppc_64-power8-linuxapp-gcc
> b/config/defconfig_ppc_64-power8-linuxapp-gcc
> index 7e248b755..9f3670ec0 100644
> --- a/config/defconfig_ppc_64-power8-linuxapp-gcc
> +++ b/config/defconfig_ppc_64-power8-linuxapp-gcc
> @@ -56,3 +56,4 @@ CONFIG_RTE_LIBRTE_ENIC_PMD=n
>  CONFIG_RTE_LIBRTE_FM10K_PMD=n
>  CONFIG_RTE_LIBRTE_SFC_EFX_PMD=n
>  CONFIG_RTE_LIBRTE_AVP_PMD=n
> +CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV=n
> diff --git a/doc/guides/rawdevs/index.rst b/doc/guides/rawdevs/index.rst
> index 7c3bd9586..0a21989e4 100644
> --- a/doc/guides/rawdevs/index.rst
> +++ b/doc/guides/rawdevs/index.rst
> @@ -14,3 +14,4 @@ application through rawdev API.
>      dpaa2_cmdif
>      dpaa2_qdma
>      ifpga_rawdev
> +    ioat_rawdev
> diff --git a/doc/guides/rawdevs/ioat_rawdev.rst
> b/doc/guides/rawdevs/ioat_rawdev.rst
> new file mode 100644
> index 000000000..0c612e73a
> --- /dev/null
> +++ b/doc/guides/rawdevs/ioat_rawdev.rst
> @@ -0,0 +1,63 @@
> +..  SPDX-License-Identifier: BSD-3-Clause
> +    Copyright(c) 2019 Intel Corporation.
> +
> +.. include:: <isonum.txt>
> +
> +IOAT Rawdev Driver for Intel\ |reg| QuickData Technology
> +===============================================================
> =======
> +
> +The ``ioat`` rawdev driver provides a poll-mode driver (PMD) for Intel\ |reg|
> +QuickData Technology, part of Intel\ |reg| I/O Acceleration Technology
> +`(Intel I/OAT)
> +<https://www.intel.com/content/www/us/en/wireless-network/accel-
> technology.html>`_.
> +This PMD, when used on supported hardware, allows data copies, for
> example,
> +cloning packet data, to be accelerated by that hardware rather than having
> to
> +be done by software, freeing up CPU cycles for other tasks.
> +
> +Hardware Requirements
> +----------------------
> +
> +On Linux, the presence of an Intel\ |reg| QuickData Technology hardware
> can
> +be detected by checking the output of the ``lspci`` command, where the
> +hardware will be often listed as "Crystal Beach DMA" or "CBDMA". For
> +example, on a system with Intel\ |reg| Xeon\ |reg| CPU E5-2699 v4
> @2.20GHz,
> +lspci shows:
> +
> +.. code-block:: console
> +
> +  # lspci | grep DMA
> +  00:04.0 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 v4/Xeon
> E3 v4/Xeon D Crystal Beach DMA Channel 0 (rev 01)
> +  00:04.1 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 v4/Xeon
> E3 v4/Xeon D Crystal Beach DMA Channel 1 (rev 01)
> +  00:04.2 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 v4/Xeon
> E3 v4/Xeon D Crystal Beach DMA Channel 2 (rev 01)
> +  00:04.3 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 v4/Xeon
> E3 v4/Xeon D Crystal Beach DMA Channel 3 (rev 01)
> +  00:04.4 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 v4/Xeon
> E3 v4/Xeon D Crystal Beach DMA Channel 4 (rev 01)
> +  00:04.5 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 v4/Xeon
> E3 v4/Xeon D Crystal Beach DMA Channel 5 (rev 01)
> +  00:04.6 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 v4/Xeon
> E3 v4/Xeon D Crystal Beach DMA Channel 6 (rev 01)
> +  00:04.7 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 v4/Xeon
> E3 v4/Xeon D Crystal Beach DMA Channel 7 (rev 01)
> +
> +On a system with Intel\ |reg| Xeon\ |reg| Gold 6154 CPU @ 3.00GHz, lspci
> +shows:
> +
> +.. code-block:: console
> +
> +  # lspci | grep DMA
> +  00:04.0 System peripheral: Intel Corporation Sky Lake-E CBDMA Registers
> (rev 04)
> +  00:04.1 System peripheral: Intel Corporation Sky Lake-E CBDMA Registers
> (rev 04)
> +  00:04.2 System peripheral: Intel Corporation Sky Lake-E CBDMA Registers
> (rev 04)
> +  00:04.3 System peripheral: Intel Corporation Sky Lake-E CBDMA Registers
> (rev 04)
> +  00:04.4 System peripheral: Intel Corporation Sky Lake-E CBDMA Registers
> (rev 04)
> +  00:04.5 System peripheral: Intel Corporation Sky Lake-E CBDMA Registers
> (rev 04)
> +  00:04.6 System peripheral: Intel Corporation Sky Lake-E CBDMA Registers
> (rev 04)
> +  00:04.7 System peripheral: Intel Corporation Sky Lake-E CBDMA Registers
> (rev 04)
> +
> +
> +Compilation
> +------------
> +
> +For builds done with ``make``, the driver compilation is enabled by the
> +``CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV`` build configuration option.
> This is
> +enabled by default in builds for x86 platforms, and disabled in other
> +configurations.
> +
> +For builds using ``meson`` and ``ninja``, the driver will be built when the
> +target platform is x86-based.
> diff --git a/doc/guides/rel_notes/release_19_08.rst
> b/doc/guides/rel_notes/release_19_08.rst
> index 3da266705..720c13f8b 100644
> --- a/doc/guides/rel_notes/release_19_08.rst
> +++ b/doc/guides/rel_notes/release_19_08.rst
> @@ -1,6 +1,8 @@
>  ..  SPDX-License-Identifier: BSD-3-Clause
>      Copyright 2019 The DPDK contributors
> 
> +.. include:: <isonum.txt>
> +
>  DPDK Release 19.08
>  ==================
> 
> @@ -99,6 +101,15 @@ New Features
>    Updated ``librte_telemetry`` to fetch the global metrics from the
>    ``librte_metrics`` library.
> 
> +* **Added Intel QuickData Technology PMD**
> +
> +  The PMD for Intel\ |reg|  QuickData Technology, part of
> +  Intel\ |reg|  I/O Acceleration Technology `(Intel I/OAT)
> +  <https://www.intel.com/content/www/us/en/wireless-network/accel-
> technology.html>`_,
> +  allows data copies to be done by hardware instead
> +  of via software, reducing cycles spent copying large blocks of data in
> +  applications.
> +
> 
>  Removed Items
>  -------------
> diff --git a/drivers/raw/Makefile b/drivers/raw/Makefile
> index 8e29b4a56..c1b85c8c7 100644
> --- a/drivers/raw/Makefile
> +++ b/drivers/raw/Makefile
> @@ -10,5 +10,6 @@ DIRS-
> $(CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV) += dpaa2_cmdif
>  DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) +=
> dpaa2_qdma
>  endif
>  DIRS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += ifpga_rawdev
> +DIRS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV) += ioat
> 
>  include $(RTE_SDK)/mk/rte.subdir.mk
> diff --git a/drivers/raw/ioat/Makefile b/drivers/raw/ioat/Makefile
> new file mode 100644
> index 000000000..7726e310a
> --- /dev/null
> +++ b/drivers/raw/ioat/Makefile
> @@ -0,0 +1,28 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2019 Intel Corporation
> +
> +include $(RTE_SDK)/mk/rte.vars.mk
> +
> +# library name
> +LIB = librte_pmd_ioat_rawdev.a
> +
> +# build flags
> +CFLAGS += -O3
> +CFLAGS += $(WERROR_FLAGS)
> +
> +LDLIBS += -lrte_eal -lrte_rawdev
> +LDLIBS += -lrte_pci -lrte_bus_pci
> +
> +# library version
> +LIBABIVER := 1
> +
> +# versioning export map
> +EXPORT_MAP := rte_pmd_ioat_version.map
> +
> +# library source files
> +SRCS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV) += ioat_rawdev.c
> +
> +# export include files
> +SYMLINK-y-include += rte_ioat_rawdev.h
> +
> +include $(RTE_SDK)/mk/rte.lib.mk
> diff --git a/drivers/raw/ioat/ioat_rawdev.c b/drivers/raw/ioat/ioat_rawdev.c
> new file mode 100644
> index 000000000..d9fc3091a
> --- /dev/null
> +++ b/drivers/raw/ioat/ioat_rawdev.c
> @@ -0,0 +1,93 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2019 Intel Corporation
> + */
> +
> +#include <rte_bus_pci.h>
> +#include <rte_rawdev_pmd.h>
> +
> +#include "rte_ioat_rawdev.h"
> +
> +/* Dynamic log type identifier */
> +int ioat_pmd_logtype;
> +
> +static struct rte_pci_driver ioat_pmd_drv;
> +
> +#define IOAT_VENDOR_ID		0x8086
> +#define IOAT_DEVICE_ID		0x2021

In the second patch, the script dpdk-devbind.py supports CBDMAs in
both Broadwell and Skylake. But you only supports Skylake here.

Thanks,
Jiayu
> +
> +#define IOAT_PMD_LOG(level, fmt, args...) rte_log(RTE_LOG_ ## level, \
> +	ioat_pmd_logtype, "%s(): " fmt "\n", __func__, ##args)
> +
> +#define IOAT_PMD_DEBUG(fmt, args...)  IOAT_PMD_LOG(DEBUG, fmt, ##
> args)
> +#define IOAT_PMD_INFO(fmt, args...)   IOAT_PMD_LOG(INFO, fmt, ## args)
> +#define IOAT_PMD_ERR(fmt, args...)    IOAT_PMD_LOG(ERR, fmt, ## args)
> +#define IOAT_PMD_WARN(fmt, args...)   IOAT_PMD_LOG(WARNING, fmt,
> ## args)
> +
> +static int
> +ioat_rawdev_create(const char *name, struct rte_pci_device *dev)
> +{
> +	RTE_SET_USED(name);
> +	RTE_SET_USED(dev);
> +	return 0;
> +}
> +
> +static int
> +ioat_rawdev_destroy(const char *name)
> +{
> +	RTE_SET_USED(name);
> +	return 0;
> +}
> +
> +static int
> +ioat_rawdev_probe(struct rte_pci_driver *drv, struct rte_pci_device *dev)
> +{
> +	char name[32];
> +	int ret = 0;
> +
> +
> +	rte_pci_device_name(&dev->addr, name, sizeof(name));
> +	IOAT_PMD_INFO("Init %s on NUMA node %d", name, dev-
> >device.numa_node);
> +
> +	dev->device.driver = &drv->driver;
> +	ret = ioat_rawdev_create(name, dev);
> +	return ret;
> +}
> +
> +static int
> +ioat_rawdev_remove(struct rte_pci_device *dev)
> +{
> +	char name[32];
> +	int ret;
> +
> +	rte_pci_device_name(&dev->addr, name, sizeof(name));
> +
> +	IOAT_PMD_INFO("Closing %s on NUMA node %d",
> +			name, dev->device.numa_node);
> +
> +	ret = ioat_rawdev_destroy(name);
> +	return ret;
> +}
> +
> +static const struct rte_pci_id pci_id_ioat_map[] = {
> +	{ RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID) },
> +	{ .vendor_id = 0, /* sentinel */ },
> +};
> +
> +static struct rte_pci_driver ioat_pmd_drv = {
> +	.id_table = pci_id_ioat_map,
> +	.drv_flags = RTE_PCI_DRV_NEED_MAPPING |
> RTE_PCI_DRV_INTR_LSC |
> +		     RTE_PCI_DRV_IOVA_AS_VA,
> +	.probe = ioat_rawdev_probe,
> +	.remove = ioat_rawdev_remove,
> +};
> +
> +RTE_PMD_REGISTER_PCI(IOAT_PMD_RAWDEV_NAME, ioat_pmd_drv);
> +RTE_PMD_REGISTER_PCI_TABLE(IOAT_PMD_RAWDEV_NAME,
> pci_id_ioat_map);
> +RTE_PMD_REGISTER_KMOD_DEP(IOAT_PMD_RAWDEV_NAME, "* igb_uio |
> uio_pci_generic");
> +
> +RTE_INIT(ioat_pmd_init_log)
> +{
> +	ioat_pmd_logtype = rte_log_register(IOAT_PMD_LOG_NAME);
> +	if (ioat_pmd_logtype >= 0)
> +		rte_log_set_level(ioat_pmd_logtype, RTE_LOG_INFO);
> +}
> diff --git a/drivers/raw/ioat/meson.build b/drivers/raw/ioat/meson.build
> new file mode 100644
> index 000000000..ba7620a68
> --- /dev/null
> +++ b/drivers/raw/ioat/meson.build
> @@ -0,0 +1,8 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright 2019 Intel Corporation
> +
> +build = dpdk_conf.has('RTE_ARCH_X86')
> +sources = files('ioat_rawdev.c')
> +deps += ['rawdev', 'bus_pci']
> +
> +install_headers('rte_ioat_rawdev.h')
> diff --git a/drivers/raw/ioat/rte_ioat_rawdev.h
> b/drivers/raw/ioat/rte_ioat_rawdev.h
> new file mode 100644
> index 000000000..e77406403
> --- /dev/null
> +++ b/drivers/raw/ioat/rte_ioat_rawdev.h
> @@ -0,0 +1,24 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2019 Intel Corporation
> + */
> +
> +#ifndef _RTE_IOAT_RAWDEV_H_
> +#define _RTE_IOAT_RAWDEV_H_
> +
> +/**
> + * @file rte_ioat_rawdev.h
> + *
> + * Definitions for using the ioat rawdev device driver
> + *
> + * @warning
> + * @b EXPERIMENTAL: these structures and APIs may change without prior
> notice
> + */
> +
> +/** Name of the device driver */
> +#define IOAT_PMD_RAWDEV_NAME rawdev_ioat
> +/** String reported as the device driver name by rte_rawdev_info_get() */
> +#define IOAT_PMD_RAWDEV_NAME_STR "rawdev_ioat"
> +/** Name used to adjust the log level for this driver */
> +#define IOAT_PMD_LOG_NAME "rawdev.ioat"
> +
> +#endif
> diff --git a/drivers/raw/ioat/rte_pmd_ioat_version.map
> b/drivers/raw/ioat/rte_pmd_ioat_version.map
> new file mode 100644
> index 000000000..9a61188cd
> --- /dev/null
> +++ b/drivers/raw/ioat/rte_pmd_ioat_version.map
> @@ -0,0 +1,4 @@
> +DPDK_19.08 {
> +
> +	local: *;
> +};
> diff --git a/drivers/raw/meson.build b/drivers/raw/meson.build
> index a61cdccef..2af8a70d4 100644
> --- a/drivers/raw/meson.build
> +++ b/drivers/raw/meson.build
> @@ -1,7 +1,9 @@
>  # SPDX-License-Identifier: BSD-3-Clause
>  # Copyright 2018 NXP
> 
> -drivers = ['skeleton_rawdev', 'dpaa2_cmdif', 'dpaa2_qdma', 'ifpga_rawdev']
> +drivers = ['dpaa2_cmdif', 'dpaa2_qdma',
> +	'ifpga_rawdev', 'ioat',
> +	'skeleton_rawdev']
>  std_deps = ['rawdev']
>  config_flag_fmt = 'RTE_LIBRTE_PMD_@0@_RAWDEV'
>  driver_name_fmt = 'rte_pmd_@0@'
> diff --git a/mk/rte.app.mk b/mk/rte.app.mk
> index 81be289a8..2a534796f 100644
> --- a/mk/rte.app.mk
> +++ b/mk/rte.app.mk
> @@ -306,6 +306,7 @@ ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -
> lrte_pmd_ifpga_rawdev
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
>  endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -
> lrte_pmd_ioat_rawdev
>  endif # CONFIG_RTE_LIBRTE_RAWDEV
> 
>  endif # !CONFIG_RTE_BUILD_SHARED_LIBS
> --
> 2.21.0