Re: [dpdk-dev] [PATCH v3 1/8] raw/ioat: add initial support for ioat rawdev driver

["Hu, Jiayu" <jiayu.hu@intel.com>]

Hi Bruce,

-----Original Message-----
From: dev <dev-bounces@dpdk.org> On Behalf Of Bruce Richardson
Sent: Thursday, June 27, 2019 6:41 PM
To: dev@dpdk.org
Cc: thomas@monjalon.net; jerinj@marvell.com; Richardson, Bruce <bruce.richardson@intel.com>
Subject: [dpdk-dev] [PATCH v3 1/8] raw/ioat: add initial support for ioat rawdev driver

Add stubs for ioat rawdev driver support in DPDK, specifically:

  * makefile and meson build hooks
  * initial public header file
  * rawdev main C file, with probe and release functions
  * release note update announcing the driver
  * initial documentation for the new section in the rawdev doc
  * unit test stubs for device unit tests

Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>

---
V2: don't create a new file for unit testing, add to existing rawdev test
    file, and place test cases in the driver selftest routine (added later
    in set)
    Add new section in document about identifying hardware using lspci
---
 MAINTAINERS                                 |  6 +-
 app/test/test_rawdev.c                      |  8 ++
 config/common_armv8a_linux                  |  1 +
 config/common_base                          |  5 ++
 config/defconfig_arm-armv7a-linuxapp-gcc    |  1 +
 config/defconfig_ppc_64-power8-linuxapp-gcc |  1 +
 doc/guides/rawdevs/index.rst                |  1 +
 doc/guides/rawdevs/ioat_rawdev.rst          | 63 ++++++++++++++
 doc/guides/rel_notes/release_19_08.rst      | 11 +++
 drivers/raw/Makefile                        |  1 +
 drivers/raw/ioat/Makefile                   | 28 +++++++
 drivers/raw/ioat/ioat_rawdev.c              | 93 +++++++++++++++++++++
 drivers/raw/ioat/meson.build                |  8 ++
 drivers/raw/ioat/rte_ioat_rawdev.h          | 24 ++++++
 drivers/raw/ioat/rte_pmd_ioat_version.map   |  4 +
 drivers/raw/meson.build                     |  4 +-
 mk/rte.app.mk                               |  1 +
 17 files changed, 258 insertions(+), 2 deletions(-)  create mode 100644 doc/guides/rawdevs/ioat_rawdev.rst
 create mode 100644 drivers/raw/ioat/Makefile  create mode 100644 drivers/raw/ioat/ioat_rawdev.c  create mode 100644 drivers/raw/ioat/meson.build  create mode 100644 drivers/raw/ioat/rte_ioat_rawdev.h
 create mode 100644 drivers/raw/ioat/rte_pmd_ioat_version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 0c3b48920..f28c526bc 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1068,6 +1068,11 @@ M: Tianfei zhang <tianfei.zhang@intel.com>
 F: drivers/raw/ifpga_rawdev/
 F: doc/guides/rawdevs/ifpga_rawdev.rst
 
+IOAT Rawdev
+M: Bruce Richardson <bruce.richardson@intel.com>
+F: drivers/raw/ioat/
+F: doc/guides/rawdevs/ioat_rawdev.rst
+
 NXP DPAA2 QDMA
 M: Nipun Gupta <nipun.gupta@nxp.com>
 F: drivers/raw/dpaa2_qdma/
@@ -1078,7 +1083,6 @@ M: Nipun Gupta <nipun.gupta@nxp.com>
 F: drivers/raw/dpaa2_cmdif/
 F: doc/guides/rawdevs/dpaa2_cmdif.rst
 
-
 Packet processing
 -----------------
 
diff --git a/app/test/test_rawdev.c b/app/test/test_rawdev.c index 043a38a13..88549fb61 100644
--- a/app/test/test_rawdev.c
+++ b/app/test/test_rawdev.c
@@ -25,3 +25,11 @@ test_rawdev_selftest_skeleton(void)
 }
 
 REGISTER_TEST_COMMAND(rawdev_autotest, test_rawdev_selftest_skeleton);
+
+static int
+test_rawdev_selftest_ioat(void)
+{
+	return TEST_SKIPPED;
+}
+
+REGISTER_TEST_COMMAND(ioat_rawdev_autotest, test_rawdev_selftest_ioat);
diff --git a/config/common_armv8a_linux b/config/common_armv8a_linux index 72091de1c..481712ebc 100644
--- a/config/common_armv8a_linux
+++ b/config/common_armv8a_linux
@@ -34,5 +34,6 @@ CONFIG_RTE_ARCH_ARM64_MEMCPY=n  CONFIG_RTE_LIBRTE_FM10K_PMD=n  CONFIG_RTE_LIBRTE_SFC_EFX_PMD=n  CONFIG_RTE_LIBRTE_AVP_PMD=n
+CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV=n
 
 CONFIG_RTE_SCHED_VECTOR=n
diff --git a/config/common_base b/config/common_base index fa1ae249a..e6b830923 100644
--- a/config/common_base
+++ b/config/common_base
@@ -747,6 +747,11 @@ CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
 #
 CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
 
+#
+# Compile PMD for Intel IOAT raw device # 
+CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV=y
+
 #
 # Compile librte_ring
 #
diff --git a/config/defconfig_arm-armv7a-linuxapp-gcc b/config/defconfig_arm-armv7a-linuxapp-gcc
index c9509b274..ee158ef9d 100644
--- a/config/defconfig_arm-armv7a-linuxapp-gcc
+++ b/config/defconfig_arm-armv7a-linuxapp-gcc
@@ -54,3 +54,4 @@ CONFIG_RTE_LIBRTE_QEDE_PMD=n  CONFIG_RTE_LIBRTE_SFC_EFX_PMD=n  CONFIG_RTE_LIBRTE_AVP_PMD=n  CONFIG_RTE_LIBRTE_NFP_PMD=n
+CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV=n
diff --git a/config/defconfig_ppc_64-power8-linuxapp-gcc b/config/defconfig_ppc_64-power8-linuxapp-gcc
index 7e248b755..9f3670ec0 100644
--- a/config/defconfig_ppc_64-power8-linuxapp-gcc
+++ b/config/defconfig_ppc_64-power8-linuxapp-gcc
@@ -56,3 +56,4 @@ CONFIG_RTE_LIBRTE_ENIC_PMD=n  CONFIG_RTE_LIBRTE_FM10K_PMD=n  CONFIG_RTE_LIBRTE_SFC_EFX_PMD=n  CONFIG_RTE_LIBRTE_AVP_PMD=n
+CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV=n
diff --git a/doc/guides/rawdevs/index.rst b/doc/guides/rawdevs/index.rst index 7c3bd9586..0a21989e4 100644
--- a/doc/guides/rawdevs/index.rst
+++ b/doc/guides/rawdevs/index.rst
@@ -14,3 +14,4 @@ application through rawdev API.
     dpaa2_cmdif
     dpaa2_qdma
     ifpga_rawdev
+    ioat_rawdev
diff --git a/doc/guides/rawdevs/ioat_rawdev.rst b/doc/guides/rawdevs/ioat_rawdev.rst
new file mode 100644
index 000000000..0c612e73a
--- /dev/null
+++ b/doc/guides/rawdevs/ioat_rawdev.rst
@@ -0,0 +1,63 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2019 Intel Corporation.
+
+.. include:: <isonum.txt>
+
+IOAT Rawdev Driver for Intel\ |reg| QuickData Technology 
+======================================================================
+
+The ``ioat`` rawdev driver provides a poll-mode driver (PMD) for Intel\ 
+|reg| QuickData Technology, part of Intel\ |reg| I/O Acceleration 
+Technology `(Intel I/OAT) 
+<https://www.intel.com/content/www/us/en/wireless-network/accel-technology.html>`_.
+This PMD, when used on supported hardware, allows data copies, for 
+example, cloning packet data, to be accelerated by that hardware rather 
+than having to be done by software, freeing up CPU cycles for other tasks.
+
+Hardware Requirements
+----------------------
+
+On Linux, the presence of an Intel\ |reg| QuickData Technology hardware 
+can be detected by checking the output of the ``lspci`` command, where 
+the hardware will be often listed as "Crystal Beach DMA" or "CBDMA". 
+For example, on a system with Intel\ |reg| Xeon\ |reg| CPU E5-2699 v4 
+@2.20GHz, lspci shows:
+
+.. code-block:: console
+
+  # lspci | grep DMA
+  00:04.0 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 
+ v4/Xeon E3 v4/Xeon D Crystal Beach DMA Channel 0 (rev 01)
+  00:04.1 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 
+ v4/Xeon E3 v4/Xeon D Crystal Beach DMA Channel 1 (rev 01)
+  00:04.2 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 
+ v4/Xeon E3 v4/Xeon D Crystal Beach DMA Channel 2 (rev 01)
+  00:04.3 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 
+ v4/Xeon E3 v4/Xeon D Crystal Beach DMA Channel 3 (rev 01)
+  00:04.4 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 
+ v4/Xeon E3 v4/Xeon D Crystal Beach DMA Channel 4 (rev 01)
+  00:04.5 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 
+ v4/Xeon E3 v4/Xeon D Crystal Beach DMA Channel 5 (rev 01)
+  00:04.6 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 
+ v4/Xeon E3 v4/Xeon D Crystal Beach DMA Channel 6 (rev 01)
+  00:04.7 System peripheral: Intel Corporation Xeon E7 v4/Xeon E5 
+ v4/Xeon E3 v4/Xeon D Crystal Beach DMA Channel 7 (rev 01)
+
+On a system with Intel\ |reg| Xeon\ |reg| Gold 6154 CPU @ 3.00GHz, 
+lspci
+shows:
+
+.. code-block:: console
+
+  # lspci | grep DMA
+  00:04.0 System peripheral: Intel Corporation Sky Lake-E CBDMA 
+ Registers (rev 04)
+  00:04.1 System peripheral: Intel Corporation Sky Lake-E CBDMA 
+ Registers (rev 04)
+  00:04.2 System peripheral: Intel Corporation Sky Lake-E CBDMA 
+ Registers (rev 04)
+  00:04.3 System peripheral: Intel Corporation Sky Lake-E CBDMA 
+ Registers (rev 04)
+  00:04.4 System peripheral: Intel Corporation Sky Lake-E CBDMA 
+ Registers (rev 04)
+  00:04.5 System peripheral: Intel Corporation Sky Lake-E CBDMA 
+ Registers (rev 04)
+  00:04.6 System peripheral: Intel Corporation Sky Lake-E CBDMA 
+ Registers (rev 04)
+  00:04.7 System peripheral: Intel Corporation Sky Lake-E CBDMA 
+ Registers (rev 04)
+
+
+Compilation
+------------
+
+For builds done with ``make``, the driver compilation is enabled by the 
+``CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV`` build configuration option. This 
+is enabled by default in builds for x86 platforms, and disabled in 
+other configurations.
+
+For builds using ``meson`` and ``ninja``, the driver will be built when 
+the target platform is x86-based.
diff --git a/doc/guides/rel_notes/release_19_08.rst b/doc/guides/rel_notes/release_19_08.rst
index 3da266705..720c13f8b 100644
--- a/doc/guides/rel_notes/release_19_08.rst
+++ b/doc/guides/rel_notes/release_19_08.rst
@@ -1,6 +1,8 @@
 ..  SPDX-License-Identifier: BSD-3-Clause
     Copyright 2019 The DPDK contributors
 
+.. include:: <isonum.txt>
+
 DPDK Release 19.08
 ==================
 
@@ -99,6 +101,15 @@ New Features
   Updated ``librte_telemetry`` to fetch the global metrics from the
   ``librte_metrics`` library.
 
+* **Added Intel QuickData Technology PMD**
+
+  The PMD for Intel\ |reg|  QuickData Technology, part of  Intel\ |reg|  
+ I/O Acceleration Technology `(Intel I/OAT)  
+ <https://www.intel.com/content/www/us/en/wireless-network/accel-techno
+ logy.html>`_,  allows data copies to be done by hardware instead  of 
+ via software, reducing cycles spent copying large blocks of data in  
+ applications.
+
 
 Removed Items
 -------------
diff --git a/drivers/raw/Makefile b/drivers/raw/Makefile index 8e29b4a56..c1b85c8c7 100644
--- a/drivers/raw/Makefile
+++ b/drivers/raw/Makefile
@@ -10,5 +10,6 @@ DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV) += dpaa2_cmdif
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += dpaa2_qdma  endif
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += ifpga_rawdev
+DIRS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV) += ioat
 
 include $(RTE_SDK)/mk/rte.subdir.mk
diff --git a/drivers/raw/ioat/Makefile b/drivers/raw/ioat/Makefile new file mode 100644 index 000000000..7726e310a
--- /dev/null
+++ b/drivers/raw/ioat/Makefile
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2019 Intel 
+Corporation
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_pmd_ioat_rawdev.a
+
+# build flags
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+
+LDLIBS += -lrte_eal -lrte_rawdev
+LDLIBS += -lrte_pci -lrte_bus_pci
+
+# library version
+LIBABIVER := 1
+
+# versioning export map
+EXPORT_MAP := rte_pmd_ioat_version.map
+
+# library source files
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV) += ioat_rawdev.c
+
+# export include files
+SYMLINK-y-include += rte_ioat_rawdev.h
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/raw/ioat/ioat_rawdev.c b/drivers/raw/ioat/ioat_rawdev.c new file mode 100644 index 000000000..d9fc3091a
--- /dev/null
+++ b/drivers/raw/ioat/ioat_rawdev.c
@@ -0,0 +1,93 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019 Intel Corporation
+ */
+
+#include <rte_bus_pci.h>
+#include <rte_rawdev_pmd.h>
+
+#include "rte_ioat_rawdev.h"
+
+/* Dynamic log type identifier */
+int ioat_pmd_logtype;
+
+static struct rte_pci_driver ioat_pmd_drv;
+
+#define IOAT_VENDOR_ID		0x8086
+#define IOAT_DEVICE_ID		0x2021

In the second patch, the script dpdk-devbind.py supports CBDMAs in
both Broadwell and Skylake. But you only supports Skylake here.

Thanks,
Jiayu
+
+#define IOAT_PMD_LOG(level, fmt, args...) rte_log(RTE_LOG_ ## level, \
+	ioat_pmd_logtype, "%s(): " fmt "\n", __func__, ##args)
+
+#define IOAT_PMD_DEBUG(fmt, args...)  IOAT_PMD_LOG(DEBUG, fmt, ## args)
+#define IOAT_PMD_INFO(fmt, args...)   IOAT_PMD_LOG(INFO, fmt, ## args)
+#define IOAT_PMD_ERR(fmt, args...)    IOAT_PMD_LOG(ERR, fmt, ## args)
+#define IOAT_PMD_WARN(fmt, args...)   IOAT_PMD_LOG(WARNING, fmt, ## args)
+
+static int
+ioat_rawdev_create(const char *name, struct rte_pci_device *dev) {
+	RTE_SET_USED(name);
+	RTE_SET_USED(dev);
+	return 0;
+}
+
+static int
+ioat_rawdev_destroy(const char *name)
+{
+	RTE_SET_USED(name);
+	return 0;
+}
+
+static int
+ioat_rawdev_probe(struct rte_pci_driver *drv, struct rte_pci_device 
+*dev) {
+	char name[32];
+	int ret = 0;
+
+
+	rte_pci_device_name(&dev->addr, name, sizeof(name));
+	IOAT_PMD_INFO("Init %s on NUMA node %d", name, dev->device.numa_node);
+
+	dev->device.driver = &drv->driver;
+	ret = ioat_rawdev_create(name, dev);
+	return ret;
+}
+
+static int
+ioat_rawdev_remove(struct rte_pci_device *dev) {
+	char name[32];
+	int ret;
+
+	rte_pci_device_name(&dev->addr, name, sizeof(name));
+
+	IOAT_PMD_INFO("Closing %s on NUMA node %d",
+			name, dev->device.numa_node);
+
+	ret = ioat_rawdev_destroy(name);
+	return ret;
+}
+
+static const struct rte_pci_id pci_id_ioat_map[] = {
+	{ RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID) },
+	{ .vendor_id = 0, /* sentinel */ },
+};
+
+static struct rte_pci_driver ioat_pmd_drv = {
+	.id_table = pci_id_ioat_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
+		     RTE_PCI_DRV_IOVA_AS_VA,
+	.probe = ioat_rawdev_probe,
+	.remove = ioat_rawdev_remove,
+};
+
+RTE_PMD_REGISTER_PCI(IOAT_PMD_RAWDEV_NAME, ioat_pmd_drv); 
+RTE_PMD_REGISTER_PCI_TABLE(IOAT_PMD_RAWDEV_NAME, pci_id_ioat_map); 
+RTE_PMD_REGISTER_KMOD_DEP(IOAT_PMD_RAWDEV_NAME, "* igb_uio | 
+uio_pci_generic");
+
+RTE_INIT(ioat_pmd_init_log)
+{
+	ioat_pmd_logtype = rte_log_register(IOAT_PMD_LOG_NAME);
+	if (ioat_pmd_logtype >= 0)
+		rte_log_set_level(ioat_pmd_logtype, RTE_LOG_INFO); }
diff --git a/drivers/raw/ioat/meson.build b/drivers/raw/ioat/meson.build new file mode 100644 index 000000000..ba7620a68
--- /dev/null
+++ b/drivers/raw/ioat/meson.build
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: BSD-3-Clause # Copyright 2019 Intel 
+Corporation
+
+build = dpdk_conf.has('RTE_ARCH_X86')
+sources = files('ioat_rawdev.c')
+deps += ['rawdev', 'bus_pci']
+
+install_headers('rte_ioat_rawdev.h')
diff --git a/drivers/raw/ioat/rte_ioat_rawdev.h b/drivers/raw/ioat/rte_ioat_rawdev.h
new file mode 100644
index 000000000..e77406403
--- /dev/null
+++ b/drivers/raw/ioat/rte_ioat_rawdev.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019 Intel Corporation
+ */
+
+#ifndef _RTE_IOAT_RAWDEV_H_
+#define _RTE_IOAT_RAWDEV_H_
+
+/**
+ * @file rte_ioat_rawdev.h
+ *
+ * Definitions for using the ioat rawdev device driver
+ *
+ * @warning
+ * @b EXPERIMENTAL: these structures and APIs may change without prior 
+notice  */
+
+/** Name of the device driver */
+#define IOAT_PMD_RAWDEV_NAME rawdev_ioat
+/** String reported as the device driver name by rte_rawdev_info_get() 
+*/ #define IOAT_PMD_RAWDEV_NAME_STR "rawdev_ioat"
+/** Name used to adjust the log level for this driver */ #define 
+IOAT_PMD_LOG_NAME "rawdev.ioat"
+
+#endif
diff --git a/drivers/raw/ioat/rte_pmd_ioat_version.map b/drivers/raw/ioat/rte_pmd_ioat_version.map
new file mode 100644
index 000000000..9a61188cd
--- /dev/null
+++ b/drivers/raw/ioat/rte_pmd_ioat_version.map
@@ -0,0 +1,4 @@
+DPDK_19.08 {
+
+	local: *;
+};
diff --git a/drivers/raw/meson.build b/drivers/raw/meson.build index a61cdccef..2af8a70d4 100644
--- a/drivers/raw/meson.build
+++ b/drivers/raw/meson.build
@@ -1,7 +1,9 @@
 # SPDX-License-Identifier: BSD-3-Clause  # Copyright 2018 NXP
 
-drivers = ['skeleton_rawdev', 'dpaa2_cmdif', 'dpaa2_qdma', 'ifpga_rawdev']
+drivers = ['dpaa2_cmdif', 'dpaa2_qdma',
+	'ifpga_rawdev', 'ioat',
+	'skeleton_rawdev']
 std_deps = ['rawdev']
 config_flag_fmt = 'RTE_LIBRTE_PMD_@0@_RAWDEV'
 driver_name_fmt = 'rte_pmd_@0@'
diff --git a/mk/rte.app.mk b/mk/rte.app.mk index 81be289a8..2a534796f 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -306,6 +306,7 @@ ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV)   += -lrte_pmd_ifpga_rawdev
 _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD)       += -lrte_pmd_ipn3ke
 endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_pmd_ioat_rawdev
 endif # CONFIG_RTE_LIBRTE_RAWDEV
 
 endif # !CONFIG_RTE_BUILD_SHARED_LIBS
--
2.21.0