[dpdk-dev] [RFC 0/4] Intel FPGA Bus

[dpdk-dev] [RFC 0/4] Intel FPGA Bus

[Rosen Xu]

With Partial Reconfigure(PR) parts of Bitstream, Field Programmable Gate Array(FPGA) not only 
provides one kinds of accelerator but also provides many types of accelerators at the same time.

How DPDK fully support FPGA?
 - We use Rawdev to provide FPGA PR
 - DPDK Driver will not bind to PCI Device it will bind to FPGA Partial-Bitstream(AFU,Accelerated Function Unit)
 - For the new Device scan, driver probe, we involve Intel FPGA Bus Module

This patchset is base on v18.02.

Rosen Xu (4):
  drivers/bus/ifpga:Intel FPGA Bus Lib Code
  lib/librte_eal/common:Add Intel FPGA Bus Running Command Parse Code
  lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be
    scanned after PCI Bus
  drivers/raw/ifpga_rawdev: Rawdev for Intel FPGA Device, it's a PCI
    Driver of FPGA Device Manager

 drivers/bus/ifpga/Makefile                         |  64 +++
 drivers/bus/ifpga/ifpga_bus.c                      | 527 +++++++++++++++++++++
 drivers/bus/ifpga/ifpga_common.c                   | 168 +++++++
 drivers/bus/ifpga/ifpga_common.h                   |  46 ++
 drivers/bus/ifpga/ifpga_logs.h                     |  59 +++
 drivers/bus/ifpga/rte_bus_ifpga.h                  | 153 ++++++
 drivers/bus/ifpga/rte_bus_ifpga_version.map        |   8 +
 drivers/raw/ifpga_rawdev/Makefile                  |  59 +++
 drivers/raw/ifpga_rawdev/ifpga_rawdev.c            | 343 ++++++++++++++
 drivers/raw/ifpga_rawdev/ifpga_rawdev.h            | 109 +++++
 drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c    | 121 +++++
 .../ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map  |   4 +
 lib/librte_eal/common/eal_common_bus.c             |  14 +-
 lib/librte_eal/common/eal_common_options.c         |   8 +-
 lib/librte_eal/common/eal_options.h                |   2 +
 15 files changed, 1683 insertions(+), 2 deletions(-)
 create mode 100644 drivers/bus/ifpga/Makefile
 create mode 100644 drivers/bus/ifpga/ifpga_bus.c
 create mode 100644 drivers/bus/ifpga/ifpga_common.c
 create mode 100644 drivers/bus/ifpga/ifpga_common.h
 create mode 100644 drivers/bus/ifpga/ifpga_logs.h
 create mode 100644 drivers/bus/ifpga/rte_bus_ifpga.h
 create mode 100644 drivers/bus/ifpga/rte_bus_ifpga_version.map
 create mode 100644 drivers/raw/ifpga_rawdev/Makefile
 create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.c
 create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.h
 create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c
 create mode 100644 drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map

-- 
1.8.3.1

[dpdk-dev] [RFC 1/4] drivers/bus/ifpga:Intel FPGA Bus Lib Code

[Rosen Xu]

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/bus/ifpga/Makefile                  |  64 ++++
 drivers/bus/ifpga/ifpga_bus.c               | 527 ++++++++++++++++++++++++++++
 drivers/bus/ifpga/ifpga_common.c            | 168 +++++++++
 drivers/bus/ifpga/ifpga_common.h            |  46 +++
 drivers/bus/ifpga/ifpga_logs.h              |  59 ++++
 drivers/bus/ifpga/rte_bus_ifpga.h           | 153 ++++++++
 drivers/bus/ifpga/rte_bus_ifpga_version.map |   8 +
 7 files changed, 1025 insertions(+)
 create mode 100644 drivers/bus/ifpga/Makefile
 create mode 100644 drivers/bus/ifpga/ifpga_bus.c
 create mode 100644 drivers/bus/ifpga/ifpga_common.c
 create mode 100644 drivers/bus/ifpga/ifpga_common.h
 create mode 100644 drivers/bus/ifpga/ifpga_logs.h
 create mode 100644 drivers/bus/ifpga/rte_bus_ifpga.h
 create mode 100644 drivers/bus/ifpga/rte_bus_ifpga_version.map

diff --git a/drivers/bus/ifpga/Makefile b/drivers/bus/ifpga/Makefile
new file mode 100644
index 0000000..c71f186
--- /dev/null
+++ b/drivers/bus/ifpga/Makefile
@@ -0,0 +1,64 @@
+#   BSD LICENSE
+#
+#   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Intel Corporation nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+LIB = librte_bus_ifpga.a
+LIBABIVER := 1
+EXPORT_MAP := rte_bus_ifpga_version.map
+
+ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT),y)
+CFLAGS += -O0 -g
+CFLAGS += "-Wno-error"
+else
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+endif
+
+CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/bus/pci
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common
+#CFLAGS += -I$(RTE_SDK)/lib/librte_rawdev
+#LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring -lrte_rawdev
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+#LDLIBS += -lrte_ethdev
+
+VPATH += $(SRCDIR)/base
+
+SRCS-y += \
+        ifpga_bus.c \
+        ifpga_common.c
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/bus/ifpga/ifpga_bus.c b/drivers/bus/ifpga/ifpga_bus.c
new file mode 100644
index 0000000..382d550
--- /dev/null
+++ b/drivers/bus/ifpga/ifpga_bus.c
@@ -0,0 +1,527 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ *   Copyright 2013-2014 6WIND S.A.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include <rte_errno.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_eal.h>
+#include <rte_common.h>
+
+#include <rte_devargs.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+#include <rte_kvargs.h>
+#include <rte_alarm.h>
+
+#include "rte_rawdev.h"
+#include "rte_rawdev_pmd.h"
+#include "rte_bus_ifpga.h"
+#include "ifpga_logs.h"
+#include "ifpga_common.h"
+
+int ifpga_bus_logtype;
+
+/*register a ifpga bus based driver */
+void rte_ifpga_driver_register(struct rte_afu_driver *driver)
+{
+	RTE_VERIFY(driver);
+
+	TAILQ_INSERT_TAIL(&rte_ifpga_bus.driver_list, driver, next);
+}
+
+/*un-register a fpga bus based driver */
+void rte_ifpga_driver_unregister(struct rte_afu_driver *driver)
+{
+	TAILQ_REMOVE(&rte_ifpga_bus.driver_list, driver, next);
+}
+
+static struct rte_afu_device *
+ifpga_find_afu_dev(const struct rte_afu_id *afu_id)
+{
+	struct rte_afu_device *afu_dev = NULL;
+	
+	TAILQ_FOREACH (afu_dev, &rte_ifpga_bus.afu_list, next) {
+
+        if(!ifpga_afu_id_cmp(&afu_dev->id, afu_id)) {
+            return afu_dev;
+        }
+	}
+	return NULL;
+}
+
+static const char *valid_args[] = {
+#define IFPGA_ARG_BDF          "bdf"
+	IFPGA_ARG_BDF,
+#define IFPGA_ARG_PORT         "port"
+	IFPGA_ARG_PORT,
+#define IFPGA_ARG_PATH         "path"
+	IFPGA_ARG_PATH,
+#define IFPGA_ARG_UUID_HIGH    "uuid_high"
+	IFPGA_ARG_UUID_HIGH,
+#define IFPGA_ARG_UUID_LOW     "uuid_low"
+	IFPGA_ARG_UUID_LOW,
+#define IFPGA_ARG_PR_ENABLE     "pr_enable"
+	IFPGA_ARG_PR_ENABLE,
+#define IFPGA_ARG_DEBUG         "debug"
+	IFPGA_ARG_DEBUG,
+	NULL
+};
+
+/*
+ * Scan the content of the FPGA bus, and the devices in the devices
+ * list
+ */
+static struct rte_afu_device *
+rte_ifpga_scan_one(struct rte_devargs *devargs)
+{
+	struct rte_kvargs *kvlist = NULL;
+	struct rte_bus *pci_bus = NULL;
+	struct rte_device *dev = NULL;
+	struct rte_rawdev *rawdev;
+	struct rte_afu_device *afu_dev = NULL;
+	struct rte_afu_pr_conf afu_pr_conf;
+	char rawdev_name[RTE_RAWDEV_NAME_MAX_LEN];
+	int ret;
+	char *path = NULL;
+	int pr_enable = 1;
+	int debug = 0;
+
+    memset((char *)(&afu_pr_conf), 0, sizeof(struct rte_afu_pr_conf));
+	
+    kvlist = rte_kvargs_parse(devargs->args, valid_args);
+    if (!kvlist) {
+    	IFPGA_BUS_ERR("error when parsing param");
+    	goto end;
+    }
+    
+    if (rte_kvargs_count(kvlist, IFPGA_ARG_BDF) == 1) {
+		if (rte_kvargs_process(kvlist, IFPGA_ARG_BDF,
+				       &ifpga_get_bdf_arg, &afu_pr_conf.afu_id.pci_addr) < 0) {
+			IFPGA_BUS_ERR("error to parse %s",
+				     IFPGA_ARG_BDF);
+			goto end;
+		}
+	} else {
+		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
+			  IFPGA_ARG_PATH);
+		goto end;
+	}
+    
+    if (rte_kvargs_count(kvlist, IFPGA_ARG_PORT) == 1) {
+		if (rte_kvargs_process(kvlist, IFPGA_ARG_PORT,
+				       &ifpga_get_integer32_arg, &afu_pr_conf.afu_id.port) < 0) {
+			IFPGA_BUS_ERR("error to parse %s",
+				     IFPGA_ARG_PORT);
+			goto end;
+		}
+	}
+	else {
+		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
+			  IFPGA_ARG_PATH);
+		goto end;
+	}
+	
+    if (rte_kvargs_count(kvlist, IFPGA_ARG_PATH) == 1) {
+		if (rte_kvargs_process(kvlist, IFPGA_ARG_PATH,
+				       &ifpga_get_string_arg, &path) < 0) {
+			IFPGA_BUS_ERR("error to parse %s",
+				     IFPGA_ARG_PATH);
+			goto end;
+		}
+	} else {
+		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
+			  IFPGA_ARG_PATH);
+		goto end;
+	}
+	
+	if (rte_kvargs_count(kvlist, IFPGA_ARG_UUID_HIGH) == 1) {
+		if (rte_kvargs_process(kvlist, IFPGA_ARG_UUID_HIGH,
+				       &ifpga_get_integer64_arg, &afu_pr_conf.afu_id.uuid_high) < 0) {
+			IFPGA_BUS_ERR("error to parse %s",
+				     IFPGA_ARG_UUID_HIGH);
+			goto end;
+		}
+	} else {
+		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
+			  IFPGA_ARG_PATH);
+		goto end;
+	}
+	
+	if (rte_kvargs_count(kvlist, IFPGA_ARG_UUID_LOW) == 1) {
+		if (rte_kvargs_process(kvlist, IFPGA_ARG_UUID_LOW,
+				       &ifpga_get_integer64_arg, &afu_pr_conf.afu_id.uuid_low) < 0) {
+			IFPGA_BUS_ERR("error to parse %s",
+				     IFPGA_ARG_UUID_LOW);
+			goto end;
+		}
+	} else {
+		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
+			  IFPGA_ARG_PATH);
+		goto end;
+	}
+	
+	if (rte_kvargs_count(kvlist, IFPGA_ARG_PR_ENABLE) == 1) {
+		if (rte_kvargs_process(kvlist, IFPGA_ARG_PR_ENABLE,
+				       &ifpga_get_integer32_arg, &pr_enable) < 0) {
+			IFPGA_BUS_ERR("error to parse %s",
+				     IFPGA_ARG_UUID_HIGH);
+			goto end;
+		}
+	}
+	
+	if (rte_kvargs_count(kvlist, IFPGA_ARG_DEBUG) == 1) {
+		if (rte_kvargs_process(kvlist, IFPGA_ARG_DEBUG,
+				       &ifpga_get_integer32_arg, &debug) < 0) {
+			IFPGA_BUS_ERR("error to parse %s",
+				     IFPGA_ARG_UUID_HIGH);
+			goto end;
+		}
+	}
+	
+	if(!debug)
+	{
+		pci_bus = rte_bus_find_by_name("pci");
+	    if (pci_bus == NULL) {
+	    	IFPGA_BUS_ERR("unable to find PCI bus\n");
+	    	goto end;
+	    }
+	    
+	    dev = pci_bus->find_device(NULL, ifpga_pci_addr_cmp, &afu_pr_conf.afu_id.pci_addr);
+	    if (dev == NULL) {
+	    	IFPGA_BUS_ERR("unable to find PCI device\n");
+	    	goto end;
+	    }
+	}
+	else
+	{
+		IFPGA_BUS_DEBUG("pci_addr domain   : %x\n",   afu_pr_conf.afu_id.pci_addr.domain);
+		IFPGA_BUS_DEBUG("pci_addr bus      : %x\n",   afu_pr_conf.afu_id.pci_addr.bus);
+		IFPGA_BUS_DEBUG("pci_addr devid    : %x\n",   afu_pr_conf.afu_id.pci_addr.devid);
+		IFPGA_BUS_DEBUG("pci_addr function : %x\n",   afu_pr_conf.afu_id.pci_addr.function);
+		
+		IFPGA_BUS_DEBUG("uuid_low          : %lx\n", afu_pr_conf.afu_id.uuid_low);
+		IFPGA_BUS_DEBUG("uuid_high         : %lx\n", afu_pr_conf.afu_id.uuid_high);
+		
+		IFPGA_BUS_DEBUG("afu port          : %x\n",   afu_pr_conf.afu_id.port);
+	}
+	
+	if (ifpga_find_afu_dev(&afu_pr_conf.afu_id))
+		goto end;
+
+	memset(rawdev_name, sizeof(rawdev_name), 0);
+	snprintf(rawdev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%x:%x", afu_pr_conf.afu_id.pci_addr.bus, afu_pr_conf.afu_id.pci_addr.devid, afu_pr_conf.afu_id.pci_addr.function);
+	rawdev = rte_rawdev_pmd_get_named_dev(rawdev_name);
+	if(!rawdev)
+		goto end;
+	
+	rawdev->dev_ops->dev_start(rawdev);
+		
+	if (pr_enable)
+	{
+		ret=rawdev->dev_ops->firmware_load(rawdev, &afu_pr_conf);
+		if (ret) {
+    		printf("do pr error\n");
+    		return NULL;
+	    }
+	}
+		
+	afu_dev = calloc(1, sizeof(*afu_dev));
+	if (!afu_dev)
+		goto end;
+
+	afu_dev->device.devargs = devargs;
+	afu_dev->device.numa_node = SOCKET_ID_ANY;
+	afu_dev->device.name = devargs->name;
+	afu_dev->rawdev = rawdev;
+	afu_dev->id.pci_addr.domain = afu_pr_conf.afu_id.pci_addr.domain;
+	afu_dev->id.pci_addr.bus = afu_pr_conf.afu_id.pci_addr.bus;
+	afu_dev->id.pci_addr.devid = afu_pr_conf.afu_id.pci_addr.devid;
+	afu_dev->id.pci_addr.function = afu_pr_conf.afu_id.pci_addr.function;
+	afu_dev->id.uuid_low  = afu_pr_conf.afu_id.uuid_low;
+	afu_dev->id.uuid_high = afu_pr_conf.afu_id.uuid_high;
+	afu_dev->id.port      = afu_pr_conf.afu_id.port;
+	
+	rawdev->dev_ops->dev_info_get(rawdev, afu_dev);
+
+    return afu_dev;
+    
+end:
+	if (kvlist)
+		rte_kvargs_free(kvlist);
+	if (path)
+		free(path);
+	
+	return NULL;
+}
+
+/*
+ * Scan the content of the FPGA bus, and the devices in the devices
+ * list
+ */
+static int
+rte_ifpga_scan(void)
+{
+	struct rte_bus *pci_bus;
+	struct rte_afu_device *afu_dev;
+	struct rte_devargs *devargs;
+
+    pci_bus = rte_bus_find_by_name("pci");
+	if (pci_bus == NULL) {
+		IFPGA_BUS_ERR("unable to find PCI bus\n");
+		return -1;
+	}
+	
+	/* for virtual devices we scan the devargs_list populated via cmdline */
+	TAILQ_FOREACH(devargs, &devargs_list, next) {
+    
+		if (devargs->bus != &rte_ifpga_bus.bus)
+			continue;
+		
+		afu_dev = rte_ifpga_scan_one(devargs);
+	    
+	    if(afu_dev)
+		    TAILQ_INSERT_TAIL(&rte_ifpga_bus.afu_list, afu_dev, next);
+	}
+
+	return 0;
+}
+
+static int
+ifpga_probe_one_driver(struct rte_afu_driver *drv,
+			struct rte_afu_device *afu_dev)
+{
+	int ret;
+
+    if((drv->id.pci_addr.bus == afu_dev->id.pci_addr.bus) &&
+       (drv->id.pci_addr.devid == afu_dev->id.pci_addr.devid) &&
+       (drv->id.pci_addr.function == afu_dev->id.pci_addr.function) &&
+       (drv->id.uuid_low == afu_dev->id.uuid_low) &&
+       (drv->id.uuid_high == afu_dev->id.uuid_high) &&
+       (drv->id.port == afu_dev->id.port)) {
+       	
+       	afu_dev->driver = drv;
+       	
+       	/* call the driver probe() function */
+		ret = drv->probe(afu_dev);
+		if (ret)
+			afu_dev->driver = NULL;
+		return ret;
+    }
+	
+	/* return positive value if driver doesn't support this device */
+	return 1;
+}
+
+static int
+ifpga_probe_all_drivers(struct rte_afu_device *afu_dev)
+{
+	const char *name;
+	struct rte_afu_driver *drv = NULL;
+	int rc;
+
+	if (afu_dev == NULL)
+		return -1;
+
+	/* Check if a driver is already loaded */
+	if (afu_dev->driver != NULL)
+		return 0;
+		
+    name = rte_ifpga_device_name(afu_dev);
+    IFPGA_BUS_DEBUG("Search driver %s to probe device %s\n", name,
+		rte_ifpga_device_name(afu_dev));
+    
+	TAILQ_FOREACH(drv, &rte_ifpga_bus.driver_list, next) {
+		rc = ifpga_probe_one_driver(drv, afu_dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return -1;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+ }
+
+/*
+ * Scan the content of the PCI bus, and call the probe() function for
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+static int
+rte_ifpga_probe(void)
+{
+	struct rte_afu_device *afu_dev = NULL;
+	int ret = 0;
+
+	TAILQ_FOREACH (afu_dev, &rte_ifpga_bus.afu_list, next) {
+
+        if (afu_dev->device.driver)
+			continue;
+			
+		ret = ifpga_probe_all_drivers(afu_dev);
+		if (ret < 0)
+		    IFPGA_BUS_ERR("failed to initialize %s device\n",
+				rte_ifpga_device_name(afu_dev));
+	}
+
+	return 0;
+}
+
+static int
+rte_ifpga_plug(struct rte_device *dev)
+{
+	return ifpga_probe_all_drivers(RTE_DEV_TO_AFU(dev));
+}
+
+static int ifpga_remove_driver(struct rte_afu_device *afu_dev)
+{
+	const char *name;
+	const struct rte_afu_driver *driver;
+
+    name = rte_ifpga_device_name(afu_dev);
+	if (!afu_dev->device.driver) {
+		IFPGA_BUS_DEBUG("no driver attach to device %s\n", name);
+		return 1;
+	}
+
+	driver = container_of(afu_dev->device.driver, const struct rte_afu_driver,
+		driver);
+	return driver->remove(afu_dev);
+}
+
+static int
+rte_ifpga_unplug(struct rte_device *dev)
+{
+	struct rte_afu_device *afu_dev;
+	struct rte_devargs *devargs;
+	int ret;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	afu_dev = RTE_DEV_TO_AFU(dev);
+	if (!dev)
+		return -ENOENT;
+
+	devargs = dev->devargs;
+
+	ret = ifpga_remove_driver(afu_dev);
+	if (ret)
+		return ret;
+
+	TAILQ_REMOVE(&rte_ifpga_bus.afu_list, afu_dev, next);
+
+	TAILQ_REMOVE(&devargs_list, devargs, next);
+
+	free(devargs->args);
+	free(devargs);
+	free(afu_dev);
+	return 0;
+
+}
+
+static struct rte_device *
+rte_ifpga_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		                                        const void *data)
+{
+	struct rte_afu_device *afu_dev;
+
+	TAILQ_FOREACH(afu_dev, &rte_ifpga_bus.afu_list, next) {
+		if (start && &afu_dev->device == start) {
+			start = NULL;
+			continue;
+		}
+		if (cmp(&afu_dev->device, data) == 0)
+			return &afu_dev->device;
+	}
+	return NULL;
+}
+static int
+rte_ifpga_parse(const char *name, void *addr)
+{
+	struct rte_afu_driver **out = addr;
+	struct rte_afu_driver *driver = NULL;
+
+	TAILQ_FOREACH(driver, &rte_ifpga_bus.driver_list, next) {
+		if (strncmp(driver->driver.name, name,
+			    strlen(driver->driver.name)) == 0)
+			break;
+		if (driver->driver.alias &&
+		    strncmp(driver->driver.alias, name,
+			    strlen(driver->driver.alias)) == 0)
+			break;
+	}
+	if (driver != NULL &&
+	    addr != NULL)
+		*out = driver;
+	return driver == NULL;
+}
+
+struct rte_ifpga_bus rte_ifpga_bus = {
+	 .bus = {
+		 .scan        = rte_ifpga_scan,
+		 .probe       = rte_ifpga_probe,
+		 .find_device = rte_ifpga_find_device,
+	     .plug        = rte_ifpga_plug,
+	     .unplug      = rte_ifpga_unplug,
+	     .parse       = rte_ifpga_parse,
+	 },
+	.afu_list    = TAILQ_HEAD_INITIALIZER(rte_ifpga_bus.afu_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(rte_ifpga_bus.driver_list),
+ };
+
+RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus.bus);
+
+RTE_INIT(ifpga_init_log)
+{
+	ifpga_bus_logtype = rte_log_register("bus.ifpga");
+	if (ifpga_bus_logtype >= 0)
+		rte_log_set_level(ifpga_bus_logtype, RTE_LOG_NOTICE);
+}
+
diff --git a/drivers/bus/ifpga/ifpga_common.c b/drivers/bus/ifpga/ifpga_common.c
new file mode 100644
index 0000000..2aa6bb0
--- /dev/null
+++ b/drivers/bus/ifpga/ifpga_common.c
@@ -0,0 +1,168 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ *   Copyright 2013-2014 6WIND S.A.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include <rte_errno.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_eal.h>
+#include <rte_common.h>
+
+#include <rte_devargs.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+#include <rte_kvargs.h>
+#include <rte_alarm.h>
+
+#include "rte_bus_ifpga.h"
+#include "ifpga_logs.h"
+#include "ifpga_common.h"
+
+int ifpga_get_string_arg(const char *key __rte_unused,
+	                     const char *value, void *extra_args)
+{
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	*(char **)extra_args = strdup(value);
+
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	return 0;
+}
+int ifpga_get_integer32_arg(const char *key __rte_unused, const char *value, void *extra_args)
+{
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	*(int *)extra_args = strtoull(value, NULL, 0);
+
+	return 0;
+}
+int ifpga_get_integer64_arg(const char *key __rte_unused, const char *value, void *extra_args)
+{
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	*(uint64_t *)extra_args = strtoull(value, NULL, 0);
+
+	return 0;
+}
+int ifpga_get_unsigned_long(const char *str, int base)
+{
+	unsigned long num;
+	char *end = NULL;
+
+	errno = 0;
+
+	num = strtoul(str, &end, base);
+	if ((str[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
+		return -1;
+
+	return num;
+
+}
+int ifpga_get_bdf_arg(const char *key __rte_unused, const char *value, void *extra_args)
+{
+#define MAX_PATH_LEN 1024
+	struct rte_pci_addr *addr;
+	int num[4];
+	char str[MAX_PATH_LEN];
+	int i, j;
+	
+	if (!value || !extra_args)
+		return -EINVAL;
+
+    addr = (struct rte_pci_addr *)extra_args;
+	strcpy(str, value);
+	memset(num, 0, 4 * sizeof(num[0]));
+	i = strlen(str) - 1;
+	j = 3;
+	while (i > 0 && j >= 0) {
+		while ((str[i - 1] != ':' && str[i - 1] != '.') && i > 0)
+			i--;
+		num[j--] = ifpga_get_unsigned_long(&str[i], 16);
+		i--;
+		if (i >= 0)
+			str[i] = '\0';
+	}
+	addr->domain = num[0];
+	addr->bus = num[1];
+	addr->devid = num[2];
+	addr->function = num[3];
+	printf("[%s]: bdf %04d:%02d:%02d.%02d\n",
+			__func__,
+			addr->domain,
+			addr->bus,
+			addr->devid,
+			addr->function);
+
+	return 0;
+}
+int ifpga_afu_id_cmp(const struct rte_afu_id *afu_id0, const struct rte_afu_id *afu_id1)
+{
+	if((afu_id0->pci_addr.bus       == afu_id1->pci_addr.bus) &&
+       (afu_id0->pci_addr.devid     == afu_id1->pci_addr.devid) &&
+       (afu_id0->pci_addr.function  == afu_id1->pci_addr.function) &&
+       (afu_id0->uuid_low           == afu_id1->uuid_low) &&
+       (afu_id0->uuid_high          == afu_id1->uuid_high) &&
+       (afu_id0->port               == afu_id1->port)) {
+       	
+        return 0;
+    }
+    else
+    	return 1;
+}
+int ifpga_pci_addr_cmp(const struct rte_device *dev, const void *_pci_addr)
+{
+	struct rte_pci_device *pdev;
+	const struct rte_pci_addr *paddr = _pci_addr;
+
+	pdev = RTE_DEV_TO_PCI(*(struct rte_device **)(void *)&dev);
+	return rte_eal_compare_pci_addr(&pdev->addr, paddr);
+}
diff --git a/drivers/bus/ifpga/ifpga_common.h b/drivers/bus/ifpga/ifpga_common.h
new file mode 100644
index 0000000..590ff27
--- /dev/null
+++ b/drivers/bus/ifpga/ifpga_common.h
@@ -0,0 +1,46 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _IFPGA_COMMON_H_
+#define _IFPGA_COMMON_H_
+
+#include <rte_pci.h>
+
+extern int ifpga_get_string_arg(const char *key __rte_unused, const char *value, void *extra_args);
+extern int ifpga_get_integer32_arg(const char *key __rte_unused, const char *value, void *extra_args);
+extern int ifpga_get_integer64_arg(const char *key __rte_unused, const char *value, void *extra_args);
+extern int ifpga_get_unsigned_long(const char *str, int base);
+extern int ifpga_get_bdf_arg(const char *key __rte_unused, const char *value, void *extra_args);
+extern int ifpga_afu_id_cmp(const struct rte_afu_id *afu_id0, const struct rte_afu_id *afu_id1);
+extern int ifpga_pci_addr_cmp(const struct rte_device *dev, const void *_pci_addr);
+
+#endif /* _IFPGA_COMMON_H_ */
diff --git a/drivers/bus/ifpga/ifpga_logs.h b/drivers/bus/ifpga/ifpga_logs.h
new file mode 100644
index 0000000..eb56393
--- /dev/null
+++ b/drivers/bus/ifpga/ifpga_logs.h
@@ -0,0 +1,59 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _IFPGA_BUS_LOGS_H_
+#define _IFPGA_BUS_LOGS_H_
+
+#include <rte_log.h>
+
+extern int ifpga_bus_logtype;
+
+#define IFPGA_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, ifpga_bus_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+#define IFPGA_BUS_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, ifpga_bus_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+#define IFPGA_BUS_FUNC_TRACE() IFPGA_BUS_LOG(DEBUG, ">>")
+
+#define IFPGA_BUS_DEBUG(fmt, args...) \
+	IFPGA_BUS_LOG(DEBUG, fmt, ## args)
+#define IFPGA_BUS_INFO(fmt, args...) \
+	IFPGA_BUS_LOG(INFO, fmt, ## args)
+#define IFPGA_BUS_ERR(fmt, args...) \
+	IFPGA_BUS_LOG(ERR, fmt, ## args)
+#define IFPGA_BUS_WARN(fmt, args...) \
+	IFPGA_BUS_LOG(WARNING, fmt, ## args)
+
+#endif /* _IFPGA_BUS_LOGS_H_ */
diff --git a/drivers/bus/ifpga/rte_bus_ifpga.h b/drivers/bus/ifpga/rte_bus_ifpga.h
new file mode 100644
index 0000000..5491a45
--- /dev/null
+++ b/drivers/bus/ifpga/rte_bus_ifpga.h
@@ -0,0 +1,153 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 RehiveTech. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of RehiveTech nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_BUS_IFPGA_H_
+#define _RTE_BUS_IFPGA_H_
+
+/**
+ * @file
+ *
+ * RTE PCI Bus Interface
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <rte_bus.h>
+#include <rte_pci.h>
+
+/** Name of Intel FPGA Bus */
+#define IFPGA_BUS_NAME ifpga
+
+/* Forward declarations */
+struct rte_afu_device;
+struct rte_afu_driver;
+
+/** List of Intel AFU devices */
+TAILQ_HEAD(rte_afu_device_list, rte_afu_device);
+/** List of AFU drivers */
+TAILQ_HEAD(rte_afu_driver_list, rte_afu_driver);
+
+#define IFPGA_BUS_BITSTREAM_PATH_MAX_LEN 256
+
+/**
+ * A structure describing an ID for a AFU driver. Each driver provides a
+ * table of these IDs for each device that it supports.
+ */
+struct rte_afu_id {
+	struct rte_pci_addr pci_addr;
+	uint64_t uuid_low;
+	uint64_t uuid_high;
+	int      port;
+}__attribute__ ((packed));
+
+/**
+ * A structure pr configuration AFU driver. 
+ */
+
+struct rte_afu_pr_conf {
+	struct rte_afu_id afu_id;
+	int pr_enable;
+	char		bs_path[IFPGA_BUS_BITSTREAM_PATH_MAX_LEN];
+};
+
+#define AFU_PRI_STR_SIZE (PCI_PRI_STR_SIZE + 8)
+
+/**
+ * A structure describing a AFU device.
+ */
+struct rte_afu_device {
+	TAILQ_ENTRY(rte_afu_device) next;       /**< Next in device list. */
+	struct rte_device device;               /**< Inherit core device */
+	struct rte_rawdev *rawdev;
+	struct rte_afu_id id;                   /**< AFU id within FPGA. */
+	uint32_t num_region;						/**< number of regions found */
+	struct rte_mem_resource mem_resource[PCI_MAX_RESOURCE];
+						/**< PCI Memory Resource */
+	struct rte_intr_handle intr_handle;     /**< Interrupt handle */
+	struct rte_afu_driver *driver;          /**< Associated driver */
+	char path[IFPGA_BUS_BITSTREAM_PATH_MAX_LEN];
+} __attribute__ ((packed));
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_afu_device.
+ */
+#define RTE_DEV_TO_AFU(ptr) \
+	container_of(ptr, struct rte_afu_device, device)
+
+/**
+ * Initialisation function for the driver called during PCI probing.
+ */
+typedef int (afu_probe_t)(struct rte_afu_device *);
+
+/**
+ * Uninitialisation function for the driver called during hotplugging.
+ */
+typedef int (afu_remove_t)(struct rte_afu_device *);
+
+/**
+ * A structure describing a PCI device.
+ */
+struct rte_afu_driver {
+	TAILQ_ENTRY(rte_afu_driver) next;       /**< Next afu driver. */
+	struct rte_driver driver;               /**< Inherit core driver. */
+	afu_probe_t *probe;                     /**< Device Probe function. */
+	afu_remove_t *remove;                   /**< Device Remove function. */
+	struct rte_afu_id id;	                /**< AFU id within FPGA. */
+	uint32_t drv_flags;                     /**< Flags contolling handling of device. */
+};
+
+/**
+ * Structure describing the Intel FPGA bus
+ */
+struct rte_ifpga_bus {
+	struct rte_bus bus;               /**< Inherit the generic class */
+	struct rte_afu_device_list afu_list;  /**< List of AFU devices */
+	struct rte_afu_driver_list driver_list;  /**< List of FPGA drivers */
+};
+
+static inline const char *
+rte_ifpga_device_name(const struct rte_afu_device *afu)
+{
+	if (afu && afu->device.name)
+		return afu->device.name;
+	return NULL;
+}
+
+extern struct rte_ifpga_bus rte_ifpga_bus;
+
+void rte_ifpga_driver_register(struct rte_afu_driver *driver);
+void rte_ifpga_driver_unregister(struct rte_afu_driver *driver);
+
+#endif /* _RTE_BUS_IFPGA_H_ */
diff --git a/drivers/bus/ifpga/rte_bus_ifpga_version.map b/drivers/bus/ifpga/rte_bus_ifpga_version.map
new file mode 100644
index 0000000..4b2380a
--- /dev/null
+++ b/drivers/bus/ifpga/rte_bus_ifpga_version.map
@@ -0,0 +1,8 @@
+DPDK_17.11 {
+	global:
+
+    rte_ifpga_driver_register;
+    rte_ifpga_driver_unregister;
+    
+	local: *;
+};
-- 
1.8.3.1

Re: [dpdk-dev] [RFC 1/4] drivers/bus/ifpga:Intel FPGA Bus Lib Code

[Shreyansh Jain]

Hello Rosen,

I have some initial (and most of them trivial) comments inline...

On Tue, Mar 6, 2018 at 7:13 AM, Rosen Xu <rosen.xu@intel.com> wrote:
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> ---
>  drivers/bus/ifpga/Makefile                  |  64 ++++
>  drivers/bus/ifpga/ifpga_bus.c               | 527 ++++++++++++++++++++++++++++
>  drivers/bus/ifpga/ifpga_common.c            | 168 +++++++++
>  drivers/bus/ifpga/ifpga_common.h            |  46 +++
>  drivers/bus/ifpga/ifpga_logs.h              |  59 ++++
>  drivers/bus/ifpga/rte_bus_ifpga.h           | 153 ++++++++
>  drivers/bus/ifpga/rte_bus_ifpga_version.map |   8 +
>  7 files changed, 1025 insertions(+)
>  create mode 100644 drivers/bus/ifpga/Makefile
>  create mode 100644 drivers/bus/ifpga/ifpga_bus.c
>  create mode 100644 drivers/bus/ifpga/ifpga_common.c
>  create mode 100644 drivers/bus/ifpga/ifpga_common.h
>  create mode 100644 drivers/bus/ifpga/ifpga_logs.h
>  create mode 100644 drivers/bus/ifpga/rte_bus_ifpga.h
>  create mode 100644 drivers/bus/ifpga/rte_bus_ifpga_version.map
>
> diff --git a/drivers/bus/ifpga/Makefile b/drivers/bus/ifpga/Makefile
> new file mode 100644
> index 0000000..c71f186
> --- /dev/null
> +++ b/drivers/bus/ifpga/Makefile
> @@ -0,0 +1,64 @@
> +#   BSD LICENSE
> +#
> +#   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
> +#   All rights reserved.
> +#
> +#   Redistribution and use in source and binary forms, with or without
> +#   modification, are permitted provided that the following conditions
> +#   are met:
> +#
> +#     * Redistributions of source code must retain the above copyright
> +#       notice, this list of conditions and the following disclaimer.
> +#     * Redistributions in binary form must reproduce the above copyright
> +#       notice, this list of conditions and the following disclaimer in
> +#       the documentation and/or other materials provided with the
> +#       distribution.
> +#     * Neither the name of Intel Corporation nor the names of its
> +#       contributors may be used to endorse or promote products derived
> +#       from this software without specific prior written permission.
> +#
> +#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

As of 18.02, I think all licensing has moved to SPDX. Maybe in formal
patch you should change to that.

> +
> +include $(RTE_SDK)/mk/rte.vars.mk
> +
> +#
> +# library name
> +#
> +LIB = librte_bus_ifpga.a
> +LIBABIVER := 1
> +EXPORT_MAP := rte_bus_ifpga_version.map
> +
> +ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT),y)

I think this is copy-paste issue - isn't it?
(CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT)
I see that you have already enabled dynamic logging - in which case
you won't need this anyway.

> +CFLAGS += -O0 -g
> +CFLAGS += "-Wno-error"
> +else
> +CFLAGS += -O3
> +CFLAGS += $(WERROR_FLAGS)
> +endif
> +
> +CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
> +CFLAGS += -I$(RTE_SDK)/drivers/bus/pci
> +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
> +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common
> +#CFLAGS += -I$(RTE_SDK)/lib/librte_rawdev
> +#LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring -lrte_rawdev
> +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
> +#LDLIBS += -lrte_ethdev
> +
> +VPATH += $(SRCDIR)/base
> +
> +SRCS-y += \
> +        ifpga_bus.c \
> +        ifpga_common.c
> +
> +include $(RTE_SDK)/mk/rte.lib.mk
> diff --git a/drivers/bus/ifpga/ifpga_bus.c b/drivers/bus/ifpga/ifpga_bus.c
> new file mode 100644
> index 0000000..382d550
> --- /dev/null
> +++ b/drivers/bus/ifpga/ifpga_bus.c
> @@ -0,0 +1,527 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
> + *   Copyright 2013-2014 6WIND S.A.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */

Same as above - SPDX identifier rather than BSD license plate.

> +
> +
> +#include <string.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/queue.h>
> +#include <sys/mman.h>
> +#include <sys/types.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +
> +#include <rte_errno.h>
> +#include <rte_bus.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_memzone.h>
> +#include <rte_eal.h>
> +#include <rte_common.h>
> +
> +#include <rte_devargs.h>
> +#include <rte_pci.h>
> +#include <rte_bus_pci.h>
> +#include <rte_kvargs.h>
> +#include <rte_alarm.h>
> +
> +#include "rte_rawdev.h"
> +#include "rte_rawdev_pmd.h"
> +#include "rte_bus_ifpga.h"
> +#include "ifpga_logs.h"
> +#include "ifpga_common.h"
> +
> +int ifpga_bus_logtype;
> +
> +/*register a ifpga bus based driver */
> +void rte_ifpga_driver_register(struct rte_afu_driver *driver)
> +{
> +       RTE_VERIFY(driver);
> +
> +       TAILQ_INSERT_TAIL(&rte_ifpga_bus.driver_list, driver, next);
> +}
> +
> +/*un-register a fpga bus based driver */
> +void rte_ifpga_driver_unregister(struct rte_afu_driver *driver)
> +{
> +       TAILQ_REMOVE(&rte_ifpga_bus.driver_list, driver, next);
> +}
> +
> +static struct rte_afu_device *
> +ifpga_find_afu_dev(const struct rte_afu_id *afu_id)
> +{
> +       struct rte_afu_device *afu_dev = NULL;
> +
> +       TAILQ_FOREACH (afu_dev, &rte_ifpga_bus.afu_list, next) {
> +
> +        if(!ifpga_afu_id_cmp(&afu_dev->id, afu_id)) {
> +            return afu_dev;
> +        }
> +       }
> +       return NULL;
> +}
> +
> +static const char *valid_args[] = {
> +#define IFPGA_ARG_BDF          "bdf"
> +       IFPGA_ARG_BDF,
> +#define IFPGA_ARG_PORT         "port"
> +       IFPGA_ARG_PORT,
> +#define IFPGA_ARG_PATH         "path"
> +       IFPGA_ARG_PATH,
> +#define IFPGA_ARG_UUID_HIGH    "uuid_high"
> +       IFPGA_ARG_UUID_HIGH,
> +#define IFPGA_ARG_UUID_LOW     "uuid_low"
> +       IFPGA_ARG_UUID_LOW,
> +#define IFPGA_ARG_PR_ENABLE     "pr_enable"
> +       IFPGA_ARG_PR_ENABLE,
> +#define IFPGA_ARG_DEBUG         "debug"
> +       IFPGA_ARG_DEBUG,
> +       NULL
> +};
> +
> +/*
> + * Scan the content of the FPGA bus, and the devices in the devices
> + * list
> + */
> +static struct rte_afu_device *
> +rte_ifpga_scan_one(struct rte_devargs *devargs)
> +{
> +       struct rte_kvargs *kvlist = NULL;
> +       struct rte_bus *pci_bus = NULL;
> +       struct rte_device *dev = NULL;
> +       struct rte_rawdev *rawdev;
> +       struct rte_afu_device *afu_dev = NULL;
> +       struct rte_afu_pr_conf afu_pr_conf;
> +       char rawdev_name[RTE_RAWDEV_NAME_MAX_LEN];
> +       int ret;
> +       char *path = NULL;
> +       int pr_enable = 1;
> +       int debug = 0;
> +
> +    memset((char *)(&afu_pr_conf), 0, sizeof(struct rte_afu_pr_conf));
> +
> +    kvlist = rte_kvargs_parse(devargs->args, valid_args);
> +    if (!kvlist) {
> +       IFPGA_BUS_ERR("error when parsing param");
> +       goto end;
> +    }
> +
> +    if (rte_kvargs_count(kvlist, IFPGA_ARG_BDF) == 1) {
> +               if (rte_kvargs_process(kvlist, IFPGA_ARG_BDF,
> +                                      &ifpga_get_bdf_arg, &afu_pr_conf.afu_id.pci_addr) < 0) {
> +                       IFPGA_BUS_ERR("error to parse %s",
> +                                    IFPGA_ARG_BDF);
> +                       goto end;
> +               }
> +       } else {
> +               IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +                         IFPGA_ARG_PATH);
> +               goto end;
> +       }
> +
> +    if (rte_kvargs_count(kvlist, IFPGA_ARG_PORT) == 1) {
> +               if (rte_kvargs_process(kvlist, IFPGA_ARG_PORT,
> +                                      &ifpga_get_integer32_arg, &afu_pr_conf.afu_id.port) < 0) {
> +                       IFPGA_BUS_ERR("error to parse %s",
> +                                    IFPGA_ARG_PORT);
> +                       goto end;
> +               }
> +       }
> +       else {
> +               IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +                         IFPGA_ARG_PATH);
> +               goto end;
> +       }
> +
> +    if (rte_kvargs_count(kvlist, IFPGA_ARG_PATH) == 1) {
> +               if (rte_kvargs_process(kvlist, IFPGA_ARG_PATH,
> +                                      &ifpga_get_string_arg, &path) < 0) {
> +                       IFPGA_BUS_ERR("error to parse %s",
> +                                    IFPGA_ARG_PATH);
> +                       goto end;
> +               }
> +       } else {
> +               IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +                         IFPGA_ARG_PATH);
> +               goto end;
> +       }
> +
> +       if (rte_kvargs_count(kvlist, IFPGA_ARG_UUID_HIGH) == 1) {
> +               if (rte_kvargs_process(kvlist, IFPGA_ARG_UUID_HIGH,
> +                                      &ifpga_get_integer64_arg, &afu_pr_conf.afu_id.uuid_high) < 0) {
> +                       IFPGA_BUS_ERR("error to parse %s",
> +                                    IFPGA_ARG_UUID_HIGH);
> +                       goto end;
> +               }
> +       } else {
> +               IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +                         IFPGA_ARG_PATH);
> +               goto end;
> +       }
> +
> +       if (rte_kvargs_count(kvlist, IFPGA_ARG_UUID_LOW) == 1) {
> +               if (rte_kvargs_process(kvlist, IFPGA_ARG_UUID_LOW,
> +                                      &ifpga_get_integer64_arg, &afu_pr_conf.afu_id.uuid_low) < 0) {
> +                       IFPGA_BUS_ERR("error to parse %s",
> +                                    IFPGA_ARG_UUID_LOW);
> +                       goto end;
> +               }
> +       } else {
> +               IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +                         IFPGA_ARG_PATH);
> +               goto end;
> +       }
> +
> +       if (rte_kvargs_count(kvlist, IFPGA_ARG_PR_ENABLE) == 1) {
> +               if (rte_kvargs_process(kvlist, IFPGA_ARG_PR_ENABLE,
> +                                      &ifpga_get_integer32_arg, &pr_enable) < 0) {
> +                       IFPGA_BUS_ERR("error to parse %s",
> +                                    IFPGA_ARG_UUID_HIGH);
> +                       goto end;
> +               }
> +       }
> +
> +       if (rte_kvargs_count(kvlist, IFPGA_ARG_DEBUG) == 1) {
> +               if (rte_kvargs_process(kvlist, IFPGA_ARG_DEBUG,
> +                                      &ifpga_get_integer32_arg, &debug) < 0) {
> +                       IFPGA_BUS_ERR("error to parse %s",
> +                                    IFPGA_ARG_UUID_HIGH);
> +                       goto end;
> +               }
> +       }
> +
> +       if(!debug)
> +       {
> +               pci_bus = rte_bus_find_by_name("pci");
> +           if (pci_bus == NULL) {
> +               IFPGA_BUS_ERR("unable to find PCI bus\n");
> +               goto end;
> +           }
> +
> +           dev = pci_bus->find_device(NULL, ifpga_pci_addr_cmp, &afu_pr_conf.afu_id.pci_addr);
> +           if (dev == NULL) {
> +               IFPGA_BUS_ERR("unable to find PCI device\n");
> +               goto end;
> +           }
> +       }

I am slightly confused here. I thought this is not on PCI bus. But,
you are searching for the iFPGA on the PCI bus. If so, you still need
a new bus?
In case the devices are still visible on the PCI bus itself, a rawdev
driver can easily identify (probe time) them and attach itself to
them.
Is my understanding wrong? (Assuming that no PCI driver will identify
the devices as its own and eventually the rawdev driver would claim
them.)

> +       else
> +       {
> +               IFPGA_BUS_DEBUG("pci_addr domain   : %x\n",   afu_pr_conf.afu_id.pci_addr.domain);
> +               IFPGA_BUS_DEBUG("pci_addr bus      : %x\n",   afu_pr_conf.afu_id.pci_addr.bus);
> +               IFPGA_BUS_DEBUG("pci_addr devid    : %x\n",   afu_pr_conf.afu_id.pci_addr.devid);
> +               IFPGA_BUS_DEBUG("pci_addr function : %x\n",   afu_pr_conf.afu_id.pci_addr.function);
> +
> +               IFPGA_BUS_DEBUG("uuid_low          : %lx\n", afu_pr_conf.afu_id.uuid_low);
> +               IFPGA_BUS_DEBUG("uuid_high         : %lx\n", afu_pr_conf.afu_id.uuid_high);
> +
> +               IFPGA_BUS_DEBUG("afu port          : %x\n",   afu_pr_conf.afu_id.port);
> +       }
> +
> +       if (ifpga_find_afu_dev(&afu_pr_conf.afu_id))
> +               goto end;
> +
> +       memset(rawdev_name, sizeof(rawdev_name), 0);
> +       snprintf(rawdev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%x:%x", afu_pr_conf.afu_id.pci_addr.bus, afu_pr_conf.afu_id.pci_addr.devid, afu_pr_conf.afu_id.pci_addr.function);
> +       rawdev = rte_rawdev_pmd_get_named_dev(rawdev_name);
> +       if(!rawdev)
> +               goto end;
> +
> +       rawdev->dev_ops->dev_start(rawdev);
> +
> +       if (pr_enable)
> +       {
> +               ret=rawdev->dev_ops->firmware_load(rawdev, &afu_pr_conf);
> +               if (ret) {
> +               printf("do pr error\n");
> +               return NULL;
> +           }
> +       }
> +
> +       afu_dev = calloc(1, sizeof(*afu_dev));
> +       if (!afu_dev)
> +               goto end;
> +
> +       afu_dev->device.devargs = devargs;
> +       afu_dev->device.numa_node = SOCKET_ID_ANY;
> +       afu_dev->device.name = devargs->name;
> +       afu_dev->rawdev = rawdev;
> +       afu_dev->id.pci_addr.domain = afu_pr_conf.afu_id.pci_addr.domain;
> +       afu_dev->id.pci_addr.bus = afu_pr_conf.afu_id.pci_addr.bus;
> +       afu_dev->id.pci_addr.devid = afu_pr_conf.afu_id.pci_addr.devid;
> +       afu_dev->id.pci_addr.function = afu_pr_conf.afu_id.pci_addr.function;
> +       afu_dev->id.uuid_low  = afu_pr_conf.afu_id.uuid_low;
> +       afu_dev->id.uuid_high = afu_pr_conf.afu_id.uuid_high;
> +       afu_dev->id.port      = afu_pr_conf.afu_id.port;
> +
> +       rawdev->dev_ops->dev_info_get(rawdev, afu_dev);
> +
> +    return afu_dev;
> +
> +end:
> +       if (kvlist)
> +               rte_kvargs_free(kvlist);
> +       if (path)
> +               free(path);
> +
> +       return NULL;
> +}
> +
> +/*
> + * Scan the content of the FPGA bus, and the devices in the devices
> + * list
> + */
> +static int
> +rte_ifpga_scan(void)
> +{
> +       struct rte_bus *pci_bus;
> +       struct rte_afu_device *afu_dev;
> +       struct rte_devargs *devargs;
> +
> +    pci_bus = rte_bus_find_by_name("pci");
> +       if (pci_bus == NULL) {
> +               IFPGA_BUS_ERR("unable to find PCI bus\n");
> +               return -1;
> +       }
> +
> +       /* for virtual devices we scan the devargs_list populated via cmdline */
> +       TAILQ_FOREACH(devargs, &devargs_list, next) {
> +
> +               if (devargs->bus != &rte_ifpga_bus.bus)
> +                       continue;
> +
> +               afu_dev = rte_ifpga_scan_one(devargs);
> +
> +           if(afu_dev)
> +                   TAILQ_INSERT_TAIL(&rte_ifpga_bus.afu_list, afu_dev, next);
> +       }
> +
> +       return 0;
> +}
> +
> +static int
> +ifpga_probe_one_driver(struct rte_afu_driver *drv,
> +                       struct rte_afu_device *afu_dev)
> +{
> +       int ret;
> +
> +    if((drv->id.pci_addr.bus == afu_dev->id.pci_addr.bus) &&
> +       (drv->id.pci_addr.devid == afu_dev->id.pci_addr.devid) &&
> +       (drv->id.pci_addr.function == afu_dev->id.pci_addr.function) &&
> +       (drv->id.uuid_low == afu_dev->id.uuid_low) &&
> +       (drv->id.uuid_high == afu_dev->id.uuid_high) &&
> +       (drv->id.port == afu_dev->id.port)) {
> +
> +               afu_dev->driver = drv;
> +
> +               /* call the driver probe() function */
> +               ret = drv->probe(afu_dev);
> +               if (ret)
> +                       afu_dev->driver = NULL;
> +               return ret;
> +    }
> +
> +       /* return positive value if driver doesn't support this device */
> +       return 1;
> +}
> +
> +static int
> +ifpga_probe_all_drivers(struct rte_afu_device *afu_dev)
> +{
> +       const char *name;
> +       struct rte_afu_driver *drv = NULL;
> +       int rc;
> +
> +       if (afu_dev == NULL)
> +               return -1;
> +
> +       /* Check if a driver is already loaded */
> +       if (afu_dev->driver != NULL)
> +               return 0;
> +
> +    name = rte_ifpga_device_name(afu_dev);
> +    IFPGA_BUS_DEBUG("Search driver %s to probe device %s\n", name,
> +               rte_ifpga_device_name(afu_dev));
> +
> +       TAILQ_FOREACH(drv, &rte_ifpga_bus.driver_list, next) {
> +               rc = ifpga_probe_one_driver(drv, afu_dev);
> +               if (rc < 0)
> +                       /* negative value is an error */
> +                       return -1;
> +               if (rc > 0)
> +                       /* positive value means driver doesn't support it */
> +                       continue;
> +               return 0;
> +       }
> +       return 1;
> + }
> +
> +/*
> + * Scan the content of the PCI bus, and call the probe() function for
> + * all registered drivers that have a matching entry in its id_table
> + * for discovered devices.
> + */
> +static int
> +rte_ifpga_probe(void)
> +{
> +       struct rte_afu_device *afu_dev = NULL;
> +       int ret = 0;
> +
> +       TAILQ_FOREACH (afu_dev, &rte_ifpga_bus.afu_list, next) {
> +
> +        if (afu_dev->device.driver)
> +                       continue;
> +
> +               ret = ifpga_probe_all_drivers(afu_dev);
> +               if (ret < 0)
> +                   IFPGA_BUS_ERR("failed to initialize %s device\n",
> +                               rte_ifpga_device_name(afu_dev));
> +       }
> +
> +       return 0;
> +}
> +
> +static int
> +rte_ifpga_plug(struct rte_device *dev)
> +{
> +       return ifpga_probe_all_drivers(RTE_DEV_TO_AFU(dev));
> +}
> +
> +static int ifpga_remove_driver(struct rte_afu_device *afu_dev)
> +{
> +       const char *name;
> +       const struct rte_afu_driver *driver;
> +
> +    name = rte_ifpga_device_name(afu_dev);
> +       if (!afu_dev->device.driver) {
> +               IFPGA_BUS_DEBUG("no driver attach to device %s\n", name);
> +               return 1;
> +       }
> +
> +       driver = container_of(afu_dev->device.driver, const struct rte_afu_driver,
> +               driver);
> +       return driver->remove(afu_dev);
> +}
> +
> +static int
> +rte_ifpga_unplug(struct rte_device *dev)
> +{
> +       struct rte_afu_device *afu_dev;
> +       struct rte_devargs *devargs;
> +       int ret;
> +
> +       if (dev == NULL)
> +               return -EINVAL;
> +
> +       afu_dev = RTE_DEV_TO_AFU(dev);
> +       if (!dev)
> +               return -ENOENT;
> +
> +       devargs = dev->devargs;
> +
> +       ret = ifpga_remove_driver(afu_dev);
> +       if (ret)
> +               return ret;
> +
> +       TAILQ_REMOVE(&rte_ifpga_bus.afu_list, afu_dev, next);
> +
> +       TAILQ_REMOVE(&devargs_list, devargs, next);
> +
> +       free(devargs->args);
> +       free(devargs);
> +       free(afu_dev);
> +       return 0;
> +
> +}
> +
> +static struct rte_device *
> +rte_ifpga_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
> +                                                       const void *data)
> +{
> +       struct rte_afu_device *afu_dev;
> +
> +       TAILQ_FOREACH(afu_dev, &rte_ifpga_bus.afu_list, next) {
> +               if (start && &afu_dev->device == start) {
> +                       start = NULL;
> +                       continue;
> +               }
> +               if (cmp(&afu_dev->device, data) == 0)
> +                       return &afu_dev->device;
> +       }
> +       return NULL;
> +}
> +static int
> +rte_ifpga_parse(const char *name, void *addr)
> +{
> +       struct rte_afu_driver **out = addr;
> +       struct rte_afu_driver *driver = NULL;
> +
> +       TAILQ_FOREACH(driver, &rte_ifpga_bus.driver_list, next) {
> +               if (strncmp(driver->driver.name, name,
> +                           strlen(driver->driver.name)) == 0)
> +                       break;
> +               if (driver->driver.alias &&
> +                   strncmp(driver->driver.alias, name,
> +                           strlen(driver->driver.alias)) == 0)
> +                       break;
> +       }
> +       if (driver != NULL &&
> +           addr != NULL)
> +               *out = driver;
> +       return driver == NULL;
> +}
> +
> +struct rte_ifpga_bus rte_ifpga_bus = {
> +        .bus = {
> +                .scan        = rte_ifpga_scan,
> +                .probe       = rte_ifpga_probe,
> +                .find_device = rte_ifpga_find_device,
> +            .plug        = rte_ifpga_plug,
> +            .unplug      = rte_ifpga_unplug,
> +            .parse       = rte_ifpga_parse,
> +        },
> +       .afu_list    = TAILQ_HEAD_INITIALIZER(rte_ifpga_bus.afu_list),
> +       .driver_list = TAILQ_HEAD_INITIALIZER(rte_ifpga_bus.driver_list),
> + };
> +
> +RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus.bus);
> +
> +RTE_INIT(ifpga_init_log)
> +{
> +       ifpga_bus_logtype = rte_log_register("bus.ifpga");
> +       if (ifpga_bus_logtype >= 0)
> +               rte_log_set_level(ifpga_bus_logtype, RTE_LOG_NOTICE);
> +}
> +
> diff --git a/drivers/bus/ifpga/ifpga_common.c b/drivers/bus/ifpga/ifpga_common.c
> new file mode 100644
> index 0000000..2aa6bb0
> --- /dev/null
> +++ b/drivers/bus/ifpga/ifpga_common.c
> @@ -0,0 +1,168 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
> + *   Copyright 2013-2014 6WIND S.A.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +
> +#include <string.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/queue.h>
> +#include <sys/mman.h>
> +#include <sys/types.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +
> +#include <rte_errno.h>
> +#include <rte_bus.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_memzone.h>
> +#include <rte_eal.h>
> +#include <rte_common.h>
> +
> +#include <rte_devargs.h>
> +#include <rte_pci.h>
> +#include <rte_bus_pci.h>
> +#include <rte_kvargs.h>
> +#include <rte_alarm.h>
> +
> +#include "rte_bus_ifpga.h"
> +#include "ifpga_logs.h"
> +#include "ifpga_common.h"
> +
> +int ifpga_get_string_arg(const char *key __rte_unused,
> +                            const char *value, void *extra_args)
> +{
> +       if (!value || !extra_args)
> +               return -EINVAL;
> +
> +       *(char **)extra_args = strdup(value);
> +
> +       if (!*(char **)extra_args)
> +               return -ENOMEM;
> +
> +       return 0;
> +}
> +int ifpga_get_integer32_arg(const char *key __rte_unused, const char *value, void *extra_args)
> +{
> +       if (!value || !extra_args)
> +               return -EINVAL;
> +
> +       *(int *)extra_args = strtoull(value, NULL, 0);
> +
> +       return 0;
> +}
> +int ifpga_get_integer64_arg(const char *key __rte_unused, const char *value, void *extra_args)
> +{
> +       if (!value || !extra_args)
> +               return -EINVAL;
> +
> +       *(uint64_t *)extra_args = strtoull(value, NULL, 0);
> +
> +       return 0;
> +}
> +int ifpga_get_unsigned_long(const char *str, int base)
> +{
> +       unsigned long num;
> +       char *end = NULL;
> +
> +       errno = 0;
> +
> +       num = strtoul(str, &end, base);
> +       if ((str[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
> +               return -1;
> +
> +       return num;
> +
> +}
> +int ifpga_get_bdf_arg(const char *key __rte_unused, const char *value, void *extra_args)
> +{
> +#define MAX_PATH_LEN 1024
> +       struct rte_pci_addr *addr;
> +       int num[4];
> +       char str[MAX_PATH_LEN];
> +       int i, j;
> +
> +       if (!value || !extra_args)
> +               return -EINVAL;
> +
> +    addr = (struct rte_pci_addr *)extra_args;
> +       strcpy(str, value);
> +       memset(num, 0, 4 * sizeof(num[0]));
> +       i = strlen(str) - 1;
> +       j = 3;
> +       while (i > 0 && j >= 0) {
> +               while ((str[i - 1] != ':' && str[i - 1] != '.') && i > 0)
> +                       i--;
> +               num[j--] = ifpga_get_unsigned_long(&str[i], 16);
> +               i--;
> +               if (i >= 0)
> +                       str[i] = '\0';
> +       }
> +       addr->domain = num[0];
> +       addr->bus = num[1];
> +       addr->devid = num[2];
> +       addr->function = num[3];
> +       printf("[%s]: bdf %04d:%02d:%02d.%02d\n",
> +                       __func__,
> +                       addr->domain,
> +                       addr->bus,
> +                       addr->devid,
> +                       addr->function);
> +
> +       return 0;
> +}
> +int ifpga_afu_id_cmp(const struct rte_afu_id *afu_id0, const struct rte_afu_id *afu_id1)
> +{
> +       if((afu_id0->pci_addr.bus       == afu_id1->pci_addr.bus) &&
> +       (afu_id0->pci_addr.devid     == afu_id1->pci_addr.devid) &&
> +       (afu_id0->pci_addr.function  == afu_id1->pci_addr.function) &&
> +       (afu_id0->uuid_low           == afu_id1->uuid_low) &&
> +       (afu_id0->uuid_high          == afu_id1->uuid_high) &&
> +       (afu_id0->port               == afu_id1->port)) {
> +
> +        return 0;
> +    }
> +    else
> +       return 1;
> +}
> +int ifpga_pci_addr_cmp(const struct rte_device *dev, const void *_pci_addr)
> +{
> +       struct rte_pci_device *pdev;
> +       const struct rte_pci_addr *paddr = _pci_addr;
> +
> +       pdev = RTE_DEV_TO_PCI(*(struct rte_device **)(void *)&dev);
> +       return rte_eal_compare_pci_addr(&pdev->addr, paddr);
> +}
> diff --git a/drivers/bus/ifpga/ifpga_common.h b/drivers/bus/ifpga/ifpga_common.h
> new file mode 100644
> index 0000000..590ff27
> --- /dev/null
> +++ b/drivers/bus/ifpga/ifpga_common.h
> @@ -0,0 +1,46 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2017 Intel Corporation. All rights reserved.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +#ifndef _IFPGA_COMMON_H_
> +#define _IFPGA_COMMON_H_
> +
> +#include <rte_pci.h>
> +
> +extern int ifpga_get_string_arg(const char *key __rte_unused, const char *value, void *extra_args);
> +extern int ifpga_get_integer32_arg(const char *key __rte_unused, const char *value, void *extra_args);
> +extern int ifpga_get_integer64_arg(const char *key __rte_unused, const char *value, void *extra_args);
> +extern int ifpga_get_unsigned_long(const char *str, int base);
> +extern int ifpga_get_bdf_arg(const char *key __rte_unused, const char *value, void *extra_args);
> +extern int ifpga_afu_id_cmp(const struct rte_afu_id *afu_id0, const struct rte_afu_id *afu_id1);
> +extern int ifpga_pci_addr_cmp(const struct rte_device *dev, const void *_pci_addr);
> +
> +#endif /* _IFPGA_COMMON_H_ */
> diff --git a/drivers/bus/ifpga/ifpga_logs.h b/drivers/bus/ifpga/ifpga_logs.h
> new file mode 100644
> index 0000000..eb56393
> --- /dev/null
> +++ b/drivers/bus/ifpga/ifpga_logs.h
> @@ -0,0 +1,59 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2017 Intel Corporation. All rights reserved.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +#ifndef _IFPGA_BUS_LOGS_H_
> +#define _IFPGA_BUS_LOGS_H_
> +
> +#include <rte_log.h>
> +
> +extern int ifpga_bus_logtype;
> +
> +#define IFPGA_LOG(level, fmt, args...) \
> +       rte_log(RTE_LOG_ ## level, ifpga_bus_logtype, "%s(): " fmt "\n", \
> +               __func__, ##args)
> +
> +#define IFPGA_BUS_LOG(level, fmt, args...) \
> +       rte_log(RTE_LOG_ ## level, ifpga_bus_logtype, "%s(): " fmt "\n", \
> +               __func__, ##args)
> +
> +#define IFPGA_BUS_FUNC_TRACE() IFPGA_BUS_LOG(DEBUG, ">>")
> +
> +#define IFPGA_BUS_DEBUG(fmt, args...) \
> +       IFPGA_BUS_LOG(DEBUG, fmt, ## args)
> +#define IFPGA_BUS_INFO(fmt, args...) \
> +       IFPGA_BUS_LOG(INFO, fmt, ## args)
> +#define IFPGA_BUS_ERR(fmt, args...) \
> +       IFPGA_BUS_LOG(ERR, fmt, ## args)
> +#define IFPGA_BUS_WARN(fmt, args...) \
> +       IFPGA_BUS_LOG(WARNING, fmt, ## args)
> +
> +#endif /* _IFPGA_BUS_LOGS_H_ */
> diff --git a/drivers/bus/ifpga/rte_bus_ifpga.h b/drivers/bus/ifpga/rte_bus_ifpga.h
> new file mode 100644
> index 0000000..5491a45
> --- /dev/null
> +++ b/drivers/bus/ifpga/rte_bus_ifpga.h
> @@ -0,0 +1,153 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2016 RehiveTech. All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of RehiveTech nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#ifndef _RTE_BUS_IFPGA_H_
> +#define _RTE_BUS_IFPGA_H_
> +
> +/**
> + * @file
> + *
> + * RTE PCI Bus Interface

Bus IFPGA

> + */
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include <rte_bus.h>
> +#include <rte_pci.h>
> +
> +/** Name of Intel FPGA Bus */
> +#define IFPGA_BUS_NAME ifpga
> +
> +/* Forward declarations */
> +struct rte_afu_device;
> +struct rte_afu_driver;
> +
> +/** List of Intel AFU devices */
> +TAILQ_HEAD(rte_afu_device_list, rte_afu_device);
> +/** List of AFU drivers */
> +TAILQ_HEAD(rte_afu_driver_list, rte_afu_driver);
> +
> +#define IFPGA_BUS_BITSTREAM_PATH_MAX_LEN 256
> +
> +/**
> + * A structure describing an ID for a AFU driver. Each driver provides a
> + * table of these IDs for each device that it supports.
> + */
> +struct rte_afu_id {
> +       struct rte_pci_addr pci_addr;
> +       uint64_t uuid_low;
> +       uint64_t uuid_high;
> +       int      port;
> +}__attribute__ ((packed));
> +
> +/**
> + * A structure pr configuration AFU driver.
> + */
> +
> +struct rte_afu_pr_conf {
> +       struct rte_afu_id afu_id;
> +       int pr_enable;
> +       char            bs_path[IFPGA_BUS_BITSTREAM_PATH_MAX_LEN];
> +};
> +
> +#define AFU_PRI_STR_SIZE (PCI_PRI_STR_SIZE + 8)
> +
> +/**
> + * A structure describing a AFU device.
> + */
> +struct rte_afu_device {
> +       TAILQ_ENTRY(rte_afu_device) next;       /**< Next in device list. */
> +       struct rte_device device;               /**< Inherit core device */
> +       struct rte_rawdev *rawdev;
> +       struct rte_afu_id id;                   /**< AFU id within FPGA. */
> +       uint32_t num_region;                                            /**< number of regions found */
> +       struct rte_mem_resource mem_resource[PCI_MAX_RESOURCE];
> +                                               /**< PCI Memory Resource */
> +       struct rte_intr_handle intr_handle;     /**< Interrupt handle */
> +       struct rte_afu_driver *driver;          /**< Associated driver */
> +       char path[IFPGA_BUS_BITSTREAM_PATH_MAX_LEN];
> +} __attribute__ ((packed));
> +
> +/**
> + * @internal
> + * Helper macro for drivers that need to convert to struct rte_afu_device.
> + */
> +#define RTE_DEV_TO_AFU(ptr) \
> +       container_of(ptr, struct rte_afu_device, device)
> +
> +/**
> + * Initialisation function for the driver called during PCI probing.
> + */
> +typedef int (afu_probe_t)(struct rte_afu_device *);
> +
> +/**
> + * Uninitialisation function for the driver called during hotplugging.
> + */
> +typedef int (afu_remove_t)(struct rte_afu_device *);
> +
> +/**
> + * A structure describing a PCI device.

Trivial: An AFU device, probably. (sorry for nitpicking - I know this
is RFC - so ignore if you already know about these changes.)

> + */
> +struct rte_afu_driver {
> +       TAILQ_ENTRY(rte_afu_driver) next;       /**< Next afu driver. */
> +       struct rte_driver driver;               /**< Inherit core driver. */
> +       afu_probe_t *probe;                     /**< Device Probe function. */
> +       afu_remove_t *remove;                   /**< Device Remove function. */
> +       struct rte_afu_id id;                   /**< AFU id within FPGA. */
> +       uint32_t drv_flags;                     /**< Flags contolling handling of device. */
> +};
> +
> +/**
> + * Structure describing the Intel FPGA bus
> + */
> +struct rte_ifpga_bus {
> +       struct rte_bus bus;               /**< Inherit the generic class */
> +       struct rte_afu_device_list afu_list;  /**< List of AFU devices */
> +       struct rte_afu_driver_list driver_list;  /**< List of FPGA drivers */
> +};
> +
> +static inline const char *
> +rte_ifpga_device_name(const struct rte_afu_device *afu)
> +{
> +       if (afu && afu->device.name)
> +               return afu->device.name;
> +       return NULL;
> +}
> +
> +extern struct rte_ifpga_bus rte_ifpga_bus;
> +
> +void rte_ifpga_driver_register(struct rte_afu_driver *driver);
> +void rte_ifpga_driver_unregister(struct rte_afu_driver *driver);
> +
> +#endif /* _RTE_BUS_IFPGA_H_ */
> diff --git a/drivers/bus/ifpga/rte_bus_ifpga_version.map b/drivers/bus/ifpga/rte_bus_ifpga_version.map
> new file mode 100644
> index 0000000..4b2380a
> --- /dev/null
> +++ b/drivers/bus/ifpga/rte_bus_ifpga_version.map
> @@ -0,0 +1,8 @@
> +DPDK_17.11 {
> +       global:
> +
> +    rte_ifpga_driver_register;
> +    rte_ifpga_driver_unregister;
> +
> +       local: *;
> +};
> --
> 1.8.3.1
>

Re: [dpdk-dev] [RFC 1/4] drivers/bus/ifpga:Intel FPGA Bus Lib Code

[Xu, Rosen]

-----Original Message-----
From: Shreyansh Jain [mailto:shreyansh.jain@nxp.com] 
Sent: Tuesday, March 06, 2018 14:10
To: Xu, Rosen <rosen.xu@intel.com>
Cc: dev@dpdk.org; Doherty, Declan <declan.doherty@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>
Subject: Re: [dpdk-dev] [RFC 1/4] drivers/bus/ifpga:Intel FPGA Bus Lib Code

Hello Rosen,

I have some initial (and most of them trivial) comments inline...

On Tue, Mar 6, 2018 at 7:13 AM, Rosen Xu <rosen.xu@intel.com> wrote:
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> ---
>  drivers/bus/ifpga/Makefile                  |  64 ++++
>  drivers/bus/ifpga/ifpga_bus.c               | 527 ++++++++++++++++++++++++++++
>  drivers/bus/ifpga/ifpga_common.c            | 168 +++++++++
>  drivers/bus/ifpga/ifpga_common.h            |  46 +++
>  drivers/bus/ifpga/ifpga_logs.h              |  59 ++++
>  drivers/bus/ifpga/rte_bus_ifpga.h           | 153 ++++++++
>  drivers/bus/ifpga/rte_bus_ifpga_version.map |   8 +
>  7 files changed, 1025 insertions(+)
>  create mode 100644 drivers/bus/ifpga/Makefile  create mode 100644 
> drivers/bus/ifpga/ifpga_bus.c  create mode 100644 
> drivers/bus/ifpga/ifpga_common.c  create mode 100644 
> drivers/bus/ifpga/ifpga_common.h  create mode 100644 
> drivers/bus/ifpga/ifpga_logs.h  create mode 100644 
> drivers/bus/ifpga/rte_bus_ifpga.h  create mode 100644 
> drivers/bus/ifpga/rte_bus_ifpga_version.map
>
> diff --git a/drivers/bus/ifpga/Makefile b/drivers/bus/ifpga/Makefile 
> new file mode 100644 index 0000000..c71f186
> --- /dev/null
> +++ b/drivers/bus/ifpga/Makefile
> @@ -0,0 +1,64 @@
> +#   BSD LICENSE
> +#
> +#   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
> +#   All rights reserved.
> +#
> +#   Redistribution and use in source and binary forms, with or without
> +#   modification, are permitted provided that the following conditions
> +#   are met:
> +#
> +#     * Redistributions of source code must retain the above copyright
> +#       notice, this list of conditions and the following disclaimer.
> +#     * Redistributions in binary form must reproduce the above copyright
> +#       notice, this list of conditions and the following disclaimer in
> +#       the documentation and/or other materials provided with the
> +#       distribution.
> +#     * Neither the name of Intel Corporation nor the names of its
> +#       contributors may be used to endorse or promote products derived
> +#       from this software without specific prior written permission.
> +#
> +#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

As of 18.02, I think all licensing has moved to SPDX. Maybe in formal patch you should change to that.
Rosen:  I will modify it

> +
> +include $(RTE_SDK)/mk/rte.vars.mk
> +
> +#
> +# library name
> +#
> +LIB = librte_bus_ifpga.a
> +LIBABIVER := 1
> +EXPORT_MAP := rte_bus_ifpga_version.map
> +
> +ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT),y)

I think this is copy-paste issue - isn't it?
Rosen: yes
(CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT)
I see that you have already enabled dynamic logging - in which case you won't need this anyway.
Rosen: ok

> +CFLAGS += -O0 -g
> +CFLAGS += "-Wno-error"
> +else
> +CFLAGS += -O3
> +CFLAGS += $(WERROR_FLAGS)
> +endif
> +
> +CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga CFLAGS += 
> +-I$(RTE_SDK)/drivers/bus/pci CFLAGS += 
> +-I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
> +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common
> +#CFLAGS += -I$(RTE_SDK)/lib/librte_rawdev #LDLIBS += -lrte_eal 
> +-lrte_mbuf -lrte_mempool -lrte_ring -lrte_rawdev LDLIBS += -lrte_eal 
> +-lrte_mbuf -lrte_mempool -lrte_ring #LDLIBS += -lrte_ethdev
> +
> +VPATH += $(SRCDIR)/base
> +
> +SRCS-y += \
> +        ifpga_bus.c \
> +        ifpga_common.c
> +
> +include $(RTE_SDK)/mk/rte.lib.mk
> diff --git a/drivers/bus/ifpga/ifpga_bus.c 
> b/drivers/bus/ifpga/ifpga_bus.c new file mode 100644 index 
> 0000000..382d550
> --- /dev/null
> +++ b/drivers/bus/ifpga/ifpga_bus.c
> @@ -0,0 +1,527 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
> + *   Copyright 2013-2014 6WIND S.A.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */

Same as above - SPDX identifier rather than BSD license plate.
Rosen: I will modify it

> +
> +
> +#include <string.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/queue.h>
> +#include <sys/mman.h>
> +#include <sys/types.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +
> +#include <rte_errno.h>
> +#include <rte_bus.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_memzone.h>
> +#include <rte_eal.h>
> +#include <rte_common.h>
> +
> +#include <rte_devargs.h>
> +#include <rte_pci.h>
> +#include <rte_bus_pci.h>
> +#include <rte_kvargs.h>
> +#include <rte_alarm.h>
> +
> +#include "rte_rawdev.h"
> +#include "rte_rawdev_pmd.h"
> +#include "rte_bus_ifpga.h"
> +#include "ifpga_logs.h"
> +#include "ifpga_common.h"
> +
> +int ifpga_bus_logtype;
> +
> +/*register a ifpga bus based driver */ void 
> +rte_ifpga_driver_register(struct rte_afu_driver *driver) {
> +       RTE_VERIFY(driver);
> +
> +       TAILQ_INSERT_TAIL(&rte_ifpga_bus.driver_list, driver, next); }
> +
> +/*un-register a fpga bus based driver */ void 
> +rte_ifpga_driver_unregister(struct rte_afu_driver *driver) {
> +       TAILQ_REMOVE(&rte_ifpga_bus.driver_list, driver, next); }
> +
> +static struct rte_afu_device *
> +ifpga_find_afu_dev(const struct rte_afu_id *afu_id) {
> +       struct rte_afu_device *afu_dev = NULL;
> +
> +       TAILQ_FOREACH (afu_dev, &rte_ifpga_bus.afu_list, next) {
> +
> +        if(!ifpga_afu_id_cmp(&afu_dev->id, afu_id)) {
> +            return afu_dev;
> +        }
> +       }
> +       return NULL;
> +}
> +
> +static const char *valid_args[] = {
> +#define IFPGA_ARG_BDF          "bdf"
> +       IFPGA_ARG_BDF,
> +#define IFPGA_ARG_PORT         "port"
> +       IFPGA_ARG_PORT,
> +#define IFPGA_ARG_PATH         "path"
> +       IFPGA_ARG_PATH,
> +#define IFPGA_ARG_UUID_HIGH    "uuid_high"
> +       IFPGA_ARG_UUID_HIGH,
> +#define IFPGA_ARG_UUID_LOW     "uuid_low"
> +       IFPGA_ARG_UUID_LOW,
> +#define IFPGA_ARG_PR_ENABLE     "pr_enable"
> +       IFPGA_ARG_PR_ENABLE,
> +#define IFPGA_ARG_DEBUG         "debug"
> +       IFPGA_ARG_DEBUG,
> +       NULL
> +};
> +
> +/*
> + * Scan the content of the FPGA bus, and the devices in the devices
> + * list
> + */
> +static struct rte_afu_device *
> +rte_ifpga_scan_one(struct rte_devargs *devargs) {
> +       struct rte_kvargs *kvlist = NULL;
> +       struct rte_bus *pci_bus = NULL;
> +       struct rte_device *dev = NULL;
> +       struct rte_rawdev *rawdev;
> +       struct rte_afu_device *afu_dev = NULL;
> +       struct rte_afu_pr_conf afu_pr_conf;
> +       char rawdev_name[RTE_RAWDEV_NAME_MAX_LEN];
> +       int ret;
> +       char *path = NULL;
> +       int pr_enable = 1;
> +       int debug = 0;
> +
> +    memset((char *)(&afu_pr_conf), 0, sizeof(struct 
> + rte_afu_pr_conf));
> +
> +    kvlist = rte_kvargs_parse(devargs->args, valid_args);
> +    if (!kvlist) {
> +       IFPGA_BUS_ERR("error when parsing param");
> +       goto end;
> +    }
> +
> +    if (rte_kvargs_count(kvlist, IFPGA_ARG_BDF) == 1) {
> +               if (rte_kvargs_process(kvlist, IFPGA_ARG_BDF,
> +                                      &ifpga_get_bdf_arg, &afu_pr_conf.afu_id.pci_addr) < 0) {
> +                       IFPGA_BUS_ERR("error to parse %s",
> +                                    IFPGA_ARG_BDF);
> +                       goto end;
> +               }
> +       } else {
> +               IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +                         IFPGA_ARG_PATH);
> +               goto end;
> +       }
> +
> +    if (rte_kvargs_count(kvlist, IFPGA_ARG_PORT) == 1) {
> +               if (rte_kvargs_process(kvlist, IFPGA_ARG_PORT,
> +                                      &ifpga_get_integer32_arg, &afu_pr_conf.afu_id.port) < 0) {
> +                       IFPGA_BUS_ERR("error to parse %s",
> +                                    IFPGA_ARG_PORT);
> +                       goto end;
> +               }
> +       }
> +       else {
> +               IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +                         IFPGA_ARG_PATH);
> +               goto end;
> +       }
> +
> +    if (rte_kvargs_count(kvlist, IFPGA_ARG_PATH) == 1) {
> +               if (rte_kvargs_process(kvlist, IFPGA_ARG_PATH,
> +                                      &ifpga_get_string_arg, &path) < 0) {
> +                       IFPGA_BUS_ERR("error to parse %s",
> +                                    IFPGA_ARG_PATH);
> +                       goto end;
> +               }
> +       } else {
> +               IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +                         IFPGA_ARG_PATH);
> +               goto end;
> +       }
> +
> +       if (rte_kvargs_count(kvlist, IFPGA_ARG_UUID_HIGH) == 1) {
> +               if (rte_kvargs_process(kvlist, IFPGA_ARG_UUID_HIGH,
> +                                      &ifpga_get_integer64_arg, &afu_pr_conf.afu_id.uuid_high) < 0) {
> +                       IFPGA_BUS_ERR("error to parse %s",
> +                                    IFPGA_ARG_UUID_HIGH);
> +                       goto end;
> +               }
> +       } else {
> +               IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +                         IFPGA_ARG_PATH);
> +               goto end;
> +       }
> +
> +       if (rte_kvargs_count(kvlist, IFPGA_ARG_UUID_LOW) == 1) {
> +               if (rte_kvargs_process(kvlist, IFPGA_ARG_UUID_LOW,
> +                                      &ifpga_get_integer64_arg, &afu_pr_conf.afu_id.uuid_low) < 0) {
> +                       IFPGA_BUS_ERR("error to parse %s",
> +                                    IFPGA_ARG_UUID_LOW);
> +                       goto end;
> +               }
> +       } else {
> +               IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +                         IFPGA_ARG_PATH);
> +               goto end;
> +       }
> +
> +       if (rte_kvargs_count(kvlist, IFPGA_ARG_PR_ENABLE) == 1) {
> +               if (rte_kvargs_process(kvlist, IFPGA_ARG_PR_ENABLE,
> +                                      &ifpga_get_integer32_arg, &pr_enable) < 0) {
> +                       IFPGA_BUS_ERR("error to parse %s",
> +                                    IFPGA_ARG_UUID_HIGH);
> +                       goto end;
> +               }
> +       }
> +
> +       if (rte_kvargs_count(kvlist, IFPGA_ARG_DEBUG) == 1) {
> +               if (rte_kvargs_process(kvlist, IFPGA_ARG_DEBUG,
> +                                      &ifpga_get_integer32_arg, &debug) < 0) {
> +                       IFPGA_BUS_ERR("error to parse %s",
> +                                    IFPGA_ARG_UUID_HIGH);
> +                       goto end;
> +               }
> +       }
> +
> +       if(!debug)
> +       {
> +               pci_bus = rte_bus_find_by_name("pci");
> +           if (pci_bus == NULL) {
> +               IFPGA_BUS_ERR("unable to find PCI bus\n");
> +               goto end;
> +           }
> +
> +           dev = pci_bus->find_device(NULL, ifpga_pci_addr_cmp, &afu_pr_conf.afu_id.pci_addr);
> +           if (dev == NULL) {
> +               IFPGA_BUS_ERR("unable to find PCI device\n");
> +               goto end;
> +           }
> +       }

I am slightly confused here. I thought this is not on PCI bus. But, you are searching for the iFPGA on the PCI bus. If so, you still need a new bus?
Rosen: it just to confirm the PCI bus is setup before FPGA BUS Scan.
In case the devices are still visible on the PCI bus itself, a rawdev driver can easily identify (probe time) them and attach itself to them.
Is my understanding wrong? (Assuming that no PCI driver will identify the devices as its own and eventually the rawdev driver would claim
them.)
Rosen: Rawdev Driver (based PCI Devide) will firstly probe before FPGA BUS Scan.

> +       else
> +       {
> +               IFPGA_BUS_DEBUG("pci_addr domain   : %x\n",   afu_pr_conf.afu_id.pci_addr.domain);
> +               IFPGA_BUS_DEBUG("pci_addr bus      : %x\n",   afu_pr_conf.afu_id.pci_addr.bus);
> +               IFPGA_BUS_DEBUG("pci_addr devid    : %x\n",   afu_pr_conf.afu_id.pci_addr.devid);
> +               IFPGA_BUS_DEBUG("pci_addr function : %x\n",   afu_pr_conf.afu_id.pci_addr.function);
> +
> +               IFPGA_BUS_DEBUG("uuid_low          : %lx\n", afu_pr_conf.afu_id.uuid_low);
> +               IFPGA_BUS_DEBUG("uuid_high         : %lx\n", afu_pr_conf.afu_id.uuid_high);
> +
> +               IFPGA_BUS_DEBUG("afu port          : %x\n",   afu_pr_conf.afu_id.port);
> +       }
> +
> +       if (ifpga_find_afu_dev(&afu_pr_conf.afu_id))
> +               goto end;
> +
> +       memset(rawdev_name, sizeof(rawdev_name), 0);
> +       snprintf(rawdev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%x:%x", afu_pr_conf.afu_id.pci_addr.bus, afu_pr_conf.afu_id.pci_addr.devid, afu_pr_conf.afu_id.pci_addr.function);
> +       rawdev = rte_rawdev_pmd_get_named_dev(rawdev_name);
> +       if(!rawdev)
> +               goto end;
> +
> +       rawdev->dev_ops->dev_start(rawdev);
> +
> +       if (pr_enable)
> +       {
> +               ret=rawdev->dev_ops->firmware_load(rawdev, &afu_pr_conf);
> +               if (ret) {
> +               printf("do pr error\n");
> +               return NULL;
> +           }
> +       }
> +
> +       afu_dev = calloc(1, sizeof(*afu_dev));
> +       if (!afu_dev)
> +               goto end;
> +
> +       afu_dev->device.devargs = devargs;
> +       afu_dev->device.numa_node = SOCKET_ID_ANY;
> +       afu_dev->device.name = devargs->name;
> +       afu_dev->rawdev = rawdev;
> +       afu_dev->id.pci_addr.domain = afu_pr_conf.afu_id.pci_addr.domain;
> +       afu_dev->id.pci_addr.bus = afu_pr_conf.afu_id.pci_addr.bus;
> +       afu_dev->id.pci_addr.devid = afu_pr_conf.afu_id.pci_addr.devid;
> +       afu_dev->id.pci_addr.function = afu_pr_conf.afu_id.pci_addr.function;
> +       afu_dev->id.uuid_low  = afu_pr_conf.afu_id.uuid_low;
> +       afu_dev->id.uuid_high = afu_pr_conf.afu_id.uuid_high;
> +       afu_dev->id.port      = afu_pr_conf.afu_id.port;
> +
> +       rawdev->dev_ops->dev_info_get(rawdev, afu_dev);
> +
> +    return afu_dev;
> +
> +end:
> +       if (kvlist)
> +               rte_kvargs_free(kvlist);
> +       if (path)
> +               free(path);
> +
> +       return NULL;
> +}
> +
> +/*
> + * Scan the content of the FPGA bus, and the devices in the devices
> + * list
> + */
> +static int
> +rte_ifpga_scan(void)
> +{
> +       struct rte_bus *pci_bus;
> +       struct rte_afu_device *afu_dev;
> +       struct rte_devargs *devargs;
> +
> +    pci_bus = rte_bus_find_by_name("pci");
> +       if (pci_bus == NULL) {
> +               IFPGA_BUS_ERR("unable to find PCI bus\n");
> +               return -1;
> +       }
> +
> +       /* for virtual devices we scan the devargs_list populated via cmdline */
> +       TAILQ_FOREACH(devargs, &devargs_list, next) {
> +
> +               if (devargs->bus != &rte_ifpga_bus.bus)
> +                       continue;
> +
> +               afu_dev = rte_ifpga_scan_one(devargs);
> +
> +           if(afu_dev)
> +                   TAILQ_INSERT_TAIL(&rte_ifpga_bus.afu_list, afu_dev, next);
> +       }
> +
> +       return 0;
> +}
> +
> +static int
> +ifpga_probe_one_driver(struct rte_afu_driver *drv,
> +                       struct rte_afu_device *afu_dev) {
> +       int ret;
> +
> +    if((drv->id.pci_addr.bus == afu_dev->id.pci_addr.bus) &&
> +       (drv->id.pci_addr.devid == afu_dev->id.pci_addr.devid) &&
> +       (drv->id.pci_addr.function == afu_dev->id.pci_addr.function) &&
> +       (drv->id.uuid_low == afu_dev->id.uuid_low) &&
> +       (drv->id.uuid_high == afu_dev->id.uuid_high) &&
> +       (drv->id.port == afu_dev->id.port)) {
> +
> +               afu_dev->driver = drv;
> +
> +               /* call the driver probe() function */
> +               ret = drv->probe(afu_dev);
> +               if (ret)
> +                       afu_dev->driver = NULL;
> +               return ret;
> +    }
> +
> +       /* return positive value if driver doesn't support this device */
> +       return 1;
> +}
> +
> +static int
> +ifpga_probe_all_drivers(struct rte_afu_device *afu_dev) {
> +       const char *name;
> +       struct rte_afu_driver *drv = NULL;
> +       int rc;
> +
> +       if (afu_dev == NULL)
> +               return -1;
> +
> +       /* Check if a driver is already loaded */
> +       if (afu_dev->driver != NULL)
> +               return 0;
> +
> +    name = rte_ifpga_device_name(afu_dev);
> +    IFPGA_BUS_DEBUG("Search driver %s to probe device %s\n", name,
> +               rte_ifpga_device_name(afu_dev));
> +
> +       TAILQ_FOREACH(drv, &rte_ifpga_bus.driver_list, next) {
> +               rc = ifpga_probe_one_driver(drv, afu_dev);
> +               if (rc < 0)
> +                       /* negative value is an error */
> +                       return -1;
> +               if (rc > 0)
> +                       /* positive value means driver doesn't support it */
> +                       continue;
> +               return 0;
> +       }
> +       return 1;
> + }
> +
> +/*
> + * Scan the content of the PCI bus, and call the probe() function for
> + * all registered drivers that have a matching entry in its id_table
> + * for discovered devices.
> + */
> +static int
> +rte_ifpga_probe(void)
> +{
> +       struct rte_afu_device *afu_dev = NULL;
> +       int ret = 0;
> +
> +       TAILQ_FOREACH (afu_dev, &rte_ifpga_bus.afu_list, next) {
> +
> +        if (afu_dev->device.driver)
> +                       continue;
> +
> +               ret = ifpga_probe_all_drivers(afu_dev);
> +               if (ret < 0)
> +                   IFPGA_BUS_ERR("failed to initialize %s device\n",
> +                               rte_ifpga_device_name(afu_dev));
> +       }
> +
> +       return 0;
> +}
> +
> +static int
> +rte_ifpga_plug(struct rte_device *dev) {
> +       return ifpga_probe_all_drivers(RTE_DEV_TO_AFU(dev));
> +}
> +
> +static int ifpga_remove_driver(struct rte_afu_device *afu_dev) {
> +       const char *name;
> +       const struct rte_afu_driver *driver;
> +
> +    name = rte_ifpga_device_name(afu_dev);
> +       if (!afu_dev->device.driver) {
> +               IFPGA_BUS_DEBUG("no driver attach to device %s\n", name);
> +               return 1;
> +       }
> +
> +       driver = container_of(afu_dev->device.driver, const struct rte_afu_driver,
> +               driver);
> +       return driver->remove(afu_dev); }
> +
> +static int
> +rte_ifpga_unplug(struct rte_device *dev) {
> +       struct rte_afu_device *afu_dev;
> +       struct rte_devargs *devargs;
> +       int ret;
> +
> +       if (dev == NULL)
> +               return -EINVAL;
> +
> +       afu_dev = RTE_DEV_TO_AFU(dev);
> +       if (!dev)
> +               return -ENOENT;
> +
> +       devargs = dev->devargs;
> +
> +       ret = ifpga_remove_driver(afu_dev);
> +       if (ret)
> +               return ret;
> +
> +       TAILQ_REMOVE(&rte_ifpga_bus.afu_list, afu_dev, next);
> +
> +       TAILQ_REMOVE(&devargs_list, devargs, next);
> +
> +       free(devargs->args);
> +       free(devargs);
> +       free(afu_dev);
> +       return 0;
> +
> +}
> +
> +static struct rte_device *
> +rte_ifpga_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
> +                                                       const void 
> +*data) {
> +       struct rte_afu_device *afu_dev;
> +
> +       TAILQ_FOREACH(afu_dev, &rte_ifpga_bus.afu_list, next) {
> +               if (start && &afu_dev->device == start) {
> +                       start = NULL;
> +                       continue;
> +               }
> +               if (cmp(&afu_dev->device, data) == 0)
> +                       return &afu_dev->device;
> +       }
> +       return NULL;
> +}
> +static int
> +rte_ifpga_parse(const char *name, void *addr) {
> +       struct rte_afu_driver **out = addr;
> +       struct rte_afu_driver *driver = NULL;
> +
> +       TAILQ_FOREACH(driver, &rte_ifpga_bus.driver_list, next) {
> +               if (strncmp(driver->driver.name, name,
> +                           strlen(driver->driver.name)) == 0)
> +                       break;
> +               if (driver->driver.alias &&
> +                   strncmp(driver->driver.alias, name,
> +                           strlen(driver->driver.alias)) == 0)
> +                       break;
> +       }
> +       if (driver != NULL &&
> +           addr != NULL)
> +               *out = driver;
> +       return driver == NULL;
> +}
> +
> +struct rte_ifpga_bus rte_ifpga_bus = {
> +        .bus = {
> +                .scan        = rte_ifpga_scan,
> +                .probe       = rte_ifpga_probe,
> +                .find_device = rte_ifpga_find_device,
> +            .plug        = rte_ifpga_plug,
> +            .unplug      = rte_ifpga_unplug,
> +            .parse       = rte_ifpga_parse,
> +        },
> +       .afu_list    = TAILQ_HEAD_INITIALIZER(rte_ifpga_bus.afu_list),
> +       .driver_list = 
> +TAILQ_HEAD_INITIALIZER(rte_ifpga_bus.driver_list),
> + };
> +
> +RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus.bus);
> +
> +RTE_INIT(ifpga_init_log)
> +{
> +       ifpga_bus_logtype = rte_log_register("bus.ifpga");
> +       if (ifpga_bus_logtype >= 0)
> +               rte_log_set_level(ifpga_bus_logtype, RTE_LOG_NOTICE); 
> +}
> +
> diff --git a/drivers/bus/ifpga/ifpga_common.c 
> b/drivers/bus/ifpga/ifpga_common.c
> new file mode 100644
> index 0000000..2aa6bb0
> --- /dev/null
> +++ b/drivers/bus/ifpga/ifpga_common.c
> @@ -0,0 +1,168 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
> + *   Copyright 2013-2014 6WIND S.A.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +
> +#include <string.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/queue.h>
> +#include <sys/mman.h>
> +#include <sys/types.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +
> +#include <rte_errno.h>
> +#include <rte_bus.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_memzone.h>
> +#include <rte_eal.h>
> +#include <rte_common.h>
> +
> +#include <rte_devargs.h>
> +#include <rte_pci.h>
> +#include <rte_bus_pci.h>
> +#include <rte_kvargs.h>
> +#include <rte_alarm.h>
> +
> +#include "rte_bus_ifpga.h"
> +#include "ifpga_logs.h"
> +#include "ifpga_common.h"
> +
> +int ifpga_get_string_arg(const char *key __rte_unused,
> +                            const char *value, void *extra_args) {
> +       if (!value || !extra_args)
> +               return -EINVAL;
> +
> +       *(char **)extra_args = strdup(value);
> +
> +       if (!*(char **)extra_args)
> +               return -ENOMEM;
> +
> +       return 0;
> +}
> +int ifpga_get_integer32_arg(const char *key __rte_unused, const char 
> +*value, void *extra_args) {
> +       if (!value || !extra_args)
> +               return -EINVAL;
> +
> +       *(int *)extra_args = strtoull(value, NULL, 0);
> +
> +       return 0;
> +}
> +int ifpga_get_integer64_arg(const char *key __rte_unused, const char 
> +*value, void *extra_args) {
> +       if (!value || !extra_args)
> +               return -EINVAL;
> +
> +       *(uint64_t *)extra_args = strtoull(value, NULL, 0);
> +
> +       return 0;
> +}
> +int ifpga_get_unsigned_long(const char *str, int base) {
> +       unsigned long num;
> +       char *end = NULL;
> +
> +       errno = 0;
> +
> +       num = strtoul(str, &end, base);
> +       if ((str[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
> +               return -1;
> +
> +       return num;
> +
> +}
> +int ifpga_get_bdf_arg(const char *key __rte_unused, const char 
> +*value, void *extra_args) { #define MAX_PATH_LEN 1024
> +       struct rte_pci_addr *addr;
> +       int num[4];
> +       char str[MAX_PATH_LEN];
> +       int i, j;
> +
> +       if (!value || !extra_args)
> +               return -EINVAL;
> +
> +    addr = (struct rte_pci_addr *)extra_args;
> +       strcpy(str, value);
> +       memset(num, 0, 4 * sizeof(num[0]));
> +       i = strlen(str) - 1;
> +       j = 3;
> +       while (i > 0 && j >= 0) {
> +               while ((str[i - 1] != ':' && str[i - 1] != '.') && i > 0)
> +                       i--;
> +               num[j--] = ifpga_get_unsigned_long(&str[i], 16);
> +               i--;
> +               if (i >= 0)
> +                       str[i] = '\0';
> +       }
> +       addr->domain = num[0];
> +       addr->bus = num[1];
> +       addr->devid = num[2];
> +       addr->function = num[3];
> +       printf("[%s]: bdf %04d:%02d:%02d.%02d\n",
> +                       __func__,
> +                       addr->domain,
> +                       addr->bus,
> +                       addr->devid,
> +                       addr->function);
> +
> +       return 0;
> +}
> +int ifpga_afu_id_cmp(const struct rte_afu_id *afu_id0, const struct 
> +rte_afu_id *afu_id1) {
> +       if((afu_id0->pci_addr.bus       == afu_id1->pci_addr.bus) &&
> +       (afu_id0->pci_addr.devid     == afu_id1->pci_addr.devid) &&
> +       (afu_id0->pci_addr.function  == afu_id1->pci_addr.function) &&
> +       (afu_id0->uuid_low           == afu_id1->uuid_low) &&
> +       (afu_id0->uuid_high          == afu_id1->uuid_high) &&
> +       (afu_id0->port               == afu_id1->port)) {
> +
> +        return 0;
> +    }
> +    else
> +       return 1;
> +}
> +int ifpga_pci_addr_cmp(const struct rte_device *dev, const void 
> +*_pci_addr) {
> +       struct rte_pci_device *pdev;
> +       const struct rte_pci_addr *paddr = _pci_addr;
> +
> +       pdev = RTE_DEV_TO_PCI(*(struct rte_device **)(void *)&dev);
> +       return rte_eal_compare_pci_addr(&pdev->addr, paddr); }
> diff --git a/drivers/bus/ifpga/ifpga_common.h 
> b/drivers/bus/ifpga/ifpga_common.h
> new file mode 100644
> index 0000000..590ff27
> --- /dev/null
> +++ b/drivers/bus/ifpga/ifpga_common.h
> @@ -0,0 +1,46 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2017 Intel Corporation. All rights reserved.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +#ifndef _IFPGA_COMMON_H_
> +#define _IFPGA_COMMON_H_
> +
> +#include <rte_pci.h>
> +
> +extern int ifpga_get_string_arg(const char *key __rte_unused, const 
> +char *value, void *extra_args); extern int 
> +ifpga_get_integer32_arg(const char *key __rte_unused, const char 
> +*value, void *extra_args); extern int ifpga_get_integer64_arg(const 
> +char *key __rte_unused, const char *value, void *extra_args); extern 
> +int ifpga_get_unsigned_long(const char *str, int base); extern int 
> +ifpga_get_bdf_arg(const char *key __rte_unused, const char *value, 
> +void *extra_args); extern int ifpga_afu_id_cmp(const struct 
> +rte_afu_id *afu_id0, const struct rte_afu_id *afu_id1); extern int 
> +ifpga_pci_addr_cmp(const struct rte_device *dev, const void 
> +*_pci_addr);
> +
> +#endif /* _IFPGA_COMMON_H_ */
> diff --git a/drivers/bus/ifpga/ifpga_logs.h 
> b/drivers/bus/ifpga/ifpga_logs.h new file mode 100644 index 
> 0000000..eb56393
> --- /dev/null
> +++ b/drivers/bus/ifpga/ifpga_logs.h
> @@ -0,0 +1,59 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2017 Intel Corporation. All rights reserved.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +#ifndef _IFPGA_BUS_LOGS_H_
> +#define _IFPGA_BUS_LOGS_H_
> +
> +#include <rte_log.h>
> +
> +extern int ifpga_bus_logtype;
> +
> +#define IFPGA_LOG(level, fmt, args...) \
> +       rte_log(RTE_LOG_ ## level, ifpga_bus_logtype, "%s(): " fmt "\n", \
> +               __func__, ##args)
> +
> +#define IFPGA_BUS_LOG(level, fmt, args...) \
> +       rte_log(RTE_LOG_ ## level, ifpga_bus_logtype, "%s(): " fmt "\n", \
> +               __func__, ##args)
> +
> +#define IFPGA_BUS_FUNC_TRACE() IFPGA_BUS_LOG(DEBUG, ">>")
> +
> +#define IFPGA_BUS_DEBUG(fmt, args...) \
> +       IFPGA_BUS_LOG(DEBUG, fmt, ## args) #define IFPGA_BUS_INFO(fmt, 
> +args...) \
> +       IFPGA_BUS_LOG(INFO, fmt, ## args) #define IFPGA_BUS_ERR(fmt, 
> +args...) \
> +       IFPGA_BUS_LOG(ERR, fmt, ## args) #define IFPGA_BUS_WARN(fmt, 
> +args...) \
> +       IFPGA_BUS_LOG(WARNING, fmt, ## args)
> +
> +#endif /* _IFPGA_BUS_LOGS_H_ */
> diff --git a/drivers/bus/ifpga/rte_bus_ifpga.h 
> b/drivers/bus/ifpga/rte_bus_ifpga.h
> new file mode 100644
> index 0000000..5491a45
> --- /dev/null
> +++ b/drivers/bus/ifpga/rte_bus_ifpga.h
> @@ -0,0 +1,153 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2016 RehiveTech. All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of RehiveTech nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#ifndef _RTE_BUS_IFPGA_H_
> +#define _RTE_BUS_IFPGA_H_
> +
> +/**
> + * @file
> + *
> + * RTE PCI Bus Interface

Bus IFPGA
Rosen: thanks you check

> + */
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include <rte_bus.h>
> +#include <rte_pci.h>
> +
> +/** Name of Intel FPGA Bus */
> +#define IFPGA_BUS_NAME ifpga
> +
> +/* Forward declarations */
> +struct rte_afu_device;
> +struct rte_afu_driver;
> +
> +/** List of Intel AFU devices */
> +TAILQ_HEAD(rte_afu_device_list, rte_afu_device);
> +/** List of AFU drivers */
> +TAILQ_HEAD(rte_afu_driver_list, rte_afu_driver);
> +
> +#define IFPGA_BUS_BITSTREAM_PATH_MAX_LEN 256
> +
> +/**
> + * A structure describing an ID for a AFU driver. Each driver 
> +provides a
> + * table of these IDs for each device that it supports.
> + */
> +struct rte_afu_id {
> +       struct rte_pci_addr pci_addr;
> +       uint64_t uuid_low;
> +       uint64_t uuid_high;
> +       int      port;
> +}__attribute__ ((packed));
> +
> +/**
> + * A structure pr configuration AFU driver.
> + */
> +
> +struct rte_afu_pr_conf {
> +       struct rte_afu_id afu_id;
> +       int pr_enable;
> +       char            bs_path[IFPGA_BUS_BITSTREAM_PATH_MAX_LEN];
> +};
> +
> +#define AFU_PRI_STR_SIZE (PCI_PRI_STR_SIZE + 8)
> +
> +/**
> + * A structure describing a AFU device.
> + */
> +struct rte_afu_device {
> +       TAILQ_ENTRY(rte_afu_device) next;       /**< Next in device list. */
> +       struct rte_device device;               /**< Inherit core device */
> +       struct rte_rawdev *rawdev;
> +       struct rte_afu_id id;                   /**< AFU id within FPGA. */
> +       uint32_t num_region;                                            /**< number of regions found */
> +       struct rte_mem_resource mem_resource[PCI_MAX_RESOURCE];
> +                                               /**< PCI Memory Resource */
> +       struct rte_intr_handle intr_handle;     /**< Interrupt handle */
> +       struct rte_afu_driver *driver;          /**< Associated driver */
> +       char path[IFPGA_BUS_BITSTREAM_PATH_MAX_LEN];
> +} __attribute__ ((packed));
> +
> +/**
> + * @internal
> + * Helper macro for drivers that need to convert to struct rte_afu_device.
> + */
> +#define RTE_DEV_TO_AFU(ptr) \
> +       container_of(ptr, struct rte_afu_device, device)
> +
> +/**
> + * Initialisation function for the driver called during PCI probing.
> + */
> +typedef int (afu_probe_t)(struct rte_afu_device *);
> +
> +/**
> + * Uninitialisation function for the driver called during hotplugging.
> + */
> +typedef int (afu_remove_t)(struct rte_afu_device *);
> +
> +/**
> + * A structure describing a PCI device.

Trivial: An AFU device, probably. (sorry for nitpicking - I know this is RFC - so ignore if you already know about these changes.)
Rosen: thanks you check

> + */
> +struct rte_afu_driver {
> +       TAILQ_ENTRY(rte_afu_driver) next;       /**< Next afu driver. */
> +       struct rte_driver driver;               /**< Inherit core driver. */
> +       afu_probe_t *probe;                     /**< Device Probe function. */
> +       afu_remove_t *remove;                   /**< Device Remove function. */
> +       struct rte_afu_id id;                   /**< AFU id within FPGA. */
> +       uint32_t drv_flags;                     /**< Flags contolling handling of device. */
> +};
> +
> +/**
> + * Structure describing the Intel FPGA bus  */ struct rte_ifpga_bus {
> +       struct rte_bus bus;               /**< Inherit the generic class */
> +       struct rte_afu_device_list afu_list;  /**< List of AFU devices */
> +       struct rte_afu_driver_list driver_list;  /**< List of FPGA 
> +drivers */ };
> +
> +static inline const char *
> +rte_ifpga_device_name(const struct rte_afu_device *afu) {
> +       if (afu && afu->device.name)
> +               return afu->device.name;
> +       return NULL;
> +}
> +
> +extern struct rte_ifpga_bus rte_ifpga_bus;
> +
> +void rte_ifpga_driver_register(struct rte_afu_driver *driver); void 
> +rte_ifpga_driver_unregister(struct rte_afu_driver *driver);
> +
> +#endif /* _RTE_BUS_IFPGA_H_ */
> diff --git a/drivers/bus/ifpga/rte_bus_ifpga_version.map 
> b/drivers/bus/ifpga/rte_bus_ifpga_version.map
> new file mode 100644
> index 0000000..4b2380a
> --- /dev/null
> +++ b/drivers/bus/ifpga/rte_bus_ifpga_version.map
> @@ -0,0 +1,8 @@
> +DPDK_17.11 {
> +       global:
> +
> +    rte_ifpga_driver_register;
> +    rte_ifpga_driver_unregister;
> +
> +       local: *;
> +};
> --
> 1.8.3.1
>

Re: [dpdk-dev] [RFC 1/4] drivers/bus/ifpga:Intel FPGA Bus Lib Code

[Gaëtan Rivet]

Hi Rosen,

A few comments inline.
(I will skip elements already pointed out by Shreyansh.)

On Tue, Mar 06, 2018 at 09:43:55AM +0800, Rosen Xu wrote:
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> ---
>  drivers/bus/ifpga/Makefile                  |  64 ++++
>  drivers/bus/ifpga/ifpga_bus.c               | 527 ++++++++++++++++++++++++++++
>  drivers/bus/ifpga/ifpga_common.c            | 168 +++++++++
>  drivers/bus/ifpga/ifpga_common.h            |  46 +++
>  drivers/bus/ifpga/ifpga_logs.h              |  59 ++++
>  drivers/bus/ifpga/rte_bus_ifpga.h           | 153 ++++++++
>  drivers/bus/ifpga/rte_bus_ifpga_version.map |   8 +
>  7 files changed, 1025 insertions(+)
>  create mode 100644 drivers/bus/ifpga/Makefile
>  create mode 100644 drivers/bus/ifpga/ifpga_bus.c
>  create mode 100644 drivers/bus/ifpga/ifpga_common.c
>  create mode 100644 drivers/bus/ifpga/ifpga_common.h
>  create mode 100644 drivers/bus/ifpga/ifpga_logs.h
>  create mode 100644 drivers/bus/ifpga/rte_bus_ifpga.h
>  create mode 100644 drivers/bus/ifpga/rte_bus_ifpga_version.map
> 
> diff --git a/drivers/bus/ifpga/Makefile b/drivers/bus/ifpga/Makefile
> new file mode 100644
> index 0000000..c71f186
> --- /dev/null
> +++ b/drivers/bus/ifpga/Makefile
> @@ -0,0 +1,64 @@
> +#   BSD LICENSE
> +#
> +#   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
> +#   All rights reserved.
> +#
> +#   Redistribution and use in source and binary forms, with or without
> +#   modification, are permitted provided that the following conditions
> +#   are met:
> +#
> +#     * Redistributions of source code must retain the above copyright
> +#       notice, this list of conditions and the following disclaimer.
> +#     * Redistributions in binary form must reproduce the above copyright
> +#       notice, this list of conditions and the following disclaimer in
> +#       the documentation and/or other materials provided with the
> +#       distribution.
> +#     * Neither the name of Intel Corporation nor the names of its
> +#       contributors may be used to endorse or promote products derived
> +#       from this software without specific prior written permission.
> +#
> +#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +
> +include $(RTE_SDK)/mk/rte.vars.mk
> +
> +#
> +# library name
> +#
> +LIB = librte_bus_ifpga.a
> +LIBABIVER := 1
> +EXPORT_MAP := rte_bus_ifpga_version.map
> +
> +ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT),y)
> +CFLAGS += -O0 -g
> +CFLAGS += "-Wno-error"
> +else
> +CFLAGS += -O3
> +CFLAGS += $(WERROR_FLAGS)
> +endif
> +
> +CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
> +CFLAGS += -I$(RTE_SDK)/drivers/bus/pci
> +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
> +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common
> +#CFLAGS += -I$(RTE_SDK)/lib/librte_rawdev
> +#LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring -lrte_rawdev
> +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
> +#LDLIBS += -lrte_ethdev
> +
> +VPATH += $(SRCDIR)/base
> +
> +SRCS-y += \
> +        ifpga_bus.c \
> +        ifpga_common.c
> +
> +include $(RTE_SDK)/mk/rte.lib.mk
> diff --git a/drivers/bus/ifpga/ifpga_bus.c b/drivers/bus/ifpga/ifpga_bus.c
> new file mode 100644
> index 0000000..382d550
> --- /dev/null
> +++ b/drivers/bus/ifpga/ifpga_bus.c
> @@ -0,0 +1,527 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
> + *   Copyright 2013-2014 6WIND S.A.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +
> +#include <string.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/queue.h>
> +#include <sys/mman.h>
> +#include <sys/types.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +
> +#include <rte_errno.h>
> +#include <rte_bus.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_memzone.h>
> +#include <rte_eal.h>
> +#include <rte_common.h>
> +
> +#include <rte_devargs.h>
> +#include <rte_pci.h>
> +#include <rte_bus_pci.h>
> +#include <rte_kvargs.h>
> +#include <rte_alarm.h>
> +
> +#include "rte_rawdev.h"
> +#include "rte_rawdev_pmd.h"
> +#include "rte_bus_ifpga.h"
> +#include "ifpga_logs.h"
> +#include "ifpga_common.h"
> +
> +int ifpga_bus_logtype;
> +
> +/*register a ifpga bus based driver */
> +void rte_ifpga_driver_register(struct rte_afu_driver *driver)
> +{
> +	RTE_VERIFY(driver);
> +
> +	TAILQ_INSERT_TAIL(&rte_ifpga_bus.driver_list, driver, next);
> +}
> +
> +/*un-register a fpga bus based driver */
> +void rte_ifpga_driver_unregister(struct rte_afu_driver *driver)
> +{
> +	TAILQ_REMOVE(&rte_ifpga_bus.driver_list, driver, next);
> +}
> +
> +static struct rte_afu_device *
> +ifpga_find_afu_dev(const struct rte_afu_id *afu_id)
> +{
> +	struct rte_afu_device *afu_dev = NULL;
> +	

You seem to have trailing spaces (here tabs), and some inconsistencies
in your file (sometimes spaces are used, sometime tabs).

You need to run checkpatch on your series.

See those errors:
http://dpdk.org/ml/archives/test-report/2018-March/043185.html

A script is available in devtools to run this before submitting.

> +	TAILQ_FOREACH (afu_dev, &rte_ifpga_bus.afu_list, next) {
> +
> +        if(!ifpga_afu_id_cmp(&afu_dev->id, afu_id)) {
> +            return afu_dev;
> +        }
> +	}
> +	return NULL;
> +}
> +
> +static const char *valid_args[] = {
> +#define IFPGA_ARG_BDF          "bdf"
> +	IFPGA_ARG_BDF,
> +#define IFPGA_ARG_PORT         "port"
> +	IFPGA_ARG_PORT,
> +#define IFPGA_ARG_PATH         "path"
> +	IFPGA_ARG_PATH,
> +#define IFPGA_ARG_UUID_HIGH    "uuid_high"
> +	IFPGA_ARG_UUID_HIGH,
> +#define IFPGA_ARG_UUID_LOW     "uuid_low"
> +	IFPGA_ARG_UUID_LOW,
> +#define IFPGA_ARG_PR_ENABLE     "pr_enable"
> +	IFPGA_ARG_PR_ENABLE,
> +#define IFPGA_ARG_DEBUG         "debug"
> +	IFPGA_ARG_DEBUG,
> +	NULL
> +};
> +
> +/*
> + * Scan the content of the FPGA bus, and the devices in the devices
> + * list
> + */
> +static struct rte_afu_device *
> +rte_ifpga_scan_one(struct rte_devargs *devargs)
> +{
> +	struct rte_kvargs *kvlist = NULL;
> +	struct rte_bus *pci_bus = NULL;
> +	struct rte_device *dev = NULL;
> +	struct rte_rawdev *rawdev;
> +	struct rte_afu_device *afu_dev = NULL;
> +	struct rte_afu_pr_conf afu_pr_conf;
> +	char rawdev_name[RTE_RAWDEV_NAME_MAX_LEN];
> +	int ret;
> +	char *path = NULL;
> +	int pr_enable = 1;
> +	int debug = 0;
> +
> +    memset((char *)(&afu_pr_conf), 0, sizeof(struct rte_afu_pr_conf));
> +	
> +    kvlist = rte_kvargs_parse(devargs->args, valid_args);
> +    if (!kvlist) {
> +    	IFPGA_BUS_ERR("error when parsing param");
> +    	goto end;
> +    }
> +    
> +    if (rte_kvargs_count(kvlist, IFPGA_ARG_BDF) == 1) {
> +		if (rte_kvargs_process(kvlist, IFPGA_ARG_BDF,
> +				       &ifpga_get_bdf_arg, &afu_pr_conf.afu_id.pci_addr) < 0) {
> +			IFPGA_BUS_ERR("error to parse %s",
> +				     IFPGA_ARG_BDF);
> +			goto end;
> +		}
> +	} else {
> +		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +			  IFPGA_ARG_PATH);
> +		goto end;
> +	}
> +    
> +    if (rte_kvargs_count(kvlist, IFPGA_ARG_PORT) == 1) {
> +		if (rte_kvargs_process(kvlist, IFPGA_ARG_PORT,
> +				       &ifpga_get_integer32_arg, &afu_pr_conf.afu_id.port) < 0) {
> +			IFPGA_BUS_ERR("error to parse %s",
> +				     IFPGA_ARG_PORT);
> +			goto end;
> +		}
> +	}
> +	else {
> +		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +			  IFPGA_ARG_PATH);
> +		goto end;
> +	}
> +	
> +    if (rte_kvargs_count(kvlist, IFPGA_ARG_PATH) == 1) {
> +		if (rte_kvargs_process(kvlist, IFPGA_ARG_PATH,
> +				       &ifpga_get_string_arg, &path) < 0) {
> +			IFPGA_BUS_ERR("error to parse %s",
> +				     IFPGA_ARG_PATH);
> +			goto end;
> +		}
> +	} else {
> +		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +			  IFPGA_ARG_PATH);
> +		goto end;
> +	}
> +	
> +	if (rte_kvargs_count(kvlist, IFPGA_ARG_UUID_HIGH) == 1) {
> +		if (rte_kvargs_process(kvlist, IFPGA_ARG_UUID_HIGH,
> +				       &ifpga_get_integer64_arg, &afu_pr_conf.afu_id.uuid_high) < 0) {
> +			IFPGA_BUS_ERR("error to parse %s",
> +				     IFPGA_ARG_UUID_HIGH);
> +			goto end;
> +		}
> +	} else {
> +		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +			  IFPGA_ARG_PATH);
> +		goto end;
> +	}
> +	
> +	if (rte_kvargs_count(kvlist, IFPGA_ARG_UUID_LOW) == 1) {
> +		if (rte_kvargs_process(kvlist, IFPGA_ARG_UUID_LOW,
> +				       &ifpga_get_integer64_arg, &afu_pr_conf.afu_id.uuid_low) < 0) {
> +			IFPGA_BUS_ERR("error to parse %s",
> +				     IFPGA_ARG_UUID_LOW);
> +			goto end;
> +		}
> +	} else {
> +		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +			  IFPGA_ARG_PATH);
> +		goto end;
> +	}
> +	
> +	if (rte_kvargs_count(kvlist, IFPGA_ARG_PR_ENABLE) == 1) {
> +		if (rte_kvargs_process(kvlist, IFPGA_ARG_PR_ENABLE,
> +				       &ifpga_get_integer32_arg, &pr_enable) < 0) {
> +			IFPGA_BUS_ERR("error to parse %s",
> +				     IFPGA_ARG_UUID_HIGH);
> +			goto end;
> +		}
> +	}
> +	
> +	if (rte_kvargs_count(kvlist, IFPGA_ARG_DEBUG) == 1) {
> +		if (rte_kvargs_process(kvlist, IFPGA_ARG_DEBUG,
> +				       &ifpga_get_integer32_arg, &debug) < 0) {
> +			IFPGA_BUS_ERR("error to parse %s",
> +				     IFPGA_ARG_UUID_HIGH);
> +			goto end;
> +		}
> +	}
> +	
> +	if(!debug)
> +	{
> +		pci_bus = rte_bus_find_by_name("pci");
> +	    if (pci_bus == NULL) {
> +	    	IFPGA_BUS_ERR("unable to find PCI bus\n");
> +	    	goto end;
> +	    }
> +	    
> +	    dev = pci_bus->find_device(NULL, ifpga_pci_addr_cmp, &afu_pr_conf.afu_id.pci_addr);
> +	    if (dev == NULL) {
> +	    	IFPGA_BUS_ERR("unable to find PCI device\n");
> +	    	goto end;
> +	    }
> +	}
> +	else
> +	{
> +		IFPGA_BUS_DEBUG("pci_addr domain   : %x\n",   afu_pr_conf.afu_id.pci_addr.domain);
> +		IFPGA_BUS_DEBUG("pci_addr bus      : %x\n",   afu_pr_conf.afu_id.pci_addr.bus);
> +		IFPGA_BUS_DEBUG("pci_addr devid    : %x\n",   afu_pr_conf.afu_id.pci_addr.devid);
> +		IFPGA_BUS_DEBUG("pci_addr function : %x\n",   afu_pr_conf.afu_id.pci_addr.function);

Alternatively, it could be interesting to have a PCI_ADDR_FMT and
PCI_ADDR_ARGS in librte_pci, to help printing PCI addresses in a
standard way. This could be leveraged in librte_bus_pci as well.

> +		
> +		IFPGA_BUS_DEBUG("uuid_low          : %lx\n", afu_pr_conf.afu_id.uuid_low);
> +		IFPGA_BUS_DEBUG("uuid_high         : %lx\n", afu_pr_conf.afu_id.uuid_high);
> +		
> +		IFPGA_BUS_DEBUG("afu port          : %x\n",   afu_pr_conf.afu_id.port);
> +	}
> +	
> +	if (ifpga_find_afu_dev(&afu_pr_conf.afu_id))
> +		goto end;
> +
> +	memset(rawdev_name, sizeof(rawdev_name), 0);
> +	snprintf(rawdev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%x:%x", afu_pr_conf.afu_id.pci_addr.bus, afu_pr_conf.afu_id.pci_addr.devid, afu_pr_conf.afu_id.pci_addr.function);
> +	rawdev = rte_rawdev_pmd_get_named_dev(rawdev_name);
> +	if(!rawdev)
> +		goto end;
> +	
> +	rawdev->dev_ops->dev_start(rawdev);
> +		
> +	if (pr_enable)
> +	{
> +		ret=rawdev->dev_ops->firmware_load(rawdev, &afu_pr_conf);
> +		if (ret) {
> +    		printf("do pr error\n");
> +    		return NULL;
> +	    }
> +	}
> +		
> +	afu_dev = calloc(1, sizeof(*afu_dev));
> +	if (!afu_dev)
> +		goto end;
> +
> +	afu_dev->device.devargs = devargs;
> +	afu_dev->device.numa_node = SOCKET_ID_ANY;
> +	afu_dev->device.name = devargs->name;
> +	afu_dev->rawdev = rawdev;
> +	afu_dev->id.pci_addr.domain = afu_pr_conf.afu_id.pci_addr.domain;
> +	afu_dev->id.pci_addr.bus = afu_pr_conf.afu_id.pci_addr.bus;
> +	afu_dev->id.pci_addr.devid = afu_pr_conf.afu_id.pci_addr.devid;
> +	afu_dev->id.pci_addr.function = afu_pr_conf.afu_id.pci_addr.function;

A pci_addr_cpy function could be written.

> +	afu_dev->id.uuid_low  = afu_pr_conf.afu_id.uuid_low;
> +	afu_dev->id.uuid_high = afu_pr_conf.afu_id.uuid_high;
> +	afu_dev->id.port      = afu_pr_conf.afu_id.port;
> +	
> +	rawdev->dev_ops->dev_info_get(rawdev, afu_dev);
> +
> +    return afu_dev;
> +    
> +end:
> +	if (kvlist)
> +		rte_kvargs_free(kvlist);
> +	if (path)
> +		free(path);
> +	
> +	return NULL;
> +}
> +
> +/*
> + * Scan the content of the FPGA bus, and the devices in the devices
> + * list
> + */
> +static int
> +rte_ifpga_scan(void)
> +{
> +	struct rte_bus *pci_bus;
> +	struct rte_afu_device *afu_dev;
> +	struct rte_devargs *devargs;
> +
> +    pci_bus = rte_bus_find_by_name("pci");
> +	if (pci_bus == NULL) {
> +		IFPGA_BUS_ERR("unable to find PCI bus\n");
> +		return -1;
> +	}
> +	
> +	/* for virtual devices we scan the devargs_list populated via cmdline */
> +	TAILQ_FOREACH(devargs, &devargs_list, next) {
> +    
> +		if (devargs->bus != &rte_ifpga_bus.bus)
> +			continue;
> +		
> +		afu_dev = rte_ifpga_scan_one(devargs);
> +	    
> +	    if(afu_dev)
> +		    TAILQ_INSERT_TAIL(&rte_ifpga_bus.afu_list, afu_dev, next);
> +	}
> +
> +	return 0;
> +}
> +
> +static int
> +ifpga_probe_one_driver(struct rte_afu_driver *drv,
> +			struct rte_afu_device *afu_dev)
> +{
> +	int ret;
> +
> +    if((drv->id.pci_addr.bus == afu_dev->id.pci_addr.bus) &&
> +       (drv->id.pci_addr.devid == afu_dev->id.pci_addr.devid) &&
> +       (drv->id.pci_addr.function == afu_dev->id.pci_addr.function) &&

Is it expected to ignore the domain here?
You should use rte_pci_addr_cmp instead.

> +       (drv->id.uuid_low == afu_dev->id.uuid_low) &&
> +       (drv->id.uuid_high == afu_dev->id.uuid_high) &&
> +       (drv->id.port == afu_dev->id.port)) {
> +       	
> +       	afu_dev->driver = drv;
> +       	
> +       	/* call the driver probe() function */
> +		ret = drv->probe(afu_dev);
> +		if (ret)
> +			afu_dev->driver = NULL;
> +		return ret;
> +    }
> +	
> +	/* return positive value if driver doesn't support this device */
> +	return 1;
> +}
> +
> +static int
> +ifpga_probe_all_drivers(struct rte_afu_device *afu_dev)
> +{
> +	const char *name;
> +	struct rte_afu_driver *drv = NULL;
> +	int rc;
> +
> +	if (afu_dev == NULL)
> +		return -1;
> +
> +	/* Check if a driver is already loaded */
> +	if (afu_dev->driver != NULL)
> +		return 0;
> +		
> +    name = rte_ifpga_device_name(afu_dev);
> +    IFPGA_BUS_DEBUG("Search driver %s to probe device %s\n", name,
> +		rte_ifpga_device_name(afu_dev));
> +    
> +	TAILQ_FOREACH(drv, &rte_ifpga_bus.driver_list, next) {
> +		rc = ifpga_probe_one_driver(drv, afu_dev);
> +		if (rc < 0)
> +			/* negative value is an error */
> +			return -1;
> +		if (rc > 0)
> +			/* positive value means driver doesn't support it */
> +			continue;
> +		return 0;
> +	}
> +	return 1;
> + }
> +
> +/*
> + * Scan the content of the PCI bus, and call the probe() function for
> + * all registered drivers that have a matching entry in its id_table
> + * for discovered devices.
> + */
> +static int
> +rte_ifpga_probe(void)
> +{
> +	struct rte_afu_device *afu_dev = NULL;
> +	int ret = 0;
> +
> +	TAILQ_FOREACH (afu_dev, &rte_ifpga_bus.afu_list, next) {
> +
> +        if (afu_dev->device.driver)
> +			continue;
> +			
> +		ret = ifpga_probe_all_drivers(afu_dev);
> +		if (ret < 0)
> +		    IFPGA_BUS_ERR("failed to initialize %s device\n",
> +				rte_ifpga_device_name(afu_dev));
> +	}
> +
> +	return 0;
> +}
> +
> +static int
> +rte_ifpga_plug(struct rte_device *dev)
> +{
> +	return ifpga_probe_all_drivers(RTE_DEV_TO_AFU(dev));
> +}
> +
> +static int ifpga_remove_driver(struct rte_afu_device *afu_dev)
> +{
> +	const char *name;
> +	const struct rte_afu_driver *driver;
> +
> +    name = rte_ifpga_device_name(afu_dev);
> +	if (!afu_dev->device.driver) {
> +		IFPGA_BUS_DEBUG("no driver attach to device %s\n", name);
> +		return 1;
> +	}
> +
> +	driver = container_of(afu_dev->device.driver, const struct rte_afu_driver,
> +		driver);
> +	return driver->remove(afu_dev);
> +}
> +
> +static int
> +rte_ifpga_unplug(struct rte_device *dev)
> +{
> +	struct rte_afu_device *afu_dev;
> +	struct rte_devargs *devargs;
> +	int ret;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	afu_dev = RTE_DEV_TO_AFU(dev);
> +	if (!dev)
> +		return -ENOENT;
> +
> +	devargs = dev->devargs;
> +
> +	ret = ifpga_remove_driver(afu_dev);
> +	if (ret)
> +		return ret;
> +
> +	TAILQ_REMOVE(&rte_ifpga_bus.afu_list, afu_dev, next);
> +
> +	TAILQ_REMOVE(&devargs_list, devargs, next);
> +
> +	free(devargs->args);
> +	free(devargs);
> +	free(afu_dev);
> +	return 0;
> +
> +}
> +
> +static struct rte_device *
> +rte_ifpga_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
> +		                                        const void *data)
> +{
> +	struct rte_afu_device *afu_dev;
> +
> +	TAILQ_FOREACH(afu_dev, &rte_ifpga_bus.afu_list, next) {
> +		if (start && &afu_dev->device == start) {
> +			start = NULL;
> +			continue;
> +		}
> +		if (cmp(&afu_dev->device, data) == 0)
> +			return &afu_dev->device;
> +	}
> +	return NULL;
> +}
> +static int
> +rte_ifpga_parse(const char *name, void *addr)
> +{
> +	struct rte_afu_driver **out = addr;
> +	struct rte_afu_driver *driver = NULL;
> +
> +	TAILQ_FOREACH(driver, &rte_ifpga_bus.driver_list, next) {
> +		if (strncmp(driver->driver.name, name,
> +			    strlen(driver->driver.name)) == 0)
> +			break;
> +		if (driver->driver.alias &&
> +		    strncmp(driver->driver.alias, name,
> +			    strlen(driver->driver.alias)) == 0)
> +			break;
> +	}
> +	if (driver != NULL &&
> +	    addr != NULL)
> +		*out = driver;
> +	return driver == NULL;
> +}
> +
> +struct rte_ifpga_bus rte_ifpga_bus = {
> +	 .bus = {
> +		 .scan        = rte_ifpga_scan,
> +		 .probe       = rte_ifpga_probe,
> +		 .find_device = rte_ifpga_find_device,

You should use only one tab each lines.

> +	     .plug        = rte_ifpga_plug,
> +	     .unplug      = rte_ifpga_unplug,
> +	     .parse       = rte_ifpga_parse,
> +	 },
> +	.afu_list    = TAILQ_HEAD_INITIALIZER(rte_ifpga_bus.afu_list),
> +	.driver_list = TAILQ_HEAD_INITIALIZER(rte_ifpga_bus.driver_list),
> + };
> +
> +RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus.bus);
> +
> +RTE_INIT(ifpga_init_log)
> +{
> +	ifpga_bus_logtype = rte_log_register("bus.ifpga");
> +	if (ifpga_bus_logtype >= 0)
> +		rte_log_set_level(ifpga_bus_logtype, RTE_LOG_NOTICE);
> +}
> +
> diff --git a/drivers/bus/ifpga/ifpga_common.c b/drivers/bus/ifpga/ifpga_common.c
> new file mode 100644
> index 0000000..2aa6bb0
> --- /dev/null
> +++ b/drivers/bus/ifpga/ifpga_common.c
> @@ -0,0 +1,168 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
> + *   Copyright 2013-2014 6WIND S.A.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +
> +#include <string.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/queue.h>
> +#include <sys/mman.h>
> +#include <sys/types.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +
> +#include <rte_errno.h>
> +#include <rte_bus.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_memzone.h>
> +#include <rte_eal.h>
> +#include <rte_common.h>
> +
> +#include <rte_devargs.h>
> +#include <rte_pci.h>
> +#include <rte_bus_pci.h>
> +#include <rte_kvargs.h>
> +#include <rte_alarm.h>
> +
> +#include "rte_bus_ifpga.h"
> +#include "ifpga_logs.h"
> +#include "ifpga_common.h"
> +
> +int ifpga_get_string_arg(const char *key __rte_unused,
> +	                     const char *value, void *extra_args)
> +{
> +	if (!value || !extra_args)
> +		return -EINVAL;
> +
> +	*(char **)extra_args = strdup(value);
> +
> +	if (!*(char **)extra_args)
> +		return -ENOMEM;
> +
> +	return 0;
> +}
> +int ifpga_get_integer32_arg(const char *key __rte_unused, const char *value, void *extra_args)
> +{
> +	if (!value || !extra_args)
> +		return -EINVAL;
> +
> +	*(int *)extra_args = strtoull(value, NULL, 0);
> +
> +	return 0;
> +}
> +int ifpga_get_integer64_arg(const char *key __rte_unused, const char *value, void *extra_args)
> +{
> +	if (!value || !extra_args)
> +		return -EINVAL;
> +
> +	*(uint64_t *)extra_args = strtoull(value, NULL, 0);
> +
> +	return 0;
> +}
> +int ifpga_get_unsigned_long(const char *str, int base)
> +{
> +	unsigned long num;
> +	char *end = NULL;
> +
> +	errno = 0;
> +
> +	num = strtoul(str, &end, base);
> +	if ((str[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
> +		return -1;
> +
> +	return num;
> +
> +}
> +int ifpga_get_bdf_arg(const char *key __rte_unused, const char *value, void *extra_args)
> +{
> +#define MAX_PATH_LEN 1024
> +	struct rte_pci_addr *addr;
> +	int num[4];
> +	char str[MAX_PATH_LEN];
> +	int i, j;
> +	
> +	if (!value || !extra_args)
> +		return -EINVAL;
> +
> +    addr = (struct rte_pci_addr *)extra_args;

Okay, you seem to be reading a PCI address here.
Why not use directly rte_pci_addr_parse?

> +	strcpy(str, value);
> +	memset(num, 0, 4 * sizeof(num[0]));
> +	i = strlen(str) - 1;
> +	j = 3;
> +	while (i > 0 && j >= 0) {
> +		while ((str[i - 1] != ':' && str[i - 1] != '.') && i > 0)
> +			i--;
> +		num[j--] = ifpga_get_unsigned_long(&str[i], 16);
> +		i--;
> +		if (i >= 0)
> +			str[i] = '\0';
> +	}
> +	addr->domain = num[0];
> +	addr->bus = num[1];
> +	addr->devid = num[2];
> +	addr->function = num[3];
> +	printf("[%s]: bdf %04d:%02d:%02d.%02d\n",
> +			__func__,
> +			addr->domain,
> +			addr->bus,
> +			addr->devid,
> +			addr->function);
> +
> +	return 0;
> +}
> +int ifpga_afu_id_cmp(const struct rte_afu_id *afu_id0, const struct rte_afu_id *afu_id1)
> +{
> +	if((afu_id0->pci_addr.bus       == afu_id1->pci_addr.bus) &&
> +       (afu_id0->pci_addr.devid     == afu_id1->pci_addr.devid) &&
> +       (afu_id0->pci_addr.function  == afu_id1->pci_addr.function) &&

Same as before: you should use rte_pci_addr_cmp here?

> +       (afu_id0->uuid_low           == afu_id1->uuid_low) &&
> +       (afu_id0->uuid_high          == afu_id1->uuid_high) &&
> +       (afu_id0->port               == afu_id1->port)) {
> +       	
> +        return 0;
> +    }
> +    else
> +    	return 1;
> +}
> +int ifpga_pci_addr_cmp(const struct rte_device *dev, const void *_pci_addr)
> +{
> +	struct rte_pci_device *pdev;
> +	const struct rte_pci_addr *paddr = _pci_addr;
> +
> +	pdev = RTE_DEV_TO_PCI(*(struct rte_device **)(void *)&dev);
> +	return rte_eal_compare_pci_addr(&pdev->addr, paddr);

This function is deprecated, rte_pci_addr_cmp must be used instead.

<snip>

Regards,
-- 
Gaëtan Rivet
6WIND

Re: [dpdk-dev] [RFC 1/4] drivers/bus/ifpga:Intel FPGA Bus Lib Code

[Xu, Rosen]

Thanks a lot Rivet, I have modified it, pls see my newest patch.

-----Original Message-----
From: Gaëtan Rivet [mailto:gaetan.rivet@6wind.com] 
Sent: Tuesday, March 06, 2018 18:06
To: Xu, Rosen <rosen.xu@intel.com>
Cc: dev@dpdk.org; Doherty, Declan <declan.doherty@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>
Subject: Re: [dpdk-dev] [RFC 1/4] drivers/bus/ifpga:Intel FPGA Bus Lib Code

Hi Rosen,

A few comments inline.
(I will skip elements already pointed out by Shreyansh.)

On Tue, Mar 06, 2018 at 09:43:55AM +0800, Rosen Xu wrote:
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> ---
>  drivers/bus/ifpga/Makefile                  |  64 ++++
>  drivers/bus/ifpga/ifpga_bus.c               | 527 ++++++++++++++++++++++++++++
>  drivers/bus/ifpga/ifpga_common.c            | 168 +++++++++
>  drivers/bus/ifpga/ifpga_common.h            |  46 +++
>  drivers/bus/ifpga/ifpga_logs.h              |  59 ++++
>  drivers/bus/ifpga/rte_bus_ifpga.h           | 153 ++++++++
>  drivers/bus/ifpga/rte_bus_ifpga_version.map |   8 +
>  7 files changed, 1025 insertions(+)
>  create mode 100644 drivers/bus/ifpga/Makefile  create mode 100644 
> drivers/bus/ifpga/ifpga_bus.c  create mode 100644 
> drivers/bus/ifpga/ifpga_common.c  create mode 100644 
> drivers/bus/ifpga/ifpga_common.h  create mode 100644 
> drivers/bus/ifpga/ifpga_logs.h  create mode 100644 
> drivers/bus/ifpga/rte_bus_ifpga.h  create mode 100644 
> drivers/bus/ifpga/rte_bus_ifpga_version.map
> 
> diff --git a/drivers/bus/ifpga/Makefile b/drivers/bus/ifpga/Makefile 
> new file mode 100644 index 0000000..c71f186
> --- /dev/null
> +++ b/drivers/bus/ifpga/Makefile
> @@ -0,0 +1,64 @@
> +#   BSD LICENSE
> +#
> +#   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
> +#   All rights reserved.
> +#
> +#   Redistribution and use in source and binary forms, with or without
> +#   modification, are permitted provided that the following conditions
> +#   are met:
> +#
> +#     * Redistributions of source code must retain the above copyright
> +#       notice, this list of conditions and the following disclaimer.
> +#     * Redistributions in binary form must reproduce the above copyright
> +#       notice, this list of conditions and the following disclaimer in
> +#       the documentation and/or other materials provided with the
> +#       distribution.
> +#     * Neither the name of Intel Corporation nor the names of its
> +#       contributors may be used to endorse or promote products derived
> +#       from this software without specific prior written permission.
> +#
> +#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +
> +include $(RTE_SDK)/mk/rte.vars.mk
> +
> +#
> +# library name
> +#
> +LIB = librte_bus_ifpga.a
> +LIBABIVER := 1
> +EXPORT_MAP := rte_bus_ifpga_version.map
> +
> +ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT),y)
> +CFLAGS += -O0 -g
> +CFLAGS += "-Wno-error"
> +else
> +CFLAGS += -O3
> +CFLAGS += $(WERROR_FLAGS)
> +endif
> +
> +CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga CFLAGS += 
> +-I$(RTE_SDK)/drivers/bus/pci CFLAGS += 
> +-I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
> +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common
> +#CFLAGS += -I$(RTE_SDK)/lib/librte_rawdev #LDLIBS += -lrte_eal 
> +-lrte_mbuf -lrte_mempool -lrte_ring -lrte_rawdev LDLIBS += -lrte_eal 
> +-lrte_mbuf -lrte_mempool -lrte_ring #LDLIBS += -lrte_ethdev
> +
> +VPATH += $(SRCDIR)/base
> +
> +SRCS-y += \
> +        ifpga_bus.c \
> +        ifpga_common.c
> +
> +include $(RTE_SDK)/mk/rte.lib.mk
> diff --git a/drivers/bus/ifpga/ifpga_bus.c 
> b/drivers/bus/ifpga/ifpga_bus.c new file mode 100644 index 
> 0000000..382d550
> --- /dev/null
> +++ b/drivers/bus/ifpga/ifpga_bus.c
> @@ -0,0 +1,527 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
> + *   Copyright 2013-2014 6WIND S.A.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +
> +#include <string.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/queue.h>
> +#include <sys/mman.h>
> +#include <sys/types.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +
> +#include <rte_errno.h>
> +#include <rte_bus.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_memzone.h>
> +#include <rte_eal.h>
> +#include <rte_common.h>
> +
> +#include <rte_devargs.h>
> +#include <rte_pci.h>
> +#include <rte_bus_pci.h>
> +#include <rte_kvargs.h>
> +#include <rte_alarm.h>
> +
> +#include "rte_rawdev.h"
> +#include "rte_rawdev_pmd.h"
> +#include "rte_bus_ifpga.h"
> +#include "ifpga_logs.h"
> +#include "ifpga_common.h"
> +
> +int ifpga_bus_logtype;
> +
> +/*register a ifpga bus based driver */ void 
> +rte_ifpga_driver_register(struct rte_afu_driver *driver) {
> +	RTE_VERIFY(driver);
> +
> +	TAILQ_INSERT_TAIL(&rte_ifpga_bus.driver_list, driver, next); }
> +
> +/*un-register a fpga bus based driver */ void 
> +rte_ifpga_driver_unregister(struct rte_afu_driver *driver) {
> +	TAILQ_REMOVE(&rte_ifpga_bus.driver_list, driver, next); }
> +
> +static struct rte_afu_device *
> +ifpga_find_afu_dev(const struct rte_afu_id *afu_id) {
> +	struct rte_afu_device *afu_dev = NULL;
> +	

You seem to have trailing spaces (here tabs), and some inconsistencies in your file (sometimes spaces are used, sometime tabs).

You need to run checkpatch on your series.

See those errors:
http://dpdk.org/ml/archives/test-report/2018-March/043185.html

A script is available in devtools to run this before submitting.

> +	TAILQ_FOREACH (afu_dev, &rte_ifpga_bus.afu_list, next) {
> +
> +        if(!ifpga_afu_id_cmp(&afu_dev->id, afu_id)) {
> +            return afu_dev;
> +        }
> +	}
> +	return NULL;
> +}
> +
> +static const char *valid_args[] = {
> +#define IFPGA_ARG_BDF          "bdf"
> +	IFPGA_ARG_BDF,
> +#define IFPGA_ARG_PORT         "port"
> +	IFPGA_ARG_PORT,
> +#define IFPGA_ARG_PATH         "path"
> +	IFPGA_ARG_PATH,
> +#define IFPGA_ARG_UUID_HIGH    "uuid_high"
> +	IFPGA_ARG_UUID_HIGH,
> +#define IFPGA_ARG_UUID_LOW     "uuid_low"
> +	IFPGA_ARG_UUID_LOW,
> +#define IFPGA_ARG_PR_ENABLE     "pr_enable"
> +	IFPGA_ARG_PR_ENABLE,
> +#define IFPGA_ARG_DEBUG         "debug"
> +	IFPGA_ARG_DEBUG,
> +	NULL
> +};
> +
> +/*
> + * Scan the content of the FPGA bus, and the devices in the devices
> + * list
> + */
> +static struct rte_afu_device *
> +rte_ifpga_scan_one(struct rte_devargs *devargs) {
> +	struct rte_kvargs *kvlist = NULL;
> +	struct rte_bus *pci_bus = NULL;
> +	struct rte_device *dev = NULL;
> +	struct rte_rawdev *rawdev;
> +	struct rte_afu_device *afu_dev = NULL;
> +	struct rte_afu_pr_conf afu_pr_conf;
> +	char rawdev_name[RTE_RAWDEV_NAME_MAX_LEN];
> +	int ret;
> +	char *path = NULL;
> +	int pr_enable = 1;
> +	int debug = 0;
> +
> +    memset((char *)(&afu_pr_conf), 0, sizeof(struct 
> + rte_afu_pr_conf));
> +	
> +    kvlist = rte_kvargs_parse(devargs->args, valid_args);
> +    if (!kvlist) {
> +    	IFPGA_BUS_ERR("error when parsing param");
> +    	goto end;
> +    }
> +    
> +    if (rte_kvargs_count(kvlist, IFPGA_ARG_BDF) == 1) {
> +		if (rte_kvargs_process(kvlist, IFPGA_ARG_BDF,
> +				       &ifpga_get_bdf_arg, &afu_pr_conf.afu_id.pci_addr) < 0) {
> +			IFPGA_BUS_ERR("error to parse %s",
> +				     IFPGA_ARG_BDF);
> +			goto end;
> +		}
> +	} else {
> +		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +			  IFPGA_ARG_PATH);
> +		goto end;
> +	}
> +    
> +    if (rte_kvargs_count(kvlist, IFPGA_ARG_PORT) == 1) {
> +		if (rte_kvargs_process(kvlist, IFPGA_ARG_PORT,
> +				       &ifpga_get_integer32_arg, &afu_pr_conf.afu_id.port) < 0) {
> +			IFPGA_BUS_ERR("error to parse %s",
> +				     IFPGA_ARG_PORT);
> +			goto end;
> +		}
> +	}
> +	else {
> +		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +			  IFPGA_ARG_PATH);
> +		goto end;
> +	}
> +	
> +    if (rte_kvargs_count(kvlist, IFPGA_ARG_PATH) == 1) {
> +		if (rte_kvargs_process(kvlist, IFPGA_ARG_PATH,
> +				       &ifpga_get_string_arg, &path) < 0) {
> +			IFPGA_BUS_ERR("error to parse %s",
> +				     IFPGA_ARG_PATH);
> +			goto end;
> +		}
> +	} else {
> +		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +			  IFPGA_ARG_PATH);
> +		goto end;
> +	}
> +	
> +	if (rte_kvargs_count(kvlist, IFPGA_ARG_UUID_HIGH) == 1) {
> +		if (rte_kvargs_process(kvlist, IFPGA_ARG_UUID_HIGH,
> +				       &ifpga_get_integer64_arg, &afu_pr_conf.afu_id.uuid_high) < 0) {
> +			IFPGA_BUS_ERR("error to parse %s",
> +				     IFPGA_ARG_UUID_HIGH);
> +			goto end;
> +		}
> +	} else {
> +		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +			  IFPGA_ARG_PATH);
> +		goto end;
> +	}
> +	
> +	if (rte_kvargs_count(kvlist, IFPGA_ARG_UUID_LOW) == 1) {
> +		if (rte_kvargs_process(kvlist, IFPGA_ARG_UUID_LOW,
> +				       &ifpga_get_integer64_arg, &afu_pr_conf.afu_id.uuid_low) < 0) {
> +			IFPGA_BUS_ERR("error to parse %s",
> +				     IFPGA_ARG_UUID_LOW);
> +			goto end;
> +		}
> +	} else {
> +		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
> +			  IFPGA_ARG_PATH);
> +		goto end;
> +	}
> +	
> +	if (rte_kvargs_count(kvlist, IFPGA_ARG_PR_ENABLE) == 1) {
> +		if (rte_kvargs_process(kvlist, IFPGA_ARG_PR_ENABLE,
> +				       &ifpga_get_integer32_arg, &pr_enable) < 0) {
> +			IFPGA_BUS_ERR("error to parse %s",
> +				     IFPGA_ARG_UUID_HIGH);
> +			goto end;
> +		}
> +	}
> +	
> +	if (rte_kvargs_count(kvlist, IFPGA_ARG_DEBUG) == 1) {
> +		if (rte_kvargs_process(kvlist, IFPGA_ARG_DEBUG,
> +				       &ifpga_get_integer32_arg, &debug) < 0) {
> +			IFPGA_BUS_ERR("error to parse %s",
> +				     IFPGA_ARG_UUID_HIGH);
> +			goto end;
> +		}
> +	}
> +	
> +	if(!debug)
> +	{
> +		pci_bus = rte_bus_find_by_name("pci");
> +	    if (pci_bus == NULL) {
> +	    	IFPGA_BUS_ERR("unable to find PCI bus\n");
> +	    	goto end;
> +	    }
> +	    
> +	    dev = pci_bus->find_device(NULL, ifpga_pci_addr_cmp, &afu_pr_conf.afu_id.pci_addr);
> +	    if (dev == NULL) {
> +	    	IFPGA_BUS_ERR("unable to find PCI device\n");
> +	    	goto end;
> +	    }
> +	}
> +	else
> +	{
> +		IFPGA_BUS_DEBUG("pci_addr domain   : %x\n",   afu_pr_conf.afu_id.pci_addr.domain);
> +		IFPGA_BUS_DEBUG("pci_addr bus      : %x\n",   afu_pr_conf.afu_id.pci_addr.bus);
> +		IFPGA_BUS_DEBUG("pci_addr devid    : %x\n",   afu_pr_conf.afu_id.pci_addr.devid);
> +		IFPGA_BUS_DEBUG("pci_addr function : %x\n",   afu_pr_conf.afu_id.pci_addr.function);

Alternatively, it could be interesting to have a PCI_ADDR_FMT and PCI_ADDR_ARGS in librte_pci, to help printing PCI addresses in a standard way. This could be leveraged in librte_bus_pci as well.

> +		
> +		IFPGA_BUS_DEBUG("uuid_low          : %lx\n", afu_pr_conf.afu_id.uuid_low);
> +		IFPGA_BUS_DEBUG("uuid_high         : %lx\n", afu_pr_conf.afu_id.uuid_high);
> +		
> +		IFPGA_BUS_DEBUG("afu port          : %x\n",   afu_pr_conf.afu_id.port);
> +	}
> +	
> +	if (ifpga_find_afu_dev(&afu_pr_conf.afu_id))
> +		goto end;
> +
> +	memset(rawdev_name, sizeof(rawdev_name), 0);
> +	snprintf(rawdev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%x:%x", afu_pr_conf.afu_id.pci_addr.bus, afu_pr_conf.afu_id.pci_addr.devid, afu_pr_conf.afu_id.pci_addr.function);
> +	rawdev = rte_rawdev_pmd_get_named_dev(rawdev_name);
> +	if(!rawdev)
> +		goto end;
> +	
> +	rawdev->dev_ops->dev_start(rawdev);
> +		
> +	if (pr_enable)
> +	{
> +		ret=rawdev->dev_ops->firmware_load(rawdev, &afu_pr_conf);
> +		if (ret) {
> +    		printf("do pr error\n");
> +    		return NULL;
> +	    }
> +	}
> +		
> +	afu_dev = calloc(1, sizeof(*afu_dev));
> +	if (!afu_dev)
> +		goto end;
> +
> +	afu_dev->device.devargs = devargs;
> +	afu_dev->device.numa_node = SOCKET_ID_ANY;
> +	afu_dev->device.name = devargs->name;
> +	afu_dev->rawdev = rawdev;
> +	afu_dev->id.pci_addr.domain = afu_pr_conf.afu_id.pci_addr.domain;
> +	afu_dev->id.pci_addr.bus = afu_pr_conf.afu_id.pci_addr.bus;
> +	afu_dev->id.pci_addr.devid = afu_pr_conf.afu_id.pci_addr.devid;
> +	afu_dev->id.pci_addr.function = 
> +afu_pr_conf.afu_id.pci_addr.function;

A pci_addr_cpy function could be written.

> +	afu_dev->id.uuid_low  = afu_pr_conf.afu_id.uuid_low;
> +	afu_dev->id.uuid_high = afu_pr_conf.afu_id.uuid_high;
> +	afu_dev->id.port      = afu_pr_conf.afu_id.port;
> +	
> +	rawdev->dev_ops->dev_info_get(rawdev, afu_dev);
> +
> +    return afu_dev;
> +    
> +end:
> +	if (kvlist)
> +		rte_kvargs_free(kvlist);
> +	if (path)
> +		free(path);
> +	
> +	return NULL;
> +}
> +
> +/*
> + * Scan the content of the FPGA bus, and the devices in the devices
> + * list
> + */
> +static int
> +rte_ifpga_scan(void)
> +{
> +	struct rte_bus *pci_bus;
> +	struct rte_afu_device *afu_dev;
> +	struct rte_devargs *devargs;
> +
> +    pci_bus = rte_bus_find_by_name("pci");
> +	if (pci_bus == NULL) {
> +		IFPGA_BUS_ERR("unable to find PCI bus\n");
> +		return -1;
> +	}
> +	
> +	/* for virtual devices we scan the devargs_list populated via cmdline */
> +	TAILQ_FOREACH(devargs, &devargs_list, next) {
> +    
> +		if (devargs->bus != &rte_ifpga_bus.bus)
> +			continue;
> +		
> +		afu_dev = rte_ifpga_scan_one(devargs);
> +	    
> +	    if(afu_dev)
> +		    TAILQ_INSERT_TAIL(&rte_ifpga_bus.afu_list, afu_dev, next);
> +	}
> +
> +	return 0;
> +}
> +
> +static int
> +ifpga_probe_one_driver(struct rte_afu_driver *drv,
> +			struct rte_afu_device *afu_dev)
> +{
> +	int ret;
> +
> +    if((drv->id.pci_addr.bus == afu_dev->id.pci_addr.bus) &&
> +       (drv->id.pci_addr.devid == afu_dev->id.pci_addr.devid) &&
> +       (drv->id.pci_addr.function == afu_dev->id.pci_addr.function) 
> + &&

Is it expected to ignore the domain here?
You should use rte_pci_addr_cmp instead.

> +       (drv->id.uuid_low == afu_dev->id.uuid_low) &&
> +       (drv->id.uuid_high == afu_dev->id.uuid_high) &&
> +       (drv->id.port == afu_dev->id.port)) {
> +       	
> +       	afu_dev->driver = drv;
> +       	
> +       	/* call the driver probe() function */
> +		ret = drv->probe(afu_dev);
> +		if (ret)
> +			afu_dev->driver = NULL;
> +		return ret;
> +    }
> +	
> +	/* return positive value if driver doesn't support this device */
> +	return 1;
> +}
> +
> +static int
> +ifpga_probe_all_drivers(struct rte_afu_device *afu_dev) {
> +	const char *name;
> +	struct rte_afu_driver *drv = NULL;
> +	int rc;
> +
> +	if (afu_dev == NULL)
> +		return -1;
> +
> +	/* Check if a driver is already loaded */
> +	if (afu_dev->driver != NULL)
> +		return 0;
> +		
> +    name = rte_ifpga_device_name(afu_dev);
> +    IFPGA_BUS_DEBUG("Search driver %s to probe device %s\n", name,
> +		rte_ifpga_device_name(afu_dev));
> +    
> +	TAILQ_FOREACH(drv, &rte_ifpga_bus.driver_list, next) {
> +		rc = ifpga_probe_one_driver(drv, afu_dev);
> +		if (rc < 0)
> +			/* negative value is an error */
> +			return -1;
> +		if (rc > 0)
> +			/* positive value means driver doesn't support it */
> +			continue;
> +		return 0;
> +	}
> +	return 1;
> + }
> +
> +/*
> + * Scan the content of the PCI bus, and call the probe() function for
> + * all registered drivers that have a matching entry in its id_table
> + * for discovered devices.
> + */
> +static int
> +rte_ifpga_probe(void)
> +{
> +	struct rte_afu_device *afu_dev = NULL;
> +	int ret = 0;
> +
> +	TAILQ_FOREACH (afu_dev, &rte_ifpga_bus.afu_list, next) {
> +
> +        if (afu_dev->device.driver)
> +			continue;
> +			
> +		ret = ifpga_probe_all_drivers(afu_dev);
> +		if (ret < 0)
> +		    IFPGA_BUS_ERR("failed to initialize %s device\n",
> +				rte_ifpga_device_name(afu_dev));
> +	}
> +
> +	return 0;
> +}
> +
> +static int
> +rte_ifpga_plug(struct rte_device *dev) {
> +	return ifpga_probe_all_drivers(RTE_DEV_TO_AFU(dev));
> +}
> +
> +static int ifpga_remove_driver(struct rte_afu_device *afu_dev) {
> +	const char *name;
> +	const struct rte_afu_driver *driver;
> +
> +    name = rte_ifpga_device_name(afu_dev);
> +	if (!afu_dev->device.driver) {
> +		IFPGA_BUS_DEBUG("no driver attach to device %s\n", name);
> +		return 1;
> +	}
> +
> +	driver = container_of(afu_dev->device.driver, const struct rte_afu_driver,
> +		driver);
> +	return driver->remove(afu_dev);
> +}
> +
> +static int
> +rte_ifpga_unplug(struct rte_device *dev) {
> +	struct rte_afu_device *afu_dev;
> +	struct rte_devargs *devargs;
> +	int ret;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	afu_dev = RTE_DEV_TO_AFU(dev);
> +	if (!dev)
> +		return -ENOENT;
> +
> +	devargs = dev->devargs;
> +
> +	ret = ifpga_remove_driver(afu_dev);
> +	if (ret)
> +		return ret;
> +
> +	TAILQ_REMOVE(&rte_ifpga_bus.afu_list, afu_dev, next);
> +
> +	TAILQ_REMOVE(&devargs_list, devargs, next);
> +
> +	free(devargs->args);
> +	free(devargs);
> +	free(afu_dev);
> +	return 0;
> +
> +}
> +
> +static struct rte_device *
> +rte_ifpga_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
> +		                                        const void *data) {
> +	struct rte_afu_device *afu_dev;
> +
> +	TAILQ_FOREACH(afu_dev, &rte_ifpga_bus.afu_list, next) {
> +		if (start && &afu_dev->device == start) {
> +			start = NULL;
> +			continue;
> +		}
> +		if (cmp(&afu_dev->device, data) == 0)
> +			return &afu_dev->device;
> +	}
> +	return NULL;
> +}
> +static int
> +rte_ifpga_parse(const char *name, void *addr) {
> +	struct rte_afu_driver **out = addr;
> +	struct rte_afu_driver *driver = NULL;
> +
> +	TAILQ_FOREACH(driver, &rte_ifpga_bus.driver_list, next) {
> +		if (strncmp(driver->driver.name, name,
> +			    strlen(driver->driver.name)) == 0)
> +			break;
> +		if (driver->driver.alias &&
> +		    strncmp(driver->driver.alias, name,
> +			    strlen(driver->driver.alias)) == 0)
> +			break;
> +	}
> +	if (driver != NULL &&
> +	    addr != NULL)
> +		*out = driver;
> +	return driver == NULL;
> +}
> +
> +struct rte_ifpga_bus rte_ifpga_bus = {
> +	 .bus = {
> +		 .scan        = rte_ifpga_scan,
> +		 .probe       = rte_ifpga_probe,
> +		 .find_device = rte_ifpga_find_device,

You should use only one tab each lines.

> +	     .plug        = rte_ifpga_plug,
> +	     .unplug      = rte_ifpga_unplug,
> +	     .parse       = rte_ifpga_parse,
> +	 },
> +	.afu_list    = TAILQ_HEAD_INITIALIZER(rte_ifpga_bus.afu_list),
> +	.driver_list = TAILQ_HEAD_INITIALIZER(rte_ifpga_bus.driver_list),
> + };
> +
> +RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus.bus);
> +
> +RTE_INIT(ifpga_init_log)
> +{
> +	ifpga_bus_logtype = rte_log_register("bus.ifpga");
> +	if (ifpga_bus_logtype >= 0)
> +		rte_log_set_level(ifpga_bus_logtype, RTE_LOG_NOTICE); }
> +
> diff --git a/drivers/bus/ifpga/ifpga_common.c 
> b/drivers/bus/ifpga/ifpga_common.c
> new file mode 100644
> index 0000000..2aa6bb0
> --- /dev/null
> +++ b/drivers/bus/ifpga/ifpga_common.c
> @@ -0,0 +1,168 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
> + *   Copyright 2013-2014 6WIND S.A.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +
> +#include <string.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/queue.h>
> +#include <sys/mman.h>
> +#include <sys/types.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +
> +#include <rte_errno.h>
> +#include <rte_bus.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_memzone.h>
> +#include <rte_eal.h>
> +#include <rte_common.h>
> +
> +#include <rte_devargs.h>
> +#include <rte_pci.h>
> +#include <rte_bus_pci.h>
> +#include <rte_kvargs.h>
> +#include <rte_alarm.h>
> +
> +#include "rte_bus_ifpga.h"
> +#include "ifpga_logs.h"
> +#include "ifpga_common.h"
> +
> +int ifpga_get_string_arg(const char *key __rte_unused,
> +	                     const char *value, void *extra_args) {
> +	if (!value || !extra_args)
> +		return -EINVAL;
> +
> +	*(char **)extra_args = strdup(value);
> +
> +	if (!*(char **)extra_args)
> +		return -ENOMEM;
> +
> +	return 0;
> +}
> +int ifpga_get_integer32_arg(const char *key __rte_unused, const char 
> +*value, void *extra_args) {
> +	if (!value || !extra_args)
> +		return -EINVAL;
> +
> +	*(int *)extra_args = strtoull(value, NULL, 0);
> +
> +	return 0;
> +}
> +int ifpga_get_integer64_arg(const char *key __rte_unused, const char 
> +*value, void *extra_args) {
> +	if (!value || !extra_args)
> +		return -EINVAL;
> +
> +	*(uint64_t *)extra_args = strtoull(value, NULL, 0);
> +
> +	return 0;
> +}
> +int ifpga_get_unsigned_long(const char *str, int base) {
> +	unsigned long num;
> +	char *end = NULL;
> +
> +	errno = 0;
> +
> +	num = strtoul(str, &end, base);
> +	if ((str[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
> +		return -1;
> +
> +	return num;
> +
> +}
> +int ifpga_get_bdf_arg(const char *key __rte_unused, const char 
> +*value, void *extra_args) { #define MAX_PATH_LEN 1024
> +	struct rte_pci_addr *addr;
> +	int num[4];
> +	char str[MAX_PATH_LEN];
> +	int i, j;
> +	
> +	if (!value || !extra_args)
> +		return -EINVAL;
> +
> +    addr = (struct rte_pci_addr *)extra_args;

Okay, you seem to be reading a PCI address here.
Why not use directly rte_pci_addr_parse?

> +	strcpy(str, value);
> +	memset(num, 0, 4 * sizeof(num[0]));
> +	i = strlen(str) - 1;
> +	j = 3;
> +	while (i > 0 && j >= 0) {
> +		while ((str[i - 1] != ':' && str[i - 1] != '.') && i > 0)
> +			i--;
> +		num[j--] = ifpga_get_unsigned_long(&str[i], 16);
> +		i--;
> +		if (i >= 0)
> +			str[i] = '\0';
> +	}
> +	addr->domain = num[0];
> +	addr->bus = num[1];
> +	addr->devid = num[2];
> +	addr->function = num[3];
> +	printf("[%s]: bdf %04d:%02d:%02d.%02d\n",
> +			__func__,
> +			addr->domain,
> +			addr->bus,
> +			addr->devid,
> +			addr->function);
> +
> +	return 0;
> +}
> +int ifpga_afu_id_cmp(const struct rte_afu_id *afu_id0, const struct 
> +rte_afu_id *afu_id1) {
> +	if((afu_id0->pci_addr.bus       == afu_id1->pci_addr.bus) &&
> +       (afu_id0->pci_addr.devid     == afu_id1->pci_addr.devid) &&
> +       (afu_id0->pci_addr.function  == afu_id1->pci_addr.function) &&

Same as before: you should use rte_pci_addr_cmp here?

> +       (afu_id0->uuid_low           == afu_id1->uuid_low) &&
> +       (afu_id0->uuid_high          == afu_id1->uuid_high) &&
> +       (afu_id0->port               == afu_id1->port)) {
> +       	
> +        return 0;
> +    }
> +    else
> +    	return 1;
> +}
> +int ifpga_pci_addr_cmp(const struct rte_device *dev, const void 
> +*_pci_addr) {
> +	struct rte_pci_device *pdev;
> +	const struct rte_pci_addr *paddr = _pci_addr;
> +
> +	pdev = RTE_DEV_TO_PCI(*(struct rte_device **)(void *)&dev);
> +	return rte_eal_compare_pci_addr(&pdev->addr, paddr);

This function is deprecated, rte_pci_addr_cmp must be used instead.

<snip>

Regards,
--
Gaëtan Rivet
6WIND

[dpdk-dev] [RFC 2/4] lib/librte_eal/common:Add Intel FPGA Bus Running Command Parse Code

[Rosen Xu]

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 lib/librte_eal/common/eal_common_options.c | 8 +++++++-
 lib/librte_eal/common/eal_options.h        | 2 ++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index 9f2f8d2..1158d21 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -73,6 +73,7 @@
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
+	{OPT_IFPGA,             1, NULL, OPT_IFPGA_NUM            },
 	{0,                     0, NULL, 0                        }
 };
 
@@ -1160,7 +1161,12 @@ static int xdigit2val(unsigned char c)
 
 		core_parsed = LCORE_OPT_MAP;
 		break;
-
+    case OPT_IFPGA_NUM:
+	    if (eal_option_device_add(RTE_DEVTYPE_VIRTUAL,
+	    		optarg) < 0) {
+	    	return -1;
+	    }
+	    break;
 	/* don't know what to do, leave this to caller */
 	default:
 		return 1;
diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h
index e86c711..bdbb2c4 100644
--- a/lib/librte_eal/common/eal_options.h
+++ b/lib/librte_eal/common/eal_options.h
@@ -55,6 +55,8 @@ enum {
 	OPT_VFIO_INTR_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
+#define OPT_IFPGA              "ifpga"
+	OPT_IFPGA_NUM,
 	OPT_LONG_MAX_NUM
 };
 
-- 
1.8.3.1

[dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus

[Rosen Xu]

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 lib/librte_eal/common/eal_common_bus.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/lib/librte_eal/common/eal_common_bus.c b/lib/librte_eal/common/eal_common_bus.c
index 3e022d5..74bfa15 100644
--- a/lib/librte_eal/common/eal_common_bus.c
+++ b/lib/librte_eal/common/eal_common_bus.c
@@ -70,15 +70,27 @@ struct rte_bus_list rte_bus_list =
 rte_bus_scan(void)
 {
 	int ret;
-	struct rte_bus *bus = NULL;
+	struct rte_bus *bus = NULL, *ifpga_bus = NULL;
 
 	TAILQ_FOREACH(bus, &rte_bus_list, next) {
+		if (!strcmp(bus->name, "ifpga")) {
+			ifpga_bus = bus;
+			continue;
+		}
+		
 		ret = bus->scan();
 		if (ret)
 			RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
 				bus->name);
 	}
 
+	if (ifpga_bus) {
+		ret = ifpga_bus->scan();
+		if (ret)
+			RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
+				ifpga_bus->name);
+	}
+
 	return 0;
 }
 
-- 
1.8.3.1

Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus

[Shreyansh Jain]

On Tue, Mar 6, 2018 at 7:13 AM, Rosen Xu <rosen.xu@intel.com> wrote:
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> ---
>  lib/librte_eal/common/eal_common_bus.c | 14 +++++++++++++-
>  1 file changed, 13 insertions(+), 1 deletion(-)
>
> diff --git a/lib/librte_eal/common/eal_common_bus.c b/lib/librte_eal/common/eal_common_bus.c
> index 3e022d5..74bfa15 100644
> --- a/lib/librte_eal/common/eal_common_bus.c
> +++ b/lib/librte_eal/common/eal_common_bus.c
> @@ -70,15 +70,27 @@ struct rte_bus_list rte_bus_list =
>  rte_bus_scan(void)
>  {
>         int ret;
> -       struct rte_bus *bus = NULL;
> +       struct rte_bus *bus = NULL, *ifpga_bus = NULL;
>
>         TAILQ_FOREACH(bus, &rte_bus_list, next) {
> +               if (!strcmp(bus->name, "ifpga")) {
> +                       ifpga_bus = bus;
> +                       continue;
> +               }
> +
>                 ret = bus->scan();
>                 if (ret)
>                         RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
>                                 bus->name);
>         }
>
> +       if (ifpga_bus) {
> +               ret = ifpga_bus->scan();
> +               if (ret)
> +                       RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
> +                               ifpga_bus->name);
> +       }
> +

You are doing this just so that PCI scans are completed *before* ifpga scans?
Well, I understand that this certainly is an issue that we can't yet
define a priority ordering of bus scans.

But, I think what you are require is a simpler:

In the file ifpga_bus.c:

+RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus.bus); <== this
...
...
#define RTE_REGISTER_BUS(nm, bus) \
RTE_INIT_PRIO(businitfn_ ##nm, 110); \

If you define your own version of RTE_REGISTER_BUS with the priority
number higher, it would be inserted later in the bus list.
rte_register_bus doesn't do any inherent ordering.
This would save the changes you are doing in the
lib/librte_eal/common/eal_common_bus.c file.

But I think there has to be a better provision of defining priority of
bus scans - I am sure when new devices come in, there would be
possibility of dependencies as in your case.

>         return 0;
>  }
>
> --
> 1.8.3.1
>

Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus

[Xu, Rosen]

-----Original Message-----
From: Shreyansh Jain [mailto:shreyansh.jain@nxp.com] 
Sent: Tuesday, March 06, 2018 14:20
To: Xu, Rosen <rosen.xu@intel.com>
Cc: dev@dpdk.org; Doherty, Declan <declan.doherty@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>
Subject: Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus

On Tue, Mar 6, 2018 at 7:13 AM, Rosen Xu <rosen.xu@intel.com> wrote:
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> ---
>  lib/librte_eal/common/eal_common_bus.c | 14 +++++++++++++-
>  1 file changed, 13 insertions(+), 1 deletion(-)
>
> diff --git a/lib/librte_eal/common/eal_common_bus.c 
> b/lib/librte_eal/common/eal_common_bus.c
> index 3e022d5..74bfa15 100644
> --- a/lib/librte_eal/common/eal_common_bus.c
> +++ b/lib/librte_eal/common/eal_common_bus.c
> @@ -70,15 +70,27 @@ struct rte_bus_list rte_bus_list =
>  rte_bus_scan(void)
>  {
>         int ret;
> -       struct rte_bus *bus = NULL;
> +       struct rte_bus *bus = NULL, *ifpga_bus = NULL;
>
>         TAILQ_FOREACH(bus, &rte_bus_list, next) {
> +               if (!strcmp(bus->name, "ifpga")) {
> +                       ifpga_bus = bus;
> +                       continue;
> +               }
> +
>                 ret = bus->scan();
>                 if (ret)
>                         RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
>                                 bus->name);
>         }
>
> +       if (ifpga_bus) {
> +               ret = ifpga_bus->scan();
> +               if (ret)
> +                       RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
> +                               ifpga_bus->name);
> +       }
> +

You are doing this just so that PCI scans are completed *before* ifpga scans?
Rosen: yes
Well, I understand that this certainly is an issue that we can't yet define a priority ordering of bus scans.

But, I think what you are require is a simpler:

In the file ifpga_bus.c:

+RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus.bus); <== this
...
...
#define RTE_REGISTER_BUS(nm, bus) \
RTE_INIT_PRIO(businitfn_ ##nm, 110); \

If you define your own version of RTE_REGISTER_BUS with the priority number higher, it would be inserted later in the bus list.
rte_register_bus doesn't do any inherent ordering.
This would save the changes you are doing in the lib/librte_eal/common/eal_common_bus.c file.

But I think there has to be a better provision of defining priority of bus scans - I am sure when new devices come in, there would be possibility of dependencies as in your case.
Rosen: is the priority scan of bus is implemented?

>         return 0;
>  }
>
> --
> 1.8.3.1
>

Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus

[Gaëtan Rivet]

On Tue, Mar 06, 2018 at 10:42:14AM +0000, Xu, Rosen wrote:
> 
> 
> -----Original Message-----
> From: Shreyansh Jain [mailto:shreyansh.jain@nxp.com] 
> Sent: Tuesday, March 06, 2018 14:20
> To: Xu, Rosen <rosen.xu@intel.com>
> Cc: dev@dpdk.org; Doherty, Declan <declan.doherty@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>
> Subject: Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus
> 
> On Tue, Mar 6, 2018 at 7:13 AM, Rosen Xu <rosen.xu@intel.com> wrote:
> > Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> > ---
> >  lib/librte_eal/common/eal_common_bus.c | 14 +++++++++++++-
> >  1 file changed, 13 insertions(+), 1 deletion(-)
> >
> > diff --git a/lib/librte_eal/common/eal_common_bus.c 
> > b/lib/librte_eal/common/eal_common_bus.c
> > index 3e022d5..74bfa15 100644
> > --- a/lib/librte_eal/common/eal_common_bus.c
> > +++ b/lib/librte_eal/common/eal_common_bus.c
> > @@ -70,15 +70,27 @@ struct rte_bus_list rte_bus_list =
> >  rte_bus_scan(void)
> >  {
> >         int ret;
> > -       struct rte_bus *bus = NULL;
> > +       struct rte_bus *bus = NULL, *ifpga_bus = NULL;
> >
> >         TAILQ_FOREACH(bus, &rte_bus_list, next) {
> > +               if (!strcmp(bus->name, "ifpga")) {
> > +                       ifpga_bus = bus;
> > +                       continue;
> > +               }
> > +
> >                 ret = bus->scan();
> >                 if (ret)
> >                         RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
> >                                 bus->name);
> >         }
> >
> > +       if (ifpga_bus) {
> > +               ret = ifpga_bus->scan();
> > +               if (ret)
> > +                       RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
> > +                               ifpga_bus->name);
> > +       }
> > +
> 
> You are doing this just so that PCI scans are completed *before* ifpga scans?
> Rosen: yes
> Well, I understand that this certainly is an issue that we can't yet define a priority ordering of bus scans.
> 
> But, I think what you are require is a simpler:
> 
> In the file ifpga_bus.c:
> 
> +RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus.bus); <== this
> ...
> ...
> #define RTE_REGISTER_BUS(nm, bus) \
> RTE_INIT_PRIO(businitfn_ ##nm, 110); \
> 
> If you define your own version of RTE_REGISTER_BUS with the priority number higher, it would be inserted later in the bus list.
> rte_register_bus doesn't do any inherent ordering.
> This would save the changes you are doing in the lib/librte_eal/common/eal_common_bus.c file.
> 
> But I think there has to be a better provision of defining priority of bus scans - I am sure when new devices come in, there would be possibility of dependencies as in your case.
> Rosen: is the priority scan of bus is implemented?

No, there is no priority set for scanning order.
However, the order in which buses are registered, will modify the order
in which scans are done.

Thus, if you change the priority of your registration, you should be
able to ensure that your scan comes last.

> 
> >         return 0;
> >  }
> >
> > --
> > 1.8.3.1
> >

-- 
Gaëtan Rivet
6WIND

Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus

[Bruce Richardson]

On Tue, Mar 06, 2018 at 11:46:22AM +0100, Gaëtan Rivet wrote:
> On Tue, Mar 06, 2018 at 10:42:14AM +0000, Xu, Rosen wrote:
> > 
> > 
> > -----Original Message-----
> > From: Shreyansh Jain [mailto:shreyansh.jain@nxp.com] 
> > Sent: Tuesday, March 06, 2018 14:20
> > To: Xu, Rosen <rosen.xu@intel.com>
> > Cc: dev@dpdk.org; Doherty, Declan <declan.doherty@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>
> > Subject: Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus
> > 
> > On Tue, Mar 6, 2018 at 7:13 AM, Rosen Xu <rosen.xu@intel.com> wrote:
> > > Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> > > ---
> > >  lib/librte_eal/common/eal_common_bus.c | 14 +++++++++++++-
> > >  1 file changed, 13 insertions(+), 1 deletion(-)
> > >
> > > diff --git a/lib/librte_eal/common/eal_common_bus.c 
> > > b/lib/librte_eal/common/eal_common_bus.c
> > > index 3e022d5..74bfa15 100644
> > > --- a/lib/librte_eal/common/eal_common_bus.c
> > > +++ b/lib/librte_eal/common/eal_common_bus.c
> > > @@ -70,15 +70,27 @@ struct rte_bus_list rte_bus_list =
> > >  rte_bus_scan(void)
> > >  {
> > >         int ret;
> > > -       struct rte_bus *bus = NULL;
> > > +       struct rte_bus *bus = NULL, *ifpga_bus = NULL;
> > >
> > >         TAILQ_FOREACH(bus, &rte_bus_list, next) {
> > > +               if (!strcmp(bus->name, "ifpga")) {
> > > +                       ifpga_bus = bus;
> > > +                       continue;
> > > +               }
> > > +
> > >                 ret = bus->scan();
> > >                 if (ret)
> > >                         RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
> > >                                 bus->name);
> > >         }
> > >
> > > +       if (ifpga_bus) {
> > > +               ret = ifpga_bus->scan();
> > > +               if (ret)
> > > +                       RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
> > > +                               ifpga_bus->name);
> > > +       }
> > > +
> > 
> > You are doing this just so that PCI scans are completed *before* ifpga scans?
> > Rosen: yes
> > Well, I understand that this certainly is an issue that we can't yet define a priority ordering of bus scans.
> > 
> > But, I think what you are require is a simpler:
> > 
> > In the file ifpga_bus.c:
> > 
> > +RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus.bus); <== this
> > ...
> > ...
> > #define RTE_REGISTER_BUS(nm, bus) \
> > RTE_INIT_PRIO(businitfn_ ##nm, 110); \
> > 
> > If you define your own version of RTE_REGISTER_BUS with the priority number higher, it would be inserted later in the bus list.
> > rte_register_bus doesn't do any inherent ordering.
> > This would save the changes you are doing in the lib/librte_eal/common/eal_common_bus.c file.
> > 
> > But I think there has to be a better provision of defining priority of bus scans - I am sure when new devices come in, there would be possibility of dependencies as in your case.
> > Rosen: is the priority scan of bus is implemented?
> 
> No, there is no priority set for scanning order.
> However, the order in which buses are registered, will modify the order
> in which scans are done.
> 
> Thus, if you change the priority of your registration, you should be
> able to ensure that your scan comes last.
> 

Can we register the bus only when a PCI device match is found at
runtime, e.g. as part of the PCI driver instance initialization?

/Bruce

Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus

[Gaëtan Rivet]

On Tue, Mar 06, 2018 at 11:36:17AM +0000, Bruce Richardson wrote:
> On Tue, Mar 06, 2018 at 11:46:22AM +0100, Gaëtan Rivet wrote:
> > On Tue, Mar 06, 2018 at 10:42:14AM +0000, Xu, Rosen wrote:
> > > 
> > > 
> > > -----Original Message-----
> > > From: Shreyansh Jain [mailto:shreyansh.jain@nxp.com] 
> > > Sent: Tuesday, March 06, 2018 14:20
> > > To: Xu, Rosen <rosen.xu@intel.com>
> > > Cc: dev@dpdk.org; Doherty, Declan <declan.doherty@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>
> > > Subject: Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus
> > > 
> > > On Tue, Mar 6, 2018 at 7:13 AM, Rosen Xu <rosen.xu@intel.com> wrote:
> > > > Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> > > > ---
> > > >  lib/librte_eal/common/eal_common_bus.c | 14 +++++++++++++-
> > > >  1 file changed, 13 insertions(+), 1 deletion(-)
> > > >
> > > > diff --git a/lib/librte_eal/common/eal_common_bus.c 
> > > > b/lib/librte_eal/common/eal_common_bus.c
> > > > index 3e022d5..74bfa15 100644
> > > > --- a/lib/librte_eal/common/eal_common_bus.c
> > > > +++ b/lib/librte_eal/common/eal_common_bus.c
> > > > @@ -70,15 +70,27 @@ struct rte_bus_list rte_bus_list =
> > > >  rte_bus_scan(void)
> > > >  {
> > > >         int ret;
> > > > -       struct rte_bus *bus = NULL;
> > > > +       struct rte_bus *bus = NULL, *ifpga_bus = NULL;
> > > >
> > > >         TAILQ_FOREACH(bus, &rte_bus_list, next) {
> > > > +               if (!strcmp(bus->name, "ifpga")) {
> > > > +                       ifpga_bus = bus;
> > > > +                       continue;
> > > > +               }
> > > > +
> > > >                 ret = bus->scan();
> > > >                 if (ret)
> > > >                         RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
> > > >                                 bus->name);
> > > >         }
> > > >
> > > > +       if (ifpga_bus) {
> > > > +               ret = ifpga_bus->scan();
> > > > +               if (ret)
> > > > +                       RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
> > > > +                               ifpga_bus->name);
> > > > +       }
> > > > +
> > > 
> > > You are doing this just so that PCI scans are completed *before* ifpga scans?
> > > Rosen: yes
> > > Well, I understand that this certainly is an issue that we can't yet define a priority ordering of bus scans.
> > > 
> > > But, I think what you are require is a simpler:
> > > 
> > > In the file ifpga_bus.c:
> > > 
> > > +RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus.bus); <== this
> > > ...
> > > ...
> > > #define RTE_REGISTER_BUS(nm, bus) \
> > > RTE_INIT_PRIO(businitfn_ ##nm, 110); \
> > > 
> > > If you define your own version of RTE_REGISTER_BUS with the priority number higher, it would be inserted later in the bus list.
> > > rte_register_bus doesn't do any inherent ordering.
> > > This would save the changes you are doing in the lib/librte_eal/common/eal_common_bus.c file.
> > > 
> > > But I think there has to be a better provision of defining priority of bus scans - I am sure when new devices come in, there would be possibility of dependencies as in your case.
> > > Rosen: is the priority scan of bus is implemented?
> > 
> > No, there is no priority set for scanning order.
> > However, the order in which buses are registered, will modify the order
> > in which scans are done.
> > 
> > Thus, if you change the priority of your registration, you should be
> > able to ensure that your scan comes last.
> > 
> 
> Can we register the bus only when a PCI device match is found at
> runtime, e.g. as part of the PCI driver instance initialization?
> 
> /Bruce

Technically, yes. You would append a new bus during rte_bus_probe, so
the linked list would simply have a new node and you would then probe
it. You would need to make sure you scan your bus first, so you would
have some weird conditions (whether you are loaded during probe or
naturally, you'd have to do your scan or not).

However, this seems like a terrible idea. You introduce an edge case
that will need to be carried over in most of the bus API implementation.

This new bus seems like a specialization of the PCI bus. Why not directly
use the PCI bus and have your driver linked to either a rawdev or a vdev,
where you could store your metadata and expose a specialized interface?

-- 
Gaëtan Rivet
6WIND

Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus

[Xu, Rosen]

-----Original Message-----
From: Gaëtan Rivet [mailto:gaetan.rivet@6wind.com] 
Sent: Tuesday, March 06, 2018 20:00
To: Richardson, Bruce <bruce.richardson@intel.com>
Cc: Xu, Rosen <rosen.xu@intel.com>; Shreyansh Jain <shreyansh.jain@nxp.com>; dev@dpdk.org; Doherty, Declan <declan.doherty@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>
Subject: Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus

On Tue, Mar 06, 2018 at 11:36:17AM +0000, Bruce Richardson wrote:
> On Tue, Mar 06, 2018 at 11:46:22AM +0100, Gaëtan Rivet wrote:
> > On Tue, Mar 06, 2018 at 10:42:14AM +0000, Xu, Rosen wrote:
> > > 
> > > 
> > > -----Original Message-----
> > > From: Shreyansh Jain [mailto:shreyansh.jain@nxp.com]
> > > Sent: Tuesday, March 06, 2018 14:20
> > > To: Xu, Rosen <rosen.xu@intel.com>
> > > Cc: dev@dpdk.org; Doherty, Declan <declan.doherty@intel.com>; 
> > > Zhang, Tianfei <tianfei.zhang@intel.com>
> > > Subject: Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel 
> > > FPGA Bus Second Scan, it should be scanned after PCI Bus
> > > 
> > > On Tue, Mar 6, 2018 at 7:13 AM, Rosen Xu <rosen.xu@intel.com> wrote:
> > > > Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> > > > ---
> > > >  lib/librte_eal/common/eal_common_bus.c | 14 +++++++++++++-
> > > >  1 file changed, 13 insertions(+), 1 deletion(-)
> > > >
> > > > diff --git a/lib/librte_eal/common/eal_common_bus.c
> > > > b/lib/librte_eal/common/eal_common_bus.c
> > > > index 3e022d5..74bfa15 100644
> > > > --- a/lib/librte_eal/common/eal_common_bus.c
> > > > +++ b/lib/librte_eal/common/eal_common_bus.c
> > > > @@ -70,15 +70,27 @@ struct rte_bus_list rte_bus_list =
> > > >  rte_bus_scan(void)
> > > >  {
> > > >         int ret;
> > > > -       struct rte_bus *bus = NULL;
> > > > +       struct rte_bus *bus = NULL, *ifpga_bus = NULL;
> > > >
> > > >         TAILQ_FOREACH(bus, &rte_bus_list, next) {
> > > > +               if (!strcmp(bus->name, "ifpga")) {
> > > > +                       ifpga_bus = bus;
> > > > +                       continue;
> > > > +               }
> > > > +
> > > >                 ret = bus->scan();
> > > >                 if (ret)
> > > >                         RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
> > > >                                 bus->name);
> > > >         }
> > > >
> > > > +       if (ifpga_bus) {
> > > > +               ret = ifpga_bus->scan();
> > > > +               if (ret)
> > > > +                       RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
> > > > +                               ifpga_bus->name);
> > > > +       }
> > > > +
> > > 
> > > You are doing this just so that PCI scans are completed *before* ifpga scans?
> > > Rosen: yes
> > > Well, I understand that this certainly is an issue that we can't yet define a priority ordering of bus scans.
> > > 
> > > But, I think what you are require is a simpler:
> > > 
> > > In the file ifpga_bus.c:
> > > 
> > > +RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus.bus); <== this
> > > ...
> > > ...
> > > #define RTE_REGISTER_BUS(nm, bus) \ RTE_INIT_PRIO(businitfn_ ##nm, 
> > > 110); \
> > > 
> > > If you define your own version of RTE_REGISTER_BUS with the priority number higher, it would be inserted later in the bus list.
> > > rte_register_bus doesn't do any inherent ordering.
> > > This would save the changes you are doing in the lib/librte_eal/common/eal_common_bus.c file.
> > > 
> > > But I think there has to be a better provision of defining priority of bus scans - I am sure when new devices come in, there would be possibility of dependencies as in your case.
> > > Rosen: is the priority scan of bus is implemented?
> > 
> > No, there is no priority set for scanning order.
> > However, the order in which buses are registered, will modify the 
> > order in which scans are done.
> > 
> > Thus, if you change the priority of your registration, you should be 
> > able to ensure that your scan comes last.
> > 
> 
> Can we register the bus only when a PCI device match is found at 
> runtime, e.g. as part of the PCI driver instance initialization?
> 
> /Bruce

Technically, yes. You would append a new bus during rte_bus_probe, so the linked list would simply have a new node and you would then probe it. You would need to make sure you scan your bus first, so you would have some weird conditions (whether you are loaded during probe or naturally, you'd have to do your scan or not).

However, this seems like a terrible idea. You introduce an edge case that will need to be carried over in most of the bus API implementation.

This new bus seems like a specialization of the PCI bus. Why not directly use the PCI bus and have your driver linked to either a rawdev or a vdev, where you could store your metadata and expose a specialized interface?
Rosen: pls see my v1 patch, in that patch we don't need to modify rte_bus_scan(), the IFPGA Bus Scen is probed by FPGA PCI Driver.
             The reason wo don't directly use PCI bus is that:
              1. One FPGA PCI Device has more than one AFU bitstream;
              2. Each AFU is a hardware device viewed by DPDK;
              3. Acceleration Driver(like Eth/Crpt) bind to AFU dirver;
              4. We also need to hotplug AFU bitstream in runtime;
--
Gaëtan Rivet
6WIND

Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus

[Xu, Rosen]

-----Original Message-----
From: Richardson, Bruce 
Sent: Tuesday, March 06, 2018 19:36
To: Ga?tan Rivet <gaetan.rivet@6wind.com>
Cc: Xu, Rosen <rosen.xu@intel.com>; Shreyansh Jain <shreyansh.jain@nxp.com>; dev@dpdk.org; Doherty, Declan <declan.doherty@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>
Subject: Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus

On Tue, Mar 06, 2018 at 11:46:22AM +0100, Gaëtan Rivet wrote:
> On Tue, Mar 06, 2018 at 10:42:14AM +0000, Xu, Rosen wrote:
> > 
> > 
> > -----Original Message-----
> > From: Shreyansh Jain [mailto:shreyansh.jain@nxp.com]
> > Sent: Tuesday, March 06, 2018 14:20
> > To: Xu, Rosen <rosen.xu@intel.com>
> > Cc: dev@dpdk.org; Doherty, Declan <declan.doherty@intel.com>; Zhang, 
> > Tianfei <tianfei.zhang@intel.com>
> > Subject: Re: [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel 
> > FPGA Bus Second Scan, it should be scanned after PCI Bus
> > 
> > On Tue, Mar 6, 2018 at 7:13 AM, Rosen Xu <rosen.xu@intel.com> wrote:
> > > Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> > > ---
> > >  lib/librte_eal/common/eal_common_bus.c | 14 +++++++++++++-
> > >  1 file changed, 13 insertions(+), 1 deletion(-)
> > >
> > > diff --git a/lib/librte_eal/common/eal_common_bus.c
> > > b/lib/librte_eal/common/eal_common_bus.c
> > > index 3e022d5..74bfa15 100644
> > > --- a/lib/librte_eal/common/eal_common_bus.c
> > > +++ b/lib/librte_eal/common/eal_common_bus.c
> > > @@ -70,15 +70,27 @@ struct rte_bus_list rte_bus_list =
> > >  rte_bus_scan(void)
> > >  {
> > >         int ret;
> > > -       struct rte_bus *bus = NULL;
> > > +       struct rte_bus *bus = NULL, *ifpga_bus = NULL;
> > >
> > >         TAILQ_FOREACH(bus, &rte_bus_list, next) {
> > > +               if (!strcmp(bus->name, "ifpga")) {
> > > +                       ifpga_bus = bus;
> > > +                       continue;
> > > +               }
> > > +
> > >                 ret = bus->scan();
> > >                 if (ret)
> > >                         RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
> > >                                 bus->name);
> > >         }
> > >
> > > +       if (ifpga_bus) {
> > > +               ret = ifpga_bus->scan();
> > > +               if (ret)
> > > +                       RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n",
> > > +                               ifpga_bus->name);
> > > +       }
> > > +
> > 
> > You are doing this just so that PCI scans are completed *before* ifpga scans?
> > Rosen: yes
> > Well, I understand that this certainly is an issue that we can't yet define a priority ordering of bus scans.
> > 
> > But, I think what you are require is a simpler:
> > 
> > In the file ifpga_bus.c:
> > 
> > +RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus.bus); <== this
> > ...
> > ...
> > #define RTE_REGISTER_BUS(nm, bus) \
> > RTE_INIT_PRIO(businitfn_ ##nm, 110); \
> > 
> > If you define your own version of RTE_REGISTER_BUS with the priority number higher, it would be inserted later in the bus list.
> > rte_register_bus doesn't do any inherent ordering.
> > This would save the changes you are doing in the lib/librte_eal/common/eal_common_bus.c file.
> > 
> > But I think there has to be a better provision of defining priority of bus scans - I am sure when new devices come in, there would be possibility of dependencies as in your case.
> > Rosen: is the priority scan of bus is implemented?
> 
> No, there is no priority set for scanning order.
> However, the order in which buses are registered, will modify the 
> order in which scans are done.
> 
> Thus, if you change the priority of your registration, you should be 
> able to ensure that your scan comes last.
> 

Can we register the bus only when a PCI device match is found at runtime, e.g. as part of the PCI driver instance initialization?
Rosen: yes, in my v1 patch it's called by FPGA PCI Driver(Rawdev)

/Bruce

[dpdk-dev] [RFC 4/4] drivers/raw/ifpga_rawdev: Rawdev for Intel FPGA Device, it's a PCI Driver of FPGA Device Manager

[Rosen Xu]

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/raw/ifpga_rawdev/Makefile                  |  59 ++++
 drivers/raw/ifpga_rawdev/ifpga_rawdev.c            | 343 +++++++++++++++++++++
 drivers/raw/ifpga_rawdev/ifpga_rawdev.h            | 109 +++++++
 drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c    | 121 ++++++++
 .../ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map  |   4 +
 5 files changed, 636 insertions(+)
 create mode 100644 drivers/raw/ifpga_rawdev/Makefile
 create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.c
 create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.h
 create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c
 create mode 100644 drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map

diff --git a/drivers/raw/ifpga_rawdev/Makefile b/drivers/raw/ifpga_rawdev/Makefile
new file mode 100644
index 0000000..3166fe2
--- /dev/null
+++ b/drivers/raw/ifpga_rawdev/Makefile
@@ -0,0 +1,59 @@
+#   BSD LICENSE
+#
+#   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Intel Corporation nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+LIB = librte_pmd_ifpga_rawdev.a
+
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga_rawdev
+LDLIBS += -lrte_eal
+LDLIBS += -lrte_rawdev
+LDLIBS += -lrte_bus_vdev
+LDLIBS += -lrte_kvargs
+
+EXPORT_MAP := rte_pmd_ifpga_rawdev_version.map
+
+LIBABIVER := 1
+
+#
+# all source are stored in SRCS-y
+#
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SKELETON_RAWDEV) += ifpga_rawdev.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_SKELETON_RAWDEV) += ifpga_rawdev_example.c
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
new file mode 100644
index 0000000..6046711
--- /dev/null
+++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
@@ -0,0 +1,343 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2016 NXP.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of NXP nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <string.h>
+#include <dirent.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_eal_memconfig.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <rte_ethdev.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+#include <rte_kvargs.h>
+#include <rte_alarm.h>
+
+#include <rte_errno.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_eal.h>
+#include <rte_common.h>
+
+#include "rte_rawdev.h"
+#include "rte_rawdev_pmd.h"
+#include "rte_bus_ifpga.h"
+#include "ifpga_rawdev.h"
+
+int ifpga_rawdev_logtype;
+
+#define PCI_VENDOR_ID_INTEL          0x8086
+/* PCI Device ID */
+#define PCIe_DEVICE_ID_RCiEP0_MCP    0xBCBD
+#define PCIe_DEVICE_ID_RCiEP0_SKX_P  0xBCC0
+#define PCIe_DEVICE_ID_RCiEP0_DCP    0x09C4
+/* VF Device */
+#define PCIe_DEVICE_ID_VF_MCP        0xBCBF
+#define PCIe_DEVICE_ID_VF_SKX_P      0xBCC1
+#define PCIe_DEVICE_ID_VF_DCP        0x09C5
+#define RTE_MAX_RAW_DEVICE           10
+
+static const struct rte_pci_id pci_ifpga_map[] = {
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_RCiEP0_MCP) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_VF_MCP) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_RCiEP0_SKX_P) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_VF_SKX_P) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_RCiEP0_DCP) },
+	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_VF_DCP) },
+	{ .vendor_id = 0, /* sentinel */ },
+};
+
+static void ifpga_rawdev_info_get(struct rte_rawdev *dev,
+				     rte_rawdev_obj_t dev_info)
+{
+	struct ifpga_rawdev *ifpga;
+	struct rte_afu_device *afu_dev;
+
+	IFPGA_RAWDEV_PMD_FUNC_TRACE();
+
+	if (!dev_info) {
+		IFPGA_RAWDEV_PMD_ERR("Invalid request");
+		return;
+	}
+
+	ifpga = ifpga_rawdev_get_priv(dev);
+
+	afu_dev = dev_info;
+	//-------------------------------
+	//Todo: fill afu_dev->rte_rawdev and afu_dev->rte_mem_resource
+	//*get afu_dev->num_region from Rawdev
+	//*get afu_dev->mem_resource from Rawdev
+	//*get afu_dev->max_vfs from Rawdev
+	//-------------------------------
+	
+}
+
+static int ifpga_rawdev_start(struct rte_rawdev *dev)
+{
+	int ret = 0;
+	struct ifpga_rawdev *ifpga;
+	enum ifpga_rawdev_device_state fpga_state;
+
+	IFPGA_RAWDEV_PMD_FUNC_TRACE();
+
+	RTE_FUNC_PTR_OR_ERR_RET(dev, -EINVAL);
+
+	ifpga = ifpga_rawdev_get_priv(dev);
+
+	fpga_state = ifpga->fpga_state;
+
+	if (fpga_state == IFPGA_IDLE) {
+		ret = ifpga_enumerate(&ifpga->hw);
+		if (ret) {
+			ifpga->fpga_state = IFPGA_ERROR;
+    		IFPGA_RAWDEV_PMD_ERR("Failed to enumerate fme: %d", ret);
+    		ret = -EINVAL;
+    		return ret;
+    	}
+    	ifpga_fme_hw_init(&ifpga->hw);
+		ifpga->fpga_state = IFPGA_READY;
+	} else {
+		IFPGA_RAWDEV_PMD_DEBUG("IFPGA is enumerated");
+	}
+
+	return ret;
+}
+
+static void ifpga_rawdev_stop(struct rte_rawdev *dev)
+{
+	
+}
+static int ifpga_rawdev_close(struct rte_rawdev *dev)
+{
+	int ret = 0;
+	
+
+	return ret;
+}
+
+static int ifpga_rawdev_reset(struct rte_rawdev *dev)
+{
+	
+
+	return 0;
+}
+static int ifpga_rawdev_pr(struct rte_rawdev *dev,
+					 rte_rawdev_obj_t pr_conf)
+{
+	struct ifpga_rawdev *ifpga;
+	struct ifpga_hw *hw;
+	struct rte_afu_pr_conf *afu_pr_conf;
+	int 	ret;
+	unsigned int     num_resource = 1;
+
+	IFPGA_RAWDEV_PMD_FUNC_TRACE();
+
+	ifpga = ifpga_rawdev_get_priv(dev);
+
+	if (!pr_conf)
+		return -EINVAL;
+
+	afu_pr_conf = pr_conf;
+
+    hw = &ifpga->hw;
+	ifpga_port_hw_init(hw, afu_pr_conf->afu_id.port);
+	ifpga_get_afu_mmio_info(hw, afu_pr_conf->afu_id.port, &ifpga->pci_dev->mem_resource[0], &num_resource);
+	ret=rte_fpga_do_pr(hw, afu_pr_conf->afu_id.port, afu_pr_conf->bs_path);
+	if (ret) {
+    	printf("do pr error\n");
+    	return ret;
+	}
+	
+	return 0;
+}
+
+static const struct rte_rawdev_ops ifpga_rawdev_ops = {
+	.dev_info_get = ifpga_rawdev_info_get,
+	.dev_configure = NULL,
+	.dev_start = ifpga_rawdev_start,
+	.dev_stop = ifpga_rawdev_stop,
+	.dev_close = ifpga_rawdev_close,
+	.dev_reset = ifpga_rawdev_reset,
+
+	.queue_def_conf = NULL,
+	.queue_setup = NULL,
+	.queue_release = NULL,
+
+	.attr_get = NULL,
+	.attr_set = NULL,
+
+	.enqueue_bufs = NULL,
+	.dequeue_bufs = NULL,
+
+	.dump = NULL,
+
+	.xstats_get = NULL,
+	.xstats_get_names = NULL,
+	.xstats_get_by_name = NULL,
+	.xstats_reset = NULL,
+
+	.firmware_status_get = NULL,
+	.firmware_version_get = NULL,
+	.firmware_load = ifpga_rawdev_pr,
+	.firmware_unload = NULL,
+
+	.dev_selftest = NULL,
+};
+
+static int
+ifpga_rawdev_create(struct rte_pci_device *pci_dev,
+		                    int socket_id)
+{
+	int ret = 0;
+	struct rte_rawdev *rawdev = NULL;
+	struct ifpga_rawdev *ifpga = NULL;
+	char name[RTE_RAWDEV_NAME_MAX_LEN];
+
+	if (!pci_dev) {
+		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+	
+	memset(name, sizeof(name), 0);
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%x:%x", pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
+	
+	IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
+
+	/* Allocate device structure */
+	rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct ifpga_rawdev),
+					 socket_id);
+	if (rawdev == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("Unable to allocate rawdevice");
+		ret = -EINVAL;
+		goto cleanup;
+	}
+
+	rawdev->dev_ops = &ifpga_rawdev_ops;
+	rawdev->device = &pci_dev->device;
+	rawdev->driver_name = pci_dev->device.driver->name;
+
+	ifpga = ifpga_rawdev_get_priv(rawdev);
+	ifpga->pci_dev = pci_dev;
+	ifpga->fpga_state = IFPGA_IDLE;
+
+	/* Initialize the shared code (base driver) */
+	ret = opae_init_shared_code(&ifpga->hw);
+	if (ret) {
+		IFPGA_RAWDEV_PMD_ERR("Failed to init shared code (base driver): %d", ret);
+		ret = -EINVAL;
+		goto cleanup;
+	}
+
+	return ret;
+
+cleanup:
+	if (rawdev)
+		rte_rawdev_pmd_release(rawdev);
+
+	return ret;
+}
+
+static int
+ifpga_rawdev_destroy(struct rte_pci_device *pci_dev)
+{
+	int ret;
+	struct rte_rawdev *rdev;
+	char name[RTE_RAWDEV_NAME_MAX_LEN];
+
+	if (!pci_dev) {
+		IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
+		ret = -EINVAL;
+		return ret;
+	}
+
+	memset(name, sizeof(name), 0);
+	snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%x:%x", pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
+
+	IFPGA_RAWDEV_PMD_INFO("Closing %s on NUMA node %d", name, rte_socket_id());
+
+	rdev = rte_rawdev_pmd_get_named_dev(name);
+	if (!rdev) {
+		IFPGA_RAWDEV_PMD_ERR("Invalid device name (%s)", name);
+		return -EINVAL;
+	}
+
+	/* rte_rawdev_close is called by pmd_release */
+	ret = rte_rawdev_pmd_release(rdev);
+	if (ret)
+		IFPGA_RAWDEV_PMD_DEBUG("Device cleanup failed");
+
+	return 0;
+}
+
+static int 
+ifpga_rawdev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+	                   struct rte_pci_device *pci_dev)
+{
+	return ifpga_rawdev_create(pci_dev, rte_socket_id());
+}
+
+static int ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return ifpga_rawdev_destroy(pci_dev);
+}
+
+static struct rte_pci_driver rte_ifpga_rawdev_pmd = {
+	.id_table  = pci_ifpga_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
+	.probe     = ifpga_rawdev_pci_probe,
+	.remove    = ifpga_rawdev_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(ifpga_rawdev_pci_driver, rte_ifpga_rawdev_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(ifpga_rawdev_pci_driver, rte_ifpga_rawdev_pmd);
+RTE_PMD_REGISTER_KMOD_DEP(ifpga_rawdev_pci_driver, "* igb_uio | uio_pci_generic | vfio-pci");
+
+RTE_INIT(ifpga_rawdev_init_log);
+static void
+ifpga_rawdev_init_log(void)
+{
+	ifpga_rawdev_logtype = rte_log_register("driver.raw.init");
+	if (ifpga_rawdev_logtype >= 0)
+		rte_log_set_level(ifpga_rawdev_logtype, RTE_LOG_NOTICE);
+}
+
+void ifpga_rawdev_test(void)
+{
+    printf("ifpga_rawdev_test \n");
+}
diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.h b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
new file mode 100644
index 0000000..dfa63c6
--- /dev/null
+++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
@@ -0,0 +1,109 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _IFPGA_RAWDEV_H_
+#define _IFPGA_RAWDEV_H_
+
+/*++++++++++++++++++++++++++++++++++These functions should be implemented by OPAE++++++++++++++++++++++++++++++++++*/
+struct ifpga_hw {
+	void                 *pci_dev; //share code will not involve any DPDK defination
+
+	int                  fme;
+	int                  port[4];
+};
+
+static int opae_init_shared_code(struct ifpga_hw *hw)
+{
+    //init ifpga_hw
+    
+    return 0;
+}
+static int ifpga_enumerate(struct ifpga_hw *hw)
+{
+	return 0;
+}
+static int ifpga_fme_hw_init(struct ifpga_hw *hw)
+{
+	return 0;
+}
+static int ifpga_port_hw_init(struct ifpga_hw *hw, int port_id)
+{
+    return 0;
+}
+static int ifpga_get_afu_mmio_info(struct ifpga_hw *hw, unsigned int port_id,
+		struct rte_mem_resource *mem_resource,
+		unsigned int *num_resource)
+{
+    return 0;
+}
+static int rte_fpga_do_pr(struct ifpga_hw *afu_dev, int port_id, const char *file_name)
+{
+    return 0;
+}
+/*++++++++++++++++++++++++++++++++++These functions should be implemented by OPAE++++++++++++++++++++++++++++++++++*/
+
+extern int ifpga_rawdev_logtype;
+
+#define IFPGA_RAWDEV_PMD_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, ifpga_rawdev_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+#define IFPGA_RAWDEV_PMD_FUNC_TRACE() IFPGA_RAWDEV_PMD_LOG(DEBUG, ">>")
+
+#define IFPGA_RAWDEV_PMD_DEBUG(fmt, args...) \
+	IFPGA_RAWDEV_PMD_LOG(DEBUG, fmt, ## args)
+#define IFPGA_RAWDEV_PMD_INFO(fmt, args...) \
+	IFPGA_RAWDEV_PMD_LOG(INFO, fmt, ## args)
+#define IFPGA_RAWDEV_PMD_ERR(fmt, args...) \
+	IFPGA_RAWDEV_PMD_LOG(ERR, fmt, ## args)
+#define IFPGA_RAWDEV_PMD_WARN(fmt, args...) \
+	IFPGA_RAWDEV_PMD_LOG(WARNING, fmt, ## args)
+
+enum ifpga_rawdev_device_state {
+	IFPGA_IDLE,
+	IFPGA_READY,
+	IFPGA_ERROR
+};
+
+struct ifpga_rawdev {
+	struct ifpga_hw hw;
+	struct rte_pci_device *pci_dev;
+	enum   ifpga_rawdev_device_state fpga_state;
+};
+
+static inline struct ifpga_rawdev *
+ifpga_rawdev_get_priv(const struct rte_rawdev *rawdev)
+{
+	return rawdev->dev_private;
+}
+
+#endif /* _IFPGA_RAWDEV_H_ */
diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c b/drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c
new file mode 100644
index 0000000..c506c0e
--- /dev/null
+++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c
@@ -0,0 +1,121 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <netinet/in.h>
+#include <setjmp.h>
+#include <stdarg.h>
+#include <ctype.h>
+#include <errno.h>
+#include <getopt.h>
+#include <signal.h>
+#include <stdbool.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_eal.h>
+#include <rte_launch.h>
+#include <rte_atomic.h>
+#include <rte_cycles.h>
+#include <rte_prefetch.h>
+#include <rte_lcore.h>
+#include <rte_per_lcore.h>
+#include <rte_branch_prediction.h>
+#include <rte_interrupts.h>
+#include <rte_random.h>
+#include <rte_debug.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_io.h>
+
+#include <rte_errno.h>
+#include <rte_bus.h>
+#include <rte_memzone.h>
+
+#include <rte_devargs.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+#include <rte_alarm.h>
+
+#include "rte_bus_ifpga.h"
+#include "ifpga_logs.h"
+
+#define RTE_PMD_REGISTER_AFU(nm, afudrv)\
+RTE_INIT(afudrvinitfn_ ##afudrv);\
+static const char *afudrvinit_ ## nm ## _alias;\
+static void afudrvinitfn_ ##afudrv(void)\
+{\
+	(afudrv).driver.name = RTE_STR(nm);\
+	(afudrv).driver.alias = afudrvinit_ ## nm ## _alias;\
+	rte_ifpga_driver_register(&afudrv);\
+} \
+RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+#define RTE_PMD_REGISTER_AFU_ALIAS(nm, alias)\
+static const char *afudrvinit_ ## nm ## _alias = RTE_STR(alias)
+
+static int afu_dev_probe(struct rte_afu_device *afu_dev)
+{
+	return 0;
+}
+static int afu_dev_remove(struct rte_afu_device *afu_dev)
+{
+	return 0;
+}
+
+static struct rte_afu_driver afu_dev_driver = {
+	.probe = afu_dev_probe,
+	.remove = afu_dev_remove,
+};
+
+RTE_PMD_REGISTER_AFU(net_afu_drv_example, afu_dev_driver);
+RTE_PMD_REGISTER_AFU_ALIAS(net_afu_drv_example, afu_dev);
+RTE_PMD_REGISTER_PARAM_STRING(net_afu_drv_example,
+	"bdf=<string> "
+	"port=<int> "
+	"uudi_high=<int64> "
+	"uuid_low=<int64> "
+	"path=<string> "
+	"pr_enable=<int>"
+	"debug=<int>");
diff --git a/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map b/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
new file mode 100644
index 0000000..179140f
--- /dev/null
+++ b/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
@@ -0,0 +1,4 @@
+DPDK_18.02 {
+
+	local: *;
+};
-- 
1.8.3.1

Re: [dpdk-dev] [RFC 4/4] drivers/raw/ifpga_rawdev: Rawdev for Intel FPGA Device, it's a PCI Driver of FPGA Device Manager

[Shreyansh Jain]

On Tue, Mar 6, 2018 at 7:13 AM, Rosen Xu <rosen.xu@intel.com> wrote:
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> ---
>  drivers/raw/ifpga_rawdev/Makefile                  |  59 ++++
>  drivers/raw/ifpga_rawdev/ifpga_rawdev.c            | 343 +++++++++++++++++++++
>  drivers/raw/ifpga_rawdev/ifpga_rawdev.h            | 109 +++++++
>  drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c    | 121 ++++++++

When rawdev skeleton driver was integrated, Thomas raised this point
of naming 'skeleton_rawdev' rather than just 'skeleton'.
So, rather than 'ifpga_rawdev' rather than 'ifpga'.
At that time I thought we could use <drivertype_driversubtype> as
model. But, frankly, to me it seems a bad choice now. Extra '_rawdev'
doesn't serve any purpose here.

So, feel free to change your naming to a more appropriate
"drivers/raw/ifpga/" or "drivers/raw/ifpga_sample" etc.

Probably I too can change the skeleton_rawdev to skeleton.

>  .../ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map  |   4 +
>  5 files changed, 636 insertions(+)
>  create mode 100644 drivers/raw/ifpga_rawdev/Makefile
>  create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.c
>  create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.h
>  create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c
>  create mode 100644 drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
>
> diff --git a/drivers/raw/ifpga_rawdev/Makefile b/drivers/raw/ifpga_rawdev/Makefile
> new file mode 100644
> index 0000000..3166fe2
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/Makefile
> @@ -0,0 +1,59 @@
> +#   BSD LICENSE
> +#
> +#   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
> +#   All rights reserved.
> +#
> +#   Redistribution and use in source and binary forms, with or without
> +#   modification, are permitted provided that the following conditions
> +#   are met:
> +#
> +#     * Redistributions of source code must retain the above copyright
> +#       notice, this list of conditions and the following disclaimer.
> +#     * Redistributions in binary form must reproduce the above copyright
> +#       notice, this list of conditions and the following disclaimer in
> +#       the documentation and/or other materials provided with the
> +#       distribution.
> +#     * Neither the name of Intel Corporation nor the names of its
> +#       contributors may be used to endorse or promote products derived
> +#       from this software without specific prior written permission.
> +#
> +#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +

SPDX identifier in place of BSD boiler-plate.

> +include $(RTE_SDK)/mk/rte.vars.mk
> +
> +#
> +# library name
> +#
> +LIB = librte_pmd_ifpga_rawdev.a
> +
> +CFLAGS += -DALLOW_EXPERIMENTAL_API
> +CFLAGS += -O3
> +CFLAGS += $(WERROR_FLAGS)
> +CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
> +CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga_rawdev
> +LDLIBS += -lrte_eal
> +LDLIBS += -lrte_rawdev
> +LDLIBS += -lrte_bus_vdev
> +LDLIBS += -lrte_kvargs
> +
> +EXPORT_MAP := rte_pmd_ifpga_rawdev_version.map
> +
> +LIBABIVER := 1
> +
> +#
> +# all source are stored in SRCS-y
> +#
> +SRCS-$(CONFIG_RTE_LIBRTE_PMD_SKELETON_RAWDEV) += ifpga_rawdev.c
> +SRCS-$(CONFIG_RTE_LIBRTE_PMD_SKELETON_RAWDEV) += ifpga_rawdev_example.c

This is a copy-paste issue - CONFIG_RTE_LIBRTE_PMD_SKELETON_RAWDEV

> +
> +include $(RTE_SDK)/mk/rte.lib.mk
> diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> new file mode 100644
> index 0000000..6046711
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> @@ -0,0 +1,343 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright 2016 NXP.

:) - should be Intel.
Even better - SPDX

> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of NXP nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +#include <string.h>
> +#include <dirent.h>
> +#include <sys/stat.h>
> +#include <unistd.h>
> +#include <sys/types.h>
> +#include <fcntl.h>
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_eal_memconfig.h>
> +#include <rte_malloc.h>
> +#include <rte_devargs.h>
> +#include <rte_memcpy.h>
> +#include <rte_ethdev.h>
> +#include <rte_pci.h>
> +#include <rte_bus_pci.h>
> +#include <rte_kvargs.h>
> +#include <rte_alarm.h>
> +
> +#include <rte_errno.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_memzone.h>
> +#include <rte_eal.h>
> +#include <rte_common.h>
> +
> +#include "rte_rawdev.h"
> +#include "rte_rawdev_pmd.h"
> +#include "rte_bus_ifpga.h"
> +#include "ifpga_rawdev.h"
> +
> +int ifpga_rawdev_logtype;
> +
> +#define PCI_VENDOR_ID_INTEL          0x8086
> +/* PCI Device ID */
> +#define PCIe_DEVICE_ID_RCiEP0_MCP    0xBCBD
> +#define PCIe_DEVICE_ID_RCiEP0_SKX_P  0xBCC0
> +#define PCIe_DEVICE_ID_RCiEP0_DCP    0x09C4
> +/* VF Device */
> +#define PCIe_DEVICE_ID_VF_MCP        0xBCBF
> +#define PCIe_DEVICE_ID_VF_SKX_P      0xBCC1
> +#define PCIe_DEVICE_ID_VF_DCP        0x09C5
> +#define RTE_MAX_RAW_DEVICE           10
> +
> +static const struct rte_pci_id pci_ifpga_map[] = {
> +       { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_RCiEP0_MCP) },
> +       { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_VF_MCP) },
> +       { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_RCiEP0_SKX_P) },
> +       { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_VF_SKX_P) },
> +       { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_RCiEP0_DCP) },
> +       { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_VF_DCP) },
> +       { .vendor_id = 0, /* sentinel */ },
> +};
> +
> +static void ifpga_rawdev_info_get(struct rte_rawdev *dev,
> +                                    rte_rawdev_obj_t dev_info)
> +{
> +       struct ifpga_rawdev *ifpga;
> +       struct rte_afu_device *afu_dev;
> +
> +       IFPGA_RAWDEV_PMD_FUNC_TRACE();
> +
> +       if (!dev_info) {
> +               IFPGA_RAWDEV_PMD_ERR("Invalid request");
> +               return;
> +       }
> +
> +       ifpga = ifpga_rawdev_get_priv(dev);
> +
> +       afu_dev = dev_info;
> +       //-------------------------------
> +       //Todo: fill afu_dev->rte_rawdev and afu_dev->rte_mem_resource
> +       //*get afu_dev->num_region from Rawdev
> +       //*get afu_dev->mem_resource from Rawdev
> +       //*get afu_dev->max_vfs from Rawdev
> +       //-------------------------------
> +
> +}
> +
> +static int ifpga_rawdev_start(struct rte_rawdev *dev)
> +{
> +       int ret = 0;
> +       struct ifpga_rawdev *ifpga;
> +       enum ifpga_rawdev_device_state fpga_state;
> +
> +       IFPGA_RAWDEV_PMD_FUNC_TRACE();
> +
> +       RTE_FUNC_PTR_OR_ERR_RET(dev, -EINVAL);
> +
> +       ifpga = ifpga_rawdev_get_priv(dev);
> +
> +       fpga_state = ifpga->fpga_state;
> +
> +       if (fpga_state == IFPGA_IDLE) {
> +               ret = ifpga_enumerate(&ifpga->hw);
> +               if (ret) {
> +                       ifpga->fpga_state = IFPGA_ERROR;
> +               IFPGA_RAWDEV_PMD_ERR("Failed to enumerate fme: %d", ret);
> +               ret = -EINVAL;
> +               return ret;
> +       }
> +       ifpga_fme_hw_init(&ifpga->hw);
> +               ifpga->fpga_state = IFPGA_READY;
> +       } else {
> +               IFPGA_RAWDEV_PMD_DEBUG("IFPGA is enumerated");
> +       }
> +
> +       return ret;
> +}
> +
> +static void ifpga_rawdev_stop(struct rte_rawdev *dev)
> +{
> +
> +}
> +static int ifpga_rawdev_close(struct rte_rawdev *dev)
> +{
> +       int ret = 0;
> +
> +
> +       return ret;
> +}
> +
> +static int ifpga_rawdev_reset(struct rte_rawdev *dev)
> +{
> +
> +
> +       return 0;
> +}
> +static int ifpga_rawdev_pr(struct rte_rawdev *dev,
> +                                        rte_rawdev_obj_t pr_conf)
> +{
> +       struct ifpga_rawdev *ifpga;
> +       struct ifpga_hw *hw;
> +       struct rte_afu_pr_conf *afu_pr_conf;
> +       int     ret;
> +       unsigned int     num_resource = 1;
> +
> +       IFPGA_RAWDEV_PMD_FUNC_TRACE();
> +
> +       ifpga = ifpga_rawdev_get_priv(dev);
> +
> +       if (!pr_conf)
> +               return -EINVAL;
> +
> +       afu_pr_conf = pr_conf;
> +
> +    hw = &ifpga->hw;
> +       ifpga_port_hw_init(hw, afu_pr_conf->afu_id.port);
> +       ifpga_get_afu_mmio_info(hw, afu_pr_conf->afu_id.port, &ifpga->pci_dev->mem_resource[0], &num_resource);
> +       ret=rte_fpga_do_pr(hw, afu_pr_conf->afu_id.port, afu_pr_conf->bs_path);
> +       if (ret) {
> +       printf("do pr error\n");
> +       return ret;
> +       }
> +
> +       return 0;
> +}
> +
> +static const struct rte_rawdev_ops ifpga_rawdev_ops = {
> +       .dev_info_get = ifpga_rawdev_info_get,
> +       .dev_configure = NULL,
> +       .dev_start = ifpga_rawdev_start,
> +       .dev_stop = ifpga_rawdev_stop,
> +       .dev_close = ifpga_rawdev_close,
> +       .dev_reset = ifpga_rawdev_reset,
> +
> +       .queue_def_conf = NULL,
> +       .queue_setup = NULL,
> +       .queue_release = NULL,
> +
> +       .attr_get = NULL,
> +       .attr_set = NULL,
> +
> +       .enqueue_bufs = NULL,
> +       .dequeue_bufs = NULL,
> +
> +       .dump = NULL,
> +
> +       .xstats_get = NULL,
> +       .xstats_get_names = NULL,
> +       .xstats_get_by_name = NULL,
> +       .xstats_reset = NULL,
> +
> +       .firmware_status_get = NULL,
> +       .firmware_version_get = NULL,
> +       .firmware_load = ifpga_rawdev_pr,
> +       .firmware_unload = NULL,
> +
> +       .dev_selftest = NULL,
> +};
> +
> +static int
> +ifpga_rawdev_create(struct rte_pci_device *pci_dev,
> +                                   int socket_id)
> +{
> +       int ret = 0;
> +       struct rte_rawdev *rawdev = NULL;
> +       struct ifpga_rawdev *ifpga = NULL;
> +       char name[RTE_RAWDEV_NAME_MAX_LEN];
> +
> +       if (!pci_dev) {
> +               IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
> +               ret = -EINVAL;
> +               goto cleanup;
> +       }
> +
> +       memset(name, sizeof(name), 0);
> +       snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%x:%x", pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
> +
> +       IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
> +
> +       /* Allocate device structure */
> +       rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct ifpga_rawdev),
> +                                        socket_id);
> +       if (rawdev == NULL) {
> +               IFPGA_RAWDEV_PMD_ERR("Unable to allocate rawdevice");
> +               ret = -EINVAL;
> +               goto cleanup;
> +       }
> +
> +       rawdev->dev_ops = &ifpga_rawdev_ops;
> +       rawdev->device = &pci_dev->device;
> +       rawdev->driver_name = pci_dev->device.driver->name;
> +
> +       ifpga = ifpga_rawdev_get_priv(rawdev);
> +       ifpga->pci_dev = pci_dev;
> +       ifpga->fpga_state = IFPGA_IDLE;
> +
> +       /* Initialize the shared code (base driver) */
> +       ret = opae_init_shared_code(&ifpga->hw);
> +       if (ret) {
> +               IFPGA_RAWDEV_PMD_ERR("Failed to init shared code (base driver): %d", ret);
> +               ret = -EINVAL;
> +               goto cleanup;
> +       }
> +
> +       return ret;
> +
> +cleanup:
> +       if (rawdev)
> +               rte_rawdev_pmd_release(rawdev);
> +
> +       return ret;
> +}
> +
> +static int
> +ifpga_rawdev_destroy(struct rte_pci_device *pci_dev)
> +{
> +       int ret;
> +       struct rte_rawdev *rdev;
> +       char name[RTE_RAWDEV_NAME_MAX_LEN];
> +
> +       if (!pci_dev) {
> +               IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
> +               ret = -EINVAL;
> +               return ret;
> +       }
> +
> +       memset(name, sizeof(name), 0);
> +       snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%x:%x", pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
> +
> +       IFPGA_RAWDEV_PMD_INFO("Closing %s on NUMA node %d", name, rte_socket_id());
> +
> +       rdev = rte_rawdev_pmd_get_named_dev(name);
> +       if (!rdev) {
> +               IFPGA_RAWDEV_PMD_ERR("Invalid device name (%s)", name);
> +               return -EINVAL;
> +       }
> +
> +       /* rte_rawdev_close is called by pmd_release */
> +       ret = rte_rawdev_pmd_release(rdev);
> +       if (ret)
> +               IFPGA_RAWDEV_PMD_DEBUG("Device cleanup failed");
> +
> +       return 0;
> +}
> +
> +static int
> +ifpga_rawdev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
> +                          struct rte_pci_device *pci_dev)
> +{
> +       return ifpga_rawdev_create(pci_dev, rte_socket_id());
> +}
> +
> +static int ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
> +{
> +       return ifpga_rawdev_destroy(pci_dev);
> +}
> +
> +static struct rte_pci_driver rte_ifpga_rawdev_pmd = {
> +       .id_table  = pci_ifpga_map,
> +       .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
> +       .probe     = ifpga_rawdev_pci_probe,
> +       .remove    = ifpga_rawdev_pci_remove,
> +};

>From above, what I understand is that ifpga is a PCI device, but there
are certain functions which are not available in PCI rte layer and
thus you need a rawdev representation.
But I somehow feel that you could have worked without adding another
bus type (drivers/bus/ifpga).

- PCI devices are scanned by PCI bus. (which is still happening in
this RFC patches)
- When probing is done, this driver (rawdev/ifpga) would identify and
attach itself to those devices (using PCI ids for matching)
  No other driver would pass the probe because of PCI_TABLE
- and then using rawdev APIs, interact with the device.

Am I missing some obvious logic here which requires a new bus?

> +
> +RTE_PMD_REGISTER_PCI(ifpga_rawdev_pci_driver, rte_ifpga_rawdev_pmd);
> +RTE_PMD_REGISTER_PCI_TABLE(ifpga_rawdev_pci_driver, rte_ifpga_rawdev_pmd);
> +RTE_PMD_REGISTER_KMOD_DEP(ifpga_rawdev_pci_driver, "* igb_uio | uio_pci_generic | vfio-pci");
> +
> +RTE_INIT(ifpga_rawdev_init_log);
> +static void
> +ifpga_rawdev_init_log(void)
> +{
> +       ifpga_rawdev_logtype = rte_log_register("driver.raw.init");
> +       if (ifpga_rawdev_logtype >= 0)
> +               rte_log_set_level(ifpga_rawdev_logtype, RTE_LOG_NOTICE);
> +}
> +
> +void ifpga_rawdev_test(void)
> +{
> +    printf("ifpga_rawdev_test \n");
> +}
> diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.h b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> new file mode 100644
> index 0000000..dfa63c6
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> @@ -0,0 +1,109 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2017 Intel Corporation. All rights reserved.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +#ifndef _IFPGA_RAWDEV_H_
> +#define _IFPGA_RAWDEV_H_
> +
> +/*++++++++++++++++++++++++++++++++++These functions should be implemented by OPAE++++++++++++++++++++++++++++++++++*/
> +struct ifpga_hw {
> +       void                 *pci_dev; //share code will not involve any DPDK defination
> +
> +       int                  fme;
> +       int                  port[4];
> +};
> +
> +static int opae_init_shared_code(struct ifpga_hw *hw)
> +{
> +    //init ifpga_hw
> +
> +    return 0;
> +}
> +static int ifpga_enumerate(struct ifpga_hw *hw)
> +{
> +       return 0;
> +}
> +static int ifpga_fme_hw_init(struct ifpga_hw *hw)
> +{
> +       return 0;
> +}
> +static int ifpga_port_hw_init(struct ifpga_hw *hw, int port_id)
> +{
> +    return 0;
> +}
> +static int ifpga_get_afu_mmio_info(struct ifpga_hw *hw, unsigned int port_id,
> +               struct rte_mem_resource *mem_resource,
> +               unsigned int *num_resource)
> +{
> +    return 0;
> +}
> +static int rte_fpga_do_pr(struct ifpga_hw *afu_dev, int port_id, const char *file_name)
> +{
> +    return 0;
> +}
> +/*++++++++++++++++++++++++++++++++++These functions should be implemented by OPAE++++++++++++++++++++++++++++++++++*/
> +
> +extern int ifpga_rawdev_logtype;
> +
> +#define IFPGA_RAWDEV_PMD_LOG(level, fmt, args...) \
> +       rte_log(RTE_LOG_ ## level, ifpga_rawdev_logtype, "%s(): " fmt "\n", \
> +               __func__, ##args)
> +
> +#define IFPGA_RAWDEV_PMD_FUNC_TRACE() IFPGA_RAWDEV_PMD_LOG(DEBUG, ">>")
> +
> +#define IFPGA_RAWDEV_PMD_DEBUG(fmt, args...) \
> +       IFPGA_RAWDEV_PMD_LOG(DEBUG, fmt, ## args)
> +#define IFPGA_RAWDEV_PMD_INFO(fmt, args...) \
> +       IFPGA_RAWDEV_PMD_LOG(INFO, fmt, ## args)
> +#define IFPGA_RAWDEV_PMD_ERR(fmt, args...) \
> +       IFPGA_RAWDEV_PMD_LOG(ERR, fmt, ## args)
> +#define IFPGA_RAWDEV_PMD_WARN(fmt, args...) \
> +       IFPGA_RAWDEV_PMD_LOG(WARNING, fmt, ## args)
> +
> +enum ifpga_rawdev_device_state {
> +       IFPGA_IDLE,
> +       IFPGA_READY,
> +       IFPGA_ERROR
> +};
> +
> +struct ifpga_rawdev {
> +       struct ifpga_hw hw;
> +       struct rte_pci_device *pci_dev;
> +       enum   ifpga_rawdev_device_state fpga_state;
> +};
> +
> +static inline struct ifpga_rawdev *
> +ifpga_rawdev_get_priv(const struct rte_rawdev *rawdev)
> +{
> +       return rawdev->dev_private;
> +}
> +
> +#endif /* _IFPGA_RAWDEV_H_ */
> diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c b/drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c
> new file mode 100644
> index 0000000..c506c0e
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c
> @@ -0,0 +1,121 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2017 Intel Corporation. All rights reserved.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include <stdint.h>
> +#include <inttypes.h>
> +#include <sys/types.h>
> +#include <sys/queue.h>
> +#include <netinet/in.h>
> +#include <setjmp.h>
> +#include <stdarg.h>
> +#include <ctype.h>
> +#include <errno.h>
> +#include <getopt.h>
> +#include <signal.h>
> +#include <stdbool.h>
> +
> +#include <rte_common.h>
> +#include <rte_log.h>
> +#include <rte_malloc.h>
> +#include <rte_memory.h>
> +#include <rte_memcpy.h>
> +#include <rte_eal.h>
> +#include <rte_launch.h>
> +#include <rte_atomic.h>
> +#include <rte_cycles.h>
> +#include <rte_prefetch.h>
> +#include <rte_lcore.h>
> +#include <rte_per_lcore.h>
> +#include <rte_branch_prediction.h>
> +#include <rte_interrupts.h>
> +#include <rte_random.h>
> +#include <rte_debug.h>
> +#include <rte_ether.h>
> +#include <rte_ethdev.h>
> +#include <rte_mempool.h>
> +#include <rte_mbuf.h>
> +#include <rte_io.h>
> +
> +#include <rte_errno.h>
> +#include <rte_bus.h>
> +#include <rte_memzone.h>
> +
> +#include <rte_devargs.h>
> +#include <rte_pci.h>
> +#include <rte_bus_pci.h>
> +#include <rte_alarm.h>
> +
> +#include "rte_bus_ifpga.h"
> +#include "ifpga_logs.h"
> +
> +#define RTE_PMD_REGISTER_AFU(nm, afudrv)\
> +RTE_INIT(afudrvinitfn_ ##afudrv);\
> +static const char *afudrvinit_ ## nm ## _alias;\
> +static void afudrvinitfn_ ##afudrv(void)\
> +{\
> +       (afudrv).driver.name = RTE_STR(nm);\
> +       (afudrv).driver.alias = afudrvinit_ ## nm ## _alias;\
> +       rte_ifpga_driver_register(&afudrv);\
> +} \
> +RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
> +
> +#define RTE_PMD_REGISTER_AFU_ALIAS(nm, alias)\
> +static const char *afudrvinit_ ## nm ## _alias = RTE_STR(alias)
> +
> +static int afu_dev_probe(struct rte_afu_device *afu_dev)
> +{
> +       return 0;
> +}
> +static int afu_dev_remove(struct rte_afu_device *afu_dev)
> +{
> +       return 0;
> +}
> +
> +static struct rte_afu_driver afu_dev_driver = {
> +       .probe = afu_dev_probe,
> +       .remove = afu_dev_remove,
> +};
> +
> +RTE_PMD_REGISTER_AFU(net_afu_drv_example, afu_dev_driver);
> +RTE_PMD_REGISTER_AFU_ALIAS(net_afu_drv_example, afu_dev);
> +RTE_PMD_REGISTER_PARAM_STRING(net_afu_drv_example,
> +       "bdf=<string> "
> +       "port=<int> "
> +       "uudi_high=<int64> "
> +       "uuid_low=<int64> "
> +       "path=<string> "
> +       "pr_enable=<int>"
> +       "debug=<int>");
> diff --git a/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map b/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> new file mode 100644
> index 0000000..179140f
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> @@ -0,0 +1,4 @@
> +DPDK_18.02 {

Will be 18.05

> +
> +       local: *;
> +};
> --
> 1.8.3.1
>

Re: [dpdk-dev] [RFC 4/4] drivers/raw/ifpga_rawdev: Rawdev for Intel FPGA Device, it's a PCI Driver of FPGA Device Manager

[Shreyansh Jain]

Just wanted to rephrase my wordings as they seem to be presenting
different meaning from what I was intending.

On Tue, Mar 6, 2018 at 12:18 PM, Shreyansh Jain <shreyansh.jain@nxp.com> wrote:
> On Tue, Mar 6, 2018 at 7:13 AM, Rosen Xu <rosen.xu@intel.com> wrote:
>> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
>> ---
>>  drivers/raw/ifpga_rawdev/Makefile                  |  59 ++++
>>  drivers/raw/ifpga_rawdev/ifpga_rawdev.c            | 343 +++++++++++++++++++++
>>  drivers/raw/ifpga_rawdev/ifpga_rawdev.h            | 109 +++++++
>>  drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c    | 121 ++++++++
>
> When rawdev skeleton driver was integrated, Thomas raised this point
> of naming 'skeleton_rawdev' rather than just 'skeleton'.

Thomas questioned why not 'skeleton' and I stuck to 'skeleton_rawdev'.
Which, in hindsight, seems to me a bad decision on my part.

> So, rather than 'ifpga_rawdev' rather than 'ifpga'.

So, rather than 'ifpga_rawdev', why not use 'ifpga'.

> At that time I thought we could use <drivertype_driversubtype> as
> model. But, frankly, to me it seems a bad choice now. Extra '_rawdev'
> doesn't serve any purpose here.
>
> So, feel free to change your naming to a more appropriate
> "drivers/raw/ifpga/" or "drivers/raw/ifpga_sample" etc.
>
> Probably I too can change the skeleton_rawdev to skeleton.

[snip]

Re: [dpdk-dev] [RFC 4/4] drivers/raw/ifpga_rawdev: Rawdev for Intel FPGA Device, it's a PCI Driver of FPGA Device Manager

[Zhang, Tianfei]

-----Original Message-----
From: Shreyansh Jain [mailto:shreyansh.jain@nxp.com] 
Sent: Tuesday, March 6, 2018 2:48 PM
To: Xu, Rosen <rosen.xu@intel.com>
Cc: dev@dpdk.org; Doherty, Declan <declan.doherty@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>
Subject: Re: [dpdk-dev] [RFC 4/4] drivers/raw/ifpga_rawdev: Rawdev for Intel FPGA Device, it's a PCI Driver of FPGA Device Manager

On Tue, Mar 6, 2018 at 7:13 AM, Rosen Xu <rosen.xu@intel.com> wrote:
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> ---
>  drivers/raw/ifpga_rawdev/Makefile                  |  59 ++++
>  drivers/raw/ifpga_rawdev/ifpga_rawdev.c            | 343 +++++++++++++++++++++
>  drivers/raw/ifpga_rawdev/ifpga_rawdev.h            | 109 +++++++
>  drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c    | 121 ++++++++

When rawdev skeleton driver was integrated, Thomas raised this point of naming 'skeleton_rawdev' rather than just 'skeleton'.
So, rather than 'ifpga_rawdev' rather than 'ifpga'.
At that time I thought we could use <drivertype_driversubtype> as model. But, frankly, to me it seems a bad choice now. Extra '_rawdev'
doesn't serve any purpose here.

So, feel free to change your naming to a more appropriate "drivers/raw/ifpga/" or "drivers/raw/ifpga_sample" etc.

Probably I too can change the skeleton_rawdev to skeleton.

>  .../ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map  |   4 +
>  5 files changed, 636 insertions(+)
>  create mode 100644 drivers/raw/ifpga_rawdev/Makefile  create mode 
> 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.c
>  create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.h
>  create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c
>  create mode 100644 
> drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
>
> diff --git a/drivers/raw/ifpga_rawdev/Makefile 
> b/drivers/raw/ifpga_rawdev/Makefile
> new file mode 100644
> index 0000000..3166fe2
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/Makefile
> @@ -0,0 +1,59 @@
> +#   BSD LICENSE
> +#
> +#   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
> +#   All rights reserved.
> +#
> +#   Redistribution and use in source and binary forms, with or without
> +#   modification, are permitted provided that the following conditions
> +#   are met:
> +#
> +#     * Redistributions of source code must retain the above copyright
> +#       notice, this list of conditions and the following disclaimer.
> +#     * Redistributions in binary form must reproduce the above copyright
> +#       notice, this list of conditions and the following disclaimer in
> +#       the documentation and/or other materials provided with the
> +#       distribution.
> +#     * Neither the name of Intel Corporation nor the names of its
> +#       contributors may be used to endorse or promote products derived
> +#       from this software without specific prior written permission.
> +#
> +#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +

SPDX identifier in place of BSD boiler-plate.

> +include $(RTE_SDK)/mk/rte.vars.mk
> +
> +#
> +# library name
> +#
> +LIB = librte_pmd_ifpga_rawdev.a
> +
> +CFLAGS += -DALLOW_EXPERIMENTAL_API
> +CFLAGS += -O3
> +CFLAGS += $(WERROR_FLAGS)
> +CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga CFLAGS += 
> +-I$(RTE_SDK)/drivers/raw/ifpga_rawdev
> +LDLIBS += -lrte_eal
> +LDLIBS += -lrte_rawdev
> +LDLIBS += -lrte_bus_vdev
> +LDLIBS += -lrte_kvargs
> +
> +EXPORT_MAP := rte_pmd_ifpga_rawdev_version.map
> +
> +LIBABIVER := 1
> +
> +#
> +# all source are stored in SRCS-y
> +#
> +SRCS-$(CONFIG_RTE_LIBRTE_PMD_SKELETON_RAWDEV) += ifpga_rawdev.c
> +SRCS-$(CONFIG_RTE_LIBRTE_PMD_SKELETON_RAWDEV) += 
> +ifpga_rawdev_example.c

This is a copy-paste issue - CONFIG_RTE_LIBRTE_PMD_SKELETON_RAWDEV

> +
> +include $(RTE_SDK)/mk/rte.lib.mk
> diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c 
> b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> new file mode 100644
> index 0000000..6046711
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> @@ -0,0 +1,343 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright 2016 NXP.

:) - should be Intel.
Even better - SPDX

> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of NXP nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +#include <string.h>
> +#include <dirent.h>
> +#include <sys/stat.h>
> +#include <unistd.h>
> +#include <sys/types.h>
> +#include <fcntl.h>
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_eal_memconfig.h>
> +#include <rte_malloc.h>
> +#include <rte_devargs.h>
> +#include <rte_memcpy.h>
> +#include <rte_ethdev.h>
> +#include <rte_pci.h>
> +#include <rte_bus_pci.h>
> +#include <rte_kvargs.h>
> +#include <rte_alarm.h>
> +
> +#include <rte_errno.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_memzone.h>
> +#include <rte_eal.h>
> +#include <rte_common.h>
> +
> +#include "rte_rawdev.h"
> +#include "rte_rawdev_pmd.h"
> +#include "rte_bus_ifpga.h"
> +#include "ifpga_rawdev.h"
> +
> +int ifpga_rawdev_logtype;
> +
> +#define PCI_VENDOR_ID_INTEL          0x8086
> +/* PCI Device ID */
> +#define PCIe_DEVICE_ID_RCiEP0_MCP    0xBCBD
> +#define PCIe_DEVICE_ID_RCiEP0_SKX_P  0xBCC0
> +#define PCIe_DEVICE_ID_RCiEP0_DCP    0x09C4
> +/* VF Device */
> +#define PCIe_DEVICE_ID_VF_MCP        0xBCBF
> +#define PCIe_DEVICE_ID_VF_SKX_P      0xBCC1
> +#define PCIe_DEVICE_ID_VF_DCP        0x09C5
> +#define RTE_MAX_RAW_DEVICE           10
> +
> +static const struct rte_pci_id pci_ifpga_map[] = {
> +       { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_RCiEP0_MCP) },
> +       { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_VF_MCP) },
> +       { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_RCiEP0_SKX_P) },
> +       { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_VF_SKX_P) },
> +       { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_RCiEP0_DCP) },
> +       { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIe_DEVICE_ID_VF_DCP) },
> +       { .vendor_id = 0, /* sentinel */ }, };
> +
> +static void ifpga_rawdev_info_get(struct rte_rawdev *dev,
> +                                    rte_rawdev_obj_t dev_info) {
> +       struct ifpga_rawdev *ifpga;
> +       struct rte_afu_device *afu_dev;
> +
> +       IFPGA_RAWDEV_PMD_FUNC_TRACE();
> +
> +       if (!dev_info) {
> +               IFPGA_RAWDEV_PMD_ERR("Invalid request");
> +               return;
> +       }
> +
> +       ifpga = ifpga_rawdev_get_priv(dev);
> +
> +       afu_dev = dev_info;
> +       //-------------------------------
> +       //Todo: fill afu_dev->rte_rawdev and afu_dev->rte_mem_resource
> +       //*get afu_dev->num_region from Rawdev
> +       //*get afu_dev->mem_resource from Rawdev
> +       //*get afu_dev->max_vfs from Rawdev
> +       //-------------------------------
> +
> +}
> +
> +static int ifpga_rawdev_start(struct rte_rawdev *dev) {
> +       int ret = 0;
> +       struct ifpga_rawdev *ifpga;
> +       enum ifpga_rawdev_device_state fpga_state;
> +
> +       IFPGA_RAWDEV_PMD_FUNC_TRACE();
> +
> +       RTE_FUNC_PTR_OR_ERR_RET(dev, -EINVAL);
> +
> +       ifpga = ifpga_rawdev_get_priv(dev);
> +
> +       fpga_state = ifpga->fpga_state;
> +
> +       if (fpga_state == IFPGA_IDLE) {
> +               ret = ifpga_enumerate(&ifpga->hw);
> +               if (ret) {
> +                       ifpga->fpga_state = IFPGA_ERROR;
> +               IFPGA_RAWDEV_PMD_ERR("Failed to enumerate fme: %d", ret);
> +               ret = -EINVAL;
> +               return ret;
> +       }
> +       ifpga_fme_hw_init(&ifpga->hw);
> +               ifpga->fpga_state = IFPGA_READY;
> +       } else {
> +               IFPGA_RAWDEV_PMD_DEBUG("IFPGA is enumerated");
> +       }
> +
> +       return ret;
> +}
> +
> +static void ifpga_rawdev_stop(struct rte_rawdev *dev) {
> +
> +}
> +static int ifpga_rawdev_close(struct rte_rawdev *dev) {
> +       int ret = 0;
> +
> +
> +       return ret;
> +}
> +
> +static int ifpga_rawdev_reset(struct rte_rawdev *dev) {
> +
> +
> +       return 0;
> +}
> +static int ifpga_rawdev_pr(struct rte_rawdev *dev,
> +                                        rte_rawdev_obj_t pr_conf) {
> +       struct ifpga_rawdev *ifpga;
> +       struct ifpga_hw *hw;
> +       struct rte_afu_pr_conf *afu_pr_conf;
> +       int     ret;
> +       unsigned int     num_resource = 1;
> +
> +       IFPGA_RAWDEV_PMD_FUNC_TRACE();
> +
> +       ifpga = ifpga_rawdev_get_priv(dev);
> +
> +       if (!pr_conf)
> +               return -EINVAL;
> +
> +       afu_pr_conf = pr_conf;
> +
> +    hw = &ifpga->hw;
> +       ifpga_port_hw_init(hw, afu_pr_conf->afu_id.port);
> +       ifpga_get_afu_mmio_info(hw, afu_pr_conf->afu_id.port, &ifpga->pci_dev->mem_resource[0], &num_resource);
> +       ret=rte_fpga_do_pr(hw, afu_pr_conf->afu_id.port, afu_pr_conf->bs_path);
> +       if (ret) {
> +       printf("do pr error\n");
> +       return ret;
> +       }
> +
> +       return 0;
> +}
> +
> +static const struct rte_rawdev_ops ifpga_rawdev_ops = {
> +       .dev_info_get = ifpga_rawdev_info_get,
> +       .dev_configure = NULL,
> +       .dev_start = ifpga_rawdev_start,
> +       .dev_stop = ifpga_rawdev_stop,
> +       .dev_close = ifpga_rawdev_close,
> +       .dev_reset = ifpga_rawdev_reset,
> +
> +       .queue_def_conf = NULL,
> +       .queue_setup = NULL,
> +       .queue_release = NULL,
> +
> +       .attr_get = NULL,
> +       .attr_set = NULL,
> +
> +       .enqueue_bufs = NULL,
> +       .dequeue_bufs = NULL,
> +
> +       .dump = NULL,
> +
> +       .xstats_get = NULL,
> +       .xstats_get_names = NULL,
> +       .xstats_get_by_name = NULL,
> +       .xstats_reset = NULL,
> +
> +       .firmware_status_get = NULL,
> +       .firmware_version_get = NULL,
> +       .firmware_load = ifpga_rawdev_pr,
> +       .firmware_unload = NULL,
> +
> +       .dev_selftest = NULL,
> +};
> +
> +static int
> +ifpga_rawdev_create(struct rte_pci_device *pci_dev,
> +                                   int socket_id) {
> +       int ret = 0;
> +       struct rte_rawdev *rawdev = NULL;
> +       struct ifpga_rawdev *ifpga = NULL;
> +       char name[RTE_RAWDEV_NAME_MAX_LEN];
> +
> +       if (!pci_dev) {
> +               IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
> +               ret = -EINVAL;
> +               goto cleanup;
> +       }
> +
> +       memset(name, sizeof(name), 0);
> +       snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%x:%x", 
> + pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
> +
> +       IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, 
> + rte_socket_id());
> +
> +       /* Allocate device structure */
> +       rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct ifpga_rawdev),
> +                                        socket_id);
> +       if (rawdev == NULL) {
> +               IFPGA_RAWDEV_PMD_ERR("Unable to allocate rawdevice");
> +               ret = -EINVAL;
> +               goto cleanup;
> +       }
> +
> +       rawdev->dev_ops = &ifpga_rawdev_ops;
> +       rawdev->device = &pci_dev->device;
> +       rawdev->driver_name = pci_dev->device.driver->name;
> +
> +       ifpga = ifpga_rawdev_get_priv(rawdev);
> +       ifpga->pci_dev = pci_dev;
> +       ifpga->fpga_state = IFPGA_IDLE;
> +
> +       /* Initialize the shared code (base driver) */
> +       ret = opae_init_shared_code(&ifpga->hw);
> +       if (ret) {
> +               IFPGA_RAWDEV_PMD_ERR("Failed to init shared code (base driver): %d", ret);
> +               ret = -EINVAL;
> +               goto cleanup;
> +       }
> +
> +       return ret;
> +
> +cleanup:
> +       if (rawdev)
> +               rte_rawdev_pmd_release(rawdev);
> +
> +       return ret;
> +}
> +
> +static int
> +ifpga_rawdev_destroy(struct rte_pci_device *pci_dev) {
> +       int ret;
> +       struct rte_rawdev *rdev;
> +       char name[RTE_RAWDEV_NAME_MAX_LEN];
> +
> +       if (!pci_dev) {
> +               IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
> +               ret = -EINVAL;
> +               return ret;
> +       }
> +
> +       memset(name, sizeof(name), 0);
> +       snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%x:%x", 
> + pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
> +
> +       IFPGA_RAWDEV_PMD_INFO("Closing %s on NUMA node %d", name, 
> + rte_socket_id());
> +
> +       rdev = rte_rawdev_pmd_get_named_dev(name);
> +       if (!rdev) {
> +               IFPGA_RAWDEV_PMD_ERR("Invalid device name (%s)", name);
> +               return -EINVAL;
> +       }
> +
> +       /* rte_rawdev_close is called by pmd_release */
> +       ret = rte_rawdev_pmd_release(rdev);
> +       if (ret)
> +               IFPGA_RAWDEV_PMD_DEBUG("Device cleanup failed");
> +
> +       return 0;
> +}
> +
> +static int
> +ifpga_rawdev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
> +                          struct rte_pci_device *pci_dev) {
> +       return ifpga_rawdev_create(pci_dev, rte_socket_id()); }
> +
> +static int ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev) {
> +       return ifpga_rawdev_destroy(pci_dev); }
> +
> +static struct rte_pci_driver rte_ifpga_rawdev_pmd = {
> +       .id_table  = pci_ifpga_map,
> +       .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
> +       .probe     = ifpga_rawdev_pci_probe,
> +       .remove    = ifpga_rawdev_pci_remove,
> +};

>From above, what I understand is that ifpga is a PCI device, but there are certain functions which are not available in PCI rte layer and thus you need a rawdev representation.
>But I somehow feel that you could have worked without adding another bus type (drivers/bus/ifpga).

In FPGA hardware view, it has a FME (FPGA management engine ) and several Ports and several AFUs.
*FME (FPGA management engine), major function include bitstream part reconfiguration and some other FPGA management 
*Port: a Bridget connect to a AFU from FME, one port link to one AFU, some management about AFU, like AUF reset.
*AFU(accelerate function unit), customer's bitsteam running on it.

So when the FPGA enumerate finished, we will find several AFUs, and use a datastruct to represent it, like afu_device, and then
add those afu_devices into ifpga_bus.

>- PCI devices are scanned by PCI bus. (which is still happening in this RFC patches)
>- When probing is done, this driver (rawdev/ifpga) would identify and attach itself to those devices (using PCI ids for matching)
>  No other driver would pass the probe because of PCI_TABLE
>- and then using rawdev APIs, interact with the device.

>Am I missing some obvious logic here which requires a new bus?

In our ifpga driver, here is our working flow coordinated with librte_rawdev:
1. we do the FPGA hardware scan or enumeration in PCI probe function which implemented in FPGA/OPAE base code.
2. rte_rawdev_info_get(),
    *we find one available AFU by port_id which pass through by DPDK application command line
    *and fill the hardware info into one AFU device
    *add this AFU device into ifpga_bus
    *trigger a hotplug to this ifpga_bus to probe available AFU driver like a Ethernet driver

3. rte_rawdev_firmware_load()
    *download bitsteam into this AFU device

4. rte_rawdev_start()
  * start this AFU device


> +
> +RTE_PMD_REGISTER_PCI(ifpga_rawdev_pci_driver, rte_ifpga_rawdev_pmd); 
> +RTE_PMD_REGISTER_PCI_TABLE(ifpga_rawdev_pci_driver, 
> +rte_ifpga_rawdev_pmd); 
> +RTE_PMD_REGISTER_KMOD_DEP(ifpga_rawdev_pci_driver, "* igb_uio | 
> +uio_pci_generic | vfio-pci");
> +
> +RTE_INIT(ifpga_rawdev_init_log);
> +static void
> +ifpga_rawdev_init_log(void)
> +{
> +       ifpga_rawdev_logtype = rte_log_register("driver.raw.init");
> +       if (ifpga_rawdev_logtype >= 0)
> +               rte_log_set_level(ifpga_rawdev_logtype, 
> +RTE_LOG_NOTICE); }
> +
> +void ifpga_rawdev_test(void)
> +{
> +    printf("ifpga_rawdev_test \n");
> +}
> diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.h 
> b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> new file mode 100644
> index 0000000..dfa63c6
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> @@ -0,0 +1,109 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2017 Intel Corporation. All rights reserved.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +#ifndef _IFPGA_RAWDEV_H_
> +#define _IFPGA_RAWDEV_H_
> +
> +/*++++++++++++++++++++++++++++++++++These functions should be 
> +implemented by OPAE++++++++++++++++++++++++++++++++++*/
> +struct ifpga_hw {
> +       void                 *pci_dev; //share code will not involve any DPDK defination
> +
> +       int                  fme;
> +       int                  port[4];
> +};
> +
> +static int opae_init_shared_code(struct ifpga_hw *hw) {
> +    //init ifpga_hw
> +
> +    return 0;
> +}
> +static int ifpga_enumerate(struct ifpga_hw *hw) {
> +       return 0;
> +}
> +static int ifpga_fme_hw_init(struct ifpga_hw *hw) {
> +       return 0;
> +}
> +static int ifpga_port_hw_init(struct ifpga_hw *hw, int port_id) {
> +    return 0;
> +}
> +static int ifpga_get_afu_mmio_info(struct ifpga_hw *hw, unsigned int port_id,
> +               struct rte_mem_resource *mem_resource,
> +               unsigned int *num_resource) {
> +    return 0;
> +}
> +static int rte_fpga_do_pr(struct ifpga_hw *afu_dev, int port_id, 
> +const char *file_name) {
> +    return 0;
> +}
> +/*++++++++++++++++++++++++++++++++++These functions should be 
> +implemented by OPAE++++++++++++++++++++++++++++++++++*/
> +
> +extern int ifpga_rawdev_logtype;
> +
> +#define IFPGA_RAWDEV_PMD_LOG(level, fmt, args...) \
> +       rte_log(RTE_LOG_ ## level, ifpga_rawdev_logtype, "%s(): " fmt "\n", \
> +               __func__, ##args)
> +
> +#define IFPGA_RAWDEV_PMD_FUNC_TRACE() IFPGA_RAWDEV_PMD_LOG(DEBUG, 
> +">>")
> +
> +#define IFPGA_RAWDEV_PMD_DEBUG(fmt, args...) \
> +       IFPGA_RAWDEV_PMD_LOG(DEBUG, fmt, ## args) #define 
> +IFPGA_RAWDEV_PMD_INFO(fmt, args...) \
> +       IFPGA_RAWDEV_PMD_LOG(INFO, fmt, ## args) #define 
> +IFPGA_RAWDEV_PMD_ERR(fmt, args...) \
> +       IFPGA_RAWDEV_PMD_LOG(ERR, fmt, ## args) #define 
> +IFPGA_RAWDEV_PMD_WARN(fmt, args...) \
> +       IFPGA_RAWDEV_PMD_LOG(WARNING, fmt, ## args)
> +
> +enum ifpga_rawdev_device_state {
> +       IFPGA_IDLE,
> +       IFPGA_READY,
> +       IFPGA_ERROR
> +};
> +
> +struct ifpga_rawdev {
> +       struct ifpga_hw hw;
> +       struct rte_pci_device *pci_dev;
> +       enum   ifpga_rawdev_device_state fpga_state;
> +};
> +
> +static inline struct ifpga_rawdev *
> +ifpga_rawdev_get_priv(const struct rte_rawdev *rawdev) {
> +       return rawdev->dev_private;
> +}
> +
> +#endif /* _IFPGA_RAWDEV_H_ */
> diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c 
> b/drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c
> new file mode 100644
> index 0000000..c506c0e
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev_example.c
> @@ -0,0 +1,121 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2017 Intel Corporation. All rights reserved.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include <stdint.h>
> +#include <inttypes.h>
> +#include <sys/types.h>
> +#include <sys/queue.h>
> +#include <netinet/in.h>
> +#include <setjmp.h>
> +#include <stdarg.h>
> +#include <ctype.h>
> +#include <errno.h>
> +#include <getopt.h>
> +#include <signal.h>
> +#include <stdbool.h>
> +
> +#include <rte_common.h>
> +#include <rte_log.h>
> +#include <rte_malloc.h>
> +#include <rte_memory.h>
> +#include <rte_memcpy.h>
> +#include <rte_eal.h>
> +#include <rte_launch.h>
> +#include <rte_atomic.h>
> +#include <rte_cycles.h>
> +#include <rte_prefetch.h>
> +#include <rte_lcore.h>
> +#include <rte_per_lcore.h>
> +#include <rte_branch_prediction.h>
> +#include <rte_interrupts.h>
> +#include <rte_random.h>
> +#include <rte_debug.h>
> +#include <rte_ether.h>
> +#include <rte_ethdev.h>
> +#include <rte_mempool.h>
> +#include <rte_mbuf.h>
> +#include <rte_io.h>
> +
> +#include <rte_errno.h>
> +#include <rte_bus.h>
> +#include <rte_memzone.h>
> +
> +#include <rte_devargs.h>
> +#include <rte_pci.h>
> +#include <rte_bus_pci.h>
> +#include <rte_alarm.h>
> +
> +#include "rte_bus_ifpga.h"
> +#include "ifpga_logs.h"
> +
> +#define RTE_PMD_REGISTER_AFU(nm, afudrv)\ RTE_INIT(afudrvinitfn_ 
> +##afudrv);\ static const char *afudrvinit_ ## nm ## _alias;\ static 
> +void afudrvinitfn_ ##afudrv(void)\ {\
> +       (afudrv).driver.name = RTE_STR(nm);\
> +       (afudrv).driver.alias = afudrvinit_ ## nm ## _alias;\
> +       rte_ifpga_driver_register(&afudrv);\
> +} \
> +RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
> +
> +#define RTE_PMD_REGISTER_AFU_ALIAS(nm, alias)\ static const char 
> +*afudrvinit_ ## nm ## _alias = RTE_STR(alias)
> +
> +static int afu_dev_probe(struct rte_afu_device *afu_dev) {
> +       return 0;
> +}
> +static int afu_dev_remove(struct rte_afu_device *afu_dev) {
> +       return 0;
> +}
> +
> +static struct rte_afu_driver afu_dev_driver = {
> +       .probe = afu_dev_probe,
> +       .remove = afu_dev_remove,
> +};
> +
> +RTE_PMD_REGISTER_AFU(net_afu_drv_example, afu_dev_driver); 
> +RTE_PMD_REGISTER_AFU_ALIAS(net_afu_drv_example, afu_dev); 
> +RTE_PMD_REGISTER_PARAM_STRING(net_afu_drv_example,
> +       "bdf=<string> "
> +       "port=<int> "
> +       "uudi_high=<int64> "
> +       "uuid_low=<int64> "
> +       "path=<string> "
> +       "pr_enable=<int>"
> +       "debug=<int>");
> diff --git a/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map 
> b/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> new file mode 100644
> index 0000000..179140f
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> @@ -0,0 +1,4 @@
> +DPDK_18.02 {

>Will be 18.05

Yes, we has rebase our code on 18.05, will send patches base on 18.05.

> +
> +       local: *;
> +};
> --
> 1.8.3.1
>