DPDK patches and discussions
 help / color / mirror / Atom feed
From: Rosen Xu <rosen.xu@intel.com>
To: dev@dpdk.org
Cc: declan.doherty@intel.com, tianfei.zhang@intel.com
Subject: [dpdk-dev] [RFC 1/4] drivers/bus/ifpga:Intel FPGA Bus Lib Code
Date: Tue,  6 Mar 2018 09:43:55 +0800	[thread overview]
Message-ID: <1520300638-134954-2-git-send-email-rosen.xu@intel.com> (raw)
In-Reply-To: <1520300638-134954-1-git-send-email-rosen.xu@intel.com>

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
---
 drivers/bus/ifpga/Makefile                  |  64 ++++
 drivers/bus/ifpga/ifpga_bus.c               | 527 ++++++++++++++++++++++++++++
 drivers/bus/ifpga/ifpga_common.c            | 168 +++++++++
 drivers/bus/ifpga/ifpga_common.h            |  46 +++
 drivers/bus/ifpga/ifpga_logs.h              |  59 ++++
 drivers/bus/ifpga/rte_bus_ifpga.h           | 153 ++++++++
 drivers/bus/ifpga/rte_bus_ifpga_version.map |   8 +
 7 files changed, 1025 insertions(+)
 create mode 100644 drivers/bus/ifpga/Makefile
 create mode 100644 drivers/bus/ifpga/ifpga_bus.c
 create mode 100644 drivers/bus/ifpga/ifpga_common.c
 create mode 100644 drivers/bus/ifpga/ifpga_common.h
 create mode 100644 drivers/bus/ifpga/ifpga_logs.h
 create mode 100644 drivers/bus/ifpga/rte_bus_ifpga.h
 create mode 100644 drivers/bus/ifpga/rte_bus_ifpga_version.map

diff --git a/drivers/bus/ifpga/Makefile b/drivers/bus/ifpga/Makefile
new file mode 100644
index 0000000..c71f186
--- /dev/null
+++ b/drivers/bus/ifpga/Makefile
@@ -0,0 +1,64 @@
+#   BSD LICENSE
+#
+#   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Intel Corporation nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+LIB = librte_bus_ifpga.a
+LIBABIVER := 1
+EXPORT_MAP := rte_bus_ifpga_version.map
+
+ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT),y)
+CFLAGS += -O0 -g
+CFLAGS += "-Wno-error"
+else
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+endif
+
+CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/bus/pci
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common
+#CFLAGS += -I$(RTE_SDK)/lib/librte_rawdev
+#LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring -lrte_rawdev
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+#LDLIBS += -lrte_ethdev
+
+VPATH += $(SRCDIR)/base
+
+SRCS-y += \
+        ifpga_bus.c \
+        ifpga_common.c
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/bus/ifpga/ifpga_bus.c b/drivers/bus/ifpga/ifpga_bus.c
new file mode 100644
index 0000000..382d550
--- /dev/null
+++ b/drivers/bus/ifpga/ifpga_bus.c
@@ -0,0 +1,527 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ *   Copyright 2013-2014 6WIND S.A.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include <rte_errno.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_eal.h>
+#include <rte_common.h>
+
+#include <rte_devargs.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+#include <rte_kvargs.h>
+#include <rte_alarm.h>
+
+#include "rte_rawdev.h"
+#include "rte_rawdev_pmd.h"
+#include "rte_bus_ifpga.h"
+#include "ifpga_logs.h"
+#include "ifpga_common.h"
+
+int ifpga_bus_logtype;
+
+/*register a ifpga bus based driver */
+void rte_ifpga_driver_register(struct rte_afu_driver *driver)
+{
+	RTE_VERIFY(driver);
+
+	TAILQ_INSERT_TAIL(&rte_ifpga_bus.driver_list, driver, next);
+}
+
+/*un-register a fpga bus based driver */
+void rte_ifpga_driver_unregister(struct rte_afu_driver *driver)
+{
+	TAILQ_REMOVE(&rte_ifpga_bus.driver_list, driver, next);
+}
+
+static struct rte_afu_device *
+ifpga_find_afu_dev(const struct rte_afu_id *afu_id)
+{
+	struct rte_afu_device *afu_dev = NULL;
+	
+	TAILQ_FOREACH (afu_dev, &rte_ifpga_bus.afu_list, next) {
+
+        if(!ifpga_afu_id_cmp(&afu_dev->id, afu_id)) {
+            return afu_dev;
+        }
+	}
+	return NULL;
+}
+
+static const char *valid_args[] = {
+#define IFPGA_ARG_BDF          "bdf"
+	IFPGA_ARG_BDF,
+#define IFPGA_ARG_PORT         "port"
+	IFPGA_ARG_PORT,
+#define IFPGA_ARG_PATH         "path"
+	IFPGA_ARG_PATH,
+#define IFPGA_ARG_UUID_HIGH    "uuid_high"
+	IFPGA_ARG_UUID_HIGH,
+#define IFPGA_ARG_UUID_LOW     "uuid_low"
+	IFPGA_ARG_UUID_LOW,
+#define IFPGA_ARG_PR_ENABLE     "pr_enable"
+	IFPGA_ARG_PR_ENABLE,
+#define IFPGA_ARG_DEBUG         "debug"
+	IFPGA_ARG_DEBUG,
+	NULL
+};
+
+/*
+ * Scan the content of the FPGA bus, and the devices in the devices
+ * list
+ */
+static struct rte_afu_device *
+rte_ifpga_scan_one(struct rte_devargs *devargs)
+{
+	struct rte_kvargs *kvlist = NULL;
+	struct rte_bus *pci_bus = NULL;
+	struct rte_device *dev = NULL;
+	struct rte_rawdev *rawdev;
+	struct rte_afu_device *afu_dev = NULL;
+	struct rte_afu_pr_conf afu_pr_conf;
+	char rawdev_name[RTE_RAWDEV_NAME_MAX_LEN];
+	int ret;
+	char *path = NULL;
+	int pr_enable = 1;
+	int debug = 0;
+
+    memset((char *)(&afu_pr_conf), 0, sizeof(struct rte_afu_pr_conf));
+	
+    kvlist = rte_kvargs_parse(devargs->args, valid_args);
+    if (!kvlist) {
+    	IFPGA_BUS_ERR("error when parsing param");
+    	goto end;
+    }
+    
+    if (rte_kvargs_count(kvlist, IFPGA_ARG_BDF) == 1) {
+		if (rte_kvargs_process(kvlist, IFPGA_ARG_BDF,
+				       &ifpga_get_bdf_arg, &afu_pr_conf.afu_id.pci_addr) < 0) {
+			IFPGA_BUS_ERR("error to parse %s",
+				     IFPGA_ARG_BDF);
+			goto end;
+		}
+	} else {
+		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
+			  IFPGA_ARG_PATH);
+		goto end;
+	}
+    
+    if (rte_kvargs_count(kvlist, IFPGA_ARG_PORT) == 1) {
+		if (rte_kvargs_process(kvlist, IFPGA_ARG_PORT,
+				       &ifpga_get_integer32_arg, &afu_pr_conf.afu_id.port) < 0) {
+			IFPGA_BUS_ERR("error to parse %s",
+				     IFPGA_ARG_PORT);
+			goto end;
+		}
+	}
+	else {
+		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
+			  IFPGA_ARG_PATH);
+		goto end;
+	}
+	
+    if (rte_kvargs_count(kvlist, IFPGA_ARG_PATH) == 1) {
+		if (rte_kvargs_process(kvlist, IFPGA_ARG_PATH,
+				       &ifpga_get_string_arg, &path) < 0) {
+			IFPGA_BUS_ERR("error to parse %s",
+				     IFPGA_ARG_PATH);
+			goto end;
+		}
+	} else {
+		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
+			  IFPGA_ARG_PATH);
+		goto end;
+	}
+	
+	if (rte_kvargs_count(kvlist, IFPGA_ARG_UUID_HIGH) == 1) {
+		if (rte_kvargs_process(kvlist, IFPGA_ARG_UUID_HIGH,
+				       &ifpga_get_integer64_arg, &afu_pr_conf.afu_id.uuid_high) < 0) {
+			IFPGA_BUS_ERR("error to parse %s",
+				     IFPGA_ARG_UUID_HIGH);
+			goto end;
+		}
+	} else {
+		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
+			  IFPGA_ARG_PATH);
+		goto end;
+	}
+	
+	if (rte_kvargs_count(kvlist, IFPGA_ARG_UUID_LOW) == 1) {
+		if (rte_kvargs_process(kvlist, IFPGA_ARG_UUID_LOW,
+				       &ifpga_get_integer64_arg, &afu_pr_conf.afu_id.uuid_low) < 0) {
+			IFPGA_BUS_ERR("error to parse %s",
+				     IFPGA_ARG_UUID_LOW);
+			goto end;
+		}
+	} else {
+		IFPGA_BUS_ERR("arg %s is mandatory for ifpga bus",
+			  IFPGA_ARG_PATH);
+		goto end;
+	}
+	
+	if (rte_kvargs_count(kvlist, IFPGA_ARG_PR_ENABLE) == 1) {
+		if (rte_kvargs_process(kvlist, IFPGA_ARG_PR_ENABLE,
+				       &ifpga_get_integer32_arg, &pr_enable) < 0) {
+			IFPGA_BUS_ERR("error to parse %s",
+				     IFPGA_ARG_UUID_HIGH);
+			goto end;
+		}
+	}
+	
+	if (rte_kvargs_count(kvlist, IFPGA_ARG_DEBUG) == 1) {
+		if (rte_kvargs_process(kvlist, IFPGA_ARG_DEBUG,
+				       &ifpga_get_integer32_arg, &debug) < 0) {
+			IFPGA_BUS_ERR("error to parse %s",
+				     IFPGA_ARG_UUID_HIGH);
+			goto end;
+		}
+	}
+	
+	if(!debug)
+	{
+		pci_bus = rte_bus_find_by_name("pci");
+	    if (pci_bus == NULL) {
+	    	IFPGA_BUS_ERR("unable to find PCI bus\n");
+	    	goto end;
+	    }
+	    
+	    dev = pci_bus->find_device(NULL, ifpga_pci_addr_cmp, &afu_pr_conf.afu_id.pci_addr);
+	    if (dev == NULL) {
+	    	IFPGA_BUS_ERR("unable to find PCI device\n");
+	    	goto end;
+	    }
+	}
+	else
+	{
+		IFPGA_BUS_DEBUG("pci_addr domain   : %x\n",   afu_pr_conf.afu_id.pci_addr.domain);
+		IFPGA_BUS_DEBUG("pci_addr bus      : %x\n",   afu_pr_conf.afu_id.pci_addr.bus);
+		IFPGA_BUS_DEBUG("pci_addr devid    : %x\n",   afu_pr_conf.afu_id.pci_addr.devid);
+		IFPGA_BUS_DEBUG("pci_addr function : %x\n",   afu_pr_conf.afu_id.pci_addr.function);
+		
+		IFPGA_BUS_DEBUG("uuid_low          : %lx\n", afu_pr_conf.afu_id.uuid_low);
+		IFPGA_BUS_DEBUG("uuid_high         : %lx\n", afu_pr_conf.afu_id.uuid_high);
+		
+		IFPGA_BUS_DEBUG("afu port          : %x\n",   afu_pr_conf.afu_id.port);
+	}
+	
+	if (ifpga_find_afu_dev(&afu_pr_conf.afu_id))
+		goto end;
+
+	memset(rawdev_name, sizeof(rawdev_name), 0);
+	snprintf(rawdev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%x:%x", afu_pr_conf.afu_id.pci_addr.bus, afu_pr_conf.afu_id.pci_addr.devid, afu_pr_conf.afu_id.pci_addr.function);
+	rawdev = rte_rawdev_pmd_get_named_dev(rawdev_name);
+	if(!rawdev)
+		goto end;
+	
+	rawdev->dev_ops->dev_start(rawdev);
+		
+	if (pr_enable)
+	{
+		ret=rawdev->dev_ops->firmware_load(rawdev, &afu_pr_conf);
+		if (ret) {
+    		printf("do pr error\n");
+    		return NULL;
+	    }
+	}
+		
+	afu_dev = calloc(1, sizeof(*afu_dev));
+	if (!afu_dev)
+		goto end;
+
+	afu_dev->device.devargs = devargs;
+	afu_dev->device.numa_node = SOCKET_ID_ANY;
+	afu_dev->device.name = devargs->name;
+	afu_dev->rawdev = rawdev;
+	afu_dev->id.pci_addr.domain = afu_pr_conf.afu_id.pci_addr.domain;
+	afu_dev->id.pci_addr.bus = afu_pr_conf.afu_id.pci_addr.bus;
+	afu_dev->id.pci_addr.devid = afu_pr_conf.afu_id.pci_addr.devid;
+	afu_dev->id.pci_addr.function = afu_pr_conf.afu_id.pci_addr.function;
+	afu_dev->id.uuid_low  = afu_pr_conf.afu_id.uuid_low;
+	afu_dev->id.uuid_high = afu_pr_conf.afu_id.uuid_high;
+	afu_dev->id.port      = afu_pr_conf.afu_id.port;
+	
+	rawdev->dev_ops->dev_info_get(rawdev, afu_dev);
+
+    return afu_dev;
+    
+end:
+	if (kvlist)
+		rte_kvargs_free(kvlist);
+	if (path)
+		free(path);
+	
+	return NULL;
+}
+
+/*
+ * Scan the content of the FPGA bus, and the devices in the devices
+ * list
+ */
+static int
+rte_ifpga_scan(void)
+{
+	struct rte_bus *pci_bus;
+	struct rte_afu_device *afu_dev;
+	struct rte_devargs *devargs;
+
+    pci_bus = rte_bus_find_by_name("pci");
+	if (pci_bus == NULL) {
+		IFPGA_BUS_ERR("unable to find PCI bus\n");
+		return -1;
+	}
+	
+	/* for virtual devices we scan the devargs_list populated via cmdline */
+	TAILQ_FOREACH(devargs, &devargs_list, next) {
+    
+		if (devargs->bus != &rte_ifpga_bus.bus)
+			continue;
+		
+		afu_dev = rte_ifpga_scan_one(devargs);
+	    
+	    if(afu_dev)
+		    TAILQ_INSERT_TAIL(&rte_ifpga_bus.afu_list, afu_dev, next);
+	}
+
+	return 0;
+}
+
+static int
+ifpga_probe_one_driver(struct rte_afu_driver *drv,
+			struct rte_afu_device *afu_dev)
+{
+	int ret;
+
+    if((drv->id.pci_addr.bus == afu_dev->id.pci_addr.bus) &&
+       (drv->id.pci_addr.devid == afu_dev->id.pci_addr.devid) &&
+       (drv->id.pci_addr.function == afu_dev->id.pci_addr.function) &&
+       (drv->id.uuid_low == afu_dev->id.uuid_low) &&
+       (drv->id.uuid_high == afu_dev->id.uuid_high) &&
+       (drv->id.port == afu_dev->id.port)) {
+       	
+       	afu_dev->driver = drv;
+       	
+       	/* call the driver probe() function */
+		ret = drv->probe(afu_dev);
+		if (ret)
+			afu_dev->driver = NULL;
+		return ret;
+    }
+	
+	/* return positive value if driver doesn't support this device */
+	return 1;
+}
+
+static int
+ifpga_probe_all_drivers(struct rte_afu_device *afu_dev)
+{
+	const char *name;
+	struct rte_afu_driver *drv = NULL;
+	int rc;
+
+	if (afu_dev == NULL)
+		return -1;
+
+	/* Check if a driver is already loaded */
+	if (afu_dev->driver != NULL)
+		return 0;
+		
+    name = rte_ifpga_device_name(afu_dev);
+    IFPGA_BUS_DEBUG("Search driver %s to probe device %s\n", name,
+		rte_ifpga_device_name(afu_dev));
+    
+	TAILQ_FOREACH(drv, &rte_ifpga_bus.driver_list, next) {
+		rc = ifpga_probe_one_driver(drv, afu_dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return -1;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+ }
+
+/*
+ * Scan the content of the PCI bus, and call the probe() function for
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+static int
+rte_ifpga_probe(void)
+{
+	struct rte_afu_device *afu_dev = NULL;
+	int ret = 0;
+
+	TAILQ_FOREACH (afu_dev, &rte_ifpga_bus.afu_list, next) {
+
+        if (afu_dev->device.driver)
+			continue;
+			
+		ret = ifpga_probe_all_drivers(afu_dev);
+		if (ret < 0)
+		    IFPGA_BUS_ERR("failed to initialize %s device\n",
+				rte_ifpga_device_name(afu_dev));
+	}
+
+	return 0;
+}
+
+static int
+rte_ifpga_plug(struct rte_device *dev)
+{
+	return ifpga_probe_all_drivers(RTE_DEV_TO_AFU(dev));
+}
+
+static int ifpga_remove_driver(struct rte_afu_device *afu_dev)
+{
+	const char *name;
+	const struct rte_afu_driver *driver;
+
+    name = rte_ifpga_device_name(afu_dev);
+	if (!afu_dev->device.driver) {
+		IFPGA_BUS_DEBUG("no driver attach to device %s\n", name);
+		return 1;
+	}
+
+	driver = container_of(afu_dev->device.driver, const struct rte_afu_driver,
+		driver);
+	return driver->remove(afu_dev);
+}
+
+static int
+rte_ifpga_unplug(struct rte_device *dev)
+{
+	struct rte_afu_device *afu_dev;
+	struct rte_devargs *devargs;
+	int ret;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	afu_dev = RTE_DEV_TO_AFU(dev);
+	if (!dev)
+		return -ENOENT;
+
+	devargs = dev->devargs;
+
+	ret = ifpga_remove_driver(afu_dev);
+	if (ret)
+		return ret;
+
+	TAILQ_REMOVE(&rte_ifpga_bus.afu_list, afu_dev, next);
+
+	TAILQ_REMOVE(&devargs_list, devargs, next);
+
+	free(devargs->args);
+	free(devargs);
+	free(afu_dev);
+	return 0;
+
+}
+
+static struct rte_device *
+rte_ifpga_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		                                        const void *data)
+{
+	struct rte_afu_device *afu_dev;
+
+	TAILQ_FOREACH(afu_dev, &rte_ifpga_bus.afu_list, next) {
+		if (start && &afu_dev->device == start) {
+			start = NULL;
+			continue;
+		}
+		if (cmp(&afu_dev->device, data) == 0)
+			return &afu_dev->device;
+	}
+	return NULL;
+}
+static int
+rte_ifpga_parse(const char *name, void *addr)
+{
+	struct rte_afu_driver **out = addr;
+	struct rte_afu_driver *driver = NULL;
+
+	TAILQ_FOREACH(driver, &rte_ifpga_bus.driver_list, next) {
+		if (strncmp(driver->driver.name, name,
+			    strlen(driver->driver.name)) == 0)
+			break;
+		if (driver->driver.alias &&
+		    strncmp(driver->driver.alias, name,
+			    strlen(driver->driver.alias)) == 0)
+			break;
+	}
+	if (driver != NULL &&
+	    addr != NULL)
+		*out = driver;
+	return driver == NULL;
+}
+
+struct rte_ifpga_bus rte_ifpga_bus = {
+	 .bus = {
+		 .scan        = rte_ifpga_scan,
+		 .probe       = rte_ifpga_probe,
+		 .find_device = rte_ifpga_find_device,
+	     .plug        = rte_ifpga_plug,
+	     .unplug      = rte_ifpga_unplug,
+	     .parse       = rte_ifpga_parse,
+	 },
+	.afu_list    = TAILQ_HEAD_INITIALIZER(rte_ifpga_bus.afu_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(rte_ifpga_bus.driver_list),
+ };
+
+RTE_REGISTER_BUS(IFPGA_BUS_NAME, rte_ifpga_bus.bus);
+
+RTE_INIT(ifpga_init_log)
+{
+	ifpga_bus_logtype = rte_log_register("bus.ifpga");
+	if (ifpga_bus_logtype >= 0)
+		rte_log_set_level(ifpga_bus_logtype, RTE_LOG_NOTICE);
+}
+
diff --git a/drivers/bus/ifpga/ifpga_common.c b/drivers/bus/ifpga/ifpga_common.c
new file mode 100644
index 0000000..2aa6bb0
--- /dev/null
+++ b/drivers/bus/ifpga/ifpga_common.c
@@ -0,0 +1,168 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ *   Copyright 2013-2014 6WIND S.A.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include <rte_errno.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_eal.h>
+#include <rte_common.h>
+
+#include <rte_devargs.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+#include <rte_kvargs.h>
+#include <rte_alarm.h>
+
+#include "rte_bus_ifpga.h"
+#include "ifpga_logs.h"
+#include "ifpga_common.h"
+
+int ifpga_get_string_arg(const char *key __rte_unused,
+	                     const char *value, void *extra_args)
+{
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	*(char **)extra_args = strdup(value);
+
+	if (!*(char **)extra_args)
+		return -ENOMEM;
+
+	return 0;
+}
+int ifpga_get_integer32_arg(const char *key __rte_unused, const char *value, void *extra_args)
+{
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	*(int *)extra_args = strtoull(value, NULL, 0);
+
+	return 0;
+}
+int ifpga_get_integer64_arg(const char *key __rte_unused, const char *value, void *extra_args)
+{
+	if (!value || !extra_args)
+		return -EINVAL;
+
+	*(uint64_t *)extra_args = strtoull(value, NULL, 0);
+
+	return 0;
+}
+int ifpga_get_unsigned_long(const char *str, int base)
+{
+	unsigned long num;
+	char *end = NULL;
+
+	errno = 0;
+
+	num = strtoul(str, &end, base);
+	if ((str[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
+		return -1;
+
+	return num;
+
+}
+int ifpga_get_bdf_arg(const char *key __rte_unused, const char *value, void *extra_args)
+{
+#define MAX_PATH_LEN 1024
+	struct rte_pci_addr *addr;
+	int num[4];
+	char str[MAX_PATH_LEN];
+	int i, j;
+	
+	if (!value || !extra_args)
+		return -EINVAL;
+
+    addr = (struct rte_pci_addr *)extra_args;
+	strcpy(str, value);
+	memset(num, 0, 4 * sizeof(num[0]));
+	i = strlen(str) - 1;
+	j = 3;
+	while (i > 0 && j >= 0) {
+		while ((str[i - 1] != ':' && str[i - 1] != '.') && i > 0)
+			i--;
+		num[j--] = ifpga_get_unsigned_long(&str[i], 16);
+		i--;
+		if (i >= 0)
+			str[i] = '\0';
+	}
+	addr->domain = num[0];
+	addr->bus = num[1];
+	addr->devid = num[2];
+	addr->function = num[3];
+	printf("[%s]: bdf %04d:%02d:%02d.%02d\n",
+			__func__,
+			addr->domain,
+			addr->bus,
+			addr->devid,
+			addr->function);
+
+	return 0;
+}
+int ifpga_afu_id_cmp(const struct rte_afu_id *afu_id0, const struct rte_afu_id *afu_id1)
+{
+	if((afu_id0->pci_addr.bus       == afu_id1->pci_addr.bus) &&
+       (afu_id0->pci_addr.devid     == afu_id1->pci_addr.devid) &&
+       (afu_id0->pci_addr.function  == afu_id1->pci_addr.function) &&
+       (afu_id0->uuid_low           == afu_id1->uuid_low) &&
+       (afu_id0->uuid_high          == afu_id1->uuid_high) &&
+       (afu_id0->port               == afu_id1->port)) {
+       	
+        return 0;
+    }
+    else
+    	return 1;
+}
+int ifpga_pci_addr_cmp(const struct rte_device *dev, const void *_pci_addr)
+{
+	struct rte_pci_device *pdev;
+	const struct rte_pci_addr *paddr = _pci_addr;
+
+	pdev = RTE_DEV_TO_PCI(*(struct rte_device **)(void *)&dev);
+	return rte_eal_compare_pci_addr(&pdev->addr, paddr);
+}
diff --git a/drivers/bus/ifpga/ifpga_common.h b/drivers/bus/ifpga/ifpga_common.h
new file mode 100644
index 0000000..590ff27
--- /dev/null
+++ b/drivers/bus/ifpga/ifpga_common.h
@@ -0,0 +1,46 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _IFPGA_COMMON_H_
+#define _IFPGA_COMMON_H_
+
+#include <rte_pci.h>
+
+extern int ifpga_get_string_arg(const char *key __rte_unused, const char *value, void *extra_args);
+extern int ifpga_get_integer32_arg(const char *key __rte_unused, const char *value, void *extra_args);
+extern int ifpga_get_integer64_arg(const char *key __rte_unused, const char *value, void *extra_args);
+extern int ifpga_get_unsigned_long(const char *str, int base);
+extern int ifpga_get_bdf_arg(const char *key __rte_unused, const char *value, void *extra_args);
+extern int ifpga_afu_id_cmp(const struct rte_afu_id *afu_id0, const struct rte_afu_id *afu_id1);
+extern int ifpga_pci_addr_cmp(const struct rte_device *dev, const void *_pci_addr);
+
+#endif /* _IFPGA_COMMON_H_ */
diff --git a/drivers/bus/ifpga/ifpga_logs.h b/drivers/bus/ifpga/ifpga_logs.h
new file mode 100644
index 0000000..eb56393
--- /dev/null
+++ b/drivers/bus/ifpga/ifpga_logs.h
@@ -0,0 +1,59 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _IFPGA_BUS_LOGS_H_
+#define _IFPGA_BUS_LOGS_H_
+
+#include <rte_log.h>
+
+extern int ifpga_bus_logtype;
+
+#define IFPGA_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, ifpga_bus_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+#define IFPGA_BUS_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, ifpga_bus_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+#define IFPGA_BUS_FUNC_TRACE() IFPGA_BUS_LOG(DEBUG, ">>")
+
+#define IFPGA_BUS_DEBUG(fmt, args...) \
+	IFPGA_BUS_LOG(DEBUG, fmt, ## args)
+#define IFPGA_BUS_INFO(fmt, args...) \
+	IFPGA_BUS_LOG(INFO, fmt, ## args)
+#define IFPGA_BUS_ERR(fmt, args...) \
+	IFPGA_BUS_LOG(ERR, fmt, ## args)
+#define IFPGA_BUS_WARN(fmt, args...) \
+	IFPGA_BUS_LOG(WARNING, fmt, ## args)
+
+#endif /* _IFPGA_BUS_LOGS_H_ */
diff --git a/drivers/bus/ifpga/rte_bus_ifpga.h b/drivers/bus/ifpga/rte_bus_ifpga.h
new file mode 100644
index 0000000..5491a45
--- /dev/null
+++ b/drivers/bus/ifpga/rte_bus_ifpga.h
@@ -0,0 +1,153 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 RehiveTech. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of RehiveTech nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_BUS_IFPGA_H_
+#define _RTE_BUS_IFPGA_H_
+
+/**
+ * @file
+ *
+ * RTE PCI Bus Interface
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <rte_bus.h>
+#include <rte_pci.h>
+
+/** Name of Intel FPGA Bus */
+#define IFPGA_BUS_NAME ifpga
+
+/* Forward declarations */
+struct rte_afu_device;
+struct rte_afu_driver;
+
+/** List of Intel AFU devices */
+TAILQ_HEAD(rte_afu_device_list, rte_afu_device);
+/** List of AFU drivers */
+TAILQ_HEAD(rte_afu_driver_list, rte_afu_driver);
+
+#define IFPGA_BUS_BITSTREAM_PATH_MAX_LEN 256
+
+/**
+ * A structure describing an ID for a AFU driver. Each driver provides a
+ * table of these IDs for each device that it supports.
+ */
+struct rte_afu_id {
+	struct rte_pci_addr pci_addr;
+	uint64_t uuid_low;
+	uint64_t uuid_high;
+	int      port;
+}__attribute__ ((packed));
+
+/**
+ * A structure pr configuration AFU driver. 
+ */
+
+struct rte_afu_pr_conf {
+	struct rte_afu_id afu_id;
+	int pr_enable;
+	char		bs_path[IFPGA_BUS_BITSTREAM_PATH_MAX_LEN];
+};
+
+#define AFU_PRI_STR_SIZE (PCI_PRI_STR_SIZE + 8)
+
+/**
+ * A structure describing a AFU device.
+ */
+struct rte_afu_device {
+	TAILQ_ENTRY(rte_afu_device) next;       /**< Next in device list. */
+	struct rte_device device;               /**< Inherit core device */
+	struct rte_rawdev *rawdev;
+	struct rte_afu_id id;                   /**< AFU id within FPGA. */
+	uint32_t num_region;						/**< number of regions found */
+	struct rte_mem_resource mem_resource[PCI_MAX_RESOURCE];
+						/**< PCI Memory Resource */
+	struct rte_intr_handle intr_handle;     /**< Interrupt handle */
+	struct rte_afu_driver *driver;          /**< Associated driver */
+	char path[IFPGA_BUS_BITSTREAM_PATH_MAX_LEN];
+} __attribute__ ((packed));
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_afu_device.
+ */
+#define RTE_DEV_TO_AFU(ptr) \
+	container_of(ptr, struct rte_afu_device, device)
+
+/**
+ * Initialisation function for the driver called during PCI probing.
+ */
+typedef int (afu_probe_t)(struct rte_afu_device *);
+
+/**
+ * Uninitialisation function for the driver called during hotplugging.
+ */
+typedef int (afu_remove_t)(struct rte_afu_device *);
+
+/**
+ * A structure describing a PCI device.
+ */
+struct rte_afu_driver {
+	TAILQ_ENTRY(rte_afu_driver) next;       /**< Next afu driver. */
+	struct rte_driver driver;               /**< Inherit core driver. */
+	afu_probe_t *probe;                     /**< Device Probe function. */
+	afu_remove_t *remove;                   /**< Device Remove function. */
+	struct rte_afu_id id;	                /**< AFU id within FPGA. */
+	uint32_t drv_flags;                     /**< Flags contolling handling of device. */
+};
+
+/**
+ * Structure describing the Intel FPGA bus
+ */
+struct rte_ifpga_bus {
+	struct rte_bus bus;               /**< Inherit the generic class */
+	struct rte_afu_device_list afu_list;  /**< List of AFU devices */
+	struct rte_afu_driver_list driver_list;  /**< List of FPGA drivers */
+};
+
+static inline const char *
+rte_ifpga_device_name(const struct rte_afu_device *afu)
+{
+	if (afu && afu->device.name)
+		return afu->device.name;
+	return NULL;
+}
+
+extern struct rte_ifpga_bus rte_ifpga_bus;
+
+void rte_ifpga_driver_register(struct rte_afu_driver *driver);
+void rte_ifpga_driver_unregister(struct rte_afu_driver *driver);
+
+#endif /* _RTE_BUS_IFPGA_H_ */
diff --git a/drivers/bus/ifpga/rte_bus_ifpga_version.map b/drivers/bus/ifpga/rte_bus_ifpga_version.map
new file mode 100644
index 0000000..4b2380a
--- /dev/null
+++ b/drivers/bus/ifpga/rte_bus_ifpga_version.map
@@ -0,0 +1,8 @@
+DPDK_17.11 {
+	global:
+
+    rte_ifpga_driver_register;
+    rte_ifpga_driver_unregister;
+    
+	local: *;
+};
-- 
1.8.3.1

  reply	other threads:[~2018-03-06  1:42 UTC|newest]

Thread overview: 19+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2018-03-06  1:43 [dpdk-dev] [RFC 0/4] Intel FPGA Bus Rosen Xu
2018-03-06  1:43 ` Rosen Xu [this message]
2018-03-06  6:09   ` [dpdk-dev] [RFC 1/4] drivers/bus/ifpga:Intel FPGA Bus Lib Code Shreyansh Jain
2018-03-06  9:24     ` Xu, Rosen
2018-03-06 10:05   ` Gaëtan Rivet
2018-03-09 15:34     ` Xu, Rosen
2018-03-06  1:43 ` [dpdk-dev] [RFC 2/4] lib/librte_eal/common:Add Intel FPGA Bus Running Command Parse Code Rosen Xu
2018-03-06  1:43 ` [dpdk-dev] [RFC 3/4] lib/librte_eal/common: Add Intel FPGA Bus Second Scan, it should be scanned after PCI Bus Rosen Xu
2018-03-06  6:20   ` Shreyansh Jain
2018-03-06 10:42     ` Xu, Rosen
2018-03-06 10:46       ` Gaëtan Rivet
2018-03-06 11:36         ` Bruce Richardson
2018-03-06 11:59           ` Gaëtan Rivet
2018-03-15  1:17             ` Xu, Rosen
2018-03-15  1:29           ` Xu, Rosen
2018-03-06  1:43 ` [dpdk-dev] [RFC 4/4] drivers/raw/ifpga_rawdev: Rawdev for Intel FPGA Device, it's a PCI Driver of FPGA Device Manager Rosen Xu
2018-03-06  6:48   ` Shreyansh Jain
2018-03-06  7:21     ` Shreyansh Jain
2018-03-07  4:10     ` Zhang, Tianfei

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1520300638-134954-2-git-send-email-rosen.xu@intel.com \
    --to=rosen.xu@intel.com \
    --cc=declan.doherty@intel.com \
    --cc=dev@dpdk.org \
    --cc=tianfei.zhang@intel.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).