[PATCH v2 04/19] net/xsc: add xsc device init and uninit

[PATCH v2 04/19] net/xsc: add xsc device init and uninit

[WanRenyong]

XSC device is a concept of low level device used to manage
hardware resource and to interact with firmware.

Signed-off-by: WanRenyong <wanry@yunsilicon.com>

---

v2:
* fix compilation error caused by missing dependency
---
 drivers/net/xsc/meson.build  |  20 +++++
 drivers/net/xsc/xsc_defs.h   |  23 +++++
 drivers/net/xsc/xsc_dev.c    | 162 +++++++++++++++++++++++++++++++++++
 drivers/net/xsc/xsc_dev.h    |  34 ++++++++
 drivers/net/xsc/xsc_ethdev.c |  22 ++++-
 drivers/net/xsc/xsc_ethdev.h |   1 +
 drivers/net/xsc/xsc_utils.c  |  96 +++++++++++++++++++++
 drivers/net/xsc/xsc_utils.h  |  14 +++
 8 files changed, 371 insertions(+), 1 deletion(-)
 create mode 100644 drivers/net/xsc/xsc_dev.c
 create mode 100644 drivers/net/xsc/xsc_dev.h
 create mode 100644 drivers/net/xsc/xsc_utils.c
 create mode 100644 drivers/net/xsc/xsc_utils.h

diff --git a/drivers/net/xsc/meson.build b/drivers/net/xsc/meson.build
index 11cdcf912b..8bf6ee7b47 100644
--- a/drivers/net/xsc/meson.build
+++ b/drivers/net/xsc/meson.build
@@ -8,6 +8,26 @@ endif
 
 sources = files(
         'xsc_ethdev.c',
+        'xsc_dev.c',
+        'xsc_utils.c',
 )
 
+libnames = ['ibverbs']
+foreach libname:libnames
+    lib = dependency('lib' + libname, required: false, method : 'pkg-config')
+    if lib.found()
+        ext_deps += lib
+    else
+        build = false
+        reason = 'missing dependency, "' + libname + '"'
+        subdir_done()
+    endif
+endforeach
 
+lib = dependency('libxscale', required: false, method : 'pkg-config')
+if lib.found()
+    ext_deps += lib
+    cflags += '-DHAVE_XSC_DV_PROVIDER=1'
+else
+    cflags += '-DHAVE_XSC_DV_PROVIDER=0'
+endif
diff --git a/drivers/net/xsc/xsc_defs.h b/drivers/net/xsc/xsc_defs.h
index b4ede6eca6..97cd61b2d1 100644
--- a/drivers/net/xsc/xsc_defs.h
+++ b/drivers/net/xsc/xsc_defs.h
@@ -8,5 +8,28 @@
 #define XSC_PCI_VENDOR_ID		0x1f67
 #define XSC_PCI_DEV_ID_MS		0x1111
 
+enum xsc_nic_mode {
+	XSC_NIC_MODE_LEGACY,
+	XSC_NIC_MODE_SWITCHDEV,
+	XSC_NIC_MODE_SOC,
+};
+
+enum xsc_pph_type {
+	XSC_PPH_NONE	= 0,
+	XSC_RX_PPH	= 0x1,
+	XSC_TX_PPH	= 0x2,
+	XSC_VFREP_PPH	= 0x4,
+	XSC_UPLINK_PPH	= 0x8,
+};
+
+enum xsc_flow_mode {
+	XSC_FLOW_OFF_HW_ONLY,
+	XSC_FLOW_ON_HW_ONLY,
+	XSC_FLOW_ON_HW_FIRST,
+	XSC_FLOW_HOTSPOT,
+	XSC_FLOW_MODE_NULL = 7,
+	XSC_FLOW_MODE_MAX,
+};
+
 #endif /* XSC_DEFS_H_ */
 
diff --git a/drivers/net/xsc/xsc_dev.c b/drivers/net/xsc/xsc_dev.c
new file mode 100644
index 0000000000..9673049628
--- /dev/null
+++ b/drivers/net/xsc/xsc_dev.c
@@ -0,0 +1,162 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2024 Yunsilicon Technology Co., Ltd.
+ */
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <sys/mman.h>
+
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+#include <rte_malloc.h>
+#include <rte_kvargs.h>
+
+#include "xsc_log.h"
+#include "xsc_defs.h"
+#include "xsc_dev.h"
+#include "xsc_utils.h"
+
+#define XSC_DEV_DEF_FLOW_MODE	XSC_FLOW_MODE_NULL
+#define XSC_DEV_CTRL_FILE_FMT	"/dev/yunsilicon/port_ctrl_" PCI_PRI_FMT
+
+static
+void xsc_dev_args_parse(struct xsc_dev *dev, struct rte_devargs *devargs)
+{
+	struct rte_kvargs *kvlist;
+	struct xsc_devargs *xdevargs = &dev->devargs;
+	const char *tmp;
+
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL)
+		return;
+
+	tmp = rte_kvargs_get(kvlist, XSC_PPH_MODE_ARG);
+	if (tmp != NULL)
+		xdevargs->pph_mode = atoi(tmp);
+	else
+		xdevargs->pph_mode = XSC_PPH_NONE;
+	tmp = rte_kvargs_get(kvlist, XSC_NIC_MODE_ARG);
+	if (tmp != NULL)
+		xdevargs->nic_mode = atoi(tmp);
+	else
+		xdevargs->nic_mode = XSC_NIC_MODE_LEGACY;
+	tmp = rte_kvargs_get(kvlist, XSC_FLOW_MODE_ARG);
+	if (tmp != NULL)
+		xdevargs->flow_mode = atoi(tmp);
+	else
+		xdevargs->flow_mode = XSC_DEV_DEF_FLOW_MODE;
+
+	rte_kvargs_free(kvlist);
+}
+
+static int
+xsc_dev_open(struct xsc_dev *dev, struct rte_pci_device *pci_dev)
+{
+	struct ibv_device *ib_dev;
+	char ctrl_file[PATH_MAX];
+	struct rte_pci_addr *pci_addr = &pci_dev->addr;
+	int ret;
+
+	ib_dev = xsc_get_ibv_device(&pci_dev->addr);
+	if (ib_dev == NULL) {
+		PMD_DRV_LOG(ERR, "Could not get ibv device");
+		return -ENODEV;
+	}
+
+	dev->ibv_ctx = ibv_open_device(ib_dev);
+	if (dev->ibv_ctx == NULL) {
+		PMD_DRV_LOG(ERR, "Could not open ibv device: %s", ib_dev->name);
+		return -ENODEV;
+	}
+
+	dev->ibv_pd = ibv_alloc_pd(dev->ibv_ctx);
+	if (dev->ibv_pd == NULL) {
+		PMD_DRV_LOG(ERR, "Failed to create pd:%s", ib_dev->name);
+		ret = -EINVAL;
+		goto alloc_pd_fail;
+	}
+
+	strcpy(dev->ibv_name, ib_dev->name);
+
+	snprintf(ctrl_file, PATH_MAX, XSC_DEV_CTRL_FILE_FMT,
+		 pci_addr->domain, pci_addr->bus, pci_addr->devid, pci_addr->function);
+
+	ret = open(ctrl_file, O_RDWR);
+	if (ret < 0) {
+		PMD_DRV_LOG(ERR, "Failed to open file: (%s) ", ctrl_file);
+		goto open_ctrl_file_fail;
+	}
+	dev->ctrl_fd = ret;
+
+	dev->bar_len = pci_dev->mem_resource[0].len;
+	dev->bar_addr = mmap(NULL, dev->bar_len, PROT_READ | PROT_WRITE,
+			     MAP_SHARED, dev->ctrl_fd, 0);
+	if (dev->bar_addr == MAP_FAILED) {
+		PMD_DRV_LOG(ERR, "Failed to mmap file: (%s) ", ctrl_file);
+		ret = -EINVAL;
+		goto mmap_fail;
+	}
+
+	return 0;
+
+mmap_fail:
+	close(dev->ctrl_fd);
+open_ctrl_file_fail:
+	ibv_dealloc_pd(dev->ibv_pd);
+alloc_pd_fail:
+	ibv_close_device(dev->ibv_ctx);
+
+	return ret;
+}
+
+static void
+xsc_dev_close(struct xsc_dev *dev)
+{
+	munmap(dev->bar_addr, dev->bar_len);
+	close(dev->ctrl_fd);
+	ibv_close_device(dev->ibv_ctx);
+}
+
+int
+xsc_dev_init(struct rte_pci_device *pci_dev, struct xsc_dev **dev)
+{
+	struct xsc_dev *d;
+	int ret;
+
+	PMD_INIT_FUNC_TRACE();
+
+	d = rte_zmalloc(NULL, sizeof(*d), RTE_CACHE_LINE_SIZE);
+	if (d == NULL) {
+		PMD_DRV_LOG(ERR, "Failed to alloc memory for xsc_dev");
+		return -ENOMEM;
+	}
+
+	xsc_dev_args_parse(d, pci_dev->device.devargs);
+
+	ret = xsc_dev_open(d, pci_dev);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "Failed to open xsc device");
+		goto dev_open_fail;
+	}
+
+	d->pci_dev = pci_dev;
+	*dev = d;
+
+	return 0;
+
+dev_open_fail:
+	rte_free(d);
+	return ret;
+}
+
+void
+xsc_dev_uninit(struct xsc_dev *dev)
+{
+	PMD_INIT_FUNC_TRACE();
+
+	xsc_dev_close(dev);
+	rte_free(dev);
+}
diff --git a/drivers/net/xsc/xsc_dev.h b/drivers/net/xsc/xsc_dev.h
new file mode 100644
index 0000000000..ce9dd65400
--- /dev/null
+++ b/drivers/net/xsc/xsc_dev.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2024 Yunsilicon Technology Co., Ltd.
+ */
+
+#ifndef _XSC_DEV_H_
+#define _XSC_DEV_H_
+
+#include <infiniband/verbs.h>
+
+#define XSC_PPH_MODE_ARG "pph_mode"
+#define XSC_NIC_MODE_ARG "nic_mode"
+#define XSC_FLOW_MODE_ARG "flow_mode"
+
+struct xsc_devargs {
+	int nic_mode;
+	int flow_mode;
+	int pph_mode;
+};
+
+struct xsc_dev {
+	struct rte_pci_device *pci_dev;
+	struct xsc_devargs devargs;
+	struct ibv_context *ibv_ctx;
+	struct ibv_pd *ibv_pd;
+	char ibv_name[IBV_SYSFS_NAME_MAX];
+	void *bar_addr;
+	uint64_t bar_len;
+	int ctrl_fd;
+};
+
+int xsc_dev_init(struct rte_pci_device *pci_dev, struct xsc_dev **dev);
+void xsc_dev_uninit(struct xsc_dev *dev);
+
+#endif /* _XSC_DEV_H_ */
diff --git a/drivers/net/xsc/xsc_ethdev.c b/drivers/net/xsc/xsc_ethdev.c
index 8f4d539848..6a33cbb2cd 100644
--- a/drivers/net/xsc/xsc_ethdev.c
+++ b/drivers/net/xsc/xsc_ethdev.c
@@ -6,27 +6,38 @@
 
 #include "xsc_log.h"
 #include "xsc_defs.h"
+#include "xsc_dev.h"
 #include "xsc_ethdev.h"
 
 static int
 xsc_ethdev_init(struct rte_eth_dev *eth_dev)
 {
 	struct xsc_ethdev_priv *priv = TO_XSC_ETHDEV_PRIV(eth_dev);
+	int ret;
 
 	PMD_INIT_FUNC_TRACE();
 
 	priv->eth_dev = eth_dev;
 	priv->pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 
+	ret = xsc_dev_init(priv->pci_dev, &priv->xdev);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "Failed to initialize xsc device");
+		return ret;
+	}
+
 	return 0;
 }
 
 static int
 xsc_ethdev_uninit(struct rte_eth_dev *eth_dev)
 {
-	RTE_SET_USED(eth_dev);
+	struct xsc_ethdev_priv *priv = TO_XSC_ETHDEV_PRIV(eth_dev);
+
 	PMD_INIT_FUNC_TRACE();
 
+	xsc_dev_uninit(priv->xdev);
+
 	return 0;
 }
 
@@ -75,8 +86,17 @@ static struct rte_pci_driver xsc_ethdev_pci_driver = {
 	.remove = xsc_ethdev_pci_remove,
 };
 
+RTE_INIT(xsc_pmd_init)
+{
+	ibv_fork_init();
+}
+
 RTE_PMD_REGISTER_PCI(net_xsc, xsc_ethdev_pci_driver);
 RTE_PMD_REGISTER_PCI_TABLE(net_xsc, xsc_ethdev_pci_id_map);
+RTE_PMD_REGISTER_PARAM_STRING(net_xsc,
+			      XSC_PPH_MODE_ARG "=<x>"
+			      XSC_NIC_MODE_ARG "=<x>"
+			      XSC_FLOW_MODE_ARG "=<x>");
 
 RTE_LOG_REGISTER_SUFFIX(xsc_logtype_init, init, NOTICE);
 RTE_LOG_REGISTER_SUFFIX(xsc_logtype_driver, driver, NOTICE);
diff --git a/drivers/net/xsc/xsc_ethdev.h b/drivers/net/xsc/xsc_ethdev.h
index 75aa34dc63..22fc462e25 100644
--- a/drivers/net/xsc/xsc_ethdev.h
+++ b/drivers/net/xsc/xsc_ethdev.h
@@ -8,6 +8,7 @@
 struct xsc_ethdev_priv {
 	struct rte_eth_dev *eth_dev;
 	struct rte_pci_device *pci_dev;
+	struct xsc_dev *xdev;
 };
 
 #define TO_XSC_ETHDEV_PRIV(dev) \
diff --git a/drivers/net/xsc/xsc_utils.c b/drivers/net/xsc/xsc_utils.c
new file mode 100644
index 0000000000..cd4e3d9bad
--- /dev/null
+++ b/drivers/net/xsc/xsc_utils.c
@@ -0,0 +1,96 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2024 Yunsilicon Technology Co., Ltd.
+ */
+
+#include <stddef.h>
+#include <unistd.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/mman.h>
+
+#include "xsc_log.h"
+#include "xsc_utils.h"
+
+static int
+xsc_get_ibdev_pci_addr(const char *dev_path, struct rte_pci_addr *pci_addr)
+{
+	FILE *file;
+	char line[32];
+	char path[PATH_MAX];
+	int ret = -ENOENT;
+
+	sprintf(path, "%s/device/uevent", dev_path);
+
+	file = fopen(path, "rb");
+	if (file == NULL) {
+		PMD_DRV_LOG(ERR, "Failed to open file: (%s) ", path);
+		return ret;
+	}
+	while (fgets(line, sizeof(line), file) == line) {
+		size_t len = strlen(line);
+
+		/* Truncate long lines. */
+		if (len == (sizeof(line) - 1)) {
+			while (line[(len - 1)] != '\n') {
+				int n = fgetc(file);
+
+				if (n == EOF)
+					goto out;
+				line[(len - 1)] = n;
+			}
+			/* No match for long lines. */
+			continue;
+		}
+		/* Extract information. */
+		if (sscanf(line,
+			   "PCI_SLOT_NAME=%04x:%hhx:%hhx.%hhx",
+			   &pci_addr->domain,
+			   &pci_addr->bus,
+			   &pci_addr->devid,
+			   &pci_addr->function) == 4) {
+			ret = 0;
+			break;
+		}
+	}
+out:
+	fclose(file);
+	return ret;
+}
+
+struct ibv_device *
+xsc_get_ibv_device(const struct rte_pci_addr *addr)
+{
+	int ibv_num, i;
+	struct ibv_device **ibv_list;
+	struct ibv_device *ibv_match = NULL;
+	struct rte_pci_addr ibv_pci_addr;
+
+	ibv_list = ibv_get_device_list(&ibv_num);
+	if (ibv_list == NULL)
+		return NULL;
+
+	for (i = 0; i < ibv_num; i++) {
+		if (xsc_get_ibdev_pci_addr(ibv_list[i]->ibdev_path, &ibv_pci_addr) != 0)
+			continue;
+		if (rte_pci_addr_cmp(addr, &ibv_pci_addr) != 0)
+			continue;
+		ibv_match = ibv_list[i];
+		PMD_DRV_LOG(DEBUG, "Finding device \"name:%s, %s, path:%s, %s\"..",
+			    ibv_list[i]->name, ibv_list[i]->dev_name,
+			    ibv_list[i]->dev_path, ibv_list[i]->ibdev_path);
+		break;
+	}
+	ibv_free_device_list(ibv_list);
+
+	if (ibv_match == NULL) {
+		PMD_DRV_LOG(WARNING,
+			    "No Verbs device matches PCI device " PCI_PRI_FMT,
+			    addr->domain, addr->bus, addr->devid, addr->function);
+	}
+
+	return ibv_match;
+}
diff --git a/drivers/net/xsc/xsc_utils.h b/drivers/net/xsc/xsc_utils.h
new file mode 100644
index 0000000000..0bc318e96a
--- /dev/null
+++ b/drivers/net/xsc/xsc_utils.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2024 Yunsilicon Technology Co., Ltd.
+ */
+
+#ifndef _XSC_UTILS_H_
+#define _XSC_UTILS_H_
+
+#include <infiniband/verbs.h>
+
+#include <ethdev_pci.h>
+
+struct ibv_device *xsc_get_ibv_device(const struct rte_pci_addr *addr);
+
+#endif /* _XSC_UTILS_H_ */
-- 
2.25.1

Re: [PATCH v2 04/19] net/xsc: add xsc device init and uninit

[Stephen Hemminger]

On Wed, 11 Sep 2024 10:07:25 +0800
"WanRenyong" <wanry@yunsilicon.com> wrote:

> +
> +	sprintf(path, "%s/device/uevent", dev_path);

better to use snprintf for safety

> +	while (fgets(line, sizeof(line), file) == line) {
> +		size_t len = strlen(line);
> +
> +		/* Truncate long lines. */
> +		if (len == (sizeof(line) - 1)) {
> +			while (line[(len - 1)] != '\n') {
> +				int n = fgetc(file);
> +
> +				if (n == EOF)
> +					goto out;
> +				line[(len - 1)] = n;
> +			}
> +			/* No match for long lines. */
> +			continue;
> +		}

If you have to deal with long lines better to use something
like getline() that handles arbitrary length.

Re: [PATCH v2 04/19] net/xsc: add xsc device init and uninit

[WanRenyong]

On 2024/9/11 11:47, Stephen Hemminger wrote:
> On Wed, 11 Sep 2024 10:07:25 +0800
> "WanRenyong" <wanry@yunsilicon.com> wrote:
>
>> +
>> +	sprintf(path, "%s/device/uevent", dev_path);
> better to use snprintf for safety
OK,  will fix it in next version.
>> +	while (fgets(line, sizeof(line), file) == line) {
>> +		size_t len = strlen(line);
>> +
>> +		/* Truncate long lines. */
>> +		if (len == (sizeof(line) - 1)) {
>> +			while (line[(len - 1)] != '\n') {
>> +				int n = fgetc(file);
>> +
>> +				if (n == EOF)
>> +					goto out;
>> +				line[(len - 1)] = n;
>> +			}
>> +			/* No match for long lines. */
>> +			continue;
>> +		}
> If you have to deal with long lines better to use something
> like getline() that handles arbitrary length.
I will think about it.

Hello Stephen,

Thanks for your review,  my reply please see above.

-- 
Thanks,
WanRenyong