[dpdk-dev] [PATCH v5 06/17] net/ionic: add basic lif support

[Alfredo Cardigliano <cardigliano@ntop.org>]

Initialize LIFs (Logical Interfaces) which represents
external connections. The NIC can multiplex many LIFs
to a single port, but in most setups, LIF0 is the
primary control for the port.
Create a device for each LIF.

Signed-off-by: Alfredo Cardigliano <cardigliano@ntop.org>
Reviewed-by: Shannon Nelson <snelson@pensando.io>
---
 drivers/net/ionic/Makefile       |   4 +
 drivers/net/ionic/ionic.h        |   5 ++
 drivers/net/ionic/ionic_dev.c    |  38 ++++++++
 drivers/net/ionic/ionic_dev.h    |   8 ++
 drivers/net/ionic/ionic_ethdev.c | 134 ++++++++++++++++++++++++++++
 drivers/net/ionic/ionic_ethdev.h |  14 +++
 drivers/net/ionic/ionic_lif.c    | 144 +++++++++++++++++++++++++++++++
 drivers/net/ionic/ionic_lif.h    |  48 +++++++++++
 drivers/net/ionic/meson.build    |   1 +
 9 files changed, 396 insertions(+)
 create mode 100644 drivers/net/ionic/ionic_ethdev.h
 create mode 100644 drivers/net/ionic/ionic_lif.c
 create mode 100644 drivers/net/ionic/ionic_lif.h

diff --git a/drivers/net/ionic/Makefile b/drivers/net/ionic/Makefile
index 31abe5b95..2dc88cdd5 100644
--- a/drivers/net/ionic/Makefile
+++ b/drivers/net/ionic/Makefile
@@ -8,6 +8,9 @@ include $(RTE_SDK)/mk/rte.vars.mk
 #
 LIB = librte_pmd_ionic.a
 
+# Required to use rte_eth_dev_create and rte_eth_dev_destroy
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 
@@ -23,6 +26,7 @@ LDLIBS += -lrte_bus_pci
 SRCS-$(CONFIG_RTE_LIBRTE_IONIC_PMD) += ionic_mac_api.c
 SRCS-$(CONFIG_RTE_LIBRTE_IONIC_PMD) += ionic_dev.c
 SRCS-$(CONFIG_RTE_LIBRTE_IONIC_PMD) += ionic_ethdev.c
+SRCS-$(CONFIG_RTE_LIBRTE_IONIC_PMD) += ionic_lif.c
 SRCS-$(CONFIG_RTE_LIBRTE_IONIC_PMD) += ionic_main.c
 
 include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/ionic/ionic.h b/drivers/net/ionic/ionic.h
index 1adea8f7d..964589ed5 100644
--- a/drivers/net/ionic/ionic.h
+++ b/drivers/net/ionic/ionic.h
@@ -50,7 +50,12 @@ struct ionic_adapter {
 	struct ionic_dev idev;
 	struct ionic_dev_bar bars[IONIC_BARS_MAX];
 	struct ionic_identity	ident;
+	struct ionic_lif *lifs[IONIC_LIFS_MAX];
 	uint32_t num_bars;
+	uint32_t nlifs;
+	uint32_t max_ntxqs_per_lif;
+	uint32_t max_nrxqs_per_lif;
+	uint32_t nintrs;
 	bool is_mgmt_nic;
 	struct rte_pci_device *pci_dev;
 	LIST_ENTRY(ionic_adapter) pci_adapters;
diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c
index d74834122..c63709ee8 100644
--- a/drivers/net/ionic/ionic_dev.c
+++ b/drivers/net/ionic/ionic_dev.c
@@ -259,3 +259,41 @@ ionic_dev_cmd_port_loopback(struct ionic_dev *idev, uint8_t loopback_mode)
 
 	ionic_dev_cmd_go(idev, &cmd);
 }
+
+/* LIF commands */
+
+void
+ionic_dev_cmd_lif_identify(struct ionic_dev *idev, uint8_t type, uint8_t ver)
+{
+	union ionic_dev_cmd cmd = {
+		.lif_identify.opcode = IONIC_CMD_LIF_IDENTIFY,
+		.lif_identify.type = type,
+		.lif_identify.ver = ver,
+	};
+
+	ionic_dev_cmd_go(idev, &cmd);
+}
+
+void
+ionic_dev_cmd_lif_init(struct ionic_dev *idev, uint16_t lif_index,
+		       rte_iova_t info_pa)
+{
+	union ionic_dev_cmd cmd = {
+		.lif_init.opcode = IONIC_CMD_LIF_INIT,
+		.lif_init.index = lif_index,
+		.lif_init.info_pa = info_pa,
+	};
+
+	ionic_dev_cmd_go(idev, &cmd);
+}
+
+void
+ionic_dev_cmd_lif_reset(struct ionic_dev *idev, uint16_t lif_index)
+{
+	union ionic_dev_cmd cmd = {
+		.lif_init.opcode = IONIC_CMD_LIF_RESET,
+		.lif_init.index = lif_index,
+	};
+
+	ionic_dev_cmd_go(idev, &cmd);
+}
diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h
index 75c50d4cb..afcfcbf56 100644
--- a/drivers/net/ionic/ionic_dev.h
+++ b/drivers/net/ionic/ionic_dev.h
@@ -9,6 +9,8 @@
 #include "ionic_if.h"
 #include "ionic_regs.h"
 
+#define IONIC_LIFS_MAX			1024
+
 #define IONIC_DEVCMD_TIMEOUT	30 /* devcmd_timeout */
 #define	IONIC_ALIGN             4096
 
@@ -142,4 +144,10 @@ void ionic_dev_cmd_port_pause(struct ionic_dev *idev, uint8_t pause_type);
 void ionic_dev_cmd_port_loopback(struct ionic_dev *idev,
 	uint8_t loopback_mode);
 
+void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, uint8_t type,
+	uint8_t ver);
+void ionic_dev_cmd_lif_init(struct ionic_dev *idev, uint16_t lif_index,
+	rte_iova_t addr);
+void ionic_dev_cmd_lif_reset(struct ionic_dev *idev, uint16_t lif_index);
+
 #endif /* _IONIC_DEV_H_ */
diff --git a/drivers/net/ionic/ionic_ethdev.c b/drivers/net/ionic/ionic_ethdev.c
index b0ae7b076..f58d2d3a6 100644
--- a/drivers/net/ionic/ionic_ethdev.c
+++ b/drivers/net/ionic/ionic_ethdev.c
@@ -7,11 +7,17 @@
 #include <rte_ethdev.h>
 #include <rte_ethdev_driver.h>
 #include <rte_malloc.h>
+#include <rte_ethdev_pci.h>
 
 #include "ionic_logs.h"
 #include "ionic.h"
 #include "ionic_dev.h"
 #include "ionic_mac_api.h"
+#include "ionic_lif.h"
+#include "ionic_ethdev.h"
+
+static int  eth_ionic_dev_init(struct rte_eth_dev *eth_dev, void *init_params);
+static int  eth_ionic_dev_uninit(struct rte_eth_dev *eth_dev);
 
 int ionic_logtype;
 
@@ -22,10 +28,81 @@ static const struct rte_pci_id pci_id_ionic_map[] = {
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
+static const struct eth_dev_ops ionic_eth_dev_ops = {
+};
+
+static int
+eth_ionic_dev_init(struct rte_eth_dev *eth_dev, void *init_params)
+{
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+	struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
+	struct ionic_adapter *adapter = (struct ionic_adapter *)init_params;
+	int err;
+
+	IONIC_PRINT_CALL();
+
+	eth_dev->dev_ops = &ionic_eth_dev_ops;
+
+	/* Multi-process not supported, primary does initialization anyway */
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	rte_eth_copy_pci_info(eth_dev, pci_dev);
+
+	lif->index = adapter->nlifs;
+	lif->eth_dev = eth_dev;
+	lif->adapter = adapter;
+	adapter->lifs[adapter->nlifs] = lif;
+
+	err = ionic_lif_alloc(lif);
+	if (err) {
+		IONIC_PRINT(ERR, "Cannot allocate LIFs: %d, aborting",
+			err);
+		goto err;
+	}
+
+	err = ionic_lif_init(lif);
+	if (err) {
+		IONIC_PRINT(ERR, "Cannot init LIFs: %d, aborting", err);
+		goto err_free_lif;
+	}
+
+	IONIC_PRINT(DEBUG, "Port %u initialized", eth_dev->data->port_id);
+
+	return 0;
+
+err_free_lif:
+	ionic_lif_free(lif);
+err:
+	return err;
+}
+
+static int
+eth_ionic_dev_uninit(struct rte_eth_dev *eth_dev)
+{
+	struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
+	struct ionic_adapter *adapter = lif->adapter;
+
+	IONIC_PRINT_CALL();
+
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	adapter->lifs[lif->index] = NULL;
+
+	ionic_lif_deinit(lif);
+	ionic_lif_free(lif);
+
+	eth_dev->dev_ops = NULL;
+
+	return 0;
+}
+
 static int
 eth_ionic_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 		struct rte_pci_device *pci_dev)
 {
+	char name[RTE_ETH_NAME_MAX_LEN];
 	struct rte_mem_resource *resource;
 	struct ionic_adapter *adapter;
 	struct ionic_hw *hw;
@@ -112,6 +189,38 @@ eth_ionic_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 		goto err_free_adapter;
 	}
 
+	/* Configure LIFs */
+	err = ionic_lif_identify(adapter);
+	if (err) {
+		IONIC_PRINT(ERR, "Cannot identify lif: %d, aborting", err);
+		goto err_free_adapter;
+	}
+
+	/* Allocate and init LIFs */
+	err = ionic_lifs_size(adapter);
+	if (err) {
+		IONIC_PRINT(ERR, "Cannot size LIFs: %d, aborting", err);
+		goto err_free_adapter;
+	}
+
+	adapter->nlifs = 0;
+	for (i = 0; i < adapter->ident.dev.nlifs; i++) {
+		snprintf(name, sizeof(name), "net_%s_lif_%lu",
+			pci_dev->device.name, i);
+
+		err = rte_eth_dev_create(&pci_dev->device, name,
+			sizeof(struct ionic_lif),
+			NULL, NULL,
+			eth_ionic_dev_init, adapter);
+		if (err) {
+			IONIC_PRINT(ERR, "Cannot create eth device for "
+				"ionic lif %s", name);
+			break;
+		}
+
+		adapter->nlifs++;
+	}
+
 	return 0;
 
 err_free_adapter:
@@ -123,6 +232,31 @@ eth_ionic_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 static int
 eth_ionic_pci_remove(struct rte_pci_device *pci_dev __rte_unused)
 {
+	char name[RTE_ETH_NAME_MAX_LEN];
+	struct ionic_adapter *adapter = NULL;
+	struct rte_eth_dev *eth_dev;
+	struct ionic_lif *lif;
+	uint32_t i;
+
+	/* Adapter lookup is using (the first) eth_dev name */
+	snprintf(name, sizeof(name), "net_%s_lif_0",
+		pci_dev->device.name);
+
+	eth_dev = rte_eth_dev_allocated(name);
+	if (eth_dev) {
+		lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
+		adapter = lif->adapter;
+	}
+
+	if (adapter) {
+		for (i = 0; i < adapter->nlifs; i++) {
+			lif = adapter->lifs[i];
+			rte_eth_dev_destroy(lif->eth_dev, eth_ionic_dev_uninit);
+		}
+
+		rte_free(adapter);
+	}
+
 	return 0;
 }
 
diff --git a/drivers/net/ionic/ionic_ethdev.h b/drivers/net/ionic/ionic_ethdev.h
new file mode 100644
index 000000000..7228d8969
--- /dev/null
+++ b/drivers/net/ionic/ionic_ethdev.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
+ * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
+ */
+
+#ifndef _IONIC_ETHDEV_H_
+#define _IONIC_ETHDEV_H_
+
+#define IONIC_ETH_DEV_TO_LIF(eth_dev) ((struct ionic_lif *) \
+	(eth_dev)->data->dev_private)
+#define IONIC_ETH_DEV_TO_ADAPTER(eth_dev) \
+	(IONIC_ETH_DEV_TO_LIF(eth_dev)->adapter)
+
+#endif /* _IONIC_ETHDEV_H_ */
+
diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c
new file mode 100644
index 000000000..2559b4c16
--- /dev/null
+++ b/drivers/net/ionic/ionic_lif.c
@@ -0,0 +1,144 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
+ * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
+ */
+
+#include <rte_malloc.h>
+#include <rte_ethdev_driver.h>
+
+#include "ionic.h"
+#include "ionic_logs.h"
+#include "ionic_lif.h"
+#include "ionic_ethdev.h"
+
+int
+ionic_lif_alloc(struct ionic_lif *lif)
+{
+	uint32_t socket_id = rte_socket_id();
+
+	snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
+
+	IONIC_PRINT(DEBUG, "Allocating Lif Info");
+
+	lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE);
+
+	lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev,
+		"lif_info", 0 /* queue_idx*/,
+		lif->info_sz, IONIC_ALIGN, socket_id);
+	if (!lif->info_z) {
+		IONIC_PRINT(ERR, "Cannot allocate lif info memory");
+		return -ENOMEM;
+	}
+
+	lif->info = lif->info_z->addr;
+	lif->info_pa = lif->info_z->iova;
+
+	return 0;
+}
+
+void
+ionic_lif_free(struct ionic_lif *lif)
+{
+	if (lif->info) {
+		rte_memzone_free(lif->info_z);
+		lif->info = NULL;
+	}
+}
+
+int
+ionic_lif_init(struct ionic_lif *lif)
+{
+	struct ionic_dev *idev = &lif->adapter->idev;
+	struct ionic_q_init_comp comp;
+	int err;
+
+	ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
+	err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
+	ionic_dev_cmd_comp(idev, &comp);
+	if (err)
+		return err;
+
+	lif->hw_index = comp.hw_index;
+
+	lif->state |= IONIC_LIF_F_INITED;
+
+	return 0;
+}
+
+void
+ionic_lif_deinit(struct ionic_lif *lif)
+{
+	if (!(lif->state & IONIC_LIF_F_INITED))
+		return;
+
+	lif->state &= ~IONIC_LIF_F_INITED;
+}
+
+int
+ionic_lif_identify(struct ionic_adapter *adapter)
+{
+	struct ionic_dev *idev = &adapter->idev;
+	struct ionic_identity *ident = &adapter->ident;
+	int err;
+	unsigned int i;
+	unsigned int lif_words = sizeof(ident->lif.words) /
+		sizeof(ident->lif.words[0]);
+	unsigned int cmd_words = sizeof(idev->dev_cmd->data) /
+		sizeof(idev->dev_cmd->data[0]);
+	unsigned int nwords;
+
+	ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC,
+		IONIC_IDENTITY_VERSION_1);
+	err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
+	if (err)
+		return (err);
+
+	nwords = RTE_MIN(lif_words, cmd_words);
+	for (i = 0; i < nwords; i++)
+		ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]);
+
+	IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ",
+		ident->lif.capabilities);
+
+	IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ",
+		ident->lif.eth.max_ucast_filters);
+	IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ",
+		ident->lif.eth.max_mcast_filters);
+
+	IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ",
+		ident->lif.eth.config.features);
+	IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ",
+		ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]);
+	IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ",
+		ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]);
+	IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ",
+		ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]);
+	IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ",
+		ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]);
+
+	return 0;
+}
+
+int
+ionic_lifs_size(struct ionic_adapter *adapter)
+{
+	struct ionic_identity *ident = &adapter->ident;
+	uint32_t nlifs = ident->dev.nlifs;
+	uint32_t nintrs, dev_nintrs = ident->dev.nintrs;
+
+	adapter->max_ntxqs_per_lif =
+		ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
+	adapter->max_nrxqs_per_lif =
+		ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
+
+	nintrs = nlifs * 1 /* notifyq */;
+
+	if (nintrs > dev_nintrs) {
+		IONIC_PRINT(ERR, "At most %d intr queues supported, minimum required is %u",
+			dev_nintrs, nintrs);
+		return -ENOSPC;
+	}
+
+	adapter->nintrs = nintrs;
+
+	return 0;
+}
diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h
new file mode 100644
index 000000000..a75e420c6
--- /dev/null
+++ b/drivers/net/ionic/ionic_lif.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
+ * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
+ */
+
+#ifndef _IONIC_LIF_H_
+#define _IONIC_LIF_H_
+
+#include <inttypes.h>
+
+#include <rte_ethdev.h>
+#include <rte_ether.h>
+
+#include "ionic_osdep.h"
+#include "ionic_dev.h"
+
+#define IONIC_LIF_F_INITED		BIT(0)
+
+#define IONIC_LIF_NAME_MAX_SZ		(32)
+
+struct ionic_lif {
+	struct ionic_adapter *adapter;
+	struct rte_eth_dev *eth_dev;
+	uint16_t port_id;  /**< Device port identifier */
+	uint32_t index;
+	uint32_t hw_index;
+	uint32_t state;
+	char name[IONIC_LIF_NAME_MAX_SZ];
+	uint32_t info_sz;
+	struct ionic_lif_info *info;
+	rte_iova_t info_pa;
+	const struct rte_memzone *info_z;
+};
+
+int ionic_lif_identify(struct ionic_adapter *adapter);
+int ionic_lifs_size(struct ionic_adapter *ionic);
+
+int ionic_lif_alloc(struct ionic_lif *lif);
+void ionic_lif_free(struct ionic_lif *lif);
+
+int ionic_lif_init(struct ionic_lif *lif);
+void ionic_lif_deinit(struct ionic_lif *lif);
+
+int ionic_lif_start(struct ionic_lif *lif);
+
+int ionic_lif_configure(struct ionic_lif *lif);
+void ionic_lif_reset(struct ionic_lif *lif);
+
+#endif /* _IONIC_LIF_H_ */
diff --git a/drivers/net/ionic/meson.build b/drivers/net/ionic/meson.build
index 4744ae126..f28016105 100644
--- a/drivers/net/ionic/meson.build
+++ b/drivers/net/ionic/meson.build
@@ -5,6 +5,7 @@ sources = files(
 	'ionic_mac_api.c',
 	'ionic_dev.c',
 	'ionic_ethdev.c',
+	'ionic_lif.c',
 	'ionic_main.c'
 )
 
-- 
2.17.1