[dpdk-dev] [PATCH] net/sfc: create virtual switch to enable VFs

[dpdk-dev] [PATCH] net/sfc: create virtual switch to enable VFs

[Andrew Rybchenko]

PF driver is responsible for vSwitch creation and vPorts allocation
for VFs.

Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Reviewed-by: Andy Moreton <amoreton@xilinx.com>
---
 doc/guides/nics/sfc_efx.rst                   |   2 +
 doc/guides/rel_notes/release_20_11.rst        |   6 +
 drivers/common/sfc_efx/efsys.h                |   2 +-
 .../sfc_efx/rte_common_sfc_efx_version.map    |   9 +
 drivers/net/sfc/meson.build                   |   1 +
 drivers/net/sfc/sfc.c                         |  36 ++++
 drivers/net/sfc/sfc.h                         |   4 +-
 drivers/net/sfc/sfc_ethdev.c                  |   2 +
 drivers/net/sfc/sfc_sriov.c                   | 168 ++++++++++++++++++
 drivers/net/sfc/sfc_sriov.h                   |  39 ++++
 10 files changed, 267 insertions(+), 2 deletions(-)
 create mode 100644 drivers/net/sfc/sfc_sriov.c
 create mode 100644 drivers/net/sfc/sfc_sriov.h

diff --git a/doc/guides/nics/sfc_efx.rst b/doc/guides/nics/sfc_efx.rst
index be1c2fe1d6..ab44ce66c8 100644
--- a/doc/guides/nics/sfc_efx.rst
+++ b/doc/guides/nics/sfc_efx.rst
@@ -71,6 +71,8 @@ SFC EFX PMD has support for:
 
 - Loopback
 
+- SR-IOV PF
+
 
 Non-supported Features
 ----------------------
diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst
index c6642f5f94..56b4ac1515 100644
--- a/doc/guides/rel_notes/release_20_11.rst
+++ b/doc/guides/rel_notes/release_20_11.rst
@@ -62,6 +62,12 @@ New Features
   * Added support for non-zero priorities for group 0 flows
   * Added support for VXLAN decap combined with VLAN pop
 
+* **Updated Solarflare network PMD.**
+
+  Updated the Solarflare ``sfc_efx`` driver with changes including:
+
+  * Added SR-IOV PF support
+
 * **Extended flow-perf application.**
 
   * Started supporting user order instead of bit mask:
diff --git a/drivers/common/sfc_efx/efsys.h b/drivers/common/sfc_efx/efsys.h
index 46b08f9789..3d769d32ae 100644
--- a/drivers/common/sfc_efx/efsys.h
+++ b/drivers/common/sfc_efx/efsys.h
@@ -156,7 +156,7 @@ prefetch_read_once(const volatile void *addr)
 
 #define EFSYS_OPT_FW_SUBVARIANT_AWARE 1
 
-#define EFSYS_OPT_EVB 0
+#define EFSYS_OPT_EVB 1
 
 #define EFSYS_OPT_MCDI_PROXY_AUTH_SERVER 0
 
diff --git a/drivers/common/sfc_efx/rte_common_sfc_efx_version.map b/drivers/common/sfc_efx/rte_common_sfc_efx_version.map
index 5e2a7ad919..1049e633c8 100644
--- a/drivers/common/sfc_efx/rte_common_sfc_efx_version.map
+++ b/drivers/common/sfc_efx/rte_common_sfc_efx_version.map
@@ -14,6 +14,15 @@ INTERNAL {
 	efx_ev_qprime;
 	efx_ev_usecs_to_ticks;
 
+	efx_evb_fini;
+	efx_evb_init;
+	efx_evb_vport_mac_set;
+	efx_evb_vport_reset;
+	efx_evb_vport_stats;
+	efx_evb_vport_vlan_set;
+	efx_evb_vswitch_create;
+	efx_evb_vswitch_destroy;
+
 	efx_evq_nbufs;
 	efx_evq_size;
 
diff --git a/drivers/net/sfc/meson.build b/drivers/net/sfc/meson.build
index 26f0323738..1c6451938a 100644
--- a/drivers/net/sfc/meson.build
+++ b/drivers/net/sfc/meson.build
@@ -39,6 +39,7 @@ sources = files(
 	'sfc_kvargs.c',
 	'sfc.c',
 	'sfc_mcdi.c',
+	'sfc_sriov.c',
 	'sfc_intr.c',
 	'sfc_ev.c',
 	'sfc_port.c',
diff --git a/drivers/net/sfc/sfc.c b/drivers/net/sfc/sfc.c
index 03ea5dc128..493a72f3ba 100644
--- a/drivers/net/sfc/sfc.c
+++ b/drivers/net/sfc/sfc.c
@@ -436,7 +436,20 @@ sfc_start(struct sfc_adapter *sa)
 
 	sa->state = SFC_ADAPTER_STARTING;
 
+	rc = 0;
 	do {
+		/*
+		 * FIXME Try to recreate vSwitch on start retry.
+		 * vSwitch is absent after MC reboot like events and
+		 * we should recreate it. May be we need proper
+		 * indication instead of guessing.
+		 */
+		if (rc != 0) {
+			sfc_sriov_vswitch_destroy(sa);
+			rc = sfc_sriov_vswitch_create(sa);
+			if (rc != 0)
+				goto fail_sriov_vswitch_create;
+		}
 		rc = sfc_try_start(sa);
 	} while ((--start_tries > 0) &&
 		 (rc == EIO || rc == EAGAIN || rc == ENOENT || rc == EINVAL));
@@ -449,6 +462,7 @@ sfc_start(struct sfc_adapter *sa)
 	return 0;
 
 fail_try_start:
+fail_sriov_vswitch_create:
 	sa->state = SFC_ADAPTER_CONFIGURED;
 fail_bad_state:
 	sfc_log_init(sa, "failed %d", rc);
@@ -727,6 +741,10 @@ sfc_attach(struct sfc_adapter *sa)
 	if (rc != 0)
 		goto fail_nic_reset;
 
+	rc = sfc_sriov_attach(sa);
+	if (rc != 0)
+		goto fail_sriov_attach;
+
 	/*
 	 * Probed NIC is sufficient for tunnel init.
 	 * Initialize tunnel support to be able to use libefx
@@ -809,11 +827,24 @@ sfc_attach(struct sfc_adapter *sa)
 
 	sfc_flow_init(sa);
 
+	/*
+	 * Create vSwitch to be able to use VFs when PF is not started yet
+	 * as DPDK port. VFs should be able to talk to each other even
+	 * if PF is down.
+	 */
+	rc = sfc_sriov_vswitch_create(sa);
+	if (rc != 0)
+		goto fail_sriov_vswitch_create;
+
 	sa->state = SFC_ADAPTER_INITIALIZED;
 
 	sfc_log_init(sa, "done");
 	return 0;
 
+fail_sriov_vswitch_create:
+	sfc_flow_fini(sa);
+	sfc_filter_detach(sa);
+
 fail_filter_attach:
 	sfc_rss_detach(sa);
 
@@ -832,7 +863,9 @@ sfc_attach(struct sfc_adapter *sa)
 fail_estimate_rsrc_limits:
 fail_tunnel_init:
 	efx_tunnel_fini(sa->nic);
+	sfc_sriov_detach(sa);
 
+fail_sriov_attach:
 fail_nic_reset:
 
 	sfc_log_init(sa, "failed %d", rc);
@@ -846,6 +879,8 @@ sfc_detach(struct sfc_adapter *sa)
 
 	SFC_ASSERT(sfc_adapter_is_locked(sa));
 
+	sfc_sriov_vswitch_destroy(sa);
+
 	sfc_flow_fini(sa);
 
 	sfc_filter_detach(sa);
@@ -854,6 +889,7 @@ sfc_detach(struct sfc_adapter *sa)
 	sfc_ev_detach(sa);
 	sfc_intr_detach(sa);
 	efx_tunnel_fini(sa->nic);
+	sfc_sriov_detach(sa);
 
 	sa->state = SFC_ADAPTER_UNINITIALIZED;
 }
diff --git a/drivers/net/sfc/sfc.h b/drivers/net/sfc/sfc.h
index fa7c79b80b..0c32527ad5 100644
--- a/drivers/net/sfc/sfc.h
+++ b/drivers/net/sfc/sfc.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- *
+*
  * Copyright(c) 2019-2020 Xilinx, Inc.
  * Copyright(c) 2016-2019 Solarflare Communications Inc.
  *
@@ -26,6 +26,7 @@
 #include "sfc_debug.h"
 #include "sfc_log.h"
 #include "sfc_filter.h"
+#include "sfc_sriov.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -224,6 +225,7 @@ struct sfc_adapter {
 	rte_atomic32_t			restart_required;
 
 	struct sfc_efx_mcdi		mcdi;
+	struct sfc_sriov		sriov;
 	struct sfc_intr			intr;
 	struct sfc_port			port;
 	struct sfc_filter		filter;
diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c
index 6c96118007..600eaa23ea 100644
--- a/drivers/net/sfc/sfc_ethdev.c
+++ b/drivers/net/sfc/sfc_ethdev.c
@@ -102,6 +102,8 @@ sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 
 	dev_info->max_rx_pktlen = EFX_MAC_PDU_MAX;
 
+	dev_info->max_vfs = sa->sriov.num_vfs;
+
 	/* Autonegotiation may be disabled */
 	dev_info->speed_capa = ETH_LINK_SPEED_FIXED;
 	if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_1000FDX))
diff --git a/drivers/net/sfc/sfc_sriov.c b/drivers/net/sfc/sfc_sriov.c
new file mode 100644
index 0000000000..6120840bb4
--- /dev/null
+++ b/drivers/net/sfc/sfc_sriov.c
@@ -0,0 +1,168 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2019-2020 Xilinx, Inc.
+ * Copyright(c) 2019 Solarflare Communications Inc.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ */
+
+#include <rte_common.h>
+#include <rte_bus_pci.h>
+
+#include "sfc.h"
+#include "sfc_log.h"
+
+#include "efx.h"
+
+
+/*
+ * Check if a MAC address is already assigned to to one of previously
+ * configured vPorts (either PF itself or one of already configured VFs).
+ *
+ * Typically the first vPort which corresponds to PF has globally
+ * administered unicast address, but it still could be locally
+ * administered if user assigned it or in the case of unconfigured NIC.
+ * So, it is safer to include it as well in uniqueness check.
+ */
+static bool
+sriov_mac_addr_assigned(const efx_vport_config_t *vport_config,
+			unsigned int num, const uint8_t *mac_addr)
+{
+	unsigned int i;
+
+	/* Check PF's MAC address as well as explained above */
+	for (i = 0; i < num; ++i) {
+		if (memcmp(mac_addr, vport_config[i].evc_mac_addr,
+			   sizeof(vport_config[i].evc_mac_addr)) == 0)
+			return true;
+	}
+
+	return false;
+}
+
+int
+sfc_sriov_attach(struct sfc_adapter *sa)
+{
+	const struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);
+	struct sfc_sriov *sriov = &sa->sriov;
+	efx_vport_config_t *vport_config;
+	unsigned int i;
+	int rc;
+
+	sfc_log_init(sa, "entry");
+
+	sriov->num_vfs = pci_dev->max_vfs;
+	if (sriov->num_vfs == 0)
+		goto done;
+
+	vport_config = calloc(sriov->num_vfs + 1, sizeof(*vport_config));
+	if (vport_config == NULL) {
+		rc = ENOMEM;
+		goto fail_alloc_vport_config;
+	}
+
+	vport_config[0].evc_function = 0xffff;
+	vport_config[0].evc_vid = EFX_VF_VID_DEFAULT;
+	vport_config[0].evc_vlan_restrict = B_FALSE;
+
+	for (i = 1; i <= sriov->num_vfs; ++i) {
+		vport_config[i].evc_function = i - 1;
+		vport_config[i].evc_vid = EFX_VF_VID_DEFAULT;
+		vport_config[i].evc_vlan_restrict = B_FALSE;
+		do {
+			rte_eth_random_addr(vport_config[i].evc_mac_addr);
+		} while (sriov_mac_addr_assigned(vport_config, i,
+						 vport_config[i].evc_mac_addr));
+	}
+
+	sriov->vport_config = vport_config;
+
+done:
+	sfc_log_init(sa, "done");
+	return 0;
+
+fail_alloc_vport_config:
+	sriov->num_vfs = 0;
+	return rc;
+}
+
+void
+sfc_sriov_detach(struct sfc_adapter *sa)
+{
+	struct sfc_sriov *sriov = &sa->sriov;
+
+	sfc_log_init(sa, "entry");
+
+	free(sriov->vport_config);
+	sriov->vport_config = NULL;
+	sriov->num_vfs = 0;
+
+	sfc_log_init(sa, "done");
+}
+
+int
+sfc_sriov_vswitch_create(struct sfc_adapter *sa)
+{
+	struct sfc_sriov *sriov = &sa->sriov;
+	efx_vport_config_t *vport_config = sriov->vport_config;
+	int rc;
+
+	sfc_log_init(sa, "entry");
+
+	if (sriov->num_vfs == 0) {
+		sfc_log_init(sa, "no VFs enabled");
+		goto done;
+	}
+
+	rc = efx_evb_init(sa->nic);
+	if (rc != 0) {
+		sfc_err(sa, "EVB init failed %d", rc);
+		goto fail_evb_init;
+	}
+
+	RTE_BUILD_BUG_ON(sizeof(sa->port.default_mac_addr) !=
+			 sizeof(vport_config[0].evc_mac_addr));
+	rte_ether_addr_copy(&sa->port.default_mac_addr,
+		(struct rte_ether_addr *)vport_config[0].evc_mac_addr);
+
+	rc = efx_evb_vswitch_create(sa->nic, sriov->num_vfs + 1,
+				    vport_config, &sriov->vswitch);
+	if (rc != 0) {
+		sfc_err(sa, "EVB vSwitch create failed %d", rc);
+		goto fail_evb_vswitch_create;
+	}
+
+done:
+	sfc_log_init(sa, "done");
+	return 0;
+
+fail_evb_vswitch_create:
+	efx_evb_fini(sa->nic);
+
+fail_evb_init:
+	return rc;
+}
+
+void
+sfc_sriov_vswitch_destroy(struct sfc_adapter *sa)
+{
+	struct sfc_sriov *sriov = &sa->sriov;
+	int rc;
+
+	sfc_log_init(sa, "entry");
+
+	if (sriov->num_vfs == 0)
+		goto done;
+
+	rc = efx_evb_vswitch_destroy(sa->nic, sriov->vswitch);
+	if (rc != 0)
+		sfc_err(sa, "efx_evb_vswitch_destroy() failed %d", rc);
+
+	sriov->vswitch = NULL;
+
+	efx_evb_fini(sa->nic);
+
+done:
+	sfc_log_init(sa, "done");
+}
diff --git a/drivers/net/sfc/sfc_sriov.h b/drivers/net/sfc/sfc_sriov.h
new file mode 100644
index 0000000000..2e2e1c69f1
--- /dev/null
+++ b/drivers/net/sfc/sfc_sriov.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2019-2020 Xilinx, Inc.
+ * Copyright(c) 2019 Solarflare Communications Inc.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ */
+
+#ifndef _SFC_SRIOV_H
+#define _SFC_SRIOV_H
+
+#include "efx.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct sfc_sriov {
+	/** Number of enabled virtual functions */
+	unsigned int			num_vfs;
+	/** PF and VFs vPorts configuration */
+	efx_vport_config_t		*vport_config;
+	/** vSwitch handle */
+	efx_vswitch_t			*vswitch;
+};
+
+struct sfc_adapter;
+
+int sfc_sriov_attach(struct sfc_adapter *sa);
+void sfc_sriov_detach(struct sfc_adapter *sa);
+
+int sfc_sriov_vswitch_create(struct sfc_adapter *sa);
+void sfc_sriov_vswitch_destroy(struct sfc_adapter *sa);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* _SFC_SRIOV_H */
-- 
2.17.1

Re: [dpdk-dev] [PATCH] net/sfc: create virtual switch to enable VFs

[Ferruh Yigit]

On 9/24/2020 2:02 PM, Andrew Rybchenko wrote:
> PF driver is responsible for vSwitch creation and vPorts allocation
> for VFs.
> 
> Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
> Reviewed-by: Andy Moreton <amoreton@xilinx.com>

Applied to dpdk-next-net/main, thanks.