[PATCH v6 10/14] net/nfp: handle link event of bond firmware creation

[Chaoyong He <chaoyong.he@corigine.com>]

From: Long Wu <long.wu@corigine.com>

If NFP physical representor port is a member port of bonding port,
its link status changed and firmware has created the bonding port,
driver will record the link status and send control message to notify
firmware.

Signed-off-by: Long Wu <long.wu@corigine.com>
Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
---
 drivers/net/nfp/flower/nfp_flower_bond.c      | 259 ++++++++++++++++++
 drivers/net/nfp/flower/nfp_flower_bond.h      |  22 ++
 .../net/nfp/flower/nfp_flower_representor.c   |  13 +
 .../net/nfp/flower/nfp_flower_representor.h   |   2 +
 drivers/net/nfp/meson.build                   |   2 +-
 drivers/net/nfp/nfp_net_common.c              |  25 ++
 drivers/net/nfp/nfp_net_common.h              |   1 -
 7 files changed, 322 insertions(+), 2 deletions(-)

diff --git a/drivers/net/nfp/flower/nfp_flower_bond.c b/drivers/net/nfp/flower/nfp_flower_bond.c
index 4ac27f117c..c814c0d4ea 100644
--- a/drivers/net/nfp/flower/nfp_flower_bond.c
+++ b/drivers/net/nfp/flower/nfp_flower_bond.c
@@ -4,8 +4,11 @@
  */
 #include "nfp_flower_bond.h"
 
+#include <eth_bond_private.h>
+#include <rte_eth_bond.h>
 #include <rte_malloc.h>
 
+#include "../nfp_logs.h"
 #include "nfp_flower_cmsg.h"
 #include "nfp_flower_representor.h"
 
@@ -43,6 +46,72 @@ nfp_flower_bond_increment_version(struct nfp_flower_bond *nfp_bond)
 		nfp_bond->batch_ver += 2;
 }
 
+static enum nfp_flower_bond_batch
+nfp_flower_bond_remove_node(struct nfp_flower_bond *nfp_bond,
+		struct nfp_bond_group *entry,
+		enum nfp_flower_bond_batch batch)
+{
+	enum nfp_flower_bond_batch batch_out;
+	struct nfp_flower_bond_cmsg_args cmsg_args;
+	struct nfp_app_fw_flower *app_flower = nfp_bond->app_fw_flower;
+
+	nfp_fl_bond_cmsg_args_init(&cmsg_args, entry, NULL, 0, batch);
+	if (nfp_flower_cmsg_bond_config_group(app_flower, &cmsg_args, &batch_out) != 0) {
+		PMD_DRV_LOG(ERR, "group remove failed.");
+		return batch_out;
+	}
+
+	entry->to_remove = false;
+	entry->offloaded = false;
+
+	if (entry->to_destroy) {
+		LIST_REMOVE(entry, next);
+		rte_free(entry);
+	}
+
+	return batch_out;
+}
+
+static void
+nfp_flower_bond_member_work_status(struct nfp_bond_group *entry,
+		struct rte_eth_dev **active_eth_devs,
+		uint32_t *active)
+{
+	uint32_t i;
+	uint32_t *flags;
+	uint32_t active_count;
+	uint16_t member_port_id;
+	struct rte_eth_dev *eth_dev;
+	struct bond_dev_private *internals;
+	struct nfp_flower_representor *repr;
+
+	active_count = 0;
+
+	internals = entry->main_dev->data->dev_private;
+	for (i = 0; i < internals->member_count; i++) {
+		member_port_id = internals->members[i].port_id;
+		if (internals->current_primary_port == member_port_id ||
+				internals->mode != BONDING_MODE_ACTIVE_BACKUP) {
+			eth_dev = &rte_eth_devices[member_port_id];
+			repr = eth_dev->data->dev_private;
+			flags = &repr->bond_port_flags;
+
+			if ((*flags & NFP_FL_BOND_PORT_CHANGED) != 0) {
+				*flags &= ~NFP_FL_BOND_PORT_CHANGED;
+				entry->dirty = true;
+			}
+
+			if ((*flags & NFP_FL_BOND_PORT_TX_ENABLED) != 0 &&
+					(*flags & NFP_FL_BOND_PORT_LINK_UP) != 0) {
+				active_eth_devs[active_count] = eth_dev;
+				active_count++;
+			}
+		}
+	}
+
+	*active = active_count;
+}
+
 int
 nfp_flower_bond_init(struct nfp_app_fw_flower *app_fw_flower)
 {
@@ -96,6 +165,132 @@ nfp_flower_bond_reset(struct nfp_flower_bond *nfp_bond)
 	return nfp_flower_cmsg_bond_config_group(app_flower, &init_args, &batch);
 }
 
+struct nfp_bond_group *
+nfp_flower_bond_find_group(struct nfp_flower_bond *nfp_bond,
+		const struct rte_eth_dev *bond_dev)
+{
+	struct nfp_bond_group *group;
+
+	LIST_FOREACH(group, &nfp_bond->group_list, next) {
+		if (group->main_dev == bond_dev)
+			return group;
+	}
+
+	return NULL;
+}
+
+void
+nfp_flower_bond_do_work(struct nfp_flower_bond *nfp_bond)
+{
+	int ret;
+	uint32_t active_count;
+	struct nfp_bond_group *entry;
+	struct nfp_app_fw_flower *app_flower;
+	struct nfp_flower_bond_cmsg_args cmsg_args;
+	struct rte_eth_dev *active_eth_devs[RTE_MAX_ETHPORTS];
+	enum nfp_flower_bond_batch batch = NFP_FLOWER_BOND_BATCH_FIRST;
+
+	app_flower = nfp_bond->app_fw_flower;
+
+	pthread_mutex_lock(&nfp_bond->mutex);
+
+	LIST_FOREACH(entry, &nfp_bond->group_list, next) {
+		if (entry->to_remove) {
+			batch = nfp_flower_bond_remove_node(nfp_bond, entry, batch);
+			continue;
+		}
+
+		nfp_flower_bond_member_work_status(entry, active_eth_devs,
+				&active_count);
+
+		if (!entry->dirty)
+			continue;
+
+		nfp_fl_bond_cmsg_args_init(&cmsg_args, entry, active_eth_devs,
+				active_count, batch);
+
+		ret = nfp_flower_cmsg_bond_config_group(app_flower, &cmsg_args, &batch);
+		if (ret == 0) {
+			entry->offloaded = true;
+			entry->dirty = false;
+		} else {
+			PMD_DRV_LOG(DEBUG, "Group config failed.");
+		}
+	}
+
+	/* End the config batch if at least one packet has been batched. */
+	if (batch == NFP_FLOWER_BOND_BATCH_MEMBER) {
+		batch = NFP_FLOWER_BOND_BATCH_FINISHED;
+		nfp_fl_bond_cmsg_args_init(&cmsg_args, NULL, NULL, 0, batch);
+		ret = nfp_flower_cmsg_bond_config_group(app_flower, &cmsg_args, &batch);
+		if (ret != 0)
+			PMD_DRV_LOG(DEBUG, "Group batch end cmsg failed");
+	}
+
+	pthread_mutex_unlock(&nfp_bond->mutex);
+
+	/*
+	 * A batch of packets rx in firmware has been set for at least two timing
+	 * units, so trigger copying at next opportunity to limit latency from
+	 * receiving data to being available for use.
+	 */
+	rte_delay_ms(1);
+}
+
+static void
+nfp_flower_bond_change_linkstatus_event(struct nfp_flower_bond *nfp_bond,
+		struct rte_eth_dev *bond_dev,
+		struct rte_eth_dev *nfp_dev)
+{
+	uint32_t *bond_port_flags;
+	struct nfp_bond_group *group;
+	struct nfp_flower_representor *repr;
+
+	if (!nfp_flower_bond_is_member_port(bond_dev, nfp_dev))
+		return;
+
+	pthread_mutex_lock(&nfp_bond->mutex);
+
+	group = nfp_flower_bond_find_group(nfp_bond, bond_dev);
+	if (group == NULL) {
+		pthread_mutex_unlock(&nfp_bond->mutex);
+		return;
+	}
+
+	pthread_mutex_unlock(&nfp_bond->mutex);
+
+	repr = nfp_dev->data->dev_private;
+	bond_port_flags = &repr->bond_port_flags;
+
+	if (repr->link.link_status == RTE_ETH_LINK_UP)
+		*bond_port_flags |= NFP_FL_BOND_PORT_LINK_UP;
+	else
+		*bond_port_flags &= ~NFP_FL_BOND_PORT_LINK_UP;
+
+	*bond_port_flags |= NFP_FL_BOND_PORT_TX_ENABLED;
+	*bond_port_flags |= NFP_FL_BOND_PORT_CHANGED;
+}
+
+int
+nfp_flower_bond_event_handle(struct nfp_flower_bond *nfp_bond,
+		struct rte_eth_dev *bond_dev,
+		struct rte_eth_dev *nfp_dev,
+		enum nfp_flower_bond_event event)
+{
+	switch (event) {
+	case NFP_FLOWER_CHANGELINKSTATUS:
+		nfp_flower_bond_change_linkstatus_event(nfp_bond, bond_dev, nfp_dev);
+		break;
+	default:
+		PMD_DRV_LOG(ERR, "Invalid bond offload event.");
+		return -1;
+	}
+
+	nfp_flower_bond_do_work(nfp_bond);
+
+	return 0;
+}
+
 enum nfp_flower_bond_batch
 nfp_flower_bond_cmsg_payload(struct nfp_flower_bond *nfp_bond,
 		struct nfp_flower_cmsg_bond_config *msg,
@@ -147,3 +342,67 @@ nfp_flower_bond_cmsg_payload(struct nfp_flower_bond *nfp_bond,
 
 	return batch;
 }
+
+bool
+nfp_flower_bond_is_member_port(struct rte_eth_dev *bond_dev,
+		struct rte_eth_dev *nfp_dev)
+{
+	uint32_t i;
+	struct bond_dev_private *internals;
+
+	if (!nfp_flower_is_phy_repr(nfp_dev))
+		return false;
+
+	internals = bond_dev->data->dev_private;
+
+	for (i = 0; i < internals->member_count; i++) {
+		if (nfp_dev == &rte_eth_devices[internals->members[i].port_id])
+			return true;
+	}
+
+	return false;
+}
+
+struct rte_eth_dev *
+nfp_flower_bond_find_bond_device(struct rte_eth_dev *nfp_dev)
+{
+	struct nfp_bond_group *group;
+	struct nfp_flower_representor *repr;
+	struct rte_eth_dev *bond_dev = NULL;
+
+	if (!nfp_flower_is_phy_repr(nfp_dev))
+		return NULL;
+
+	repr = nfp_dev->data->dev_private;
+
+	pthread_mutex_lock(&repr->app_fw_flower->nfp_bond->mutex);
+	LIST_FOREACH(group, &repr->app_fw_flower->nfp_bond->group_list, next) {
+		if (nfp_flower_bond_is_member_port(group->main_dev, nfp_dev)) {
+			bond_dev = group->main_dev;
+			break;
+		}
+	}
+	pthread_mutex_unlock(&repr->app_fw_flower->nfp_bond->mutex);
+
+	return bond_dev;
+}
+
+bool
+nfp_flower_bond_all_member_are_phyrepr(struct rte_eth_dev *bond_dev)
+{
+	uint32_t i;
+	struct rte_eth_dev *eth_dev;
+	struct bond_dev_private *internals;
+
+	if (bond_dev == NULL)
+		return false;
+
+	internals = bond_dev->data->dev_private;
+	for (i = 0; i < internals->member_count; i++) {
+		eth_dev = &rte_eth_devices[internals->members[i].port_id];
+		if (!nfp_flower_is_phy_repr(eth_dev))
+			return false;
+	}
+
+	return true;
+}
diff --git a/drivers/net/nfp/flower/nfp_flower_bond.h b/drivers/net/nfp/flower/nfp_flower_bond.h
index e15d9e09d5..e4a09b3427 100644
--- a/drivers/net/nfp/flower/nfp_flower_bond.h
+++ b/drivers/net/nfp/flower/nfp_flower_bond.h
@@ -36,12 +36,23 @@
 #define NFP_FL_BOND_SWITCH             RTE_BIT32(6)
 #define NFP_FL_BOND_RESET              RTE_BIT32(7)
 
+/* BOND port state flags. */
+#define NFP_FL_BOND_PORT_LINK_UP       RTE_BIT32(0)
+#define NFP_FL_BOND_PORT_TX_ENABLED    RTE_BIT32(1)
+#define NFP_FL_BOND_PORT_CHANGED       RTE_BIT32(2)
+
 enum nfp_flower_bond_batch {
 	NFP_FLOWER_BOND_BATCH_FIRST,
 	NFP_FLOWER_BOND_BATCH_MEMBER,
 	NFP_FLOWER_BOND_BATCH_FINISHED
 };
 
+enum nfp_flower_bond_event {
+	NFP_FLOWER_CHANGEUPPER,
+	NFP_FLOWER_CHANGELINKSTATUS,
+	NFP_FLOWER_UNREGISTER
+};
+
 /* Control message payload for bond config */
 struct nfp_flower_cmsg_bond_config {
 	/** Configuration flags */
@@ -115,5 +126,16 @@ int nfp_flower_bond_reset(struct nfp_flower_bond *nfp_bond);
 enum nfp_flower_bond_batch nfp_flower_bond_cmsg_payload(struct nfp_flower_bond *nfp_bond,
 		struct nfp_flower_cmsg_bond_config *msg,
 		struct nfp_flower_bond_cmsg_args *init_args);
+struct rte_eth_dev *nfp_flower_bond_find_bond_device(struct rte_eth_dev *nfp_dev);
+bool nfp_flower_bond_is_member_port(struct rte_eth_dev *bond_dev,
+		struct rte_eth_dev *nfp_dev);
+struct nfp_bond_group *nfp_flower_bond_find_group(struct nfp_flower_bond *nfp_bond,
+		const struct rte_eth_dev *bond_dev);
+int nfp_flower_bond_event_handle(struct nfp_flower_bond *nfp_bond,
+		struct rte_eth_dev *bond_dev,
+		struct rte_eth_dev *nfp_dev,
+		enum nfp_flower_bond_event event);
+void nfp_flower_bond_do_work(struct nfp_flower_bond *nfp_bond);
+bool nfp_flower_bond_all_member_are_phyrepr(struct rte_eth_dev *bond_dev);
 
 #endif /* __NFP_FLOWER_BOND_H__ */
diff --git a/drivers/net/nfp/flower/nfp_flower_representor.c b/drivers/net/nfp/flower/nfp_flower_representor.c
index 7d8c055b80..2810a7a271 100644
--- a/drivers/net/nfp/flower/nfp_flower_representor.c
+++ b/drivers/net/nfp/flower/nfp_flower_representor.c
@@ -827,3 +827,16 @@ nfp_flower_repr_create(struct nfp_app_fw_flower *app_fw_flower)
 
 	return ret;
 }
+
+bool
+nfp_flower_is_phy_repr(struct rte_eth_dev *eth_dev)
+{
+	struct nfp_flower_representor *repr;
+
+	if ((eth_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR) != 0) {
+		repr = eth_dev->data->dev_private;
+		return repr->repr_type == NFP_REPR_TYPE_PHYS_PORT;
+	}
+
+	return false;
+}
diff --git a/drivers/net/nfp/flower/nfp_flower_representor.h b/drivers/net/nfp/flower/nfp_flower_representor.h
index 8053617562..164e2f1322 100644
--- a/drivers/net/nfp/flower/nfp_flower_representor.h
+++ b/drivers/net/nfp/flower/nfp_flower_representor.h
@@ -14,6 +14,7 @@ struct nfp_flower_representor {
 	uint32_t repr_type;
 	uint32_t port_id;
 	uint32_t nfp_idx;    /**< Only valid for the repr of physical port */
+	uint32_t bond_port_flags;
 	char name[RTE_ETH_NAME_MAX_LEN];
 	struct rte_ether_addr mac_addr;
 	struct nfp_app_fw_flower *app_fw_flower;
@@ -24,5 +25,6 @@ struct nfp_flower_representor {
 };
 
 int nfp_flower_repr_create(struct nfp_app_fw_flower *app_fw_flower);
+bool nfp_flower_is_phy_repr(struct rte_eth_dev *eth_dev);
 
 #endif /* __NFP_FLOWER_REPRESENTOR_H__ */
diff --git a/drivers/net/nfp/meson.build b/drivers/net/nfp/meson.build
index f0d4bbcecb..c309505c83 100644
--- a/drivers/net/nfp/meson.build
+++ b/drivers/net/nfp/meson.build
@@ -42,4 +42,4 @@ sources = files(
         'nfp_rxtx.c',
 )
 
-deps += ['hash', 'security', 'common_nfp']
+deps += ['hash', 'security', 'common_nfp', 'net_bond']
diff --git a/drivers/net/nfp/nfp_net_common.c b/drivers/net/nfp/nfp_net_common.c
index a438eb5871..810903c033 100644
--- a/drivers/net/nfp/nfp_net_common.c
+++ b/drivers/net/nfp/nfp_net_common.c
@@ -9,6 +9,7 @@
 
 #include <rte_alarm.h>
 
+#include "flower/nfp_flower.h"
 #include "flower/nfp_flower_representor.h"
 #include "nfd3/nfp_nfd3.h"
 #include "nfdk/nfp_nfdk.h"
@@ -719,6 +720,23 @@ nfp_net_speed_aneg_update(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static void
+nfp_net_bond_link_event_notify(struct rte_eth_dev *dev)
+{
+	struct rte_eth_dev *bond_dev;
+	struct nfp_flower_representor *repr = dev->data->dev_private;
+
+	if (!nfp_flower_support_bond_offload(repr->app_fw_flower))
+		return;
+
+	bond_dev = nfp_flower_bond_find_bond_device(dev);
+	if (!nfp_flower_bond_all_member_are_phyrepr(bond_dev))
+		return;
+
+	nfp_flower_bond_event_handle(repr->app_fw_flower->nfp_bond, bond_dev,
+			dev, NFP_FLOWER_CHANGELINKSTATUS);
+}
+
 int
 nfp_net_link_update_common(struct rte_eth_dev *dev,
 		struct nfp_net_hw *hw,
@@ -753,6 +771,13 @@ nfp_net_link_update_common(struct rte_eth_dev *dev,
 			PMD_DRV_LOG(INFO, "NIC Link is Up");
 		else
 			PMD_DRV_LOG(INFO, "NIC Link is Down");
+
+		/*
+		 * Link status changed and if repr is a member port in bond device,
+		 * we need to call func to do something special.
+		 */
+		if ((dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR) != 0)
+			nfp_net_bond_link_event_notify(dev);
 	}
 
 	return ret;
diff --git a/drivers/net/nfp/nfp_net_common.h b/drivers/net/nfp/nfp_net_common.h
index 66c900e3b8..9d6cf4b016 100644
--- a/drivers/net/nfp/nfp_net_common.h
+++ b/drivers/net/nfp/nfp_net_common.h
@@ -295,7 +295,6 @@ int nfp_net_fec_get(struct rte_eth_dev *dev,
 		uint32_t *fec_capa);
 int nfp_net_fec_set(struct rte_eth_dev *dev,
 		uint32_t fec_capa);
-
 #define NFP_PRIV_TO_APP_FW_NIC(app_fw_priv)\
 	((struct nfp_app_fw_nic *)app_fw_priv)
 
-- 
2.39.1