[PATCH v6 08/14] net/nfp: add bond firmware creation initialization

[Chaoyong He <chaoyong.he@corigine.com>]

From: Long Wu <long.wu@corigine.com>

Firmware supports several features and bond firmware creation is one
of the features. Driver notifies firmware that driver supports bond
firmware creation feature by CPP bus write. If write successfully,
initialize driver configuration.

Signed-off-by: Long Wu <long.wu@corigine.com>
Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
---
 drivers/net/nfp/flower/nfp_flower.c      | 64 ++++++++++++++++++++++-
 drivers/net/nfp/flower/nfp_flower.h      | 13 +++++
 drivers/net/nfp/flower/nfp_flower_bond.c | 59 +++++++++++++++++++++
 drivers/net/nfp/flower/nfp_flower_bond.h | 65 ++++++++++++++++++++++++
 drivers/net/nfp/meson.build              |  1 +
 5 files changed, 200 insertions(+), 2 deletions(-)
 create mode 100644 drivers/net/nfp/flower/nfp_flower_bond.c
 create mode 100644 drivers/net/nfp/flower/nfp_flower_bond.h

diff --git a/drivers/net/nfp/flower/nfp_flower.c b/drivers/net/nfp/flower/nfp_flower.c
index 94b50611f0..195960e00d 100644
--- a/drivers/net/nfp/flower/nfp_flower.c
+++ b/drivers/net/nfp/flower/nfp_flower.c
@@ -16,6 +16,7 @@
 #include "../nfp_cpp_bridge.h"
 #include "../nfp_logs.h"
 #include "../nfp_mtr.h"
+#include "nfp_flower_bond.h"
 #include "nfp_flower_ctrl.h"
 #include "nfp_flower_representor.h"
 
@@ -228,6 +229,42 @@ nfp_flower_init_vnic_common(struct nfp_net_hw *hw,
 	return 0;
 }
 
+static int
+nfp_flower_bond_feature_init(struct nfp_app_fw_flower *app_fw_flower)
+{
+	int ret;
+
+	/* Notify hardware that driver supports hardware creation of bonding port */
+	ret = nfp_rtsym_write_le(app_fw_flower->pf_hw->pf_dev->sym_tbl,
+			"_abi_flower_balance_sync_enable", 1);
+	if (ret != 0) {
+		PMD_INIT_LOG(ERR, "Enable bonding port hardware creation failed.");
+		return ret;
+	}
+
+	ret = nfp_flower_bond_init(app_fw_flower);
+	if (ret != 0) {
+		PMD_INIT_LOG(ERR, "Initialize bonding structure failed.");
+		return ret;
+	}
+
+	app_fw_flower->flower_en_feats |= NFP_FL_ENABLE_BOND;
+
+	return 0;
+}
+
+static int
+nfp_flower_sync_feature_bits(struct nfp_app_fw_flower *app_fw_flower)
+{
+	int ret;
+
+	ret = nfp_flower_bond_feature_init(app_fw_flower);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
+
 static int
 nfp_flower_init_ctrl_vnic(struct nfp_net_hw *hw)
 {
@@ -538,6 +575,20 @@ nfp_flower_cleanup_ctrl_vnic(struct nfp_net_hw *hw)
 	rte_free(eth_dev);
 }
 
+static void
+nfp_flower_bond_feature_cleanup(struct nfp_app_fw_flower *app_fw_flower)
+{
+	nfp_flower_bond_cleanup(app_fw_flower);
+	app_fw_flower->flower_en_feats &= ~NFP_FL_ENABLE_BOND;
+}
+
+static void
+nfp_flower_sync_feature_cleanup(struct nfp_app_fw_flower *app_fw_flower)
+{
+	if (nfp_flower_support_bond_offload(app_fw_flower))
+		nfp_flower_bond_feature_cleanup(app_fw_flower);
+}
+
 static int
 nfp_flower_start_ctrl_vnic(struct nfp_net_hw *net_hw)
 {
@@ -768,22 +819,31 @@ nfp_init_app_fw_flower(struct nfp_pf_dev *pf_dev,
 		goto ctrl_vnic_cleanup;
 	}
 
+	/* Synchronize the features of driver and hardware */
+	ret = nfp_flower_sync_feature_bits(app_fw_flower);
+	if (ret != 0) {
+		PMD_INIT_LOG(ERR, "Sync feature bits failed");
+		goto ctrl_vnic_cleanup;
+	}
+
 	/* Start up flower services */
 	ret = nfp_flower_enable_services(app_fw_flower);
 	if (ret != 0) {
 		PMD_INIT_LOG(ERR, "Could not enable flower services");
 		ret = -ESRCH;
-		goto ctrl_vnic_cleanup;
+		goto sync_feature_cleanup;
 	}
 
 	ret = nfp_flower_repr_create(app_fw_flower);
 	if (ret != 0) {
 		PMD_INIT_LOG(ERR, "Could not create representor ports");
-		goto ctrl_vnic_cleanup;
+		goto sync_feature_cleanup;
 	}
 
 	return 0;
 
+sync_feature_cleanup:
+	nfp_flower_sync_feature_cleanup(app_fw_flower);
 ctrl_vnic_cleanup:
 	nfp_flower_cleanup_ctrl_vnic(app_fw_flower->ctrl_hw);
 ctrl_cpp_area_cleanup:
diff --git a/drivers/net/nfp/flower/nfp_flower.h b/drivers/net/nfp/flower/nfp_flower.h
index 8393de66c5..30443cd568 100644
--- a/drivers/net/nfp/flower/nfp_flower.h
+++ b/drivers/net/nfp/flower/nfp_flower.h
@@ -7,6 +7,7 @@
 #define __NFP_FLOWER_H__
 
 #include "../nfp_net_common.h"
+#include "nfp_flower_bond.h"
 
 /* Extra features bitmap. */
 #define NFP_FL_FEATS_GENEVE             RTE_BIT64(0)
@@ -61,6 +62,9 @@ struct nfp_app_fw_flower {
 	/** Number of phyport representors */
 	uint8_t num_phyport_reprs;
 
+	/** Bitmap of features enabled by HW */
+	uint8_t flower_en_feats;
+
 	/** Pointer to the PF vNIC */
 	struct nfp_net_hw *pf_hw;
 
@@ -96,6 +100,9 @@ struct nfp_app_fw_flower {
 
 	/** Function pointers for different NFD version */
 	struct nfp_flower_nfd_func nfd_func;
+
+	/** Link bond data block */
+	struct nfp_flower_bond *nfp_bond;
 };
 
 static inline bool
@@ -104,6 +111,12 @@ nfp_flower_support_decap_v2(const struct nfp_app_fw_flower *app_fw_flower)
 	return app_fw_flower->ext_features & NFP_FL_FEATS_DECAP_V2;
 }
 
+static inline bool
+nfp_flower_support_bond_offload(const struct nfp_app_fw_flower *app_fw_flower)
+{
+	return app_fw_flower->flower_en_feats & NFP_FL_ENABLE_BOND;
+}
+
 int nfp_init_app_fw_flower(struct nfp_pf_dev *pf_dev,
 		const struct nfp_dev_info *dev_info);
 void nfp_uninit_app_fw_flower(struct nfp_pf_dev *pf_dev);
diff --git a/drivers/net/nfp/flower/nfp_flower_bond.c b/drivers/net/nfp/flower/nfp_flower_bond.c
new file mode 100644
index 0000000000..bbd2818e68
--- /dev/null
+++ b/drivers/net/nfp/flower/nfp_flower_bond.c
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 Corigine, Inc.
+ * All rights reserved.
+ */
+#include "nfp_flower_bond.h"
+
+#include <rte_malloc.h>
+
+#include "nfp_flower_representor.h"
+
+static void
+nfp_flower_bond_increment_version(struct nfp_flower_bond *nfp_bond)
+{
+	/* LSB is not considered by firmware so add 2 for each increment. */
+	nfp_bond->batch_ver += 2;
+	nfp_bond->batch_ver &= NFP_FL_BOND_VERSION_MASK;
+
+	/* Zero is reserved by firmware. */
+	if (nfp_bond->batch_ver == 0)
+		nfp_bond->batch_ver += 2;
+}
+
+int
+nfp_flower_bond_init(struct nfp_app_fw_flower *app_fw_flower)
+{
+	struct nfp_flower_bond *nfp_bond;
+
+	app_fw_flower->nfp_bond = rte_zmalloc("nfp_bond",
+			sizeof(struct nfp_flower_bond), RTE_CACHE_LINE_SIZE);
+	if (app_fw_flower->nfp_bond == NULL)
+		return -ENOMEM;
+
+	nfp_bond = app_fw_flower->nfp_bond;
+	pthread_mutex_init(&nfp_bond->mutex, NULL);
+	LIST_INIT(&nfp_bond->group_list);
+	nfp_flower_bond_increment_version(nfp_bond);
+	nfp_bond->app_fw_flower = app_fw_flower;
+
+	return 0;
+}
+
+void
+nfp_flower_bond_cleanup(struct nfp_app_fw_flower *app_fw_flower)
+{
+	struct nfp_bond_group *entry;
+	struct nfp_flower_bond *nfp_bond = app_fw_flower->nfp_bond;
+
+	pthread_mutex_lock(&nfp_bond->mutex);
+	LIST_FOREACH(entry, &nfp_bond->group_list, next) {
+		LIST_REMOVE(entry, next);
+		rte_free(entry);
+	}
+	pthread_mutex_unlock(&nfp_bond->mutex);
+
+	pthread_mutex_destroy(&nfp_bond->mutex);
+
+	rte_free(nfp_bond);
+	app_fw_flower->nfp_bond = NULL;
+}
diff --git a/drivers/net/nfp/flower/nfp_flower_bond.h b/drivers/net/nfp/flower/nfp_flower_bond.h
new file mode 100644
index 0000000000..be79764a23
--- /dev/null
+++ b/drivers/net/nfp/flower/nfp_flower_bond.h
@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Corigine, Inc.
+ * All rights reserved.
+ */
+
+#ifndef __NFP_FLOWER_BOND_H__
+#define __NFP_FLOWER_BOND_H__
+
+#include <pthread.h>
+#include <stdbool.h>
+#include <sys/queue.h>
+
+/* The batch version of bond offload packets between firmware and driver */
+#define NFP_FL_BOND_VERSION_MASK       0x007fffff    /* [0, 22] */
+
+#define NFP_FL_ENABLE_BOND             RTE_BIT32(1)
+
+/* ID 0 reserved and IDs 1 to 31 are valid */
+#define NFP_FL_BOND_GROUP_MIN          1
+#define NFP_FL_BOND_GROUP_MAX          32
+
+/* List entry for each bond group */
+struct nfp_bond_group {
+	/** List entry */
+	LIST_ENTRY(nfp_bond_group) next;
+	/** Marked if the group needs synced to HW */
+	bool dirty;
+	/** Marked if the group is currently offloaded to NIC */
+	bool offloaded;
+	/** Marked if the group should be removed from NIC */
+	bool to_remove;
+	/** Marked if the group should be removed from driver */
+	bool to_destroy;
+	/** Assigned group ID for host/kernel sync */
+	uint32_t group_id;
+	/** Number of members in group */
+	uint32_t member_cnt;
+	/** Group instance in case of ID reuse */
+	uint32_t group_inst;
+	/** Group main Netdev */
+	struct rte_eth_dev *main_dev;
+};
+
+/* Flower APP priv data for bond offload */
+struct nfp_flower_bond {
+	/** Marker to reset firmware bond config */
+	bool rst_cfg;
+	/** List of all main/member groups offloaded */
+	LIST_HEAD(, nfp_bond_group) group_list;
+	/** Lock to protect bond group_list */
+	pthread_mutex_t mutex;
+	/** Incremented for each batch of config packets */
+	uint32_t batch_ver;
+	/** Instance allocator for groups */
+	uint32_t global_inst;
+	/** Incremented for each config packet sent */
+	uint32_t pkt_num;
+	/** Pointer to the flower app */
+	struct nfp_app_fw_flower *app_fw_flower;
+};
+
+int nfp_flower_bond_init(struct nfp_app_fw_flower *app_fw_flower);
+void nfp_flower_bond_cleanup(struct nfp_app_fw_flower *app_fw_flower);
+
+#endif /* __NFP_FLOWER_BOND_H__ */
diff --git a/drivers/net/nfp/meson.build b/drivers/net/nfp/meson.build
index 46be6f60cd..f0d4bbcecb 100644
--- a/drivers/net/nfp/meson.build
+++ b/drivers/net/nfp/meson.build
@@ -13,6 +13,7 @@ sources = files(
         'flower/nfp_flower_ctrl.c',
         'flower/nfp_flower_flow.c',
         'flower/nfp_flower_representor.c',
+        'flower/nfp_flower_bond.c',
         'nfd3/nfp_nfd3_dp.c',
         'nfdk/nfp_nfdk_dp.c',
         'nfpcore/nfp_cppcore.c',
-- 
2.39.1