[dpdk-dev] [PATCH v2 53/58] net/bnxt: add HA support in ULP

[Ajit Khaparde <ajit.khaparde@broadcom.com>]

[-- Attachment #1: Type: text/plain, Size: 27927 bytes --]

From: Mike Baucom <michael.baucom@broadcom.com>

Add the ability for cooperative applications to share resources and
use the high availability functionality in the HW.

Signed-off-by: Mike Baucom <michael.baucom@broadcom.com>
Signed-off-by: Venkat Duvvuru <venkatkumar.duvvuru@broadcom.com>
Reviewed-by: Shahaji Bhosle <sbhosle@broadcom.com>
Reviewed-by: Randy Schacher <stuart.schacher@broadcom.com>
Reviewed-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
---
 drivers/net/bnxt/tf_ulp/bnxt_ulp.c      |  59 +++
 drivers/net/bnxt/tf_ulp/bnxt_ulp.h      |  10 +
 drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c |  15 +-
 drivers/net/bnxt/tf_ulp/meson.build     |   1 +
 drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c    | 551 ++++++++++++++++++++++++
 drivers/net/bnxt/tf_ulp/ulp_ha_mgr.h    |  67 +++
 drivers/net/bnxt/tf_ulp/ulp_mapper.c    |  52 ++-
 7 files changed, 745 insertions(+), 10 deletions(-)
 create mode 100644 drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c
 create mode 100644 drivers/net/bnxt/tf_ulp/ulp_ha_mgr.h

diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp.c b/drivers/net/bnxt/tf_ulp/bnxt_ulp.c
index 0daa8e4c29..972bf8b992 100644
--- a/drivers/net/bnxt/tf_ulp/bnxt_ulp.c
+++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp.c
@@ -23,6 +23,7 @@
 #include "ulp_mapper.h"
 #include "ulp_port_db.h"
 #include "ulp_tun.h"
+#include "ulp_ha_mgr.h"
 
 /* Linked list of all TF sessions. */
 STAILQ_HEAD(, bnxt_ulp_session_state) bnxt_ulp_session_list =
@@ -315,6 +316,9 @@ bnxt_ulp_cntxt_app_caps_init(struct bnxt_ulp_context *ulp_ctx,
 		if (info[i].flags & BNXT_ULP_APP_CAP_SHARED_EN)
 			ulp_ctx->cfg_data->ulp_flags |=
 				BNXT_ULP_SHARED_SESSION_ENABLED;
+		if (info[i].flags & BNXT_ULP_APP_CAP_HOT_UPGRADE_EN)
+			ulp_ctx->cfg_data->ulp_flags |=
+				BNXT_ULP_HIGH_AVAIL_ENABLED;
 	}
 	if (!found) {
 		BNXT_TF_DBG(ERR, "APP ID %d, Device ID: 0x%x not supported.\n",
@@ -1137,9 +1141,18 @@ static void
 bnxt_ulp_deinit(struct bnxt *bp,
 		struct bnxt_ulp_session_state *session)
 {
+	bool ha_enabled;
+
 	if (!bp->ulp_ctx || !bp->ulp_ctx->cfg_data)
 		return;
 
+	ha_enabled = bnxt_ulp_cntxt_ha_enabled(bp->ulp_ctx);
+	if (ha_enabled && session->session_opened) {
+		int32_t rc = ulp_ha_mgr_close(bp->ulp_ctx);
+		if (rc)
+			BNXT_TF_DBG(ERR, "Failed to close HA (%d)\n", rc);
+	}
+
 	/* clean up default flows */
 	bnxt_ulp_destroy_df_rules(bp, true);
 
@@ -1179,6 +1192,9 @@ bnxt_ulp_deinit(struct bnxt *bp,
 	/* free the flow db lock */
 	pthread_mutex_destroy(&bp->ulp_ctx->cfg_data->flow_db_lock);
 
+	if (ha_enabled)
+		ulp_ha_mgr_deinit(bp->ulp_ctx);
+
 	/* Delete the ulp context and tf session and free the ulp context */
 	ulp_ctx_deinit(bp, session);
 	BNXT_TF_DBG(DEBUG, "ulp ctx has been deinitialized\n");
@@ -1275,6 +1291,19 @@ bnxt_ulp_init(struct bnxt *bp,
 		BNXT_TF_DBG(ERR, "Failed to set tx global configuration\n");
 		goto jump_to_error;
 	}
+
+	if (bnxt_ulp_cntxt_ha_enabled(bp->ulp_ctx)) {
+		rc = ulp_ha_mgr_init(bp->ulp_ctx);
+		if (rc) {
+			BNXT_TF_DBG(ERR, "Failed to initialize HA %d\n", rc);
+			goto jump_to_error;
+		}
+		rc = ulp_ha_mgr_open(bp->ulp_ctx);
+		if (rc) {
+			BNXT_TF_DBG(ERR, "Failed to Process HA Open %d\n", rc);
+			goto jump_to_error;
+		}
+	}
 	BNXT_TF_DBG(DEBUG, "ulp ctx has been initialized\n");
 	return rc;
 
@@ -1828,3 +1857,33 @@ bnxt_ulp_cntxt_release_fdb_lock(struct bnxt_ulp_context	*ulp_ctx)
 
 	pthread_mutex_unlock(&ulp_ctx->cfg_data->flow_db_lock);
 }
+
+/* Function to set the ha info into the context */
+int32_t
+bnxt_ulp_cntxt_ptr2_ha_info_set(struct bnxt_ulp_context *ulp_ctx,
+				struct bnxt_ulp_ha_mgr_info *ulp_ha_info)
+{
+	if (ulp_ctx == NULL || ulp_ctx->cfg_data == NULL) {
+		BNXT_TF_DBG(ERR, "Invalid ulp context data\n");
+		return -EINVAL;
+	}
+	ulp_ctx->cfg_data->ha_info = ulp_ha_info;
+	return 0;
+}
+
+/* Function to retrieve the ha info from the context. */
+struct bnxt_ulp_ha_mgr_info *
+bnxt_ulp_cntxt_ptr2_ha_info_get(struct bnxt_ulp_context *ulp_ctx)
+{
+	if (ulp_ctx == NULL || ulp_ctx->cfg_data == NULL)
+		return NULL;
+	return ulp_ctx->cfg_data->ha_info;
+}
+
+bool
+bnxt_ulp_cntxt_ha_enabled(struct bnxt_ulp_context *ulp_ctx)
+{
+	if (ulp_ctx == NULL || ulp_ctx->cfg_data == NULL)
+		return false;
+	return !!ULP_HIGH_AVAIL_IS_ENABLED(ulp_ctx->cfg_data->ulp_flags);
+}
diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp.h b/drivers/net/bnxt/tf_ulp/bnxt_ulp.h
index 1ba67ed9f6..b1f090a5cb 100644
--- a/drivers/net/bnxt/tf_ulp/bnxt_ulp.h
+++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp.h
@@ -67,6 +67,7 @@ struct bnxt_ulp_data {
 	void				*mapper_data;
 	struct bnxt_ulp_port_db		*port_db;
 	struct bnxt_ulp_fc_info		*fc_info;
+	struct bnxt_ulp_ha_mgr_info	*ha_info;
 	uint32_t			ulp_flags;
 	struct bnxt_ulp_df_rule_info	df_rule_info[RTE_MAX_ETHPORTS];
 	struct bnxt_ulp_vfr_rule_info	vfr_rule_info[RTE_MAX_ETHPORTS];
@@ -275,4 +276,13 @@ bnxt_ulp_cntxt_app_caps_init(struct bnxt_ulp_context *ulp_ctx,
 struct bnxt_ulp_resource_resv_info *
 bnxt_ulp_resource_resv_list_get(uint32_t *num_entries);
 
+int32_t
+bnxt_ulp_cntxt_ptr2_ha_info_set(struct bnxt_ulp_context *ulp_ctx,
+				struct bnxt_ulp_ha_mgr_info *ulp_ha_info);
+
+struct bnxt_ulp_ha_mgr_info *
+bnxt_ulp_cntxt_ptr2_ha_info_get(struct bnxt_ulp_context *ulp_ctx);
+
+bool
+bnxt_ulp_cntxt_ha_enabled(struct bnxt_ulp_context *ulp_ctx);
 #endif /* _BNXT_ULP_H_ */
diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c b/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c
index 9c27217573..96e6a76270 100644
--- a/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c
+++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c
@@ -11,6 +11,7 @@
 #include "ulp_mapper.h"
 #include "ulp_fc_mgr.h"
 #include "ulp_port_db.h"
+#include "ulp_ha_mgr.h"
 #include <rte_malloc.h>
 
 static int32_t
@@ -112,11 +113,17 @@ bnxt_ulp_init_mapper_params(struct bnxt_ulp_mapper_create_parms *mapper_cparms,
 	/* update the WC Priority flag */
 	if (!bnxt_ulp_cntxt_ptr2_ulp_flags_get(params->ulp_ctx, &ulp_flags) &&
 	    ULP_HIGH_AVAIL_IS_ENABLED(ulp_flags)) {
-		/* TBD: read the state and Set the WC priority */
-		ULP_COMP_FLD_IDX_WR(params,
-				    BNXT_ULP_CF_IDX_WC_IS_HA_HIGH_REG, 1);
+		enum ulp_ha_mgr_region region = ULP_HA_REGION_LOW;
+		int32_t rc;
+
+		rc = ulp_ha_mgr_region_get(params->ulp_ctx, &region);
+		if (rc)
+			BNXT_TF_DBG(ERR, "Unable to get WC region\n");
+		if (region == ULP_HA_REGION_HI)
+			ULP_COMP_FLD_IDX_WR(params,
+					    BNXT_ULP_CF_IDX_WC_IS_HA_HIGH_REG,
+					    1);
 	}
-
 }
 
 /* Function to create the rte flow. */
diff --git a/drivers/net/bnxt/tf_ulp/meson.build b/drivers/net/bnxt/tf_ulp/meson.build
index 1bb93d4938..40479c5936 100644
--- a/drivers/net/bnxt/tf_ulp/meson.build
+++ b/drivers/net/bnxt/tf_ulp/meson.build
@@ -24,6 +24,7 @@ sources += files(
         'ulp_tun.c',
         'ulp_gen_tbl.c',
 	'ulp_gen_hash.c',
+	'ulp_ha_mgr.c',
 	'ulp_rte_handler_tbl.c',
         'ulp_template_db_wh_plus_act.c',
         'ulp_template_db_wh_plus_class.c',
diff --git a/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c b/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c
new file mode 100644
index 0000000000..dc71054f46
--- /dev/null
+++ b/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c
@@ -0,0 +1,551 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2021 Broadcom
+ * All rights reserved.
+ */
+
+#include <rte_common.h>
+#include <rte_cycles.h>
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_alarm.h>
+#include "bnxt.h"
+#include "bnxt_ulp.h"
+#include "bnxt_tf_common.h"
+#include "ulp_ha_mgr.h"
+#include "ulp_flow_db.h"
+
+/* Local only MACROs and defines that aren't exported */
+#define ULP_HA_TIMER_THREAD	(1 << 0)
+#define ULP_HA_TIMER_IS_RUNNING(info) (!!((info)->flags & ULP_HA_TIMER_THREAD))
+#define ULP_HA_TIMER_SEC 1
+#define ULP_HA_WAIT_TIME (MS_PER_S / 10)
+#define ULP_HA_WAIT_TIMEOUT (MS_PER_S * 2)
+
+#define ULP_HA_IF_TBL_DIR	TF_DIR_RX
+#define ULP_HA_IF_TBL_TYPE	TF_IF_TBL_TYPE_PROF_PARIF_ERR_ACT_REC_PTR
+#define ULP_HA_IF_TBL_IDX 10
+
+static void ulp_ha_mgr_timer_cancel(struct bnxt_ulp_context *ulp_ctx);
+static int32_t ulp_ha_mgr_timer_start(struct bnxt_ulp_context *ulp_ctx);
+static void ulp_ha_mgr_timer_cb(void *arg);
+static int32_t ulp_ha_mgr_app_type_set(struct bnxt_ulp_context *ulp_ctx,
+				enum ulp_ha_mgr_app_type app_type);
+static int32_t
+ulp_ha_mgr_region_set(struct bnxt_ulp_context *ulp_ctx,
+		      enum ulp_ha_mgr_region region);
+static int32_t
+ulp_ha_mgr_state_set(struct bnxt_ulp_context *ulp_ctx,
+		     enum ulp_ha_mgr_state state);
+
+static int32_t
+ulp_ha_mgr_state_set(struct bnxt_ulp_context *ulp_ctx,
+		     enum ulp_ha_mgr_state state)
+{
+	struct tf_set_if_tbl_entry_parms set_parms = { 0 };
+	struct tf *tfp;
+	uint32_t val = 0;
+	int32_t rc = 0;
+
+	if (ulp_ctx == NULL) {
+		BNXT_TF_DBG(ERR, "Invalid parms in state get.\n");
+		return -EINVAL;
+	}
+	tfp = bnxt_ulp_cntxt_tfp_get(ulp_ctx, BNXT_ULP_SHARED_SESSION_NO);
+	if (tfp == NULL) {
+		BNXT_TF_DBG(ERR, "Unable to get the TFP.\n");
+		return -EINVAL;
+	}
+
+	val = (uint32_t)state;
+
+	set_parms.dir = ULP_HA_IF_TBL_DIR;
+	set_parms.type = ULP_HA_IF_TBL_TYPE;
+	set_parms.data = (uint8_t *)&val;
+	set_parms.data_sz_in_bytes = sizeof(val);
+	set_parms.idx = ULP_HA_IF_TBL_IDX;
+
+	rc = tf_set_if_tbl_entry(tfp, &set_parms);
+	if (rc)
+		BNXT_TF_DBG(ERR, "Failed to write the HA state\n");
+
+	return rc;
+}
+
+static int32_t
+ulp_ha_mgr_region_set(struct bnxt_ulp_context *ulp_ctx,
+		      enum ulp_ha_mgr_region region)
+{
+	struct bnxt_ulp_ha_mgr_info *ha_info;
+
+	if (ulp_ctx == NULL) {
+		BNXT_TF_DBG(ERR, "Invalid params in ha region get.\n");
+		return -EINVAL;
+	}
+
+	ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+	if (ha_info == NULL) {
+		BNXT_TF_DBG(ERR, "Unable to get ha info\n");
+		return -EINVAL;
+	}
+	ha_info->region = region;
+
+	return 0;
+}
+
+static int32_t
+ulp_ha_mgr_app_type_set(struct bnxt_ulp_context *ulp_ctx,
+			enum ulp_ha_mgr_app_type app_type)
+{
+	struct bnxt_ulp_ha_mgr_info *ha_info;
+
+	if (ulp_ctx == NULL) {
+		BNXT_TF_DBG(ERR, "Invalid Parms.\n");
+		return -EINVAL;
+	}
+
+	ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+	if (ha_info == NULL) {
+		BNXT_TF_DBG(ERR, "Unable to get the ha info.\n");
+		return -EINVAL;
+	}
+	ha_info->app_type = app_type;
+
+	return 0;
+}
+
+/*
+ * When a secondary opens, the timer is started and periodically checks for a
+ * close of the primary (state moved to SEC_TIMER_COPY).
+ * In SEC_TIMER_COPY:
+ * - The flow db must be locked to prevent flows from being added to the high
+ *   region during a move.
+ * - Move the high entries to low
+ * - Set the region to low for subsequent flows
+ * - Switch our persona to Primary
+ * - Set the state to Primary Run
+ * - Release the flow db lock for flows to continue
+ */
+static void
+ulp_ha_mgr_timer_cb(void *arg)
+{
+	struct tf_move_tcam_shared_entries_parms mparms = { 0 };
+	struct bnxt_ulp_context *ulp_ctx;
+	enum ulp_ha_mgr_state curr_state;
+	struct tf *tfp;
+	int32_t rc;
+
+	ulp_ctx = (struct bnxt_ulp_context *)arg;
+	rc = ulp_ha_mgr_state_get(ulp_ctx, &curr_state);
+	if (rc) {
+		/*
+		 * This shouldn't happen, if it does, resetart the timer
+		 * and try again next time.
+		 */
+		BNXT_TF_DBG(ERR, "On HA CB:Failed(%d) to get state.\n", rc);
+		goto cb_restart;
+	}
+	if (curr_state != ULP_HA_STATE_SEC_TIMER_COPY)
+		goto cb_restart;
+
+	/* Protect the flow database during the copy */
+	if (bnxt_ulp_cntxt_acquire_fdb_lock(ulp_ctx)) {
+		/* Should not fail, if we do, restart timer and try again */
+		BNXT_TF_DBG(ERR, "Flow db lock acquire failed\n");
+		goto cb_restart;
+	}
+	/* All paths after this point must release the fdb lock */
+
+	/* The Primary has issued a close and we are in the timer copy
+	 * phase.  Become the new Primary, Set state to Primary Run and
+	 * move WC entries to Low Region.
+	 */
+	BNXT_TF_DBG(INFO, "On HA CB: Moving entries HI to LOW\n");
+	mparms.dir = TF_DIR_RX;
+	mparms.tcam_tbl_type = TF_TCAM_TBL_TYPE_WC_TCAM_HIGH;
+	tfp = bnxt_ulp_cntxt_tfp_get(ulp_ctx, BNXT_ULP_SHARED_SESSION_YES);
+	if (tfp == NULL) {
+		BNXT_TF_DBG(ERR, "On HA CB: Unable to get the TFP.\n");
+		goto unlock;
+	}
+
+	rc = tf_move_tcam_shared_entries(tfp, &mparms);
+	if (rc) {
+		BNXT_TF_DBG(ERR, "On HA_CB: Failed to move entries\n");
+		goto unlock;
+	}
+
+	ulp_ha_mgr_region_set(ulp_ctx, ULP_HA_REGION_LOW);
+	ulp_ha_mgr_app_type_set(ulp_ctx, ULP_HA_APP_TYPE_PRIM);
+	ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_RUN);
+	BNXT_TF_DBG(INFO, "On HA CB: SEC[SEC_TIMER_COPY] => PRIM[PRIM_RUN]\n");
+unlock:
+	bnxt_ulp_cntxt_release_fdb_lock(ulp_ctx);
+	return;
+cb_restart:
+	ulp_ha_mgr_timer_start(ulp_ctx);
+}
+
+static int32_t
+ulp_ha_mgr_timer_start(struct bnxt_ulp_context *ulp_ctx)
+{
+	struct bnxt_ulp_ha_mgr_info *ha_info;
+
+	if (ulp_ctx == NULL) {
+		BNXT_TF_DBG(ERR, "Invalid parmsi for ha timer start.\n");
+		return -EINVAL;
+	}
+
+	ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+
+	if (ha_info == NULL) {
+		BNXT_TF_DBG(ERR, "Unable to get HA Info in timer start.\n");
+		return -EINVAL;
+	}
+	ha_info->flags |= ULP_HA_TIMER_THREAD;
+	rte_eal_alarm_set(US_PER_S * ULP_HA_TIMER_SEC,
+			  ulp_ha_mgr_timer_cb,
+			  (void *)ulp_ctx);
+	return 0;
+}
+
+static void
+ulp_ha_mgr_timer_cancel(struct bnxt_ulp_context *ulp_ctx)
+{
+	struct bnxt_ulp_ha_mgr_info *ha_info;
+
+	ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+	if (ha_info == NULL) {
+		BNXT_TF_DBG(ERR, "Unable to get ha info\n");
+		return;
+	}
+
+	ha_info->flags &= ~ULP_HA_TIMER_THREAD;
+	rte_eal_alarm_cancel(ulp_ha_mgr_timer_cb, (void *)ulp_ctx);
+}
+
+int32_t
+ulp_ha_mgr_init(struct bnxt_ulp_context *ulp_ctx)
+{
+	struct bnxt_ulp_ha_mgr_info *ha_info;
+	int32_t rc;
+	ha_info = rte_zmalloc("ulp_ha_mgr_info", sizeof(*ha_info), 0);
+	if (!ha_info)
+		return -ENOMEM;
+
+	/* Add the HA info tbl to the ulp context. */
+	bnxt_ulp_cntxt_ptr2_ha_info_set(ulp_ctx, ha_info);
+
+	rc = pthread_mutex_init(&ha_info->ha_lock, NULL);
+	if (rc) {
+		PMD_DRV_LOG(ERR, "Failed to initialize ha mutex\n");
+		goto cleanup;
+	}
+
+	return 0;
+cleanup:
+	if (ha_info != NULL)
+		ulp_ha_mgr_deinit(ulp_ctx);
+	return -ENOMEM;
+}
+
+void
+ulp_ha_mgr_deinit(struct bnxt_ulp_context *ulp_ctx)
+{
+	struct bnxt_ulp_ha_mgr_info *ha_info;
+
+	ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+	if (ha_info == NULL) {
+		BNXT_TF_DBG(ERR, "Unable to get HA Info for deinit.\n");
+		return;
+	}
+
+	pthread_mutex_destroy(&ha_info->ha_lock);
+	rte_free(ha_info);
+
+	bnxt_ulp_cntxt_ptr2_ha_info_set(ulp_ctx, NULL);
+}
+
+int32_t
+ulp_ha_mgr_app_type_get(struct bnxt_ulp_context *ulp_ctx,
+			enum ulp_ha_mgr_app_type *app_type)
+{
+	struct bnxt_ulp_ha_mgr_info *ha_info;
+
+	if (ulp_ctx == NULL || app_type == NULL) {
+		BNXT_TF_DBG(ERR, "Invalid Parms.\n");
+		return -EINVAL;
+	}
+
+	ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+	if (ha_info == NULL) {
+		BNXT_TF_DBG(ERR, "Unable to get the HA info.\n");
+		return -EINVAL;
+	}
+	*app_type = ha_info->app_type;
+
+	return 0;
+}
+
+int32_t
+ulp_ha_mgr_state_get(struct bnxt_ulp_context *ulp_ctx,
+		     enum ulp_ha_mgr_state *state)
+{
+	struct tf_get_if_tbl_entry_parms get_parms = { 0 };
+	struct tf *tfp;
+	uint32_t val = 0;
+	int32_t rc = 0;
+
+	if (ulp_ctx == NULL || state == NULL) {
+		BNXT_TF_DBG(ERR, "Invalid parms in state get.\n");
+		return -EINVAL;
+	}
+	tfp = bnxt_ulp_cntxt_tfp_get(ulp_ctx, BNXT_ULP_SHARED_SESSION_NO);
+	if (tfp == NULL) {
+		BNXT_TF_DBG(ERR, "Unable to get the TFP.\n");
+		return -EINVAL;
+	}
+
+	get_parms.dir = ULP_HA_IF_TBL_DIR;
+	get_parms.type = ULP_HA_IF_TBL_TYPE;
+	get_parms.idx = ULP_HA_IF_TBL_IDX;
+	get_parms.data = (uint8_t *)&val;
+	get_parms.data_sz_in_bytes = sizeof(val);
+
+	rc = tf_get_if_tbl_entry(tfp, &get_parms);
+	if (rc)
+		BNXT_TF_DBG(ERR, "Failed to read the HA state\n");
+
+	*state = val;
+	return rc;
+}
+
+int32_t
+ulp_ha_mgr_open(struct bnxt_ulp_context *ulp_ctx)
+{
+	enum ulp_ha_mgr_state curr_state;
+	int32_t rc;
+
+	rc = ulp_ha_mgr_state_get(ulp_ctx, &curr_state);
+	if (rc) {
+		BNXT_TF_DBG(ERR, "Failed to get HA state on Open (%d)\n", rc);
+		return -EINVAL;
+	}
+
+	/*
+	 * An Open can only occur during the Init and Primary Run states. During
+	 * Init, the system attempting to Open will become the only system
+	 * running. During Primary Run, the system attempting to Open will
+	 * become the secondary system temporarily, and should eventually be
+	 * transitioned to the primary system.
+	 */
+	switch (curr_state) {
+	case ULP_HA_STATE_INIT:
+		/*
+		 * No system is running, as we are the primary.  Since no other
+		 * system is running, we start writing into the low region.  By
+		 * writing into the low region, we save room for the secondary
+		 * system to override our entries by using the high region.
+		 */
+		ulp_ha_mgr_app_type_set(ulp_ctx, ULP_HA_APP_TYPE_PRIM);
+		ulp_ha_mgr_region_set(ulp_ctx, ULP_HA_REGION_LOW);
+		rc = ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_RUN);
+		if (rc) {
+			BNXT_TF_DBG(ERR, "On Open: Failed to set PRIM_RUN.\n");
+			return -EINVAL;
+		}
+
+		BNXT_TF_DBG(INFO, "On Open: [INIT] => PRIM[PRIM_RUN]\n");
+		break;
+	case ULP_HA_STATE_PRIM_RUN:
+		/*
+		 * The secondary system is starting in order to take over.
+		 * The current primary is expected to eventually close and pass
+		 * full control to this system;however, until the primary closes
+		 * both are operational.
+		 *
+		 * The timer is started in order to determine when the
+		 * primary has closed.
+		 */
+		ulp_ha_mgr_app_type_set(ulp_ctx, ULP_HA_APP_TYPE_SEC);
+		ulp_ha_mgr_region_set(ulp_ctx, ULP_HA_REGION_HI);
+
+		/*
+		 * TODO:
+		 * Clear the high region so the secondary can begin overriding
+		 * the current entries.
+		 */
+		rc = ulp_ha_mgr_timer_start(ulp_ctx);
+		if (rc) {
+			BNXT_TF_DBG(ERR, "Unable to start timer on HA Open.\n");
+			return -EINVAL;
+		}
+
+		rc = ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_SEC_RUN);
+		if (rc) {
+			BNXT_TF_DBG(ERR, "On Open: Failed to set PRIM_SEC_RUN\n");
+			return -EINVAL;
+		}
+		BNXT_TF_DBG(INFO, "On Open: [PRIM_RUN] => [PRIM_SEC_RUN]\n");
+		break;
+	default:
+		BNXT_TF_DBG(ERR, "On Open: Unknown state 0x%x\n", curr_state);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int32_t
+ulp_ha_mgr_close(struct bnxt_ulp_context *ulp_ctx)
+{
+	enum ulp_ha_mgr_state curr_state, next_state, poll_state;
+	enum ulp_ha_mgr_app_type app_type;
+	int32_t timeout;
+	int32_t rc;
+
+	rc = ulp_ha_mgr_state_get(ulp_ctx, &curr_state);
+	if (rc) {
+		BNXT_TF_DBG(ERR, "On Close: Failed(%d) to get HA state\n", rc);
+		return -EINVAL;
+	}
+
+	rc = ulp_ha_mgr_app_type_get(ulp_ctx, &app_type);
+	if (rc) {
+		BNXT_TF_DBG(ERR, "On Close: Failed to get the app type.\n");
+		return -EINVAL;
+	}
+
+	if (curr_state == ULP_HA_STATE_PRIM_RUN &&
+	    app_type == ULP_HA_APP_TYPE_PRIM) {
+		/*
+		 * Only the primary is running, so a close effectively moves the
+		 * system back to INIT.
+		 */
+		next_state = ULP_HA_STATE_INIT;
+		ulp_ha_mgr_state_set(ulp_ctx, next_state);
+		BNXT_TF_DBG(INFO, "On Close: PRIM[PRIM_RUN] => [INIT]\n");
+	} else if (curr_state == ULP_HA_STATE_PRIM_SEC_RUN &&
+		  app_type == ULP_HA_APP_TYPE_PRIM) {
+		/*
+		 * While both are running, the primary received a close.
+		 * Cleanup the flows, set the COPY state, and wait for the
+		 * secondary to become the Primary.
+		 */
+		BNXT_TF_DBG(INFO,
+			    "On Close: PRIM[PRIM_SEC_RUN] flushing flows.\n");
+
+		ulp_flow_db_flush_flows(ulp_ctx, BNXT_ULP_FDB_TYPE_REGULAR);
+		ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_SEC_TIMER_COPY);
+
+		/*
+		 * TODO: This needs to be bounded in case the other system does
+		 * not move to PRIM_RUN.
+		 */
+		BNXT_TF_DBG(INFO,
+			    "On Close: PRIM[PRIM_SEC_RUN] => [Copy], enter wait.\n");
+		timeout = ULP_HA_WAIT_TIMEOUT;
+		do {
+			rte_delay_ms(ULP_HA_WAIT_TIME);
+			rc = ulp_ha_mgr_state_get(ulp_ctx, &poll_state);
+			if (rc) {
+				BNXT_TF_DBG(ERR,
+					    "Failed to get HA state on Close (%d)\n",
+					    rc);
+				goto cleanup;
+			}
+			timeout -= ULP_HA_WAIT_TIME;
+			BNXT_TF_DBG(INFO,
+				    "On Close: Waiting %d ms for PRIM_RUN\n",
+				    timeout);
+		} while (poll_state != ULP_HA_STATE_PRIM_RUN && timeout > 0);
+
+		if (timeout <= 0) {
+			BNXT_TF_DBG(ERR, "On Close: SEC[COPY] Timed out\n");
+			goto cleanup;
+		}
+
+		BNXT_TF_DBG(INFO, "On Close: PRIM[PRIM_SEC_RUN] => [COPY]\n");
+	} else if (curr_state == ULP_HA_STATE_PRIM_SEC_RUN &&
+		   app_type == ULP_HA_APP_TYPE_SEC) {
+		/*
+		 * While both are running, the secondary unexpectedly received a
+		 * close.  Cancel the timer, set the state to Primary RUN since
+		 * it is the only one running.
+		 */
+		ulp_ha_mgr_timer_cancel(ulp_ctx);
+		ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_RUN);
+
+		BNXT_TF_DBG(INFO, "On Close: SEC[PRIM_SEC_RUN] => [PRIM_RUN]\n");
+	} else if (curr_state == ULP_HA_STATE_SEC_TIMER_COPY &&
+		   app_type == ULP_HA_APP_TYPE_SEC) {
+		/*
+		 * While both were running and the Secondary went into copy,
+		 * secondary received a close.  Wait until the former Primary
+		 * clears the copy stage, close, and set to INIT.
+		 */
+		BNXT_TF_DBG(INFO, "On Close: SEC[COPY] wait for PRIM_RUN\n");
+
+		timeout = ULP_HA_WAIT_TIMEOUT;
+		do {
+			rte_delay_ms(ULP_HA_WAIT_TIME);
+			rc = ulp_ha_mgr_state_get(ulp_ctx, &poll_state);
+			if (rc) {
+				BNXT_TF_DBG(ERR,
+					    "Failed to get HA state on Close (%d)\n",
+					    rc);
+				goto cleanup;
+			}
+
+			timeout -= ULP_HA_WAIT_TIME;
+			BNXT_TF_DBG(INFO,
+				    "On Close: Waiting %d ms for PRIM_RUN\n",
+				    timeout);
+		} while (poll_state != ULP_HA_STATE_PRIM_RUN &&
+			 timeout >= 0);
+
+		if (timeout <= 0) {
+			BNXT_TF_DBG(ERR,
+				    "On Close: SEC[COPY] Timed out\n");
+			goto cleanup;
+		}
+
+		next_state = ULP_HA_STATE_INIT;
+		rc = ulp_ha_mgr_state_set(ulp_ctx, next_state);
+		if (rc) {
+			BNXT_TF_DBG(ERR,
+				    "On Close: Failed to set state to INIT(%x)\n",
+				    rc);
+			goto cleanup;
+		}
+
+		BNXT_TF_DBG(INFO,
+			    "On Close: SEC[COPY] => [INIT] after %d ms\n",
+			    ULP_HA_WAIT_TIMEOUT - timeout);
+	} else {
+		BNXT_TF_DBG(ERR, "On Close: Invalid type/state %d/%d\n",
+			    curr_state, app_type);
+	}
+cleanup:
+	return rc;
+}
+
+int32_t
+ulp_ha_mgr_region_get(struct bnxt_ulp_context *ulp_ctx,
+		      enum ulp_ha_mgr_region *region)
+{
+	struct bnxt_ulp_ha_mgr_info *ha_info;
+
+	if (ulp_ctx == NULL || region == NULL) {
+		BNXT_TF_DBG(ERR, "Invalid params in ha region get.\n");
+		return -EINVAL;
+	}
+
+	ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+	if (ha_info == NULL) {
+		BNXT_TF_DBG(ERR, "Unable to get ha info\n");
+		return -EINVAL;
+	}
+	*region = ha_info->region;
+
+	return 0;
+}
diff --git a/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.h b/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.h
new file mode 100644
index 0000000000..793511564a
--- /dev/null
+++ b/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.h
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2014-2021 Broadcom
+ * All rights reserved.
+ */
+
+#ifndef _ULP_HA_MGR_H_
+#define _ULP_HA_MGR_H_
+
+#include "bnxt_ulp.h"
+
+enum ulp_ha_mgr_state {
+	ULP_HA_STATE_INIT,
+	ULP_HA_STATE_PRIM_RUN,
+	ULP_HA_STATE_PRIM_SEC_RUN,
+	ULP_HA_STATE_SEC_TIMER_COPY,
+	ULP_HA_PRIM_CLOSE
+};
+
+enum ulp_ha_mgr_app_type {
+	ULP_HA_APP_TYPE_NONE,
+	ULP_HA_APP_TYPE_PRIM,
+	ULP_HA_APP_TYPE_SEC
+};
+
+enum ulp_ha_mgr_region {
+	ULP_HA_REGION_LOW,
+	ULP_HA_REGION_HI
+};
+
+struct bnxt_ulp_ha_mgr_info {
+	enum ulp_ha_mgr_app_type app_type;
+	enum ulp_ha_mgr_region region;
+	uint32_t flags;
+	pthread_mutex_t ha_lock;
+};
+
+bool
+ulp_ha_mgr_is_enabled(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_enable(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_init(struct bnxt_ulp_context *ulp_ctx);
+
+void
+ulp_ha_mgr_deinit(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_app_type_get(struct bnxt_ulp_context *ulp_ctx,
+			enum ulp_ha_mgr_app_type *app_type);
+
+int32_t
+ulp_ha_mgr_state_get(struct bnxt_ulp_context *ulp_ctx,
+		     enum ulp_ha_mgr_state *state);
+
+int32_t
+ulp_ha_mgr_open(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_close(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_region_get(struct bnxt_ulp_context *ulp_ctx,
+		      enum ulp_ha_mgr_region *region);
+
+#endif /* _ULP_HA_MGR_H_*/
diff --git a/drivers/net/bnxt/tf_ulp/ulp_mapper.c b/drivers/net/bnxt/tf_ulp/ulp_mapper.c
index e2404c392b..05a43b6dc5 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_mapper.c
+++ b/drivers/net/bnxt/tf_ulp/ulp_mapper.c
@@ -19,6 +19,7 @@
 #include "tf_util.h"
 #include "ulp_template_db_tbl.h"
 #include "ulp_port_db.h"
+#include "ulp_ha_mgr.h"
 
 static uint8_t mapper_fld_zeros[16] = { 0 };
 
@@ -419,7 +420,7 @@ ulp_mapper_ident_fields_get(struct bnxt_ulp_mapper_parms *mparms,
 }
 
 static inline int32_t
-ulp_mapper_tcam_entry_free(struct bnxt_ulp_context *ulp  __rte_unused,
+ulp_mapper_tcam_entry_free(struct bnxt_ulp_context *ulp,
 			   struct tf *tfp,
 			   struct ulp_flow_db_res_params *res)
 {
@@ -429,6 +430,30 @@ ulp_mapper_tcam_entry_free(struct bnxt_ulp_context *ulp  __rte_unused,
 		.idx		= (uint16_t)res->resource_hndl
 	};
 
+	/* If HA is enabled, we may have to remap the TF Type */
+	if (bnxt_ulp_cntxt_ha_enabled(ulp)) {
+		enum ulp_ha_mgr_region region;
+		int32_t rc;
+
+		switch (res->resource_type) {
+		case TF_TCAM_TBL_TYPE_WC_TCAM_HIGH:
+		case TF_TCAM_TBL_TYPE_WC_TCAM_LOW:
+			rc = ulp_ha_mgr_region_get(ulp, &region);
+			if (rc)
+				/* Log this, but assume region is correct */
+				BNXT_TF_DBG(ERR,
+					    "Unable to get HA region (%d)\n",
+					    rc);
+			else
+				fparms.tcam_tbl_type =
+					(region == ULP_HA_REGION_LOW) ?
+					TF_TCAM_TBL_TYPE_WC_TCAM_LOW :
+					TF_TCAM_TBL_TYPE_WC_TCAM_HIGH;
+			break;
+		default:
+			break;
+		}
+	}
 	return tf_free_tcam_entry(tfp, &fparms);
 }
 
@@ -2904,10 +2929,12 @@ static int32_t
 ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx,
 				      struct bnxt_ulp_mapper_data *mapper_data)
 {
+	struct tf_get_shared_tbl_increment_parms iparms;
 	struct bnxt_ulp_glb_resource_info *glb_res;
 	struct tf_get_session_info_parms sparms;
 	uint32_t num_entries, i, dev_id, res;
 	struct tf_resource_info *res_info;
+	uint32_t addend;
 	uint64_t regval;
 	enum tf_dir dir;
 	int32_t rc = 0;
@@ -2915,13 +2942,11 @@ ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx,
 	uint8_t app_id;
 
 	memset(&sparms, 0, sizeof(sparms));
-
 	glb_res = bnxt_ulp_app_glb_resource_info_list_get(&num_entries);
 	if (!glb_res || !num_entries) {
 		BNXT_TF_DBG(ERR, "Invalid Arguments\n");
 		return -EINVAL;
 	}
-
 	tfp = bnxt_ulp_cntxt_shared_tfp_get(ulp_ctx);
 	if (!tfp) {
 		BNXT_TF_DBG(ERR, "Failed to get tfp for app global init");
@@ -2958,12 +2983,29 @@ ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx,
 			continue;
 		dir = glb_res[i].direction;
 		res = glb_res[i].resource_type;
+		addend = 1;
 
 		switch (glb_res[i].resource_func) {
 		case BNXT_ULP_RESOURCE_FUNC_IDENTIFIER:
 			res_info = &sparms.session_info.ident[dir].info[res];
 			break;
 		case BNXT_ULP_RESOURCE_FUNC_INDEX_TABLE:
+			/*
+			 * Tables may have various strides for the allocations.
+			 * Need to account.
+			 */
+			memset(&iparms, 0, sizeof(iparms));
+			iparms.dir = dir;
+			iparms.type = res;
+			rc = tf_get_shared_tbl_increment(tfp, &iparms);
+			if (rc) {
+				BNXT_TF_DBG(ERR,
+					    "Failed to get addend for %s[%s] rc=(%d)\n",
+					    tf_tbl_type_2_str(res),
+					    tf_dir_2_str(dir), rc);
+				return rc;
+			}
+			addend = iparms.increment_cnt;
 			res_info = &sparms.session_info.tbl[dir].info[res];
 			break;
 		case BNXT_ULP_RESOURCE_FUNC_TCAM_TABLE:
@@ -2977,10 +3019,8 @@ ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx,
 				    glb_res[i].resource_func);
 			continue;
 		}
-
 		regval = tfp_cpu_to_be_64((uint64_t)res_info->start);
-		res_info->start++;
-
+		res_info->start += addend;
 		/*
 		 * All resources written to the global regfile are shared for
 		 * this function.
-- 
2.21.1 (Apple Git-122.3)