[dpdk-dev] [PATCH v3 16/34] net/bnxt: add support for ULP session manager init

[Venkat Duvvuru <venkatkumar.duvvuru@broadcom.com>]

A ULP session will contain all the resources needed to support
rte flow offloads. A session is initialized as part of rte_eth_device
start. A DPDK application can have multiple interfaces which
means rte_eth_device start will be called for each of these devices.
ULP session manager will make sure that a single ULP session is only
initialized once. Apart from this, it also initializes MARK database,
EEM table & flow database. ULP session manager also manages a list of
all opened ULP sessions.

Signed-off-by: Venkat Duvvuru <venkatkumar.duvvuru@broadcom.com>
Signed-off-by: Mike Baucom <michael.baucom@broadcom.com>
Reviewed-by: Lance Richardson <lance.richardson@broadcom.com>
Reviewed-by: Ajit Kumar Khaparde <ajit.khaparde@broadcom.com>
---
 drivers/net/bnxt/Makefile                     |   6 +-
 drivers/net/bnxt/bnxt.h                       |   5 +
 drivers/net/bnxt/bnxt_ethdev.c                |   4 +
 drivers/net/bnxt/tf_ulp/bnxt_tf_common.h      |  35 ++
 drivers/net/bnxt/tf_ulp/bnxt_ulp.c            | 527 ++++++++++++++++++++++++++
 drivers/net/bnxt/tf_ulp/bnxt_ulp.h            | 100 +++++
 drivers/net/bnxt/tf_ulp/ulp_flow_db.c         | 187 +++++++++
 drivers/net/bnxt/tf_ulp/ulp_flow_db.h         |  77 ++++
 drivers/net/bnxt/tf_ulp/ulp_mark_mgr.c        |  94 +++++
 drivers/net/bnxt/tf_ulp/ulp_mark_mgr.h        |  49 +++
 drivers/net/bnxt/tf_ulp/ulp_template_db.c     |  27 ++
 drivers/net/bnxt/tf_ulp/ulp_template_db.h     |  35 ++
 drivers/net/bnxt/tf_ulp/ulp_template_struct.h |  40 ++
 13 files changed, 1185 insertions(+), 1 deletion(-)
 create mode 100644 drivers/net/bnxt/tf_ulp/bnxt_tf_common.h
 create mode 100644 drivers/net/bnxt/tf_ulp/bnxt_ulp.c
 create mode 100644 drivers/net/bnxt/tf_ulp/bnxt_ulp.h
 create mode 100644 drivers/net/bnxt/tf_ulp/ulp_flow_db.c
 create mode 100644 drivers/net/bnxt/tf_ulp/ulp_flow_db.h
 create mode 100644 drivers/net/bnxt/tf_ulp/ulp_mark_mgr.c
 create mode 100644 drivers/net/bnxt/tf_ulp/ulp_mark_mgr.h
 create mode 100644 drivers/net/bnxt/tf_ulp/ulp_template_db.c
 create mode 100644 drivers/net/bnxt/tf_ulp/ulp_template_db.h
 create mode 100644 drivers/net/bnxt/tf_ulp/ulp_template_struct.h

diff --git a/drivers/net/bnxt/Makefile b/drivers/net/bnxt/Makefile
index 4c95847..bb9b888 100644
--- a/drivers/net/bnxt/Makefile
+++ b/drivers/net/bnxt/Makefile
@@ -44,7 +44,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_rxtx_vec_sse.c
 endif
 
 ifeq ($(CONFIG_RTE_LIBRTE_BNXT_PMD), y)
-CFLAGS += -I$(SRCDIR) -I$(SRCDIR)/tf_core
+CFLAGS += -I$(SRCDIR) -I$(SRCDIR)/tf_core -I$(SRCDIR)/tf_ulp
 endif
 
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_core.c
@@ -57,6 +57,10 @@ SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_rm.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_tbl.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tfp.c
 
+SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_ulp/bnxt_ulp.c
+SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_ulp/ulp_mark_mgr.c
+SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_ulp/ulp_flow_db.c
+SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_ulp/ulp_template_db.c
 #
 # Export include files
 #
diff --git a/drivers/net/bnxt/bnxt.h b/drivers/net/bnxt/bnxt.h
index cd84ebd..cd20740 100644
--- a/drivers/net/bnxt/bnxt.h
+++ b/drivers/net/bnxt/bnxt.h
@@ -22,6 +22,7 @@
 #include "bnxt_util.h"
 
 #include "tf_core.h"
+#include "bnxt_ulp.h"
 
 /* Vendor ID */
 #define PCI_VENDOR_ID_BROADCOM		0x14E4
@@ -687,6 +688,7 @@ struct bnxt {
 	uint16_t		port_svif;
 
 	struct tf		tfp;
+	struct bnxt_ulp_context	ulp_ctx;
 	uint8_t			truflow;
 };
 
@@ -729,6 +731,9 @@ extern int bnxt_logtype_driver;
 #define PMD_DRV_LOG(level, fmt, args...) \
 	  PMD_DRV_LOG_RAW(level, fmt, ## args)
 
+int32_t bnxt_ulp_init(struct bnxt *bp);
+void bnxt_ulp_deinit(struct bnxt *bp);
+
 uint16_t bnxt_get_vnic_id(uint16_t port);
 uint16_t bnxt_get_svif(uint16_t port_id, bool func_svif);
 
diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c
index c4bbf1d..1703ce3 100644
--- a/drivers/net/bnxt/bnxt_ethdev.c
+++ b/drivers/net/bnxt/bnxt_ethdev.c
@@ -904,6 +904,10 @@ static int bnxt_dev_start_op(struct rte_eth_dev *eth_dev)
 	pthread_mutex_lock(&bp->def_cp_lock);
 	bnxt_schedule_fw_health_check(bp);
 	pthread_mutex_unlock(&bp->def_cp_lock);
+
+	if (bp->truflow)
+		bnxt_ulp_init(bp);
+
 	return 0;
 
 error:
diff --git a/drivers/net/bnxt/tf_ulp/bnxt_tf_common.h b/drivers/net/bnxt/tf_ulp/bnxt_tf_common.h
new file mode 100644
index 0000000..3516df4
--- /dev/null
+++ b/drivers/net/bnxt/tf_ulp/bnxt_tf_common.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2014-2019 Broadcom
+ * All rights reserved.
+ */
+
+#ifndef _BNXT_TF_COMMON_H_
+#define _BNXT_TF_COMMON_H_
+
+#define BNXT_TF_DBG(lvl, fmt, args...)	PMD_DRV_LOG(lvl, fmt, ## args)
+
+#define BNXT_ULP_EM_FLOWS			8192
+#define BNXT_ULP_1M_FLOWS			1000000
+#define BNXT_EEM_RX_GLOBAL_ID_MASK		(BNXT_ULP_1M_FLOWS - 1)
+#define BNXT_EEM_TX_GLOBAL_ID_MASK		(BNXT_ULP_1M_FLOWS - 1)
+#define BNXT_EEM_HASH_KEY2_USED			0x8000000
+#define BNXT_EEM_RX_HW_HASH_KEY2_BIT		BNXT_ULP_1M_FLOWS
+#define	BNXT_ULP_DFLT_RX_MAX_KEY		512
+#define	BNXT_ULP_DFLT_RX_MAX_ACTN_ENTRY		256
+#define	BNXT_ULP_DFLT_RX_MEM			0
+#define	BNXT_ULP_RX_NUM_FLOWS			32
+#define	BNXT_ULP_RX_TBL_IF_ID			0
+#define	BNXT_ULP_DFLT_TX_MAX_KEY		512
+#define	BNXT_ULP_DFLT_TX_MAX_ACTN_ENTRY		256
+#define	BNXT_ULP_DFLT_TX_MEM			0
+#define	BNXT_ULP_TX_NUM_FLOWS			32
+#define	BNXT_ULP_TX_TBL_IF_ID			0
+
+struct bnxt_ulp_mark_tbl *
+bnxt_ulp_cntxt_ptr2_mark_db_get(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+bnxt_ulp_cntxt_ptr2_mark_db_set(struct bnxt_ulp_context *ulp_ctx,
+				struct bnxt_ulp_mark_tbl *mark_tbl);
+
+#endif /* _BNXT_TF_COMMON_H_ */
diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp.c b/drivers/net/bnxt/tf_ulp/bnxt_ulp.c
new file mode 100644
index 0000000..7afc6bf
--- /dev/null
+++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp.c
@@ -0,0 +1,527 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2020 Broadcom
+ * All rights reserved.
+ */
+
+#include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_flow.h>
+#include <rte_flow_driver.h>
+#include <rte_tailq.h>
+
+#include "bnxt_ulp.h"
+#include "bnxt_tf_common.h"
+#include "bnxt.h"
+#include "tf_core.h"
+#include "tf_ext_flow_handle.h"
+
+#include "ulp_template_db.h"
+#include "ulp_template_struct.h"
+#include "ulp_mark_mgr.h"
+#include "ulp_flow_db.h"
+
+/* Linked list of all TF sessions. */
+STAILQ_HEAD(, bnxt_ulp_session_state) bnxt_ulp_session_list =
+			STAILQ_HEAD_INITIALIZER(bnxt_ulp_session_list);
+
+/* Mutex to synchronize bnxt_ulp_session_list operations. */
+static pthread_mutex_t bnxt_ulp_global_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+/*
+ * Initialize an ULP session.
+ * An ULP session will contain all the resources needed to support rte flow
+ * offloads. A session is initialized as part of rte_eth_device start.
+ * A single vswitch instance can have multiple uplinks which means
+ * rte_eth_device start will be called for each of these devices.
+ * ULP session manager will make sure that a single ULP session is only
+ * initialized once. Apart from this, it also initializes MARK database,
+ * EEM table & flow database. ULP session manager also manages a list of
+ * all opened ULP sessions.
+ */
+static int32_t
+ulp_ctx_session_open(struct bnxt *bp,
+		     struct bnxt_ulp_session_state *session)
+{
+	struct rte_eth_dev		*ethdev = bp->eth_dev;
+	int32_t				rc = 0;
+	struct tf_open_session_parms	params;
+
+	memset(&params, 0, sizeof(params));
+
+	rc = rte_eth_dev_get_name_by_port(ethdev->data->port_id,
+					  params.ctrl_chan_name);
+	if (rc) {
+		BNXT_TF_DBG(ERR, "Invalid port %d, rc = %d\n",
+			    ethdev->data->port_id, rc);
+		return rc;
+	}
+
+	rc = tf_open_session(&bp->tfp, &params);
+	if (rc) {
+		BNXT_TF_DBG(ERR, "Failed to open TF session - %s, rc = %d\n",
+			    params.ctrl_chan_name, rc);
+		return -EINVAL;
+	}
+	session->session_opened = 1;
+	session->g_tfp = &bp->tfp;
+	return rc;
+}
+
+static void
+bnxt_init_tbl_scope_parms(struct bnxt *bp,
+			  struct tf_alloc_tbl_scope_parms *params)
+{
+	struct bnxt_ulp_device_params	*dparms;
+	uint32_t dev_id;
+	int rc;
+
+	rc = bnxt_ulp_cntxt_dev_id_get(&bp->ulp_ctx, &dev_id);
+	if (rc)
+		/* TBD: For now, just use default. */
+		dparms = 0;
+	else
+		dparms = bnxt_ulp_device_params_get(dev_id);
+
+	if (!dparms) {
+		params->rx_max_key_sz_in_bits = BNXT_ULP_DFLT_RX_MAX_KEY;
+		params->rx_max_action_entry_sz_in_bits =
+			BNXT_ULP_DFLT_RX_MAX_ACTN_ENTRY;
+		params->rx_mem_size_in_mb = BNXT_ULP_DFLT_RX_MEM;
+		params->rx_num_flows_in_k = BNXT_ULP_RX_NUM_FLOWS;
+		params->rx_tbl_if_id = BNXT_ULP_RX_TBL_IF_ID;
+
+		params->tx_max_key_sz_in_bits = BNXT_ULP_DFLT_TX_MAX_KEY;
+		params->tx_max_action_entry_sz_in_bits =
+			BNXT_ULP_DFLT_TX_MAX_ACTN_ENTRY;
+		params->tx_mem_size_in_mb = BNXT_ULP_DFLT_TX_MEM;
+		params->tx_num_flows_in_k = BNXT_ULP_TX_NUM_FLOWS;
+		params->tx_tbl_if_id = BNXT_ULP_TX_TBL_IF_ID;
+	} else {
+		params->rx_max_key_sz_in_bits = BNXT_ULP_DFLT_RX_MAX_KEY;
+		params->rx_max_action_entry_sz_in_bits =
+			BNXT_ULP_DFLT_RX_MAX_ACTN_ENTRY;
+		params->rx_mem_size_in_mb = BNXT_ULP_DFLT_RX_MEM;
+		params->rx_num_flows_in_k = dparms->num_flows / (1024);
+		params->rx_tbl_if_id = BNXT_ULP_RX_TBL_IF_ID;
+
+		params->tx_max_key_sz_in_bits = BNXT_ULP_DFLT_TX_MAX_KEY;
+		params->tx_max_action_entry_sz_in_bits =
+			BNXT_ULP_DFLT_TX_MAX_ACTN_ENTRY;
+		params->tx_mem_size_in_mb = BNXT_ULP_DFLT_TX_MEM;
+		params->tx_num_flows_in_k = dparms->num_flows / (1024);
+		params->tx_tbl_if_id = BNXT_ULP_TX_TBL_IF_ID;
+	}
+}
+
+/* Initialize Extended Exact Match host memory. */
+static int32_t
+ulp_eem_tbl_scope_init(struct bnxt *bp)
+{
+	struct tf_alloc_tbl_scope_parms params = {0};
+	int rc;
+
+	bnxt_init_tbl_scope_parms(bp, &params);
+
+	rc = tf_alloc_tbl_scope(&bp->tfp, &params);
+	if (rc) {
+		BNXT_TF_DBG(ERR, "Unable to allocate eem table scope rc = %d\n",
+			    rc);
+		return rc;
+	}
+
+	rc = bnxt_ulp_cntxt_tbl_scope_id_set(&bp->ulp_ctx, params.tbl_scope_id);
+	if (rc) {
+		BNXT_TF_DBG(ERR, "Unable to set table scope id\n");
+		return rc;
+	}
+
+	return 0;
+}
+
+/* The function to free and deinit the ulp context data. */
+static int32_t
+ulp_ctx_deinit(struct bnxt *bp,
+	       struct bnxt_ulp_session_state *session)
+{
+	if (!session || !bp) {
+		BNXT_TF_DBG(ERR, "Invalid Arguments\n");
+		return -EINVAL;
+	}
+
+	/* Free the contents */
+	if (session->cfg_data) {
+		rte_free(session->cfg_data);
+		bp->ulp_ctx.cfg_data = NULL;
+		session->cfg_data = NULL;
+	}
+	return 0;
+}
+
+/* The function to allocate and initialize the ulp context data. */
+static int32_t
+ulp_ctx_init(struct bnxt *bp,
+	     struct bnxt_ulp_session_state *session)
+{
+	struct bnxt_ulp_data	*ulp_data;
+	int32_t			rc = 0;
+
+	if (!session || !bp) {
+		BNXT_TF_DBG(ERR, "Invalid Arguments\n");
+		return -EINVAL;
+	}
+
+	/* Allocate memory to hold ulp context data. */
+	ulp_data = rte_zmalloc("bnxt_ulp_data",
+			       sizeof(struct bnxt_ulp_data), 0);
+	if (!ulp_data) {
+		BNXT_TF_DBG(ERR, "Failed to allocate memory for ulp data\n");
+		return -ENOMEM;
+	}
+
+	/* Increment the ulp context data reference count usage. */
+	bp->ulp_ctx.cfg_data = ulp_data;
+	session->cfg_data = ulp_data;
+	ulp_data->ref_cnt++;
+
+	/* Open the ulp session. */
+	rc = ulp_ctx_session_open(bp, session);
+	if (rc) {
+		(void)ulp_ctx_deinit(bp, session);
+		return rc;
+	}
+	bnxt_ulp_cntxt_tfp_set(&bp->ulp_ctx, session->g_tfp);
+	return rc;
+}
+
+static int32_t
+ulp_ctx_attach(struct bnxt_ulp_context *ulp_ctx,
+	       struct bnxt_ulp_session_state *session)
+{
+	if (!ulp_ctx || !session) {
+		BNXT_TF_DBG(ERR, "Invalid Arguments\n");
+		return -EINVAL;
+	}
+
+	/* Increment the ulp context data reference count usage. */
+	ulp_ctx->cfg_data = session->cfg_data;
+	ulp_ctx->cfg_data->ref_cnt++;
+
+	/* TBD call TF_session_attach. */
+	ulp_ctx->g_tfp = session->g_tfp;
+	return 0;
+}
+
+/*
+ * Initialize the state of an ULP session.
+ * If the state of an ULP session is not initialized, set it's state to
+ * initialized. If the state is already initialized, do nothing.
+ */
+static void
+ulp_context_initialized(struct bnxt_ulp_session_state *session, bool *init)
+{
+	pthread_mutex_lock(&session->bnxt_ulp_mutex);
+
+	if (!session->bnxt_ulp_init) {
+		session->bnxt_ulp_init = true;
+		*init = false;
+	} else {
+		*init = true;
+	}
+
+	pthread_mutex_unlock(&session->bnxt_ulp_mutex);
+}
+
+/*
+ * Check if an ULP session is already allocated for a specific PCI
+ * domain & bus. If it is already allocated simply return the session
+ * pointer, otherwise allocate a new session.
+ */
+static struct bnxt_ulp_session_state *
+ulp_get_session(struct rte_pci_addr *pci_addr)
+{
+	struct bnxt_ulp_session_state *session;
+
+	STAILQ_FOREACH(session, &bnxt_ulp_session_list, next) {
+		if (session->pci_info.domain == pci_addr->domain &&
+		    session->pci_info.bus == pci_addr->bus) {
+			return session;
+		}
+	}
+	return NULL;
+}
+
+/*
+ * Allocate and Initialize an ULP session and set it's state to INITIALIZED.
+ * If it's already initialized simply return the already existing session.
+ */
+static struct bnxt_ulp_session_state *
+ulp_session_init(struct bnxt *bp,
+		 bool *init)
+{
+	struct rte_pci_device		*pci_dev;
+	struct rte_pci_addr		*pci_addr;
+	struct bnxt_ulp_session_state	*session;
+
+	if (!bp)
+		return NULL;
+
+	pci_dev = RTE_DEV_TO_PCI(bp->eth_dev->device);
+	pci_addr = &pci_dev->addr;
+
+	pthread_mutex_lock(&bnxt_ulp_global_mutex);
+
+	session = ulp_get_session(pci_addr);
+	if (!session) {
+		/* Not Found the session  Allocate a new one */
+		session = rte_zmalloc("bnxt_ulp_session",
+				      sizeof(struct bnxt_ulp_session_state),
+				      0);
+		if (!session) {
+			BNXT_TF_DBG(ERR,
+				    "Allocation failed for bnxt_ulp_session\n");
+			pthread_mutex_unlock(&bnxt_ulp_global_mutex);
+			return NULL;
+
+		} else {
+			/* Add it to the queue */
+			session->pci_info.domain = pci_addr->domain;
+			session->pci_info.bus = pci_addr->bus;
+			pthread_mutex_init(&session->bnxt_ulp_mutex, NULL);
+			STAILQ_INSERT_TAIL(&bnxt_ulp_session_list,
+					   session, next);
+		}
+	}
+	ulp_context_initialized(session, init);
+	pthread_mutex_unlock(&bnxt_ulp_global_mutex);
+	return session;
+}
+
+/*
+ * When a port is initialized by dpdk. This functions is called
+ * and this function initializes the ULP context and rest of the
+ * infrastructure associated with it.
+ */
+int32_t
+bnxt_ulp_init(struct bnxt *bp)
+{
+	struct bnxt_ulp_session_state *session;
+	bool init;
+	int rc;
+
+	/*
+	 * Multiple uplink ports can be associated with a single vswitch.
+	 * Make sure only the port that is started first will initialize
+	 * the TF session.
+	 */
+	session = ulp_session_init(bp, &init);
+	if (!session) {
+		BNXT_TF_DBG(ERR, "Failed to initialize the tf session\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * If ULP is already initialized for a specific domain then simply
+	 * assign the ulp context to this rte_eth_dev.
+	 */
+	if (init) {
+		rc = ulp_ctx_attach(&bp->ulp_ctx, session);
+		if (rc) {
+			BNXT_TF_DBG(ERR,
+				    "Failed to attach the ulp context\n");
+		}
+		return rc;
+	}
+
+	/* Allocate and Initialize the ulp context. */
+	rc = ulp_ctx_init(bp, session);
+	if (rc) {
+		BNXT_TF_DBG(ERR, "Failed to create the ulp context\n");
+		goto jump_to_error;
+	}
+
+	/* Create the Mark database. */
+	rc = ulp_mark_db_init(&bp->ulp_ctx);
+	if (rc) {
+		BNXT_TF_DBG(ERR, "Failed to create the mark database\n");
+		goto jump_to_error;
+	}
+
+	/* Create the flow database. */
+	rc = ulp_flow_db_init(&bp->ulp_ctx);
+	if (rc) {
+		BNXT_TF_DBG(ERR, "Failed to create the flow database\n");
+		goto jump_to_error;
+	}
+
+	/* Create the eem table scope. */
+	rc = ulp_eem_tbl_scope_init(bp);
+	if (rc) {
+		BNXT_TF_DBG(ERR, "Failed to create the eem scope table\n");
+		goto jump_to_error;
+	}
+
+	return rc;
+
+jump_to_error:
+	return -ENOMEM;
+}
+
+/* Below are the access functions to access internal data of ulp context. */
+
+/* Function to set the Mark DB into the context. */
+int32_t
+bnxt_ulp_cntxt_ptr2_mark_db_set(struct bnxt_ulp_context *ulp_ctx,
+				struct bnxt_ulp_mark_tbl *mark_tbl)
+{
+	if (!ulp_ctx || !ulp_ctx->cfg_data) {
+		BNXT_TF_DBG(ERR, "Invalid ulp context data\n");
+		return -EINVAL;
+	}
+
+	ulp_ctx->cfg_data->mark_tbl = mark_tbl;
+
+	return 0;
+}
+
+/* Function to retrieve the Mark DB from the context. */
+struct bnxt_ulp_mark_tbl *
+bnxt_ulp_cntxt_ptr2_mark_db_get(struct bnxt_ulp_context *ulp_ctx)
+{
+	if (!ulp_ctx || !ulp_ctx->cfg_data)
+		return NULL;
+
+	return ulp_ctx->cfg_data->mark_tbl;
+}
+
+/* Function to set the device id of the hardware. */
+int32_t
+bnxt_ulp_cntxt_dev_id_set(struct bnxt_ulp_context *ulp_ctx,
+			  uint32_t dev_id)
+{
+	if (ulp_ctx && ulp_ctx->cfg_data) {
+		ulp_ctx->cfg_data->dev_id = dev_id;
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+/* Function to get the device id of the hardware. */
+int32_t
+bnxt_ulp_cntxt_dev_id_get(struct bnxt_ulp_context *ulp_ctx,
+			  uint32_t *dev_id)
+{
+	if (ulp_ctx && ulp_ctx->cfg_data) {
+		*dev_id = ulp_ctx->cfg_data->dev_id;
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+/* Function to get the table scope id of the EEM table. */
+int32_t
+bnxt_ulp_cntxt_tbl_scope_id_get(struct bnxt_ulp_context *ulp_ctx,
+				uint32_t *tbl_scope_id)
+{
+	if (ulp_ctx && ulp_ctx->cfg_data) {
+		*tbl_scope_id = ulp_ctx->cfg_data->tbl_scope_id;
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+/* Function to set the table scope id of the EEM table. */
+int32_t
+bnxt_ulp_cntxt_tbl_scope_id_set(struct bnxt_ulp_context *ulp_ctx,
+				uint32_t tbl_scope_id)
+{
+	if (ulp_ctx && ulp_ctx->cfg_data) {
+		ulp_ctx->cfg_data->tbl_scope_id = tbl_scope_id;
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+/* Function to set the tfp session details from the ulp context. */
+int32_t
+bnxt_ulp_cntxt_tfp_set(struct bnxt_ulp_context *ulp, struct tf *tfp)
+{
+	if (!ulp) {
+		BNXT_TF_DBG(ERR, "Invalid arguments\n");
+		return -EINVAL;
+	}
+
+	/* TBD The tfp should be removed once tf_attach is implemented. */
+	ulp->g_tfp = tfp;
+	return 0;
+}
+
+/* Function to get the tfp session details from the ulp context. */
+struct tf *
+bnxt_ulp_cntxt_tfp_get(struct bnxt_ulp_context *ulp)
+{
+	if (!ulp) {
+		BNXT_TF_DBG(ERR, "Invalid arguments\n");
+		return NULL;
+	}
+	/* TBD The tfp should be removed once tf_attach is implemented. */
+	return ulp->g_tfp;
+}
+
+/*
+ * Get the device table entry based on the device id.
+ *
+ * dev_id [in] The device id of the hardware
+ *
+ * Returns the pointer to the device parameters.
+ */
+struct bnxt_ulp_device_params *
+bnxt_ulp_device_params_get(uint32_t dev_id)
+{
+	if (dev_id < BNXT_ULP_MAX_NUM_DEVICES)
+		return &ulp_device_params[dev_id];
+	return NULL;
+}
+
+/* Function to set the flow database to the ulp context. */
+int32_t
+bnxt_ulp_cntxt_ptr2_flow_db_set(struct bnxt_ulp_context	*ulp_ctx,
+				struct bnxt_ulp_flow_db	*flow_db)
+{
+	if (!ulp_ctx || !ulp_ctx->cfg_data) {
+		BNXT_TF_DBG(ERR, "Invalid ulp context data\n");
+		return -EINVAL;
+	}
+
+	ulp_ctx->cfg_data->flow_db = flow_db;
+	return 0;
+}
+
+/* Function to get the flow database from the ulp context. */
+struct bnxt_ulp_flow_db	*
+bnxt_ulp_cntxt_ptr2_flow_db_get(struct bnxt_ulp_context	*ulp_ctx)
+{
+	if (!ulp_ctx || !ulp_ctx->cfg_data) {
+		BNXT_TF_DBG(ERR, "Invalid ulp context data\n");
+		return NULL;
+	}
+
+	return ulp_ctx->cfg_data->flow_db;
+}
+
+/* Function to get the ulp context from eth device. */
+struct bnxt_ulp_context	*
+bnxt_ulp_eth_dev_ptr2_cntxt_get(struct rte_eth_dev	*dev)
+{
+	struct bnxt	*bp;
+
+	bp = (struct bnxt *)dev->data->dev_private;
+	if (!bp) {
+		BNXT_TF_DBG(ERR, "Bnxt private data is not initialized\n");
+		return NULL;
+	}
+	return &bp->ulp_ctx;
+}
diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp.h b/drivers/net/bnxt/tf_ulp/bnxt_ulp.h
new file mode 100644
index 0000000..d88225f
--- /dev/null
+++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp.h
@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2020 Broadcom
+ * All rights reserved.
+ */
+
+#ifndef _BNXT_ULP_H_
+#define _BNXT_ULP_H_
+
+#include <inttypes.h>
+#include <stdbool.h>
+#include <sys/queue.h>
+
+#include "rte_ethdev.h"
+
+struct bnxt_ulp_data {
+	uint32_t			tbl_scope_id;
+	struct bnxt_ulp_mark_tbl	*mark_tbl;
+	uint32_t			dev_id; /* Hardware device id */
+	uint32_t			ref_cnt;
+	struct bnxt_ulp_flow_db		*flow_db;
+};
+
+struct bnxt_ulp_context {
+	struct bnxt_ulp_data	*cfg_data;
+	/* TBD The tfp should be removed once tf_attach is implemented. */
+	struct tf		*g_tfp;
+};
+
+struct bnxt_ulp_pci_info {
+	uint32_t	domain;
+	uint8_t		bus;
+};
+
+struct bnxt_ulp_session_state {
+	STAILQ_ENTRY(bnxt_ulp_session_state)	next;
+	bool					bnxt_ulp_init;
+	pthread_mutex_t				bnxt_ulp_mutex;
+	struct bnxt_ulp_pci_info		pci_info;
+	struct bnxt_ulp_data			*cfg_data;
+	/* TBD The tfp should be removed once tf_attach is implemented. */
+	struct tf				*g_tfp;
+	uint32_t				session_opened;
+};
+
+/* ULP flow id structure */
+struct rte_tf_flow {
+	uint32_t	flow_id;
+};
+
+/* Function to set the device id of the hardware. */
+int32_t
+bnxt_ulp_cntxt_dev_id_set(struct bnxt_ulp_context *ulp_ctx, uint32_t dev_id);
+
+/* Function to get the device id of the hardware. */
+int32_t
+bnxt_ulp_cntxt_dev_id_get(struct bnxt_ulp_context *ulp_ctx, uint32_t *dev_id);
+
+/* Function to set the table scope id of the EEM table. */
+int32_t
+bnxt_ulp_cntxt_tbl_scope_id_set(struct bnxt_ulp_context *ulp_ctx,
+				uint32_t tbl_scope_id);
+
+/* Function to get the table scope id of the EEM table. */
+int32_t
+bnxt_ulp_cntxt_tbl_scope_id_get(struct bnxt_ulp_context *ulp_ctx,
+				uint32_t *tbl_scope_id);
+
+/* Function to set the tfp session details in the ulp context. */
+int32_t
+bnxt_ulp_cntxt_tfp_set(struct bnxt_ulp_context *ulp, struct tf *tfp);
+
+/* Function to get the tfp session details from ulp context. */
+struct tf *
+bnxt_ulp_cntxt_tfp_get(struct bnxt_ulp_context *ulp);
+
+/* Get the device table entry based on the device id. */
+struct bnxt_ulp_device_params *
+bnxt_ulp_device_params_get(uint32_t dev_id);
+
+int32_t
+bnxt_ulp_ctxt_ptr2_mark_db_set(struct bnxt_ulp_context *ulp_ctx,
+			       struct bnxt_ulp_mark_tbl *mark_tbl);
+
+struct bnxt_ulp_mark_tbl *
+bnxt_ulp_ctxt_ptr2_mark_db_get(struct bnxt_ulp_context *ulp_ctx);
+
+/* Function to set the flow database to the ulp context. */
+int32_t
+bnxt_ulp_cntxt_ptr2_flow_db_set(struct bnxt_ulp_context	*ulp_ctx,
+				struct bnxt_ulp_flow_db	*flow_db);
+
+/* Function to get the flow database from the ulp context. */
+struct bnxt_ulp_flow_db	*
+bnxt_ulp_cntxt_ptr2_flow_db_get(struct bnxt_ulp_context	*ulp_ctx);
+
+/* Function to get the ulp context from eth device. */
+struct bnxt_ulp_context	*
+bnxt_ulp_eth_dev_ptr2_cntxt_get(struct rte_eth_dev *dev);
+
+#endif /* _BNXT_ULP_H_ */
diff --git a/drivers/net/bnxt/tf_ulp/ulp_flow_db.c b/drivers/net/bnxt/tf_ulp/ulp_flow_db.c
new file mode 100644
index 0000000..3dd39c1
--- /dev/null
+++ b/drivers/net/bnxt/tf_ulp/ulp_flow_db.c
@@ -0,0 +1,187 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2014-2020 Broadcom
+ * All rights reserved.
+ */
+
+#include <rte_malloc.h>
+#include "bnxt.h"
+#include "bnxt_tf_common.h"
+#include "ulp_flow_db.h"
+#include "ulp_template_struct.h"
+
+/*
+ * Helper function to allocate the flow table and initialize
+ * the stack for allocation operations.
+ *
+ * flow_db [in] Ptr to flow database structure
+ * tbl_idx [in] The index to table creation.
+ *
+ * Returns 0 on success or negative number on failure.
+ */
+static int32_t
+ulp_flow_db_alloc_resource(struct bnxt_ulp_flow_db *flow_db,
+			   enum bnxt_ulp_flow_db_tables tbl_idx)
+{
+	uint32_t			idx = 0;
+	struct bnxt_ulp_flow_tbl	*flow_tbl;
+	uint32_t			size;
+
+	flow_tbl = &flow_db->flow_tbl[tbl_idx];
+
+	size = sizeof(struct ulp_fdb_resource_info) * flow_tbl->num_resources;
+	flow_tbl->flow_resources =
+			rte_zmalloc("ulp_fdb_resource_info", size, 0);
+
+	if (!flow_tbl->flow_resources) {
+		BNXT_TF_DBG(ERR, "Failed to alloc memory for flow table\n");
+		return -ENOMEM;
+	}
+	size = sizeof(uint32_t) * flow_tbl->num_resources;
+	flow_tbl->flow_tbl_stack = rte_zmalloc("flow_tbl_stack", size, 0);
+	if (!flow_tbl->flow_tbl_stack) {
+		BNXT_TF_DBG(ERR, "Failed to alloc memory flow tbl stack\n");
+		return -ENOMEM;
+	}
+	size = (flow_tbl->num_flows / sizeof(uint64_t)) + 1;
+	flow_tbl->active_flow_tbl = rte_zmalloc("active flow tbl", size, 0);
+	if (!flow_tbl->active_flow_tbl) {
+		BNXT_TF_DBG(ERR, "Failed to alloc memory active tbl\n");
+		return -ENOMEM;
+	}
+
+	/* Initialize the stack table. */
+	for (idx = 0; idx < flow_tbl->num_resources; idx++)
+		flow_tbl->flow_tbl_stack[idx] = idx;
+
+	/* Ignore the first element in the list. */
+	flow_tbl->head_index = 1;
+	/* Tail points to the last entry in the list. */
+	flow_tbl->tail_index = flow_tbl->num_resources - 1;
+	return 0;
+}
+
+/*
+ * Helper function to de allocate the flow table.
+ *
+ * flow_db [in] Ptr to flow database structure
+ * tbl_idx [in] The index to table creation.
+ *
+ * Returns none.
+ */
+static void
+ulp_flow_db_dealloc_resource(struct bnxt_ulp_flow_db *flow_db,
+			     enum bnxt_ulp_flow_db_tables tbl_idx)
+{
+	struct bnxt_ulp_flow_tbl	*flow_tbl;
+
+	flow_tbl = &flow_db->flow_tbl[tbl_idx];
+
+	/* Free all the allocated tables in the flow table. */
+	if (flow_tbl->active_flow_tbl) {
+		rte_free(flow_tbl->active_flow_tbl);
+		flow_tbl->active_flow_tbl = NULL;
+	}
+
+	if (flow_tbl->flow_tbl_stack) {
+		rte_free(flow_tbl->flow_tbl_stack);
+		flow_tbl->flow_tbl_stack = NULL;
+	}
+
+	if (flow_tbl->flow_resources) {
+		rte_free(flow_tbl->flow_resources);
+		flow_tbl->flow_resources = NULL;
+	}
+}
+
+/*
+ * Initialize the flow database. Memory is allocated in this
+ * call and assigned to the flow database.
+ *
+ * ulp_ctxt [in] Ptr to ulp context
+ *
+ * Returns 0 on success or negative number on failure.
+ */
+int32_t	ulp_flow_db_init(struct bnxt_ulp_context *ulp_ctxt)
+{
+	struct bnxt_ulp_device_params		*dparms;
+	struct bnxt_ulp_flow_tbl		*flow_tbl;
+	struct bnxt_ulp_flow_db			*flow_db;
+	uint32_t				dev_id;
+
+	/* Get the dev specific number of flows that needed to be supported. */
+	if (bnxt_ulp_cntxt_dev_id_get(ulp_ctxt, &dev_id)) {
+		BNXT_TF_DBG(ERR, "Invalid device id\n");
+		return -EINVAL;
+	}
+
+	dparms = bnxt_ulp_device_params_get(dev_id);
+	if (!dparms) {
+		BNXT_TF_DBG(ERR, "could not fetch the device params\n");
+		return -ENODEV;
+	}
+
+	flow_db = rte_zmalloc("bnxt_ulp_flow_db",
+			      sizeof(struct bnxt_ulp_flow_db), 0);
+	if (!flow_db) {
+		BNXT_TF_DBG(ERR,
+			    "Failed to allocate memory for flow table ptr\n");
+		goto error_free;
+	}
+
+	/* Attach the flow database to the ulp context. */
+	bnxt_ulp_cntxt_ptr2_flow_db_set(ulp_ctxt, flow_db);
+
+	/* Populate the regular flow table limits. */
+	flow_tbl = &flow_db->flow_tbl[BNXT_ULP_REGULAR_FLOW_TABLE];
+	flow_tbl->num_flows = dparms->num_flows + 1;
+	flow_tbl->num_resources = (flow_tbl->num_flows *
+				   dparms->num_resources_per_flow);
+
+	/* Populate the default flow table limits. */
+	flow_tbl = &flow_db->flow_tbl[BNXT_ULP_DEFAULT_FLOW_TABLE];
+	flow_tbl->num_flows = BNXT_FLOW_DB_DEFAULT_NUM_FLOWS + 1;
+	flow_tbl->num_resources = (flow_tbl->num_flows *
+				   BNXT_FLOW_DB_DEFAULT_NUM_RESOURCES);
+
+	/* Allocate the resource for the regular flow table. */
+	if (ulp_flow_db_alloc_resource(flow_db, BNXT_ULP_REGULAR_FLOW_TABLE))
+		goto error_free;
+	if (ulp_flow_db_alloc_resource(flow_db, BNXT_ULP_DEFAULT_FLOW_TABLE))
+		goto error_free;
+
+	/* All good so return. */
+	return 0;
+error_free:
+	ulp_flow_db_deinit(ulp_ctxt);
+	return -ENOMEM;
+}
+
+/*
+ * Deinitialize the flow database. Memory is deallocated in
+ * this call and all flows should have been purged before this
+ * call.
+ *
+ * ulp_ctxt [in] Ptr to ulp context
+ *
+ * Returns 0 on success.
+ */
+int32_t	ulp_flow_db_deinit(struct bnxt_ulp_context *ulp_ctxt)
+{
+	struct bnxt_ulp_flow_db			*flow_db;
+
+	flow_db = bnxt_ulp_cntxt_ptr2_flow_db_get(ulp_ctxt);
+	if (!flow_db) {
+		BNXT_TF_DBG(ERR, "Invalid Arguments\n");
+		return -EINVAL;
+	}
+
+	/* Detach the flow database from the ulp context. */
+	bnxt_ulp_cntxt_ptr2_flow_db_set(ulp_ctxt, NULL);
+
+	/* Free up all the memory. */
+	ulp_flow_db_dealloc_resource(flow_db, BNXT_ULP_REGULAR_FLOW_TABLE);
+	ulp_flow_db_dealloc_resource(flow_db, BNXT_ULP_DEFAULT_FLOW_TABLE);
+	rte_free(flow_db);
+
+	return 0;
+}
diff --git a/drivers/net/bnxt/tf_ulp/ulp_flow_db.h b/drivers/net/bnxt/tf_ulp/ulp_flow_db.h
new file mode 100644
index 0000000..a2ee8fa
--- /dev/null
+++ b/drivers/net/bnxt/tf_ulp/ulp_flow_db.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2014-2019 Broadcom
+ * All rights reserved.
+ */
+
+#ifndef _ULP_FLOW_DB_H_
+#define _ULP_FLOW_DB_H_
+
+#include "bnxt_ulp.h"
+#include "ulp_template_db.h"
+
+#define BNXT_FLOW_DB_DEFAULT_NUM_FLOWS		128
+#define BNXT_FLOW_DB_DEFAULT_NUM_RESOURCES	5
+
+/* Structure for the flow database resource information. */
+struct ulp_fdb_resource_info {
+	/* Points to next resource in the chained list. */
+	uint32_t	nxt_resource_idx;
+	union {
+		uint64_t	resource_em_handle;
+		struct {
+			uint32_t	resource_type;
+			uint32_t	resource_hndl;
+		};
+	};
+};
+
+/* Structure for the flow database resource information. */
+struct bnxt_ulp_flow_tbl {
+	/* Flow tbl is the resource object list for each flow id. */
+	struct ulp_fdb_resource_info	*flow_resources;
+
+	/* Flow table stack to track free list of resources. */
+	uint32_t	*flow_tbl_stack;
+	uint32_t	head_index;
+	uint32_t	tail_index;
+
+	/* Table to track the active flows. */
+	uint64_t	*active_flow_tbl;
+	uint32_t	num_flows;
+	uint32_t	num_resources;
+};
+
+/* Flow database supports two tables. */
+enum bnxt_ulp_flow_db_tables {
+	BNXT_ULP_REGULAR_FLOW_TABLE,
+	BNXT_ULP_DEFAULT_FLOW_TABLE,
+	BNXT_ULP_FLOW_TABLE_MAX
+};
+
+/* Structure for the flow database resource information. */
+struct bnxt_ulp_flow_db {
+	struct bnxt_ulp_flow_tbl	flow_tbl[BNXT_ULP_FLOW_TABLE_MAX];
+};
+
+/*
+ * Initialize the flow database. Memory is allocated in this
+ * call and assigned to the flow database.
+ *
+ * ulp_ctxt [in] Ptr to ulp context
+ *
+ * Returns 0 on success or negative number on failure.
+ */
+int32_t	ulp_flow_db_init(struct bnxt_ulp_context *ulp_ctxt);
+
+/*
+ * Deinitialize the flow database. Memory is deallocated in
+ * this call and all flows should have been purged before this
+ * call.
+ *
+ * ulp_ctxt [in] Ptr to ulp context
+ *
+ * Returns 0 on success.
+ */
+int32_t	ulp_flow_db_deinit(struct bnxt_ulp_context *ulp_ctxt);
+
+#endif /* _ULP_FLOW_DB_H_ */
diff --git a/drivers/net/bnxt/tf_ulp/ulp_mark_mgr.c b/drivers/net/bnxt/tf_ulp/ulp_mark_mgr.c
new file mode 100644
index 0000000..3f28a73
--- /dev/null
+++ b/drivers/net/bnxt/tf_ulp/ulp_mark_mgr.c
@@ -0,0 +1,94 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2014-2020 Broadcom
+ * All rights reserved.
+ */
+
+#include <rte_common.h>
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include "bnxt_ulp.h"
+#include "tf_ext_flow_handle.h"
+#include "ulp_mark_mgr.h"
+#include "bnxt_tf_common.h"
+#include "../bnxt.h"
+#include "ulp_template_db.h"
+#include "ulp_template_struct.h"
+
+/*
+ * Allocate and Initialize all Mark Manager resources for this ulp context.
+ *
+ * ctxt [in] The ulp context for the mark manager.
+ *
+ */
+int32_t
+ulp_mark_db_init(struct bnxt_ulp_context *ctxt)
+{
+	struct bnxt_ulp_device_params *dparms;
+	struct bnxt_ulp_mark_tbl *mark_tbl = NULL;
+	uint32_t dev_id;
+
+	if (!ctxt) {
+		BNXT_TF_DBG(DEBUG, "Invalid ULP CTXT\n");
+		return -EINVAL;
+	}
+
+	if (bnxt_ulp_cntxt_dev_id_get(ctxt, &dev_id)) {
+		BNXT_TF_DBG(DEBUG, "Failed to get device id\n");
+		return -EINVAL;
+	}
+
+	dparms = bnxt_ulp_device_params_get(dev_id);
+	if (!dparms) {
+		BNXT_TF_DBG(DEBUG, "Failed to device parms\n");
+		return -EINVAL;
+	}
+
+	mark_tbl = rte_zmalloc("ulp_rx_mark_tbl_ptr",
+			       sizeof(struct bnxt_ulp_mark_tbl), 0);
+	if (!mark_tbl)
+		goto mem_error;
+
+	/* Need to allocate 2 * Num flows to account for hash type bit. */
+	mark_tbl->lfid_tbl = rte_zmalloc("ulp_rx_em_flow_mark_table",
+					 dparms->lfid_entries *
+					    sizeof(struct bnxt_lfid_mark_info),
+					 0);
+
+	if (!mark_tbl->lfid_tbl)
+		goto mem_error;
+
+	/* Need to allocate 2 * Num flows to account for hash type bit. */
+	mark_tbl->gfid_tbl = rte_zmalloc("ulp_rx_eem_flow_mark_table",
+					 2 * dparms->num_flows *
+					    sizeof(struct bnxt_gfid_mark_info),
+					 0);
+	if (!mark_tbl->gfid_tbl)
+		goto mem_error;
+
+	/*
+	 * TBD: This needs to be generalized for better mark handling
+	 * These values are used to compress the FID to the allowable index
+	 * space.  The FID from hw may be the full hash.
+	 */
+	mark_tbl->gfid_max	= dparms->gfid_entries - 1;
+	mark_tbl->gfid_mask	= (dparms->gfid_entries / 2) - 1;
+	mark_tbl->gfid_type_bit = (dparms->gfid_entries / 2);
+
+	BNXT_TF_DBG(DEBUG, "GFID Max = 0x%08x\nGFID MASK = 0x%08x\n",
+		    mark_tbl->gfid_max,
+		    mark_tbl->gfid_mask);
+
+	/* Add the mart tbl to the ulp context. */
+	bnxt_ulp_cntxt_ptr2_mark_db_set(ctxt, mark_tbl);
+
+	return 0;
+
+mem_error:
+	rte_free(mark_tbl->gfid_tbl);
+	rte_free(mark_tbl->lfid_tbl);
+	rte_free(mark_tbl);
+	BNXT_TF_DBG(DEBUG,
+		    "Failed to allocate memory for mark mgr\n");
+
+	return -ENOMEM;
+}
diff --git a/drivers/net/bnxt/tf_ulp/ulp_mark_mgr.h b/drivers/net/bnxt/tf_ulp/ulp_mark_mgr.h
new file mode 100644
index 0000000..b175abd
--- /dev/null
+++ b/drivers/net/bnxt/tf_ulp/ulp_mark_mgr.h
@@ -0,0 +1,49 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2014-2019 Broadcom
+ * All rights reserved.
+ */
+
+#ifndef _ULP_MARK_MGR_H_
+#define _ULP_MARK_MGR_H_
+
+#include "bnxt_ulp.h"
+
+#define ULP_MARK_INVALID (0)
+struct bnxt_lfid_mark_info {
+	uint16_t	mark_id;
+	bool		valid;
+};
+
+struct bnxt_gfid_mark_info {
+	uint32_t	mark_id;
+	bool		valid;
+};
+
+struct bnxt_ulp_mark_tbl {
+	struct bnxt_lfid_mark_info	*lfid_tbl;
+	struct bnxt_gfid_mark_info	*gfid_tbl;
+	uint32_t			gfid_mask;
+	uint32_t			gfid_type_bit;
+	uint32_t			gfid_max;
+};
+
+/*
+ * Allocate and Initialize all Mark Manager resources for this ulp context.
+ *
+ * Initialize MARK database for GFID & LFID tables
+ * GFID: Global flow id which is based on EEM hash id.
+ * LFID: Local flow id which is the CFA action pointer.
+ * GFID is used for EEM flows, LFID is used for EM flows.
+ *
+ * Flow mapper modules adds mark_id in the MARK database.
+ *
+ * BNXT PMD receive handler extracts the hardware flow id from the
+ * received completion record. Fetches mark_id from the MARK
+ * database using the flow id. Injects mark_id into the packet's mbuf.
+ *
+ * ctxt [in] The ulp context for the mark manager.
+ */
+int32_t
+ulp_mark_db_init(struct bnxt_ulp_context *ctxt);
+
+#endif /* _ULP_MARK_MGR_H_ */
diff --git a/drivers/net/bnxt/tf_ulp/ulp_template_db.c b/drivers/net/bnxt/tf_ulp/ulp_template_db.c
new file mode 100644
index 0000000..9670635
--- /dev/null
+++ b/drivers/net/bnxt/tf_ulp/ulp_template_db.c
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2014-2019 Broadcom
+ * All rights reserved.
+ */
+
+/*
+ * date: Mon Mar  9 02:37:53 2020
+ * version: 0.0
+ */
+
+#include "ulp_template_db.h"
+#include "ulp_template_struct.h"
+
+struct bnxt_ulp_device_params ulp_device_params[] = {
+	[BNXT_ULP_DEVICE_ID_WH_PLUS] = {
+		.global_fid_enable       = BNXT_ULP_SYM_YES,
+		.byte_order              = (enum bnxt_ulp_byte_order)
+						BNXT_ULP_SYM_LITTLE_ENDIAN,
+		.encap_byte_swap         = 1,
+		.lfid_entries            = 16384,
+		.lfid_entry_size         = 4,
+		.gfid_entries            = 65536,
+		.gfid_entry_size         = 4,
+		.num_flows               = 32768,
+		.num_resources_per_flow  = 8
+	}
+};
diff --git a/drivers/net/bnxt/tf_ulp/ulp_template_db.h b/drivers/net/bnxt/tf_ulp/ulp_template_db.h
new file mode 100644
index 0000000..ba2a101
--- /dev/null
+++ b/drivers/net/bnxt/tf_ulp/ulp_template_db.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2014-2019 Broadcom
+ * All rights reserved.
+ */
+
+/*
+ * date: Mon Mar  9 02:37:53 2020
+ * version: 0.0
+ */
+
+#ifndef ULP_TEMPLATE_DB_H_
+#define ULP_TEMPLATE_DB_H_
+
+#define BNXT_ULP_MAX_NUM_DEVICES 4
+
+enum bnxt_ulp_byte_order {
+	BNXT_ULP_BYTE_ORDER_BE,
+	BNXT_ULP_BYTE_ORDER_LE,
+	BNXT_ULP_BYTE_ORDER_LAST
+};
+
+enum bnxt_ulp_device_id {
+	BNXT_ULP_DEVICE_ID_WH_PLUS,
+	BNXT_ULP_DEVICE_ID_THOR,
+	BNXT_ULP_DEVICE_ID_STINGRAY,
+	BNXT_ULP_DEVICE_ID_STINGRAY2,
+	BNXT_ULP_DEVICE_ID_LAST
+};
+
+enum bnxt_ulp_sym {
+	BNXT_ULP_SYM_LITTLE_ENDIAN = 1,
+	BNXT_ULP_SYM_YES = 1
+};
+
+#endif /* _ULP_TEMPLATE_DB_H_ */
diff --git a/drivers/net/bnxt/tf_ulp/ulp_template_struct.h b/drivers/net/bnxt/tf_ulp/ulp_template_struct.h
new file mode 100644
index 0000000..4b9d0b2
--- /dev/null
+++ b/drivers/net/bnxt/tf_ulp/ulp_template_struct.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2014-2019 Broadcom
+ * All rights reserved.
+ */
+
+#ifndef _ULP_TEMPLATE_STRUCT_H_
+#define _ULP_TEMPLATE_STRUCT_H_
+
+#include <stdint.h>
+#include "rte_ether.h"
+#include "rte_icmp.h"
+#include "rte_ip.h"
+#include "rte_tcp.h"
+#include "rte_udp.h"
+#include "rte_esp.h"
+#include "rte_sctp.h"
+#include "rte_flow.h"
+#include "tf_core.h"
+
+/* Device specific parameters. */
+struct bnxt_ulp_device_params {
+	uint8_t				description[16];
+	uint32_t			global_fid_enable;
+	enum bnxt_ulp_byte_order	byte_order;
+	uint8_t				encap_byte_swap;
+	uint32_t			lfid_entries;
+	uint32_t			lfid_entry_size;
+	uint64_t			gfid_entries;
+	uint32_t			gfid_entry_size;
+	uint64_t			num_flows;
+	uint32_t			num_resources_per_flow;
+};
+
+/*
+ * The ulp_device_params is indexed by the dev_id.
+ * This table maintains the device specific parameters.
+ */
+extern struct bnxt_ulp_device_params ulp_device_params[];
+
+#endif /* _ULP_TEMPLATE_STRUCT_H_ */
-- 
2.7.4