[dpdk-dev] [PATCH v3 20/51] net/bnxt: update RM with residual checker

[Ajit Khaparde <ajit.khaparde@broadcom.com>]

From: Michael Wildt <michael.wildt@broadcom.com>

- Add residual checker to the TF Host RM as well as new RM APIs. On
  close it will scan the DB and check of any remaining elements. If
  found they will be logged and FW msg sent for FW to scrub that
  specific type of resources.
- Update the module bind to be aware of the module type, for each of
  the modules.
- Add additional type 2 string util functions.
- Fix the device naming to be in compliance with TF.
- Update the device unbind order as to assure TCAMs gets flushed
  first.
- Update the close functionality such that the session gets
  closed after the device is unbound.

Signed-off-by: Michael Wildt <michael.wildt@broadcom.com>
Signed-off-by: Venkat Duvvuru <venkatkumar.duvvuru@broadcom.com>
Reviewed-by: Randy Schacher <stuart.schacher@broadcom.com>
Reviewed-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
---
 drivers/net/bnxt/tf_core/tf_device.c     |  53 +++--
 drivers/net/bnxt/tf_core/tf_device.h     |  25 +-
 drivers/net/bnxt/tf_core/tf_device_p4.h  |   1 -
 drivers/net/bnxt/tf_core/tf_identifier.c |  10 +-
 drivers/net/bnxt/tf_core/tf_msg.c        |  67 +++++-
 drivers/net/bnxt/tf_core/tf_msg.h        |   7 +
 drivers/net/bnxt/tf_core/tf_rm_new.c     | 287 +++++++++++++++++++++--
 drivers/net/bnxt/tf_core/tf_rm_new.h     |  45 +++-
 drivers/net/bnxt/tf_core/tf_session.c    |  58 +++--
 drivers/net/bnxt/tf_core/tf_tbl_type.c   |   5 +-
 drivers/net/bnxt/tf_core/tf_tcam.c       |   5 +-
 drivers/net/bnxt/tf_core/tf_tcam.h       |   4 +
 drivers/net/bnxt/tf_core/tf_util.c       |  55 ++++-
 drivers/net/bnxt/tf_core/tf_util.h       |  32 +++
 14 files changed, 561 insertions(+), 93 deletions(-)

diff --git a/drivers/net/bnxt/tf_core/tf_device.c b/drivers/net/bnxt/tf_core/tf_device.c
index b474e8c25..441d0c678 100644
--- a/drivers/net/bnxt/tf_core/tf_device.c
+++ b/drivers/net/bnxt/tf_core/tf_device.c
@@ -10,7 +10,7 @@
 struct tf;
 
 /* Forward declarations */
-static int dev_unbind_p4(struct tf *tfp);
+static int tf_dev_unbind_p4(struct tf *tfp);
 
 /**
  * Device specific bind function, WH+
@@ -32,10 +32,10 @@ static int dev_unbind_p4(struct tf *tfp);
  *   - (-EINVAL) on parameter or internal failure.
  */
 static int
-dev_bind_p4(struct tf *tfp,
-	    bool shadow_copy,
-	    struct tf_session_resources *resources,
-	    struct tf_dev_info *dev_handle)
+tf_dev_bind_p4(struct tf *tfp,
+	       bool shadow_copy,
+	       struct tf_session_resources *resources,
+	       struct tf_dev_info *dev_handle)
 {
 	int rc;
 	int frc;
@@ -93,7 +93,7 @@ dev_bind_p4(struct tf *tfp,
 
  fail:
 	/* Cleanup of already created modules */
-	frc = dev_unbind_p4(tfp);
+	frc = tf_dev_unbind_p4(tfp);
 	if (frc)
 		return frc;
 
@@ -111,7 +111,7 @@ dev_bind_p4(struct tf *tfp,
  *   - (-EINVAL) on failure.
  */
 static int
-dev_unbind_p4(struct tf *tfp)
+tf_dev_unbind_p4(struct tf *tfp)
 {
 	int rc = 0;
 	bool fail = false;
@@ -119,25 +119,28 @@ dev_unbind_p4(struct tf *tfp)
 	/* Unbind all the support modules. As this is only done on
 	 * close we only report errors as everything has to be cleaned
 	 * up regardless.
+	 *
+	 * In case of residuals TCAMs are cleaned up first as to
+	 * invalidate the pipeline in a clean manner.
 	 */
-	rc = tf_ident_unbind(tfp);
+	rc = tf_tcam_unbind(tfp);
 	if (rc) {
 		TFP_DRV_LOG(ERR,
-			    "Device unbind failed, Identifier\n");
+			    "Device unbind failed, TCAM\n");
 		fail = true;
 	}
 
-	rc = tf_tbl_unbind(tfp);
+	rc = tf_ident_unbind(tfp);
 	if (rc) {
 		TFP_DRV_LOG(ERR,
-			    "Device unbind failed, Table Type\n");
+			    "Device unbind failed, Identifier\n");
 		fail = true;
 	}
 
-	rc = tf_tcam_unbind(tfp);
+	rc = tf_tbl_unbind(tfp);
 	if (rc) {
 		TFP_DRV_LOG(ERR,
-			    "Device unbind failed, TCAM\n");
+			    "Device unbind failed, Table Type\n");
 		fail = true;
 	}
 
@@ -148,18 +151,18 @@ dev_unbind_p4(struct tf *tfp)
 }
 
 int
-dev_bind(struct tf *tfp __rte_unused,
-	 enum tf_device_type type,
-	 bool shadow_copy,
-	 struct tf_session_resources *resources,
-	 struct tf_dev_info *dev_handle)
+tf_dev_bind(struct tf *tfp __rte_unused,
+	    enum tf_device_type type,
+	    bool shadow_copy,
+	    struct tf_session_resources *resources,
+	    struct tf_dev_info *dev_handle)
 {
 	switch (type) {
 	case TF_DEVICE_TYPE_WH:
-		return dev_bind_p4(tfp,
-				   shadow_copy,
-				   resources,
-				   dev_handle);
+		return tf_dev_bind_p4(tfp,
+				      shadow_copy,
+				      resources,
+				      dev_handle);
 	default:
 		TFP_DRV_LOG(ERR,
 			    "No such device\n");
@@ -168,12 +171,12 @@ dev_bind(struct tf *tfp __rte_unused,
 }
 
 int
-dev_unbind(struct tf *tfp,
-	   struct tf_dev_info *dev_handle)
+tf_dev_unbind(struct tf *tfp,
+	      struct tf_dev_info *dev_handle)
 {
 	switch (dev_handle->type) {
 	case TF_DEVICE_TYPE_WH:
-		return dev_unbind_p4(tfp);
+		return tf_dev_unbind_p4(tfp);
 	default:
 		TFP_DRV_LOG(ERR,
 			    "No such device\n");
diff --git a/drivers/net/bnxt/tf_core/tf_device.h b/drivers/net/bnxt/tf_core/tf_device.h
index c31bf2357..c8feac55d 100644
--- a/drivers/net/bnxt/tf_core/tf_device.h
+++ b/drivers/net/bnxt/tf_core/tf_device.h
@@ -14,6 +14,17 @@
 struct tf;
 struct tf_session;
 
+/**
+ *
+ */
+enum tf_device_module_type {
+	TF_DEVICE_MODULE_TYPE_IDENTIFIER,
+	TF_DEVICE_MODULE_TYPE_TABLE,
+	TF_DEVICE_MODULE_TYPE_TCAM,
+	TF_DEVICE_MODULE_TYPE_EM,
+	TF_DEVICE_MODULE_TYPE_MAX
+};
+
 /**
  * The Device module provides a general device template. A supported
  * device type should implement one or more of the listed function
@@ -60,11 +71,11 @@ struct tf_dev_info {
  *   - (-EINVAL) parameter failure.
  *   - (-ENODEV) no such device supported.
  */
-int dev_bind(struct tf *tfp,
-	     enum tf_device_type type,
-	     bool shadow_copy,
-	     struct tf_session_resources *resources,
-	     struct tf_dev_info *dev_handle);
+int tf_dev_bind(struct tf *tfp,
+		enum tf_device_type type,
+		bool shadow_copy,
+		struct tf_session_resources *resources,
+		struct tf_dev_info *dev_handle);
 
 /**
  * Device release handles cleanup of the device specific information.
@@ -80,8 +91,8 @@ int dev_bind(struct tf *tfp,
  *   - (-EINVAL) parameter failure.
  *   - (-ENODEV) no such device supported.
  */
-int dev_unbind(struct tf *tfp,
-	       struct tf_dev_info *dev_handle);
+int tf_dev_unbind(struct tf *tfp,
+		  struct tf_dev_info *dev_handle);
 
 /**
  * Truflow device specific function hooks structure
diff --git a/drivers/net/bnxt/tf_core/tf_device_p4.h b/drivers/net/bnxt/tf_core/tf_device_p4.h
index 235d81f96..411e21637 100644
--- a/drivers/net/bnxt/tf_core/tf_device_p4.h
+++ b/drivers/net/bnxt/tf_core/tf_device_p4.h
@@ -77,5 +77,4 @@ struct tf_rm_element_cfg tf_tbl_p4[TF_TBL_TYPE_MAX] = {
 	/* CFA_RESOURCE_TYPE_P4_EXT */
 	{ TF_RM_ELEM_CFG_NULL, CFA_RESOURCE_TYPE_INVALID }
 };
-
 #endif /* _TF_DEVICE_P4_H_ */
diff --git a/drivers/net/bnxt/tf_core/tf_identifier.c b/drivers/net/bnxt/tf_core/tf_identifier.c
index ee07a6aea..b197bb271 100644
--- a/drivers/net/bnxt/tf_core/tf_identifier.c
+++ b/drivers/net/bnxt/tf_core/tf_identifier.c
@@ -39,12 +39,12 @@ tf_ident_bind(struct tf *tfp,
 		return -EINVAL;
 	}
 
+	db_cfg.type = TF_DEVICE_MODULE_TYPE_IDENTIFIER;
 	db_cfg.num_elements = parms->num_elements;
+	db_cfg.cfg = parms->cfg;
 
 	for (i = 0; i < TF_DIR_MAX; i++) {
 		db_cfg.dir = i;
-		db_cfg.num_elements = parms->num_elements;
-		db_cfg.cfg = parms->cfg;
 		db_cfg.alloc_cnt = parms->resources->ident_cnt[i].cnt;
 		db_cfg.rm_db = &ident_db[i];
 		rc = tf_rm_create_db(tfp, &db_cfg);
@@ -86,8 +86,10 @@ tf_ident_unbind(struct tf *tfp __rte_unused)
 		fparms.dir = i;
 		fparms.rm_db = ident_db[i];
 		rc = tf_rm_free_db(tfp, &fparms);
-		if (rc)
-			return rc;
+		if (rc) {
+			TFP_DRV_LOG(ERR,
+				    "rm free failed on unbind\n");
+		}
 
 		ident_db[i] = NULL;
 	}
diff --git a/drivers/net/bnxt/tf_core/tf_msg.c b/drivers/net/bnxt/tf_core/tf_msg.c
index a2e3840f0..c015b0ce2 100644
--- a/drivers/net/bnxt/tf_core/tf_msg.c
+++ b/drivers/net/bnxt/tf_core/tf_msg.c
@@ -1110,6 +1110,69 @@ tf_msg_session_resc_alloc(struct tf *tfp,
 	return rc;
 }
 
+int
+tf_msg_session_resc_flush(struct tf *tfp,
+			  enum tf_dir dir,
+			  uint16_t size,
+			  struct tf_rm_resc_entry *resv)
+{
+	int rc;
+	int i;
+	struct tfp_send_msg_parms parms = { 0 };
+	struct hwrm_tf_session_resc_flush_input req = { 0 };
+	struct hwrm_tf_session_resc_flush_output resp = { 0 };
+	uint8_t fw_session_id;
+	struct tf_msg_dma_buf resv_buf = { 0 };
+	struct tf_rm_resc_entry *resv_data;
+	int dma_size;
+
+	TF_CHECK_PARMS2(tfp, resv);
+
+	rc = tf_session_get_fw_session_id(tfp, &fw_session_id);
+	if (rc) {
+		TFP_DRV_LOG(ERR,
+			    "%s: Unable to lookup FW id, rc:%s\n",
+			    tf_dir_2_str(dir),
+			    strerror(-rc));
+		return rc;
+	}
+
+	/* Prepare DMA buffers */
+	dma_size = size * sizeof(struct tf_rm_resc_entry);
+	rc = tf_msg_alloc_dma_buf(&resv_buf, dma_size);
+	if (rc)
+		return rc;
+
+	/* Populate the request */
+	req.fw_session_id = tfp_cpu_to_le_32(fw_session_id);
+	req.flags = tfp_cpu_to_le_16(dir);
+	req.flush_size = size;
+
+	resv_data = (struct tf_rm_resc_entry *)resv_buf.va_addr;
+	for (i = 0; i < size; i++) {
+		resv_data[i].type = tfp_cpu_to_le_32(resv[i].type);
+		resv_data[i].start = tfp_cpu_to_le_16(resv[i].start);
+		resv_data[i].stride = tfp_cpu_to_le_16(resv[i].stride);
+	}
+
+	req.flush_addr = tfp_cpu_to_le_64(resv_buf.pa_addr);
+
+	parms.tf_type = HWRM_TF_SESSION_RESC_FLUSH;
+	parms.req_data = (uint32_t *)&req;
+	parms.req_size = sizeof(req);
+	parms.resp_data = (uint32_t *)&resp;
+	parms.resp_size = sizeof(resp);
+	parms.mailbox = TF_KONG_MB;
+
+	rc = tfp_send_msg_direct(tfp, &parms);
+	if (rc)
+		return rc;
+
+	tf_msg_free_dma_buf(&resv_buf);
+
+	return rc;
+}
+
 /**
  * Sends EM mem register request to Firmware
  */
@@ -1512,9 +1575,7 @@ tf_msg_tcam_entry_set(struct tf *tfp,
 	uint8_t *data = NULL;
 	int data_size = 0;
 
-	rc = tf_tcam_tbl_2_hwrm(parms->type, &req.type);
-	if (rc != 0)
-		return rc;
+	req.type = parms->type;
 
 	req.idx = tfp_cpu_to_le_16(parms->idx);
 	if (parms->dir == TF_DIR_TX)
diff --git a/drivers/net/bnxt/tf_core/tf_msg.h b/drivers/net/bnxt/tf_core/tf_msg.h
index fb635f6dc..1ff1044e8 100644
--- a/drivers/net/bnxt/tf_core/tf_msg.h
+++ b/drivers/net/bnxt/tf_core/tf_msg.h
@@ -181,6 +181,13 @@ int tf_msg_session_resc_alloc(struct tf *tfp,
 			      struct tf_rm_resc_req_entry *request,
 			      struct tf_rm_resc_entry *resv);
 
+/**
+ * Sends session resource flush request to TF Firmware
+ */
+int tf_msg_session_resc_flush(struct tf *tfp,
+			      enum tf_dir dir,
+			      uint16_t size,
+			      struct tf_rm_resc_entry *resv);
 /**
  * Sends EM internal insert request to Firmware
  */
diff --git a/drivers/net/bnxt/tf_core/tf_rm_new.c b/drivers/net/bnxt/tf_core/tf_rm_new.c
index 6abf79aa1..02b4b5c8f 100644
--- a/drivers/net/bnxt/tf_core/tf_rm_new.c
+++ b/drivers/net/bnxt/tf_core/tf_rm_new.c
@@ -60,6 +60,11 @@ struct tf_rm_new_db {
 	 */
 	enum tf_dir dir;
 
+	/**
+	 * Module type, used for logging purposes.
+	 */
+	enum tf_device_module_type type;
+
 	/**
 	 * The DB consists of an array of elements
 	 */
@@ -167,6 +172,178 @@ tf_rm_adjust_index(struct tf_rm_element *db,
 	return rc;
 }
 
+/**
+ * Logs an array of found residual entries to the console.
+ *
+ * [in] dir
+ *   Receive or transmit direction
+ *
+ * [in] type
+ *   Type of Device Module
+ *
+ * [in] count
+ *   Number of entries in the residual array
+ *
+ * [in] residuals
+ *   Pointer to an array of residual entries. Array is index same as
+ *   the DB in which this function is used. Each entry holds residual
+ *   value for that entry.
+ */
+static void
+tf_rm_log_residuals(enum tf_dir dir,
+		    enum tf_device_module_type type,
+		    uint16_t count,
+		    uint16_t *residuals)
+{
+	int i;
+
+	/* Walk the residual array and log the types that wasn't
+	 * cleaned up to the console.
+	 */
+	for (i = 0; i < count; i++) {
+		if (residuals[i] != 0)
+			TFP_DRV_LOG(ERR,
+				"%s, %s was not cleaned up, %d outstanding\n",
+				tf_dir_2_str(dir),
+				tf_device_module_type_subtype_2_str(type, i),
+				residuals[i]);
+	}
+}
+
+/**
+ * Performs a check of the passed in DB for any lingering elements. If
+ * a resource type was found to not have been cleaned up by the caller
+ * then its residual values are recorded, logged and passed back in an
+ * allocate reservation array that the caller can pass to the FW for
+ * cleanup.
+ *
+ * [in] db
+ *   Pointer to the db, used for the lookup
+ *
+ * [out] resv_size
+ *   Pointer to the reservation size of the generated reservation
+ *   array.
+ *
+ * [in/out] resv
+ *   Pointer Pointer to a reservation array. The reservation array is
+ *   allocated after the residual scan and holds any found residual
+ *   entries. Thus it can be smaller than the DB that the check was
+ *   performed on. Array must be freed by the caller.
+ *
+ * [out] residuals_present
+ *   Pointer to a bool flag indicating if residual was present in the
+ *   DB
+ *
+ * Returns:
+ *     0          - Success
+ *   - EOPNOTSUPP - Operation not supported
+ */
+static int
+tf_rm_check_residuals(struct tf_rm_new_db *rm_db,
+		      uint16_t *resv_size,
+		      struct tf_rm_resc_entry **resv,
+		      bool *residuals_present)
+{
+	int rc;
+	int i;
+	int f;
+	uint16_t count;
+	uint16_t found;
+	uint16_t *residuals = NULL;
+	uint16_t hcapi_type;
+	struct tf_rm_get_inuse_count_parms iparms;
+	struct tf_rm_get_alloc_info_parms aparms;
+	struct tf_rm_get_hcapi_parms hparms;
+	struct tf_rm_alloc_info info;
+	struct tfp_calloc_parms cparms;
+	struct tf_rm_resc_entry *local_resv = NULL;
+
+	/* Create array to hold the entries that have residuals */
+	cparms.nitems = rm_db->num_entries;
+	cparms.size = sizeof(uint16_t);
+	cparms.alignment = 0;
+	rc = tfp_calloc(&cparms);
+	if (rc)
+		return rc;
+
+	residuals = (uint16_t *)cparms.mem_va;
+
+	/* Traverse the DB and collect any residual elements */
+	iparms.rm_db = rm_db;
+	iparms.count = &count;
+	for (i = 0, found = 0; i < rm_db->num_entries; i++) {
+		iparms.db_index = i;
+		rc = tf_rm_get_inuse_count(&iparms);
+		/* Not a device supported entry, just skip */
+		if (rc == -ENOTSUP)
+			continue;
+		if (rc)
+			goto cleanup_residuals;
+
+		if (count) {
+			found++;
+			residuals[i] = count;
+			*residuals_present = true;
+		}
+	}
+
+	if (*residuals_present) {
+		/* Populate a reduced resv array with only the entries
+		 * that have residuals.
+		 */
+		cparms.nitems = found;
+		cparms.size = sizeof(struct tf_rm_resc_entry);
+		cparms.alignment = 0;
+		rc = tfp_calloc(&cparms);
+		if (rc)
+			return rc;
+
+		local_resv = (struct tf_rm_resc_entry *)cparms.mem_va;
+
+		aparms.rm_db = rm_db;
+		hparms.rm_db = rm_db;
+		hparms.hcapi_type = &hcapi_type;
+		for (i = 0, f = 0; i < rm_db->num_entries; i++) {
+			if (residuals[i] == 0)
+				continue;
+			aparms.db_index = i;
+			aparms.info = &info;
+			rc = tf_rm_get_info(&aparms);
+			if (rc)
+				goto cleanup_all;
+
+			hparms.db_index = i;
+			rc = tf_rm_get_hcapi_type(&hparms);
+			if (rc)
+				goto cleanup_all;
+
+			local_resv[f].type = hcapi_type;
+			local_resv[f].start = info.entry.start;
+			local_resv[f].stride = info.entry.stride;
+			f++;
+		}
+		*resv_size = found;
+	}
+
+	tf_rm_log_residuals(rm_db->dir,
+			    rm_db->type,
+			    rm_db->num_entries,
+			    residuals);
+
+	tfp_free((void *)residuals);
+	*resv = local_resv;
+
+	return 0;
+
+ cleanup_all:
+	tfp_free((void *)local_resv);
+	*resv = NULL;
+ cleanup_residuals:
+	tfp_free((void *)residuals);
+
+	return rc;
+}
+
 int
 tf_rm_create_db(struct tf *tfp,
 		struct tf_rm_create_db_parms *parms)
@@ -373,6 +550,7 @@ tf_rm_create_db(struct tf *tfp,
 
 	rm_db->num_entries = i;
 	rm_db->dir = parms->dir;
+	rm_db->type = parms->type;
 	*parms->rm_db = (void *)rm_db;
 
 	tfp_free((void *)req);
@@ -392,20 +570,69 @@ tf_rm_create_db(struct tf *tfp,
 }
 
 int
-tf_rm_free_db(struct tf *tfp __rte_unused,
+tf_rm_free_db(struct tf *tfp,
 	      struct tf_rm_free_db_parms *parms)
 {
-	int rc = 0;
+	int rc;
 	int i;
+	uint16_t resv_size = 0;
 	struct tf_rm_new_db *rm_db;
+	struct tf_rm_resc_entry *resv;
+	bool residuals_found = false;
 
-	TF_CHECK_PARMS1(parms);
+	TF_CHECK_PARMS2(parms, parms->rm_db);
 
-	/* Traverse the DB and clear each pool.
-	 * NOTE:
-	 *   Firmware is not cleared. It will be cleared on close only.
+	/* Device unbind happens when the TF Session is closed and the
+	 * session ref count is 0. Device unbind will cleanup each of
+	 * its support modules, i.e. Identifier, thus we're ending up
+	 * here to close the DB.
+	 *
+	 * On TF Session close it is assumed that the session has already
+	 * cleaned up all its resources, individually, while
+	 * destroying its flows.
+	 *
+	 * To assist in the 'cleanup checking' the DB is checked for any
+	 * remaining elements and logged if found to be the case.
+	 *
+	 * Any such elements will need to be 'cleared' ahead of
+	 * returning the resources to the HCAPI RM.
+	 *
+	 * RM will signal FW to flush the DB resources. FW will
+	 * perform the invalidation. TF Session close will return the
+	 * previous allocated elements to the RM and then close the
+	 * HCAPI RM registration. That then saves several 'free' msgs
+	 * from being required.
 	 */
+
 	rm_db = (struct tf_rm_new_db *)parms->rm_db;
+
+	/* Check for residuals that the client didn't clean up */
+	rc = tf_rm_check_residuals(rm_db,
+				   &resv_size,
+				   &resv,
+				   &residuals_found);
+	if (rc)
+		return rc;
+
+	/* Invalidate any residuals followed by a DB traversal for
+	 * pool cleanup.
+	 */
+	if (residuals_found) {
+		rc = tf_msg_session_resc_flush(tfp,
+					       parms->dir,
+					       resv_size,
+					       resv);
+		tfp_free((void *)resv);
+		/* On failure we still have to cleanup so we can only
+		 * log that FW failed.
+		 */
+		if (rc)
+			TFP_DRV_LOG(ERR,
+				    "%s: Internal Flush error, module:%s\n",
+				    tf_dir_2_str(parms->dir),
+				    tf_device_module_type_2_str(rm_db->type));
+	}
+
 	for (i = 0; i < rm_db->num_entries; i++)
 		tfp_free((void *)rm_db->db[i].pool);
 
@@ -417,7 +644,7 @@ tf_rm_free_db(struct tf *tfp __rte_unused,
 int
 tf_rm_allocate(struct tf_rm_allocate_parms *parms)
 {
-	int rc = 0;
+	int rc;
 	int id;
 	uint32_t index;
 	struct tf_rm_new_db *rm_db;
@@ -446,11 +673,12 @@ tf_rm_allocate(struct tf_rm_allocate_parms *parms)
 
 	id = ba_alloc(rm_db->db[parms->db_index].pool);
 	if (id == BA_FAIL) {
+		rc = -ENOMEM;
 		TFP_DRV_LOG(ERR,
 			    "%s: Allocation failed, rc:%s\n",
 			    tf_dir_2_str(rm_db->dir),
 			    strerror(-rc));
-		return -ENOMEM;
+		return rc;
 	}
 
 	/* Adjust for any non zero start value */
@@ -475,7 +703,7 @@ tf_rm_allocate(struct tf_rm_allocate_parms *parms)
 int
 tf_rm_free(struct tf_rm_free_parms *parms)
 {
-	int rc = 0;
+	int rc;
 	uint32_t adj_index;
 	struct tf_rm_new_db *rm_db;
 	enum tf_rm_elem_cfg_type cfg_type;
@@ -521,7 +749,7 @@ tf_rm_free(struct tf_rm_free_parms *parms)
 int
 tf_rm_is_allocated(struct tf_rm_is_allocated_parms *parms)
 {
-	int rc = 0;
+	int rc;
 	uint32_t adj_index;
 	struct tf_rm_new_db *rm_db;
 	enum tf_rm_elem_cfg_type cfg_type;
@@ -565,7 +793,6 @@ tf_rm_is_allocated(struct tf_rm_is_allocated_parms *parms)
 int
 tf_rm_get_info(struct tf_rm_get_alloc_info_parms *parms)
 {
-	int rc = 0;
 	struct tf_rm_new_db *rm_db;
 	enum tf_rm_elem_cfg_type cfg_type;
 
@@ -579,15 +806,16 @@ tf_rm_get_info(struct tf_rm_get_alloc_info_parms *parms)
 	    cfg_type != TF_RM_ELEM_CFG_PRIVATE)
 		return -ENOTSUP;
 
-	parms->info = &rm_db->db[parms->db_index].alloc;
+	memcpy(parms->info,
+	       &rm_db->db[parms->db_index].alloc,
+	       sizeof(struct tf_rm_alloc_info));
 
-	return rc;
+	return 0;
 }
 
 int
 tf_rm_get_hcapi_type(struct tf_rm_get_hcapi_parms *parms)
 {
-	int rc = 0;
 	struct tf_rm_new_db *rm_db;
 	enum tf_rm_elem_cfg_type cfg_type;
 
@@ -603,5 +831,36 @@ tf_rm_get_hcapi_type(struct tf_rm_get_hcapi_parms *parms)
 
 	*parms->hcapi_type = rm_db->db[parms->db_index].hcapi_type;
 
+	return 0;
+}
+
+int
+tf_rm_get_inuse_count(struct tf_rm_get_inuse_count_parms *parms)
+{
+	int rc = 0;
+	struct tf_rm_new_db *rm_db;
+	enum tf_rm_elem_cfg_type cfg_type;
+
+	TF_CHECK_PARMS2(parms, parms->rm_db);
+
+	rm_db = (struct tf_rm_new_db *)parms->rm_db;
+	cfg_type = rm_db->db[parms->db_index].cfg_type;
+
+	/* Bail out if not controlled by RM */
+	if (cfg_type != TF_RM_ELEM_CFG_HCAPI &&
+	    cfg_type != TF_RM_ELEM_CFG_PRIVATE)
+		return -ENOTSUP;
+
+	/* Bail silently (no logging), if the pool is not valid there
+	 * was no elements allocated for it.
+	 */
+	if (rm_db->db[parms->db_index].pool == NULL) {
+		*parms->count = 0;
+		return 0;
+	}
+
+	*parms->count = ba_inuse_count(rm_db->db[parms->db_index].pool);
+
 	return rc;
+
 }
diff --git a/drivers/net/bnxt/tf_core/tf_rm_new.h b/drivers/net/bnxt/tf_core/tf_rm_new.h
index ebf38c411..a40296ed2 100644
--- a/drivers/net/bnxt/tf_core/tf_rm_new.h
+++ b/drivers/net/bnxt/tf_core/tf_rm_new.h
@@ -8,6 +8,7 @@
 
 #include "tf_core.h"
 #include "bitalloc.h"
+#include "tf_device.h"
 
 struct tf;
 
@@ -57,9 +58,9 @@ struct tf_rm_new_entry {
 enum tf_rm_elem_cfg_type {
 	/** No configuration */
 	TF_RM_ELEM_CFG_NULL,
-	/** HCAPI 'controlled' */
+	/** HCAPI 'controlled', uses a Pool for internal storage */
 	TF_RM_ELEM_CFG_HCAPI,
-	/** Private thus not HCAPI 'controlled' */
+	/** Private thus not HCAPI 'controlled', creates a Pool for storage */
 	TF_RM_ELEM_CFG_PRIVATE,
 	/**
 	 * Shared element thus it belongs to a shared FW Session and
@@ -123,7 +124,11 @@ struct tf_rm_alloc_info {
  */
 struct tf_rm_create_db_parms {
 	/**
-	 * [in] Receive or transmit direction
+	 * [in] Device module type. Used for logging purposes.
+	 */
+	enum tf_device_module_type type;
+	/**
+	 * [in] Receive or transmit direction.
 	 */
 	enum tf_dir dir;
 	/**
@@ -263,6 +268,25 @@ struct tf_rm_get_hcapi_parms {
 	uint16_t *hcapi_type;
 };
 
+/**
+ * Get InUse count parameters for single element
+ */
+struct tf_rm_get_inuse_count_parms {
+	/**
+	 * [in] RM DB Handle
+	 */
+	void *rm_db;
+	/**
+	 * [in] DB Index, indicates which DB entry to perform the
+	 * action on.
+	 */
+	uint16_t db_index;
+	/**
+	 * [out] Pointer to the inuse count for the specified db_index
+	 */
+	uint16_t *count;
+};
+
 /**
  * @page rm Resource Manager
  *
@@ -279,6 +303,8 @@ struct tf_rm_get_hcapi_parms {
  * @ref tf_rm_get_info
  *
  * @ref tf_rm_get_hcapi_type
+ *
+ * @ref tf_rm_get_inuse_count
  */
 
 /**
@@ -396,4 +422,17 @@ int tf_rm_get_info(struct tf_rm_get_alloc_info_parms *parms);
  */
 int tf_rm_get_hcapi_type(struct tf_rm_get_hcapi_parms *parms);
 
+/**
+ * Performs a lookup in the Resource Manager DB and retrives the
+ * requested HCAPI RM type inuse count.
+ *
+ * [in] parms
+ *   Pointer to get inuse parameters
+ *
+ * Returns
+ *   - (0) if successful.
+ *   - (-EINVAL) on failure.
+ */
+int tf_rm_get_inuse_count(struct tf_rm_get_inuse_count_parms *parms);
+
 #endif /* TF_RM_NEW_H_ */
diff --git a/drivers/net/bnxt/tf_core/tf_session.c b/drivers/net/bnxt/tf_core/tf_session.c
index 3a602618c..b08d06306 100644
--- a/drivers/net/bnxt/tf_core/tf_session.c
+++ b/drivers/net/bnxt/tf_core/tf_session.c
@@ -91,11 +91,11 @@ tf_session_open_session(struct tf *tfp,
 		   parms->open_cfg->ctrl_chan_name,
 		   TF_SESSION_NAME_MAX);
 
-	rc = dev_bind(tfp,
-		      parms->open_cfg->device_type,
-		      session->shadow_copy,
-		      &parms->open_cfg->resources,
-		      &session->dev);
+	rc = tf_dev_bind(tfp,
+			 parms->open_cfg->device_type,
+			 session->shadow_copy,
+			 &parms->open_cfg->resources,
+			 &session->dev);
 	/* Logging handled by dev_bind */
 	if (rc)
 		return rc;
@@ -151,6 +151,8 @@ tf_session_close_session(struct tf *tfp,
 		return rc;
 	}
 
+	tfs->ref_count--;
+
 	/* Record the session we're closing so the caller knows the
 	 * details.
 	 */
@@ -164,6 +166,32 @@ tf_session_close_session(struct tf *tfp,
 		return rc;
 	}
 
+	if (tfs->ref_count > 0) {
+		/* In case we're attached only the session client gets
+		 * closed.
+		 */
+		rc = tf_msg_session_close(tfp);
+		if (rc) {
+			/* Log error */
+			TFP_DRV_LOG(ERR,
+				    "FW Session close failed, rc:%s\n",
+				    strerror(-rc));
+		}
+
+		return 0;
+	}
+
+	/* Final cleanup as we're last user of the session */
+
+	/* Unbind the device */
+	rc = tf_dev_unbind(tfp, tfd);
+	if (rc) {
+		/* Log error */
+		TFP_DRV_LOG(ERR,
+			    "Device unbind failed, rc:%s\n",
+			    strerror(-rc));
+	}
+
 	/* In case we're attached only the session client gets closed */
 	rc = tf_msg_session_close(tfp);
 	if (rc) {
@@ -173,23 +201,9 @@ tf_session_close_session(struct tf *tfp,
 			    strerror(-rc));
 	}
 
-	tfs->ref_count--;
-
-	/* Final cleanup as we're last user of the session */
-	if (tfs->ref_count == 0) {
-		/* Unbind the device */
-		rc = dev_unbind(tfp, tfd);
-		if (rc) {
-			/* Log error */
-			TFP_DRV_LOG(ERR,
-				    "Device unbind failed, rc:%s\n",
-				    strerror(-rc));
-		}
-
-		tfp_free(tfp->session->core_data);
-		tfp_free(tfp->session);
-		tfp->session = NULL;
-	}
+	tfp_free(tfp->session->core_data);
+	tfp_free(tfp->session);
+	tfp->session = NULL;
 
 	return 0;
 }
diff --git a/drivers/net/bnxt/tf_core/tf_tbl_type.c b/drivers/net/bnxt/tf_core/tf_tbl_type.c
index 51f8f0740..bdf7d2089 100644
--- a/drivers/net/bnxt/tf_core/tf_tbl_type.c
+++ b/drivers/net/bnxt/tf_core/tf_tbl_type.c
@@ -51,11 +51,12 @@ tf_tbl_bind(struct tf *tfp,
 	}
 
 	db_cfg.num_elements = parms->num_elements;
+	db_cfg.type = TF_DEVICE_MODULE_TYPE_TABLE;
+	db_cfg.num_elements = parms->num_elements;
+	db_cfg.cfg = parms->cfg;
 
 	for (i = 0; i < TF_DIR_MAX; i++) {
 		db_cfg.dir = i;
-		db_cfg.num_elements = parms->num_elements;
-		db_cfg.cfg = parms->cfg;
 		db_cfg.alloc_cnt = parms->resources->tbl_cnt[i].cnt;
 		db_cfg.rm_db = &tbl_db[i];
 		rc = tf_rm_create_db(tfp, &db_cfg);
diff --git a/drivers/net/bnxt/tf_core/tf_tcam.c b/drivers/net/bnxt/tf_core/tf_tcam.c
index e0fac31f2..2f4441de8 100644
--- a/drivers/net/bnxt/tf_core/tf_tcam.c
+++ b/drivers/net/bnxt/tf_core/tf_tcam.c
@@ -54,11 +54,12 @@ tf_tcam_bind(struct tf *tfp,
 	}
 
 	db_cfg.num_elements = parms->num_elements;
+	db_cfg.type = TF_DEVICE_MODULE_TYPE_TCAM;
+	db_cfg.num_elements = parms->num_elements;
+	db_cfg.cfg = parms->cfg;
 
 	for (i = 0; i < TF_DIR_MAX; i++) {
 		db_cfg.dir = i;
-		db_cfg.num_elements = parms->num_elements;
-		db_cfg.cfg = parms->cfg;
 		db_cfg.alloc_cnt = parms->resources->tcam_cnt[i].cnt;
 		db_cfg.rm_db = &tcam_db[i];
 		rc = tf_rm_create_db(tfp, &db_cfg);
diff --git a/drivers/net/bnxt/tf_core/tf_tcam.h b/drivers/net/bnxt/tf_core/tf_tcam.h
index 67c3bcb49..5090dfd9f 100644
--- a/drivers/net/bnxt/tf_core/tf_tcam.h
+++ b/drivers/net/bnxt/tf_core/tf_tcam.h
@@ -146,6 +146,10 @@ struct tf_tcam_set_parms {
 	 * [in] Type of object to set
 	 */
 	enum tf_tcam_tbl_type type;
+	/**
+	 * [in] Type of HCAPI
+	 */
+	uint16_t hcapi_type;
 	/**
 	 * [in] Entry index to write to
 	 */
diff --git a/drivers/net/bnxt/tf_core/tf_util.c b/drivers/net/bnxt/tf_core/tf_util.c
index a9010543d..16c43eb67 100644
--- a/drivers/net/bnxt/tf_core/tf_util.c
+++ b/drivers/net/bnxt/tf_core/tf_util.c
@@ -7,8 +7,8 @@
 
 #include "tf_util.h"
 
-const char
-*tf_dir_2_str(enum tf_dir dir)
+const char *
+tf_dir_2_str(enum tf_dir dir)
 {
 	switch (dir) {
 	case TF_DIR_RX:
@@ -20,8 +20,8 @@ const char
 	}
 }
 
-const char
-*tf_ident_2_str(enum tf_identifier_type id_type)
+const char *
+tf_ident_2_str(enum tf_identifier_type id_type)
 {
 	switch (id_type) {
 	case TF_IDENT_TYPE_L2_CTXT:
@@ -39,8 +39,8 @@ const char
 	}
 }
 
-const char
-*tf_tcam_tbl_2_str(enum tf_tcam_tbl_type tcam_type)
+const char *
+tf_tcam_tbl_2_str(enum tf_tcam_tbl_type tcam_type)
 {
 	switch (tcam_type) {
 	case TF_TCAM_TBL_TYPE_L2_CTXT_TCAM:
@@ -60,8 +60,8 @@ const char
 	}
 }
 
-const char
-*tf_tbl_type_2_str(enum tf_tbl_type tbl_type)
+const char *
+tf_tbl_type_2_str(enum tf_tbl_type tbl_type)
 {
 	switch (tbl_type) {
 	case TF_TBL_TYPE_FULL_ACT_RECORD:
@@ -131,8 +131,8 @@ const char
 	}
 }
 
-const char
-*tf_em_tbl_type_2_str(enum tf_em_tbl_type em_type)
+const char *
+tf_em_tbl_type_2_str(enum tf_em_tbl_type em_type)
 {
 	switch (em_type) {
 	case TF_EM_TBL_TYPE_EM_RECORD:
@@ -143,3 +143,38 @@ const char
 		return "Invalid EM type";
 	}
 }
+
+const char *
+tf_device_module_type_subtype_2_str(enum tf_device_module_type dm_type,
+				    uint16_t mod_type)
+{
+	switch (dm_type) {
+	case TF_DEVICE_MODULE_TYPE_IDENTIFIER:
+		return tf_ident_2_str(mod_type);
+	case TF_DEVICE_MODULE_TYPE_TABLE:
+		return tf_tcam_tbl_2_str(mod_type);
+	case TF_DEVICE_MODULE_TYPE_TCAM:
+		return tf_tbl_type_2_str(mod_type);
+	case TF_DEVICE_MODULE_TYPE_EM:
+		return tf_em_tbl_type_2_str(mod_type);
+	default:
+		return "Invalid Device Module type";
+	}
+}
+
+const char *
+tf_device_module_type_2_str(enum tf_device_module_type dm_type)
+{
+	switch (dm_type) {
+	case TF_DEVICE_MODULE_TYPE_IDENTIFIER:
+		return "Identifer";
+	case TF_DEVICE_MODULE_TYPE_TABLE:
+		return "Table";
+	case TF_DEVICE_MODULE_TYPE_TCAM:
+		return "TCAM";
+	case TF_DEVICE_MODULE_TYPE_EM:
+		return "EM";
+	default:
+		return "Invalid Device Module type";
+	}
+}
diff --git a/drivers/net/bnxt/tf_core/tf_util.h b/drivers/net/bnxt/tf_core/tf_util.h
index ad8edaf30..c97e2a66a 100644
--- a/drivers/net/bnxt/tf_core/tf_util.h
+++ b/drivers/net/bnxt/tf_core/tf_util.h
@@ -7,6 +7,7 @@
 #define _TF_UTIL_H_
 
 #include "tf_core.h"
+#include "tf_device.h"
 
 /**
  * Helper function converting direction to text string
@@ -63,4 +64,35 @@ const char *tf_tbl_type_2_str(enum tf_tbl_type tbl_type);
  */
 const char *tf_em_tbl_type_2_str(enum tf_em_tbl_type em_type);
 
+/**
+ * Helper function converting device module type and module type to
+ * text string.
+ *
+ * [in] dm_type
+ *   Device Module type
+ *
+ * [in] mod_type
+ *   Module specific type
+ *
+ * Returns:
+ *   Pointer to a char string holding the string for the EM type
+ */
+const char *tf_device_module_type_subtype_2_str
+					(enum tf_device_module_type dm_type,
+					 uint16_t mod_type);
+
+/**
+ * Helper function converting device module type to text string
+ *
+ * [in] dm_type
+ *   Device Module type
+ *
+ * [in] mod_type
+ *   Module specific type
+ *
+ * Returns:
+ *   Pointer to a char string holding the string for the EM type
+ */
+const char *tf_device_module_type_2_str(enum tf_device_module_type dm_type);
+
 #endif /* _TF_UTIL_H_ */
-- 
2.21.1 (Apple Git-122.3)