DPDK patches and discussions
 help / color / mirror / Atom feed
From: Ajit Khaparde <ajit.khaparde@broadcom.com>
To: dev@dpdk.org
Cc: Kishore Padmanabha <kishore.padmanabha@broadcom.com>,
	Shahaji Bhosle <sbhosle@broadcom.com>
Subject: [dpdk-dev] [PATCH v4 06/15] net/bnxt: add hierarchical flow counters
Date: Sun, 25 Oct 2020 20:56:07 -0700	[thread overview]
Message-ID: <20201026035616.19264-7-ajit.khaparde@broadcom.com> (raw)
In-Reply-To: <20201026035616.19264-1-ajit.khaparde@broadcom.com>

From: Kishore Padmanabha <kishore.padmanabha@broadcom.com>

Add support for hierarchical flow counter accumulation.
In case of hierarchical flows, involving parent and child flows,
the child flow counters are aggregated to get the parent flow counter
information. This should help in cases where one ore more flows
is related to a previously offloaded flow.

Signed-off-by: Kishore Padmanabha <kishore.padmanabha@broadcom.com>
Reviewed-by: Shahaji Bhosle <sbhosle@broadcom.com>
Reviewed-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
---
 drivers/net/bnxt/tf_ulp/ulp_fc_mgr.c          |  92 ++++-
 drivers/net/bnxt/tf_ulp/ulp_fc_mgr.h          |  19 +
 drivers/net/bnxt/tf_ulp/ulp_flow_db.c         | 382 ++++++++++++++----
 drivers/net/bnxt/tf_ulp/ulp_flow_db.h         |  44 ++
 .../net/bnxt/tf_ulp/ulp_template_db_enum.h    |   3 +-
 5 files changed, 447 insertions(+), 93 deletions(-)

diff --git a/drivers/net/bnxt/tf_ulp/ulp_fc_mgr.c b/drivers/net/bnxt/tf_ulp/ulp_fc_mgr.c
index 41736a80df..734b419986 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_fc_mgr.c
+++ b/drivers/net/bnxt/tf_ulp/ulp_fc_mgr.c
@@ -21,13 +21,13 @@ static int
 ulp_fc_mgr_shadow_mem_alloc(struct hw_fc_mem_info *parms, int size)
 {
 	/* Allocate memory*/
-	if (parms == NULL)
+	if (!parms)
 		return -EINVAL;
 
 	parms->mem_va = rte_zmalloc("ulp_fc_info",
 				    RTE_CACHE_LINE_ROUNDUP(size),
 				    4096);
-	if (parms->mem_va == NULL) {
+	if (!parms->mem_va) {
 		BNXT_TF_DBG(ERR, "Allocate failed mem_va\n");
 		return -ENOMEM;
 	}
@@ -149,7 +149,6 @@ ulp_fc_mgr_deinit(struct bnxt_ulp_context *ctxt)
 	for (i = 0; i < TF_DIR_MAX; i++)
 		ulp_fc_mgr_shadow_mem_free(&ulp_fc_info->shadow_hw_tbl[i]);
 
-
 	rte_free(ulp_fc_info);
 
 	/* Safe to ignore on deinit */
@@ -254,7 +253,7 @@ ulp_bulk_get_flow_stats(struct tf *tfp,
 	stats = (uint64_t *)fc_info->shadow_hw_tbl[dir].mem_va;
 	parms.physical_mem_addr = (uintptr_t)fc_info->shadow_hw_tbl[dir].mem_pa;
 
-	if (stats == NULL) {
+	if (!stats) {
 		PMD_DRV_LOG(ERR,
 			    "BULK: Memory not initialized id:0x%x dir:%d\n",
 			    parms.starting_idx, dir);
@@ -274,7 +273,8 @@ ulp_bulk_get_flow_stats(struct tf *tfp,
 		sw_acc_tbl_entry = &fc_info->sw_acc_tbl[dir][i];
 		if (!sw_acc_tbl_entry->valid)
 			continue;
-		sw_acc_tbl_entry->pkt_count += FLOW_CNTR_PKTS(stats[i], dparms);
+		sw_acc_tbl_entry->pkt_count += FLOW_CNTR_PKTS(stats[i],
+							      dparms);
 		sw_acc_tbl_entry->byte_count += FLOW_CNTR_BYTES(stats[i],
 								dparms);
 	}
@@ -282,7 +282,8 @@ ulp_bulk_get_flow_stats(struct tf *tfp,
 	return rc;
 }
 
-static int ulp_get_single_flow_stat(struct tf *tfp,
+static int ulp_get_single_flow_stat(struct bnxt_ulp_context *ctxt,
+				    struct tf *tfp,
 				    struct bnxt_ulp_fc_info *fc_info,
 				    enum tf_dir dir,
 				    uint32_t hw_cntr_id,
@@ -291,7 +292,7 @@ static int ulp_get_single_flow_stat(struct tf *tfp,
 	int rc = 0;
 	struct tf_get_tbl_entry_parms parms = { 0 };
 	enum tf_tbl_type stype = TF_TBL_TYPE_ACT_STATS_64;  /* TBD:Template? */
-	struct sw_acc_counter *sw_acc_tbl_entry = NULL;
+	struct sw_acc_counter *sw_acc_tbl_entry = NULL, *t_sw;
 	uint64_t stats = 0;
 	uint32_t sw_cntr_indx = 0;
 
@@ -318,6 +319,18 @@ static int ulp_get_single_flow_stat(struct tf *tfp,
 	sw_acc_tbl_entry->pkt_count = FLOW_CNTR_PKTS(stats, dparms);
 	sw_acc_tbl_entry->byte_count = FLOW_CNTR_BYTES(stats, dparms);
 
+	/* Update the parent counters if it is child flow */
+	if (sw_acc_tbl_entry->parent_flow_id) {
+		/* Update the parent counters */
+		t_sw = sw_acc_tbl_entry;
+		if (ulp_flow_db_parent_flow_count_update(ctxt,
+							 t_sw->parent_flow_id,
+							 t_sw->pkt_count,
+							 t_sw->byte_count)) {
+			PMD_DRV_LOG(ERR, "Error updating parent counters\n");
+		}
+	}
+
 	return rc;
 }
 
@@ -384,13 +397,17 @@ ulp_fc_mgr_alarm_cb(void *arg)
 			break;
 	}
 	*/
+
+	/* reset the parent accumulation counters before accumulation if any */
+	ulp_flow_db_parent_flow_count_reset(ctxt);
+
 	num_entries = dparms->flow_count_db_entries / 2;
 	for (i = 0; i < TF_DIR_MAX; i++) {
 		for (j = 0; j < num_entries; j++) {
 			if (!ulp_fc_info->sw_acc_tbl[i][j].valid)
 				continue;
 			hw_cntr_id = ulp_fc_info->sw_acc_tbl[i][j].hw_cntr_id;
-			rc = ulp_get_single_flow_stat(tfp, ulp_fc_info, i,
+			rc = ulp_get_single_flow_stat(ctxt, tfp, ulp_fc_info, i,
 						      hw_cntr_id, dparms);
 			if (rc)
 				break;
@@ -573,11 +590,12 @@ int ulp_fc_mgr_query_count_get(struct bnxt_ulp_context *ctxt,
 		     (params.resource_sub_type ==
 		      BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_INT_COUNT ||
 		      params.resource_sub_type ==
-		      BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_EXT_COUNT)) {
+		      BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_EXT_COUNT ||
+		      params.resource_sub_type ==
+		      BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_INT_COUNT_ACC)) {
 			found_cntr_resource = true;
 			break;
 		}
-
 	} while (!rc && nxt_resource_index);
 
 	bnxt_ulp_cntxt_release_fdb_lock(ctxt);
@@ -587,12 +605,12 @@ int ulp_fc_mgr_query_count_get(struct bnxt_ulp_context *ctxt,
 
 	dir = params.direction;
 	hw_cntr_id = params.resource_hndl;
-	sw_cntr_idx = hw_cntr_id -
-		ulp_fc_info->shadow_hw_tbl[dir].start_idx;
-	sw_acc_tbl_entry = &ulp_fc_info->sw_acc_tbl[dir][sw_cntr_idx];
 	if (params.resource_sub_type ==
 			BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_INT_COUNT) {
 		pthread_mutex_lock(&ulp_fc_info->fc_lock);
+		sw_cntr_idx = hw_cntr_id -
+			ulp_fc_info->shadow_hw_tbl[dir].start_idx;
+		sw_acc_tbl_entry = &ulp_fc_info->sw_acc_tbl[dir][sw_cntr_idx];
 		if (sw_acc_tbl_entry->pkt_count) {
 			count->hits_set = 1;
 			count->bytes_set = 1;
@@ -604,6 +622,15 @@ int ulp_fc_mgr_query_count_get(struct bnxt_ulp_context *ctxt,
 			sw_acc_tbl_entry->byte_count = 0;
 		}
 		pthread_mutex_unlock(&ulp_fc_info->fc_lock);
+	} else if (params.resource_sub_type ==
+			BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_INT_COUNT_ACC) {
+		/* Get the stats from the parent child table */
+		ulp_flow_db_parent_flow_count_get(ctxt,
+						  flow_id,
+						  &count->hits,
+						  &count->bytes);
+		count->hits_set = 1;
+		count->bytes_set = 1;
 	} else {
 		/* TBD: Handle External counters */
 		rc = -EINVAL;
@@ -611,3 +638,42 @@ int ulp_fc_mgr_query_count_get(struct bnxt_ulp_context *ctxt,
 
 	return rc;
 }
+
+/*
+ * Set the parent flow if it is SW accumulation counter entry.
+ *
+ * ctxt [in] The ulp context for the flow counter manager
+ *
+ * dir [in] The direction of the flow
+ *
+ * hw_cntr_id [in] The HW flow counter ID
+ *
+ * fid [in] parent flow id
+ *
+ */
+int32_t ulp_fc_mgr_cntr_parent_flow_set(struct bnxt_ulp_context *ctxt,
+					enum tf_dir dir,
+					uint32_t hw_cntr_id,
+					uint32_t fid)
+{
+	struct bnxt_ulp_fc_info *ulp_fc_info;
+	uint32_t sw_cntr_idx;
+	int32_t rc = 0;
+
+	ulp_fc_info = bnxt_ulp_cntxt_ptr2_fc_info_get(ctxt);
+	if (!ulp_fc_info)
+		return -EIO;
+
+	pthread_mutex_lock(&ulp_fc_info->fc_lock);
+	sw_cntr_idx = hw_cntr_id - ulp_fc_info->shadow_hw_tbl[dir].start_idx;
+	if (ulp_fc_info->sw_acc_tbl[dir][sw_cntr_idx].valid) {
+		ulp_fc_info->sw_acc_tbl[dir][sw_cntr_idx].parent_flow_id = fid;
+	} else {
+		BNXT_TF_DBG(ERR, "Failed to set parent flow id %x:%x\n",
+			    hw_cntr_id, fid);
+		rc = -ENOENT;
+	}
+	pthread_mutex_unlock(&ulp_fc_info->fc_lock);
+
+	return rc;
+}
diff --git a/drivers/net/bnxt/tf_ulp/ulp_fc_mgr.h b/drivers/net/bnxt/tf_ulp/ulp_fc_mgr.h
index 0cb880d4bc..de4d3dfe95 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_fc_mgr.h
+++ b/drivers/net/bnxt/tf_ulp/ulp_fc_mgr.h
@@ -26,6 +26,7 @@ struct sw_acc_counter {
 	uint64_t byte_count;
 	bool	valid;
 	uint32_t hw_cntr_id;
+	uint32_t parent_flow_id;
 };
 
 struct hw_fc_mem_info {
@@ -163,4 +164,22 @@ bool ulp_fc_mgr_thread_isstarted(struct bnxt_ulp_context *ctxt);
 int ulp_fc_mgr_query_count_get(struct bnxt_ulp_context *ulp_ctx,
 			       uint32_t flow_id,
 			       struct rte_flow_query_count *count);
+
+/*
+ * Set the parent flow if in the SW accumulator table entry
+ *
+ * ctxt [in] The ulp context for the flow counter manager
+ *
+ * dir [in] The direction of the flow
+ *
+ * hw_cntr_id [in] The HW flow counter ID
+ *
+ * fid [in] parent flow id
+ *
+ */
+int32_t ulp_fc_mgr_cntr_parent_flow_set(struct bnxt_ulp_context *ctxt,
+					enum tf_dir dir,
+					uint32_t hw_cntr_id,
+					uint32_t fid);
+
 #endif /* _ULP_FC_MGR_H_ */
diff --git a/drivers/net/bnxt/tf_ulp/ulp_flow_db.c b/drivers/net/bnxt/tf_ulp/ulp_flow_db.c
index 3be7489083..8780c01cc7 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_flow_db.c
+++ b/drivers/net/bnxt/tf_ulp/ulp_flow_db.c
@@ -1058,12 +1058,12 @@ ulp_flow_db_validate_flow_func(struct bnxt_ulp_context *ulp_ctx,
  * the first match.
  */
 static int32_t
-ulp_flow_db_resource_hndl_get(struct bnxt_ulp_context *ulp_ctx,
-			      enum bnxt_ulp_fdb_type flow_type,
-			      uint32_t flow_id,
-			      uint32_t resource_func,
-			      uint32_t res_subtype,
-			      uint64_t *res_hndl)
+ulp_flow_db_resource_params_get(struct bnxt_ulp_context *ulp_ctx,
+				enum bnxt_ulp_fdb_type flow_type,
+				uint32_t flow_id,
+				uint32_t resource_func,
+				uint32_t res_subtype,
+				struct ulp_flow_db_res_params *params)
 {
 	struct bnxt_ulp_flow_db *flow_db;
 	struct bnxt_ulp_flow_tbl *flow_tbl;
@@ -1076,6 +1076,11 @@ ulp_flow_db_resource_hndl_get(struct bnxt_ulp_context *ulp_ctx,
 		return -EINVAL;
 	}
 
+	if (!params) {
+		BNXT_TF_DBG(ERR, "invalid argument\n");
+		return -EINVAL;
+	}
+
 	if (flow_type > BNXT_ULP_FDB_TYPE_DEFAULT) {
 		BNXT_TF_DBG(ERR, "Invalid flow type\n");
 		return -EINVAL;
@@ -1096,12 +1101,14 @@ ulp_flow_db_resource_hndl_get(struct bnxt_ulp_context *ulp_ctx,
 	}
 	/* Iterate the resource to get the resource handle */
 	res_id =  flow_id;
+	memset(params, 0, sizeof(struct ulp_flow_db_res_params));
 	while (res_id) {
 		fid_res = &flow_tbl->flow_resources[res_id];
 		if (ulp_flow_db_resource_func_get(fid_res) == resource_func) {
 			if (resource_func & ULP_FLOW_DB_RES_FUNC_NEED_LOWER) {
 				if (res_subtype == fid_res->resource_sub_type) {
-					*res_hndl = fid_res->resource_hndl;
+					ulp_flow_db_res_info_to_params(fid_res,
+								       params);
 					return 0;
 				}
 
@@ -1109,7 +1116,8 @@ ulp_flow_db_resource_hndl_get(struct bnxt_ulp_context *ulp_ctx,
 				   BNXT_ULP_RESOURCE_FUNC_EXT_EM_TABLE ||
 				   resource_func ==
 				   BNXT_ULP_RESOURCE_FUNC_INT_EM_TABLE) {
-				*res_hndl = fid_res->resource_em_handle;
+				ulp_flow_db_res_info_to_params(fid_res,
+							       params);
 				return 0;
 			}
 		}
@@ -1134,23 +1142,51 @@ ulp_default_flow_db_cfa_action_get(struct bnxt_ulp_context *ulp_ctx,
 				   uint16_t *cfa_action)
 {
 	uint8_t sub_type = BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_VFR_CFA_ACTION;
-	uint64_t hndl;
+	struct ulp_flow_db_res_params params;
 	int32_t rc;
 
-	rc = ulp_flow_db_resource_hndl_get(ulp_ctx,
-					   BNXT_ULP_FDB_TYPE_DEFAULT,
-					   flow_id,
-					   BNXT_ULP_RESOURCE_FUNC_INDEX_TABLE,
-					   sub_type, &hndl);
+	rc = ulp_flow_db_resource_params_get(ulp_ctx,
+					     BNXT_ULP_FDB_TYPE_DEFAULT,
+					     flow_id,
+					     BNXT_ULP_RESOURCE_FUNC_INDEX_TABLE,
+					     sub_type, &params);
 	if (rc) {
 		BNXT_TF_DBG(ERR, "CFA Action ptr not found for flow id %u\n",
 			    flow_id);
 		return -ENOENT;
 	}
-	*cfa_action = hndl;
+	*cfa_action = params.resource_hndl;
 	return 0;
 }
 
+/* internal validation function for parent flow tbl */
+static struct bnxt_ulp_flow_db *
+ulp_flow_db_parent_arg_validation(struct bnxt_ulp_context *ulp_ctxt,
+				  uint32_t fid)
+{
+	struct bnxt_ulp_flow_db *flow_db;
+
+	flow_db = bnxt_ulp_cntxt_ptr2_flow_db_get(ulp_ctxt);
+	if (!flow_db) {
+		BNXT_TF_DBG(ERR, "Invalid Arguments\n");
+		return NULL;
+	}
+
+	/* check for max flows */
+	if (fid >= flow_db->flow_tbl.num_flows || !fid) {
+		BNXT_TF_DBG(ERR, "Invalid flow index\n");
+		return NULL;
+	}
+
+	/* No support for parent child db then just exit */
+	if (!flow_db->parent_child_db.entries_count) {
+		BNXT_TF_DBG(ERR, "parent child db not supported\n");
+		return NULL;
+	}
+
+	return flow_db;
+}
+
 /*
  * Allocate the entry in the parent-child database
  *
@@ -1167,26 +1203,15 @@ ulp_flow_db_parent_flow_alloc(struct bnxt_ulp_context *ulp_ctxt,
 	struct ulp_fdb_parent_child_db *p_pdb;
 	uint32_t idx, free_idx = 0;
 
-	flow_db = bnxt_ulp_cntxt_ptr2_flow_db_get(ulp_ctxt);
+	/* validate the arguments */
+	flow_db = ulp_flow_db_parent_arg_validation(ulp_ctxt, fid);
 	if (!flow_db) {
-		BNXT_TF_DBG(ERR, "Invalid Arguments\n");
-		return -EINVAL;
-	}
-
-	/* check for max flows */
-	if (fid >= flow_db->flow_tbl.num_flows || !fid) {
-		BNXT_TF_DBG(ERR, "Invalid flow index\n");
-		return -EINVAL;
-	}
-
-	/* No support for parent child db then just exit */
-	if (!flow_db->parent_child_db.entries_count) {
-		BNXT_TF_DBG(ERR, "parent child db not supported\n");
+		BNXT_TF_DBG(ERR, "parent child db validation failed\n");
 		return -EINVAL;
 	}
 
 	p_pdb = &flow_db->parent_child_db;
-	for (idx = 0; idx <= p_pdb->entries_count; idx++) {
+	for (idx = 0; idx < p_pdb->entries_count; idx++) {
 		if (p_pdb->parent_flow_tbl[idx].parent_fid == fid) {
 			BNXT_TF_DBG(ERR, "fid is already allocated\n");
 			return -EINVAL;
@@ -1222,26 +1247,15 @@ ulp_flow_db_parent_flow_free(struct bnxt_ulp_context *ulp_ctxt,
 	struct ulp_fdb_parent_child_db *p_pdb;
 	uint32_t idx;
 
-	flow_db = bnxt_ulp_cntxt_ptr2_flow_db_get(ulp_ctxt);
+	/* validate the arguments */
+	flow_db = ulp_flow_db_parent_arg_validation(ulp_ctxt, fid);
 	if (!flow_db) {
-		BNXT_TF_DBG(ERR, "Invalid Arguments\n");
-		return -EINVAL;
-	}
-
-	/* check for max flows */
-	if (fid >= flow_db->flow_tbl.num_flows || !fid) {
-		BNXT_TF_DBG(ERR, "Invalid flow index\n");
-		return -EINVAL;
-	}
-
-	/* No support for parent child db then just exit */
-	if (!flow_db->parent_child_db.entries_count) {
-		BNXT_TF_DBG(ERR, "parent child db not supported\n");
+		BNXT_TF_DBG(ERR, "parent child db validation failed\n");
 		return -EINVAL;
 	}
 
 	p_pdb = &flow_db->parent_child_db;
-	for (idx = 0; idx <= p_pdb->entries_count; idx++) {
+	for (idx = 0; idx < p_pdb->entries_count; idx++) {
 		if (p_pdb->parent_flow_tbl[idx].parent_fid == fid) {
 			/* free the contents */
 			p_pdb->parent_flow_tbl[idx].parent_fid = 0;
@@ -1275,15 +1289,10 @@ ulp_flow_db_parent_child_flow_set(struct bnxt_ulp_context *ulp_ctxt,
 	uint32_t idx, a_idx;
 	uint64_t *t;
 
-	flow_db = bnxt_ulp_cntxt_ptr2_flow_db_get(ulp_ctxt);
+	/* validate the arguments */
+	flow_db = ulp_flow_db_parent_arg_validation(ulp_ctxt, parent_fid);
 	if (!flow_db) {
-		BNXT_TF_DBG(ERR, "Invalid Arguments\n");
-		return -EINVAL;
-	}
-
-	/* check for fid validity */
-	if (parent_fid >= flow_db->flow_tbl.num_flows || !parent_fid) {
-		BNXT_TF_DBG(ERR, "Invalid parent flow index %x\n", parent_fid);
+		BNXT_TF_DBG(ERR, "parent child db validation failed\n");
 		return -EINVAL;
 	}
 
@@ -1293,15 +1302,9 @@ ulp_flow_db_parent_child_flow_set(struct bnxt_ulp_context *ulp_ctxt,
 		return -EINVAL;
 	}
 
-	/* No support for parent child db then just exit */
-	if (!flow_db->parent_child_db.entries_count) {
-		BNXT_TF_DBG(ERR, "parent child db not supported\n");
-		return -EINVAL;
-	}
-
 	p_pdb = &flow_db->parent_child_db;
 	a_idx = child_fid / ULP_INDEX_BITMAP_SIZE;
-	for (idx = 0; idx <= p_pdb->entries_count; idx++) {
+	for (idx = 0; idx < p_pdb->entries_count; idx++) {
 		if (p_pdb->parent_flow_tbl[idx].parent_fid == parent_fid) {
 			t = p_pdb->parent_flow_tbl[idx].child_fid_bitset;
 			if (set_flag)
@@ -1334,26 +1337,15 @@ ulp_flow_db_parent_flow_idx_get(struct bnxt_ulp_context *ulp_ctxt,
 	struct ulp_fdb_parent_child_db *p_pdb;
 	uint32_t idx;
 
-	flow_db = bnxt_ulp_cntxt_ptr2_flow_db_get(ulp_ctxt);
+	/* validate the arguments */
+	flow_db = ulp_flow_db_parent_arg_validation(ulp_ctxt, parent_fid);
 	if (!flow_db) {
-		BNXT_TF_DBG(ERR, "Invalid Arguments\n");
-		return -EINVAL;
-	}
-
-	/* check for fid validity */
-	if (parent_fid >= flow_db->flow_tbl.num_flows || !parent_fid) {
-		BNXT_TF_DBG(ERR, "Invalid parent flow index %x\n", parent_fid);
-		return -EINVAL;
-	}
-
-	/* No support for parent child db then just exit */
-	if (!flow_db->parent_child_db.entries_count) {
-		BNXT_TF_DBG(ERR, "parent child db not supported\n");
+		BNXT_TF_DBG(ERR, "parent child db validation failed\n");
 		return -EINVAL;
 	}
 
 	p_pdb = &flow_db->parent_child_db;
-	for (idx = 0; idx <= p_pdb->entries_count; idx++) {
+	for (idx = 0; idx < p_pdb->entries_count; idx++) {
 		if (p_pdb->parent_flow_tbl[idx].parent_fid == parent_fid) {
 			*parent_idx = idx;
 			return 0;
@@ -1425,6 +1417,73 @@ ulp_flow_db_parent_child_flow_next_entry_get(struct bnxt_ulp_flow_db *flow_db,
 	return 0;
 }
 
+/*
+ * Set the counter accumulation in the parent flow
+ *
+ * ulp_ctxt [in] Ptr to ulp_context
+ * parent_idx [in] The parent index of the parent flow entry
+ *
+ * returns index on success and negative on failure.
+ */
+static int32_t
+ulp_flow_db_parent_flow_count_accum_set(struct bnxt_ulp_context *ulp_ctxt,
+					uint32_t parent_idx)
+{
+	struct bnxt_ulp_flow_db *flow_db;
+	struct ulp_fdb_parent_child_db *p_pdb;
+
+	flow_db = bnxt_ulp_cntxt_ptr2_flow_db_get(ulp_ctxt);
+	if (!flow_db) {
+		BNXT_TF_DBG(ERR, "Invalid Arguments\n");
+		return -EINVAL;
+	}
+
+	/* check for parent idx validity */
+	p_pdb = &flow_db->parent_child_db;
+	if (parent_idx >= p_pdb->entries_count ||
+	    !p_pdb->parent_flow_tbl[parent_idx].parent_fid) {
+		BNXT_TF_DBG(ERR, "Invalid parent flow index %x\n", parent_idx);
+		return -EINVAL;
+	}
+
+	p_pdb->parent_flow_tbl[parent_idx].counter_acc = 1;
+	return 0;
+}
+
+/*
+ * Get the counter accumulation in the parent flow
+ *
+ * ulp_ctxt [in] Ptr to ulp_context
+ * parent_fid [in] The flow id of the parent flow entry
+ *
+ * returns 0 if counter accum is set else -1.
+ */
+static int32_t
+ulp_flow_db_parent_flow_count_accum_get(struct bnxt_ulp_context *ulp_ctxt,
+					uint32_t parent_fid)
+{
+	struct bnxt_ulp_flow_db *flow_db;
+	struct ulp_fdb_parent_child_db *p_pdb;
+	uint32_t idx;
+
+	/* validate the arguments */
+	flow_db = ulp_flow_db_parent_arg_validation(ulp_ctxt, parent_fid);
+	if (!flow_db) {
+		BNXT_TF_DBG(ERR, "parent child db validation failed\n");
+		return -EINVAL;
+	}
+
+	p_pdb = &flow_db->parent_child_db;
+	for (idx = 0; idx < p_pdb->entries_count; idx++) {
+		if (p_pdb->parent_flow_tbl[idx].parent_fid == parent_fid) {
+			if (p_pdb->parent_flow_tbl[idx].counter_acc)
+				return 0;
+			break;
+		}
+	}
+	return -1;
+}
+
 /*
  * Orphan the child flow entry
  * This is called only for child flows that have
@@ -1498,6 +1557,8 @@ int32_t
 ulp_flow_db_parent_flow_create(struct bnxt_ulp_mapper_parms *parms)
 {
 	struct ulp_flow_db_res_params fid_parms;
+	uint32_t sub_type = BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_INT_COUNT_ACC;
+	struct ulp_flow_db_res_params res_params;
 	int32_t fid_idx;
 
 	/* create the child flow entry in parent flow table */
@@ -1519,6 +1580,22 @@ ulp_flow_db_parent_flow_create(struct bnxt_ulp_mapper_parms *parms)
 			    parms->fid);
 		return -1;
 	}
+
+	/* check of the flow has internal counter accumulation enabled */
+	if (!ulp_flow_db_resource_params_get(parms->ulp_ctx,
+					     BNXT_ULP_FDB_TYPE_REGULAR,
+					     parms->fid,
+					     BNXT_ULP_RESOURCE_FUNC_INDEX_TABLE,
+					     sub_type,
+					     &res_params)) {
+		/* Enable the counter accumulation in parent entry */
+		if (ulp_flow_db_parent_flow_count_accum_set(parms->ulp_ctx,
+							    fid_idx)) {
+			BNXT_TF_DBG(ERR, "Error in setting counter acc %x\n",
+				    parms->fid);
+			return -1;
+		}
+	}
 	return 0;
 }
 
@@ -1533,6 +1610,10 @@ int32_t
 ulp_flow_db_child_flow_create(struct bnxt_ulp_mapper_parms *parms)
 {
 	struct ulp_flow_db_res_params fid_parms;
+	uint32_t sub_type = BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_INT_COUNT;
+	enum bnxt_ulp_resource_func res_fun;
+	struct ulp_flow_db_res_params res_p;
+	uint32_t parent_fid = parms->parent_fid;
 	int32_t rc;
 
 	/* create the parent flow entry in parent flow table */
@@ -1541,7 +1622,7 @@ ulp_flow_db_child_flow_create(struct bnxt_ulp_mapper_parms *parms)
 					       parms->fid, 1);
 	if (rc) {
 		BNXT_TF_DBG(ERR, "Error in setting child fid %x\n", parms->fid);
-		return -1;
+		return rc;
 	}
 
 	/* Add the parent details in the resource list of the flow */
@@ -1549,11 +1630,154 @@ ulp_flow_db_child_flow_create(struct bnxt_ulp_mapper_parms *parms)
 	fid_parms.resource_func	= BNXT_ULP_RESOURCE_FUNC_CHILD_FLOW;
 	fid_parms.resource_hndl	= parms->parent_fid;
 	fid_parms.critical_resource = BNXT_ULP_CRITICAL_RESOURCE_NO;
-	if (ulp_flow_db_resource_add(parms->ulp_ctx, BNXT_ULP_FDB_TYPE_REGULAR,
-				     parms->fid, &fid_parms)) {
+	rc  = ulp_flow_db_resource_add(parms->ulp_ctx,
+				       BNXT_ULP_FDB_TYPE_REGULAR,
+				       parms->fid, &fid_parms);
+	if (rc) {
 		BNXT_TF_DBG(ERR, "Error in adding flow res for fid %x\n",
 			    parms->fid);
-		return -1;
+		return rc;
+	}
+
+	/* check if accumulation count is set for parent flow */
+	rc = ulp_flow_db_parent_flow_count_accum_get(parms->ulp_ctx,
+						     parms->parent_fid);
+	if (!rc) {
+		/* check if internal count action included for this flow.*/
+		res_fun = BNXT_ULP_RESOURCE_FUNC_INDEX_TABLE;
+		rc = ulp_flow_db_resource_params_get(parms->ulp_ctx,
+						     BNXT_ULP_FDB_TYPE_REGULAR,
+						     parms->fid,
+						     res_fun,
+						     sub_type,
+						     &res_p);
+		if (!rc) {
+			/* update the counter manager to include parent fid */
+			if (ulp_fc_mgr_cntr_parent_flow_set(parms->ulp_ctx,
+							    res_p.direction,
+							    res_p.resource_hndl,
+							    parent_fid)) {
+				BNXT_TF_DBG(ERR, "Error in setting child %x\n",
+					    parms->fid);
+				return -1;
+			}
+		}
 	}
+	/* return success */
 	return 0;
 }
+
+/*
+ * Update the parent counters
+ *
+ * ulp_ctxt [in] Ptr to ulp_context
+ * parent_fid [in] The flow id of the parent flow entry
+ * packet_count [in] - packet count
+ * byte_count [in] - byte count
+ *
+ * returns 0 on success
+ */
+int32_t
+ulp_flow_db_parent_flow_count_update(struct bnxt_ulp_context *ulp_ctxt,
+				     uint32_t parent_fid,
+				     uint64_t packet_count,
+				     uint64_t byte_count)
+{
+	struct bnxt_ulp_flow_db *flow_db;
+	struct ulp_fdb_parent_child_db *p_pdb;
+	uint32_t idx;
+
+	/* validate the arguments */
+	flow_db = ulp_flow_db_parent_arg_validation(ulp_ctxt, parent_fid);
+	if (!flow_db) {
+		BNXT_TF_DBG(ERR, "parent child db validation failed\n");
+		return -EINVAL;
+	}
+
+	p_pdb = &flow_db->parent_child_db;
+	for (idx = 0; idx < p_pdb->entries_count; idx++) {
+		if (p_pdb->parent_flow_tbl[idx].parent_fid == parent_fid) {
+			if (p_pdb->parent_flow_tbl[idx].counter_acc) {
+				p_pdb->parent_flow_tbl[idx].pkt_count +=
+					packet_count;
+				p_pdb->parent_flow_tbl[idx].byte_count +=
+					byte_count;
+			}
+			return 0;
+		}
+	}
+	return -ENOENT;
+}
+
+/*
+ * Get the parent accumulation counters
+ *
+ * ulp_ctxt [in] Ptr to ulp_context
+ * parent_fid [in] The flow id of the parent flow entry
+ * packet_count [out] - packet count
+ * byte_count [out] - byte count
+ *
+ * returns 0 on success
+ */
+int32_t
+ulp_flow_db_parent_flow_count_get(struct bnxt_ulp_context *ulp_ctxt,
+				  uint32_t parent_fid,
+				  uint64_t *packet_count,
+				  uint64_t *byte_count)
+{
+	struct bnxt_ulp_flow_db *flow_db;
+	struct ulp_fdb_parent_child_db *p_pdb;
+	uint32_t idx;
+
+	/* validate the arguments */
+	flow_db = ulp_flow_db_parent_arg_validation(ulp_ctxt, parent_fid);
+	if (!flow_db) {
+		BNXT_TF_DBG(ERR, "parent child db validation failed\n");
+		return -EINVAL;
+	}
+
+	p_pdb = &flow_db->parent_child_db;
+	for (idx = 0; idx < p_pdb->entries_count; idx++) {
+		if (p_pdb->parent_flow_tbl[idx].parent_fid == parent_fid) {
+			if (p_pdb->parent_flow_tbl[idx].counter_acc) {
+				*packet_count =
+					p_pdb->parent_flow_tbl[idx].pkt_count;
+				*byte_count =
+					p_pdb->parent_flow_tbl[idx].byte_count;
+			}
+			return 0;
+		}
+	}
+	return -ENOENT;
+}
+
+/*
+ * reset the parent accumulation counters
+ *
+ * ulp_ctxt [in] Ptr to ulp_context
+ *
+ * returns none
+ */
+void
+ulp_flow_db_parent_flow_count_reset(struct bnxt_ulp_context *ulp_ctxt)
+{
+	struct bnxt_ulp_flow_db *flow_db;
+	struct ulp_fdb_parent_child_db *p_pdb;
+	uint32_t idx;
+
+	/* validate the arguments */
+	flow_db = ulp_flow_db_parent_arg_validation(ulp_ctxt, 1);
+	if (!flow_db) {
+		BNXT_TF_DBG(ERR, "parent child db validation failed\n");
+		return;
+	}
+
+	p_pdb = &flow_db->parent_child_db;
+	for (idx = 0; idx < p_pdb->entries_count; idx++) {
+		if (p_pdb->parent_flow_tbl[idx].parent_fid &&
+		    p_pdb->parent_flow_tbl[idx].counter_acc) {
+			p_pdb->parent_flow_tbl[idx].pkt_count = 0;
+			p_pdb->parent_flow_tbl[idx].byte_count = 0;
+		}
+	}
+}
diff --git a/drivers/net/bnxt/tf_ulp/ulp_flow_db.h b/drivers/net/bnxt/tf_ulp/ulp_flow_db.h
index 95fd1992d6..10e69bae45 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_flow_db.h
+++ b/drivers/net/bnxt/tf_ulp/ulp_flow_db.h
@@ -56,6 +56,9 @@ struct bnxt_ulp_flow_tbl {
 /* Structure to maintain parent-child flow relationships */
 struct ulp_fdb_parent_info {
 	uint32_t	parent_fid;
+	uint32_t	counter_acc;
+	uint64_t	pkt_count;
+	uint64_t	byte_count;
 	uint64_t	*child_fid_bitset;
 };
 
@@ -356,4 +359,45 @@ ulp_flow_db_parent_flow_create(struct bnxt_ulp_mapper_parms *parms);
 int32_t
 ulp_flow_db_child_flow_create(struct bnxt_ulp_mapper_parms *parms);
 
+/*
+ * Update the parent counters
+ *
+ * ulp_ctxt [in] Ptr to ulp_context
+ * parent_fid [in] The flow id of the parent flow entry
+ * packet_count [in] - packet count
+ * byte_count [in] - byte count
+ *
+ * returns 0 on success
+ */
+int32_t
+ulp_flow_db_parent_flow_count_update(struct bnxt_ulp_context *ulp_ctxt,
+				     uint32_t parent_fid,
+				     uint64_t packet_count,
+				     uint64_t byte_count);
+/*
+ * Get the parent accumulation counters
+ *
+ * ulp_ctxt [in] Ptr to ulp_context
+ * parent_fid [in] The flow id of the parent flow entry
+ * packet_count [out] - packet count
+ * byte_count [out] - byte count
+ *
+ * returns 0 on success
+ */
+int32_t
+ulp_flow_db_parent_flow_count_get(struct bnxt_ulp_context *ulp_ctxt,
+				  uint32_t parent_fid,
+				  uint64_t *packet_count,
+				  uint64_t *byte_count);
+
+/*
+ * reset the parent accumulation counters
+ *
+ * ulp_ctxt [in] Ptr to ulp_context
+ *
+ * returns none
+ */
+void
+ulp_flow_db_parent_flow_count_reset(struct bnxt_ulp_context *ulp_ctxt);
+
 #endif /* _ULP_FLOW_DB_H_ */
diff --git a/drivers/net/bnxt/tf_ulp/ulp_template_db_enum.h b/drivers/net/bnxt/tf_ulp/ulp_template_db_enum.h
index 168e308c2b..6dade9afdb 100644
--- a/drivers/net/bnxt/tf_ulp/ulp_template_db_enum.h
+++ b/drivers/net/bnxt/tf_ulp/ulp_template_db_enum.h
@@ -332,7 +332,8 @@ enum bnxt_ulp_resource_sub_type {
 	BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_NORMAL = 0,
 	BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_VFR_CFA_ACTION = 1,
 	BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_INT_COUNT = 2,
-	BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_EXT_COUNT = 3,
+	BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_INT_COUNT_ACC = 3,
+	BNXT_ULP_RESOURCE_SUB_TYPE_INDEX_TYPE_EXT_COUNT = 4,
 	BNXT_ULP_RESOURCE_SUB_TYPE_CACHE_TYPE_L2_CNTXT_TCAM = 0,
 	BNXT_ULP_RESOURCE_SUB_TYPE_CACHE_TYPE_PROFILE_TCAM = 1
 };
-- 
2.21.1 (Apple Git-122.3)


  parent reply	other threads:[~2020-10-26  3:58 UTC|newest]

Thread overview: 64+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-10-17  6:27 [dpdk-dev] [PATCH 00/14] bnxt patches Venkat Duvvuru
2020-10-17  6:27 ` [dpdk-dev] [PATCH 01/14] net/bnxt: device cleanup of FW Venkat Duvvuru
2020-10-17  6:27 ` [dpdk-dev] [PATCH 02/14] net/bnxt: add stingray support Venkat Duvvuru
2020-10-17  6:27 ` [dpdk-dev] [PATCH 03/14] net/bnxt: changes to support 2 table scopes Venkat Duvvuru
2020-10-17  6:27 ` [dpdk-dev] [PATCH 04/14] net/bnxt: map table scope API Venkat Duvvuru
2020-10-17  6:28 ` [dpdk-dev] [PATCH 05/14] net/bnxt: table scope to PF Mapping for SR and Wh+ Venkat Duvvuru
2020-10-17  6:28 ` [dpdk-dev] [PATCH 06/14] net/bnxt: add build option for EM slot allocation Venkat Duvvuru
2020-10-17  6:28 ` [dpdk-dev] [PATCH 07/14] net/bnxt: update SR ULP resource counts Venkat Duvvuru
2020-10-17  6:28 ` [dpdk-dev] [PATCH 08/14] net/bnxt: fix infinite loop in flow query count API Venkat Duvvuru
2020-10-17  6:28 ` [dpdk-dev] [PATCH 09/14] net/bnxt: add support for parent flow accumulation counters Venkat Duvvuru
2020-10-17  6:28 ` [dpdk-dev] [PATCH 10/14] net/bnxt: use cfa pair alloc for configuring reps Venkat Duvvuru
2020-10-17  6:28 ` [dpdk-dev] [PATCH 11/14] net/bnxt: add mapper support for wildcard TCAM entry Venkat Duvvuru
2020-10-17  6:28 ` [dpdk-dev] [PATCH 12/14] net/bnxt: refactor flow id allocation Venkat Duvvuru
2020-10-17  6:28 ` [dpdk-dev] [PATCH 13/14] net/bnxt: add support for VXLAN decap templates Venkat Duvvuru
2020-10-17  6:28 ` [dpdk-dev] [PATCH 14/14] net/bnxt: add VXLAN decap offload support Venkat Duvvuru
2020-10-20 21:55 ` [dpdk-dev] [PATCH v2 00/11] bnxt fixes and enhancements to TRUFLOW support Ajit Khaparde
2020-10-20 21:55   ` [dpdk-dev] [PATCH v2 01/11] net/bnxt: add stingray support to core layer Ajit Khaparde
2020-10-21 18:07     ` Ferruh Yigit
2020-10-21 18:11       ` Ajit Khaparde
2020-10-22  9:11         ` Ferruh Yigit
2020-10-23  5:10           ` Ajit Khaparde
2020-10-20 21:55   ` [dpdk-dev] [PATCH v2 02/11] net/bnxt: changes to support two table scopes Ajit Khaparde
2020-10-20 21:55   ` [dpdk-dev] [PATCH v2 03/11] net/bnxt: add table scope to PF Mapping Ajit Khaparde
2020-10-20 21:55   ` [dpdk-dev] [PATCH v2 04/11] net/bnxt: update ULP resource counts Ajit Khaparde
2020-10-20 21:55   ` [dpdk-dev] [PATCH v2 05/11] net/bnxt: fix infinite loop in flow query count Ajit Khaparde
2020-10-20 21:55   ` [dpdk-dev] [PATCH v2 06/11] net/bnxt: add support for flow counter accumulation Ajit Khaparde
2020-10-20 21:55   ` [dpdk-dev] [PATCH v2 07/11] net/bnxt: change HWRM command to create reps Ajit Khaparde
2020-10-20 21:55   ` [dpdk-dev] [PATCH v2 08/11] net/bnxt: add mapper support for wildcard TCAM Ajit Khaparde
2020-10-20 21:55   ` [dpdk-dev] [PATCH v2 09/11] net/bnxt: refactor flow id allocation Ajit Khaparde
2020-10-20 21:55   ` [dpdk-dev] [PATCH v2 10/11] net/bnxt: add support for VXLAN decap templates Ajit Khaparde
2020-10-20 21:55   ` [dpdk-dev] [PATCH v2 11/11] net/bnxt: add VXLAN decap offload support Ajit Khaparde
2020-10-21  5:31   ` [dpdk-dev] [PATCH v2 00/11] bnxt fixes and enhancements to TRUFLOW support Ajit Khaparde
2020-10-22 22:05   ` [dpdk-dev] [PATCH v3 " Ajit Khaparde
2020-10-23  5:08     ` Ajit Khaparde
2020-10-26  3:56       ` [dpdk-dev] [PATCH v4 00/15] bnxt fixes and enhancements Ajit Khaparde
2020-10-26  3:56         ` [dpdk-dev] [PATCH v4 01/15] net/bnxt: add stingray support to core layer Ajit Khaparde
2020-10-26  3:56         ` [dpdk-dev] [PATCH v4 02/15] net/bnxt: support two table scopes Ajit Khaparde
2020-10-26  3:56         ` [dpdk-dev] [PATCH v4 03/15] net/bnxt: add table scope to PF Mapping Ajit Khaparde
2020-10-26  3:56         ` [dpdk-dev] [PATCH v4 04/15] net/bnxt: update ULP resource counts Ajit Khaparde
2020-10-26  3:56         ` [dpdk-dev] [PATCH v4 05/15] net/bnxt: fix flow query count Ajit Khaparde
2020-10-26  3:56         ` Ajit Khaparde [this message]
2020-10-26  3:56         ` [dpdk-dev] [PATCH v4 07/15] net/bnxt: modify HWRM command to create reps Ajit Khaparde
2020-10-26  3:56         ` [dpdk-dev] [PATCH v4 08/15] net/bnxt: add mapper support for wildcard TCAM Ajit Khaparde
2020-10-26  3:56         ` [dpdk-dev] [PATCH v4 09/15] net/bnxt: refactor flow id allocation Ajit Khaparde
2020-10-26  3:56         ` [dpdk-dev] [PATCH v4 10/15] net/bnxt: add VXLAN decap templates Ajit Khaparde
2020-10-26  3:56         ` [dpdk-dev] [PATCH v4 11/15] net/bnxt: add VXLAN decap offload support Ajit Khaparde
2020-10-26  3:56         ` [dpdk-dev] [PATCH v4 12/15] net/bnxt: increase the size of Rx CQ Ajit Khaparde
2020-10-26  3:56         ` [dpdk-dev] [PATCH v4 13/15] net/bnxt: fix to reset mbuf data offset Ajit Khaparde
2020-10-26  3:56         ` [dpdk-dev] [PATCH v4 14/15] net/bnxt: set thread safe flow ops flag Ajit Khaparde
2020-10-26  3:56         ` [dpdk-dev] [PATCH v4 15/15] net/bnxt: fix Rx performance by removing spinlock Ajit Khaparde
2020-10-26 17:42         ` [dpdk-dev] [PATCH v4 00/15] bnxt fixes and enhancements Ajit Khaparde
2020-10-22 22:05   ` [dpdk-dev] [PATCH v3 01/11] net/bnxt: add stingray support to core layer Ajit Khaparde
2020-10-23 10:54     ` Ferruh Yigit
2020-10-23 16:32       ` Ajit Khaparde
2020-10-22 22:05   ` [dpdk-dev] [PATCH v3 02/11] net/bnxt: changes to support two table scopes Ajit Khaparde
2020-10-22 22:05   ` [dpdk-dev] [PATCH v3 03/11] net/bnxt: add table scope to PF Mapping Ajit Khaparde
2020-10-22 22:05   ` [dpdk-dev] [PATCH v3 04/11] net/bnxt: update ULP resource counts Ajit Khaparde
2020-10-22 22:05   ` [dpdk-dev] [PATCH v3 05/11] net/bnxt: fix infinite loop in flow query count Ajit Khaparde
2020-10-22 22:05   ` [dpdk-dev] [PATCH v3 06/11] net/bnxt: add support for flow counter accumulation Ajit Khaparde
2020-10-22 22:05   ` [dpdk-dev] [PATCH v3 07/11] net/bnxt: change HWRM command to create reps Ajit Khaparde
2020-10-22 22:05   ` [dpdk-dev] [PATCH v3 08/11] net/bnxt: add mapper support for wildcard TCAM Ajit Khaparde
2020-10-22 22:05   ` [dpdk-dev] [PATCH v3 09/11] net/bnxt: refactor flow id allocation Ajit Khaparde
2020-10-22 22:05   ` [dpdk-dev] [PATCH v3 10/11] net/bnxt: add support for VXLAN decap templates Ajit Khaparde
2020-10-22 22:05   ` [dpdk-dev] [PATCH v3 11/11] net/bnxt: add VXLAN decap offload support Ajit Khaparde

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20201026035616.19264-7-ajit.khaparde@broadcom.com \
    --to=ajit.khaparde@broadcom.com \
    --cc=dev@dpdk.org \
    --cc=kishore.padmanabha@broadcom.com \
    --cc=sbhosle@broadcom.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).