[PATCH] net/mlx5: fix the unneeded stub table allocation

[PATCH] net/mlx5: fix the unneeded stub table allocation

[Bing Zhao]

The HWS non-template flow API is reusing some implementation of
template API to unifiy code logic. So for each rule creation, a stub
/ temporary table is used in order to reuse the actions construction.

Since this is temporary and used only internally, there is no need to
save the table permanently. Only parts of them are mandatory, so the
allocation / free from the heap of RTE memory is a waste and causes
a lot of overhead. By using the pre-allocated workspace and set the
needed fields expliticly will save the overhead and help to speed up
the rule insertion rate.

Fixes: 27d171b88031 ("net/mlx5: abstract flow action and enable reconfigure")
Cc: mkashani@nvidia.com

Signed-off-by: Bing Zhao <bingz@nvidia.com>
---
 drivers/net/mlx5/mlx5_flow.c    | 11 +++++++++--
 drivers/net/mlx5/mlx5_flow.h    |  3 +++
 drivers/net/mlx5/mlx5_flow_hw.c | 19 +++++++++++--------
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 16ddd05448..9203643300 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -8270,14 +8270,21 @@ flow_alloc_thread_workspace(void)
 {
 	size_t data_size = RTE_ALIGN(sizeof(struct mlx5_flow_workspace), sizeof(long));
 	size_t rss_queue_array_size = sizeof(uint16_t) * RTE_ETH_RSS_RETA_SIZE_512;
-	struct mlx5_flow_workspace *data = calloc(1, data_size +
-						     rss_queue_array_size);
+	size_t alloc_size = data_size + rss_queue_array_size;
+#ifdef HAVE_MLX5_HWS_SUPPORT
+	/* Dummy table size for the non-template API. */
+	alloc_size += sizeof(struct rte_flow_template_table);
+#endif
+	struct mlx5_flow_workspace *data = calloc(1, alloc_size);
 
 	if (!data) {
 		DRV_LOG(ERR, "Failed to allocate flow workspace memory.");
 		return NULL;
 	}
 	data->rss_desc.queue = RTE_PTR_ADD(data, data_size);
+#ifdef HAVE_MLX5_HWS_SUPPORT
+	data->table = RTE_PTR_ADD(data->rss_desc.queue, rss_queue_array_size);
+#endif
 	return data;
 }
 
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index bcc2782460..757bbf73c1 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1919,6 +1919,9 @@ struct mlx5_flow_workspace {
 	/* The meter policy used by meter in flow. */
 	struct mlx5_flow_meter_policy *final_policy;
 	/* The final policy when meter policy is hierarchy. */
+#ifdef HAVE_MLX5_HWS_SUPPORT
+	struct rte_flow_template_table *table;
+#endif
 	uint32_t skip_matcher_reg:1;
 	/* Indicates if need to skip matcher register in translate. */
 	uint32_t mark:1; /* Indicates if flow contains mark action. */
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index 7233ac46c4..f383d87337 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -13517,7 +13517,6 @@ flow_hw_translate_flow_actions(struct rte_eth_dev *dev,
 	int ret = 0;
 	uint32_t src_group = 0;
 	enum mlx5dr_table_type table_type;
-	struct rte_flow_template_table *table = NULL;
 	struct mlx5_flow_group grp;
 	struct rte_flow_actions_template *at = NULL;
 	struct rte_flow_actions_template_attr template_attr = {
@@ -13531,6 +13530,10 @@ flow_hw_translate_flow_actions(struct rte_eth_dev *dev,
 	RTE_SET_USED(action_flags);
 	memset(masks, 0, sizeof(masks));
 	memset(mask_conf, 0, sizeof(mask_conf));
+	/* Only set the needed fields explicitly. */
+	struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace();
+	struct rte_flow_template_table *table;
+
 	/*
 	 * Notice All direct actions will be unmasked,
 	 * except for modify header and encap,
@@ -13540,6 +13543,12 @@ flow_hw_translate_flow_actions(struct rte_eth_dev *dev,
 	 * shared actions will be parsed as part of template translation
 	 * and not during action construct.
 	 */
+	if (!wks)
+		return rte_flow_error_set(error, ENOMEM,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "failed to push flow workspace");
+	table = wks->table;
 	flow_nta_build_template_mask(actions, masks, mask_conf);
 	/* The group in the attribute translation was done in advance. */
 	ret = __translate_group(dev, attr, external, attr->group, &src_group, error);
@@ -13551,11 +13560,6 @@ flow_hw_translate_flow_actions(struct rte_eth_dev *dev,
 		table_type = MLX5DR_TABLE_TYPE_NIC_TX;
 	else
 		table_type = MLX5DR_TABLE_TYPE_NIC_RX;
-	/* TODO: consider to reuse the workspace per thread. */
-	table = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*table), 0, SOCKET_ID_ANY);
-	if (!table)
-		return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_ACTION,
-					  actions, "Failed to allocate dummy table");
 	at = __flow_hw_actions_template_create(dev, &template_attr, actions, masks, true, error);
 	if (!at) {
 		ret = -rte_errno;
@@ -13592,10 +13596,9 @@ flow_hw_translate_flow_actions(struct rte_eth_dev *dev,
 		__flow_hw_action_template_destroy(dev, hw_acts);
 	else
 		__flow_hw_act_data_flush(dev, hw_acts);
-	if (table)
-		mlx5_free(table);
 	if (at)
 		mlx5_free(at);
+	mlx5_flow_pop_thread_workspace();
 	return ret;
 }
 
-- 
2.34.1

RE: [PATCH] net/mlx5: fix the unneeded stub table allocation

[Dariusz Sosnowski]

> -----Original Message-----
> From: Bing Zhao <bingz@nvidia.com>
> Sent: Tuesday, November 26, 2024 10:26
> To: Dariusz Sosnowski <dsosnowski@nvidia.com>; Slava Ovsiienko
> <viacheslavo@nvidia.com>; dev@dpdk.org; Raslan Darawsheh
> <rasland@nvidia.com>
> Cc: Ori Kam <orika@nvidia.com>; Suanming Mou <suanmingm@nvidia.com>;
> Matan Azrad <matan@nvidia.com>; Maayan Kashani
> <mkashani@nvidia.com>
> Subject: [PATCH] net/mlx5: fix the unneeded stub table allocation
> 
> The HWS non-template flow API is reusing some implementation of template
> API to unifiy code logic. So for each rule creation, a stub / temporary table is
> used in order to reuse the actions construction.
> 
> Since this is temporary and used only internally, there is no need to save the
> table permanently. Only parts of them are mandatory, so the allocation / free
> from the heap of RTE memory is a waste and causes a lot of overhead. By
> using the pre-allocated workspace and set the needed fields expliticly will save
> the overhead and help to speed up the rule insertion rate.
> 
> Fixes: 27d171b88031 ("net/mlx5: abstract flow action and enable
> reconfigure")
> Cc: mkashani@nvidia.com
> 
> Signed-off-by: Bing Zhao <bingz@nvidia.com>

Acked-by: Dariusz Sosnowski <dsosnowski@nvidia.com>

Best regards,
Dariusz Sosnowski