[dpdk-dev] [PATCH 08/10] net/mlx5: allocate metadata object from indexed pool

[Suanming Mou <suanmingm@mellanox.com>]

Allocate metadata object from indexed pool helps rte flow saves the 4
bytes index instead of 8 bytes pointer. For metadata object itself, it
helps save MALLOC_ELEM_OVERHEAD bytes from rte_malloc().

Signed-off-by: Suanming Mou <suanmingm@mellanox.com>
---
 drivers/net/mlx5/mlx5.c      | 11 +++++++++++
 drivers/net/mlx5/mlx5.h      |  1 +
 drivers/net/mlx5/mlx5_flow.c | 40 ++++++++++++++++++++++++++++------------
 drivers/net/mlx5/mlx5_flow.h |  5 +++--
 4 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 81a2e94..767fd9c 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -267,6 +267,17 @@ struct mlx5_dev_spawn_data {
 		.type = "mlx5_meter_ipool",
 	},
 	{
+		.size = sizeof(struct mlx5_flow_meter),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 0,
+		.release_mem_en = 1,
+		.malloc = rte_malloc_socket,
+		.free = rte_free,
+		.type = "mlx5_mcp_ipool",
+	},
+	{
 		.size = (sizeof(struct mlx5_hrxq) + MLX5_RSS_HASH_KEY_LEN),
 		.trunk_size = 64,
 		.grow_trunk = 3,
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index cf785f5..1141935 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -60,6 +60,7 @@ enum mlx5_ipool_index {
 	MLX5_IPOOL_JUMP, /* Pool for jump resource. */
 #endif
 	MLX5_IPOOL_MTR, /* Pool for meter resource. */
+	MLX5_IPOOL_MCP, /* Pool for metadata resource. */
 	MLX5_IPOOL_HRXQ, /* Pool for hrxq resource. */
 	MLX5_IPOOL_MLX5_FLOW, /* Pool for mlx5 flow handle. */
 	MLX5_IPOOL_RTE_FLOW, /* Pool for rte_flow. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index cc2b207..0d2e3df 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -2961,6 +2961,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 		[3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
 	};
 	struct mlx5_flow_mreg_copy_resource *mcp_res;
+	uint32_t idx = 0;
 	int ret;
 
 	/* Fill the register fileds in the flow. */
@@ -3029,11 +3030,12 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 		};
 	}
 	/* Build a new entry. */
-	mcp_res = rte_zmalloc(__func__, sizeof(*mcp_res), 0);
+	mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
 	if (!mcp_res) {
 		rte_errno = ENOMEM;
 		return NULL;
 	}
+	mcp_res->idx = idx;
 	/*
 	 * The copy Flows are not included in any list. There
 	 * ones are referenced from other Flows and can not
@@ -3055,7 +3057,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 error:
 	if (mcp_res->flow)
 		flow_list_destroy(dev, NULL, mcp_res->flow);
-	rte_free(mcp_res);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
 	return NULL;
 }
 
@@ -3071,9 +3073,13 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 flow_mreg_del_copy_action(struct rte_eth_dev *dev,
 			  struct rte_flow *flow)
 {
-	struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy;
+	struct mlx5_flow_mreg_copy_resource *mcp_res;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
+	if (!flow->mreg_copy)
+		return;
+	mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
+				 flow->mreg_copy);
 	if (!mcp_res || !priv->mreg_cp_tbl)
 		return;
 	if (flow->copy_applied) {
@@ -3092,8 +3098,8 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 	MLX5_ASSERT(mcp_res->flow);
 	flow_list_destroy(dev, NULL, mcp_res->flow);
 	mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
-	rte_free(mcp_res);
-	flow->mreg_copy = NULL;
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
+	flow->mreg_copy = 0;
 }
 
 /**
@@ -3111,10 +3117,15 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 flow_mreg_start_copy_action(struct rte_eth_dev *dev,
 			    struct rte_flow *flow)
 {
-	struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy;
+	struct mlx5_flow_mreg_copy_resource *mcp_res;
+	struct mlx5_priv *priv = dev->data->dev_private;
 	int ret;
 
-	if (!mcp_res || flow->copy_applied)
+	if (!flow->mreg_copy || flow->copy_applied)
+		return 0;
+	mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
+				 flow->mreg_copy);
+	if (!mcp_res)
 		return 0;
 	if (!mcp_res->appcnt) {
 		ret = flow_drv_apply(dev, mcp_res->flow, NULL);
@@ -3138,9 +3149,14 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 flow_mreg_stop_copy_action(struct rte_eth_dev *dev,
 			   struct rte_flow *flow)
 {
-	struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy;
+	struct mlx5_flow_mreg_copy_resource *mcp_res;
+	struct mlx5_priv *priv = dev->data->dev_private;
 
-	if (!mcp_res || !flow->copy_applied)
+	if (!flow->mreg_copy || !flow->copy_applied)
+		return;
+	mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP],
+				 flow->mreg_copy);
+	if (!mcp_res)
 		return;
 	MLX5_ASSERT(mcp_res->appcnt);
 	--mcp_res->appcnt;
@@ -3171,7 +3187,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 	MLX5_ASSERT(mcp_res->flow);
 	flow_list_destroy(dev, NULL, mcp_res->flow);
 	mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
-	rte_free(mcp_res);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
 }
 
 /**
@@ -3263,7 +3279,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 				(dev, MLX5_FLOW_MARK_DEFAULT, error);
 			if (!mcp_res)
 				return -rte_errno;
-			flow->mreg_copy = mcp_res;
+			flow->mreg_copy = mcp_res->idx;
 			if (dev->data->dev_started) {
 				mcp_res->appcnt++;
 				flow->copy_applied = 1;
@@ -3276,7 +3292,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 				flow_mreg_add_copy_action(dev, mark->id, error);
 			if (!mcp_res)
 				return -rte_errno;
-			flow->mreg_copy = mcp_res;
+			flow->mreg_copy = mcp_res->idx;
 			if (dev->data->dev_started) {
 				mcp_res->appcnt++;
 				flow->copy_applied = 1;
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index c8b2d20..7611410 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -460,6 +460,7 @@ struct mlx5_flow_mreg_copy_resource {
 	/* List entry for device flows. */
 	uint32_t refcnt; /* Reference counter. */
 	uint32_t appcnt; /* Apply/Remove counter. */
+	uint32_t idx;
 	struct rte_flow *flow; /* Built flow for copy. */
 };
 
@@ -758,8 +759,8 @@ struct rte_flow {
 	enum mlx5_flow_drv_type drv_type; /**< Driver type. */
 	struct mlx5_flow_rss_queue rss; /**< RSS queue context. */
 	uint32_t counter; /**< Holds flow counter. */
-	struct mlx5_flow_mreg_copy_resource *mreg_copy;
-	/**< pointer to metadata register copy table resource. */
+	uint32_t mreg_copy;
+	/**< Index to metadata register copy table resource. */
 	uint16_t meter; /**< Holds flow meter id. */
 	uint32_t dev_handles;
 	/**< Device flow handles that are part of the flow. */
-- 
1.8.3.1