[dpdk-dev] [PATCH v4 14/14] net/mlx5: allow multiple flow tables on the same level

[Li Zhang <lizh@nvidia.com>]

The driver devices support creation of multiple flow tables.
Jump action can be used in order to move the packet steering
to different flow table.
Table 0 is always the root table for packet steering.

Jumping between tables may cause endless loops in steering mechanism,
that's why each table has level attribute,
the driver sub-system may not allow jumping to table with
equal or lower level than the current table.

Currently, in the driver, the table ID and level are always identical.

Allow multiple flow table creation with the same level attribute.

This patch adds the table id in flow table data entry, while
allocates the flow table, if the table level is same but the
different table id, the new table will be allocated with new
table object id. It supports 4M multiple flow tables on the
same level.

Signed-off-by: Li Zhang <lizh@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5.c         |  9 ++--
 drivers/net/mlx5/mlx5.h         | 10 ++--
 drivers/net/mlx5/mlx5_flow.c    | 17 ++++---
 drivers/net/mlx5/mlx5_flow.h    | 12 +++--
 drivers/net/mlx5/mlx5_flow_dv.c | 84 +++++++++++++++++++--------------
 5 files changed, 79 insertions(+), 53 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index dac71fed95..572db1bc67 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1269,9 +1269,12 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 	 * because DV expect to see them even if they cannot be created by
 	 * RDMA-CORE.
 	 */
-	if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, NULL, 0, 1, &error) ||
-	    !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0, NULL, 0, 1, &error) ||
-	    !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0, NULL, 0, 1, &error)) {
+	if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0,
+		NULL, 0, 1, 0, &error) ||
+	    !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0,
+		NULL, 0, 1, 0, &error) ||
+	    !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0,
+		NULL, 0, 1, 0, &error)) {
 		err = ENOMEM;
 		goto error;
 	}
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 61957d0017..9a02aa4488 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -770,10 +770,12 @@ struct mlx5_aso_mtr_pools_mng {
 union mlx5_flow_tbl_key {
 	struct {
 		/* Table ID should be at the lowest address. */
-		uint32_t table_id;	/**< ID of the table. */
-		uint16_t dummy;		/**< Dummy table for DV API. */
-		uint8_t domain;		/**< 1 - FDB, 0 - NIC TX/RX. */
-		uint8_t direction;	/**< 1 - egress, 0 - ingress. */
+		uint32_t level;	/**< Level of the table. */
+		uint32_t id:22;	/**< ID of the table. */
+		uint32_t dummy:1;	/**< Dummy table for DV API. */
+		uint32_t is_fdb:1;	/**< 1 - FDB, 0 - NIC TX/RX. */
+		uint32_t is_egress:1;	/**< 1 - egress, 0 - ingress. */
+		uint32_t reserved:7;	/**< must be zero for comparison. */
 	};
 	uint64_t v64;			/**< full 64bits value of key */
 };
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index e97f68d2f5..242c6f2288 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -4342,6 +4342,9 @@ flow_create_split_inner(struct rte_eth_dev *dev,
 		dev_flow->handle->mark = 1;
 	if (sub_flow)
 		*sub_flow = dev_flow;
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	dev_flow->dv.table_id = flow_split_info->table_id;
+#endif
 	return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
 }
 
@@ -5489,8 +5492,7 @@ flow_create_split_sample(struct rte_eth_dev *dev,
 						struct mlx5_flow_tbl_data_entry,
 						tbl);
 			sfx_attr.group = sfx_attr.transfer ?
-						(sfx_tbl_data->table_id - 1) :
-						sfx_tbl_data->table_id;
+			(sfx_tbl_data->level - 1) : sfx_tbl_data->level;
 		} else {
 			MLX5_ASSERT(attr->transfer);
 			sfx_attr.group = jump_table;
@@ -5690,7 +5692,8 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 		.skip_scale = 0,
 		.flow_idx = 0,
 		.prefix_mark = 0,
-		.prefix_layers = 0
+		.prefix_layers = 0,
+		.table_id = 0
 	};
 	int ret;
 
@@ -7743,10 +7746,12 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
 	union tunnel_offload_mark mbits = { .val = mark };
 	union mlx5_flow_tbl_key table_key = {
 		{
-			.table_id = tunnel_id_to_flow_tbl(mbits.table_id),
+			.level = tunnel_id_to_flow_tbl(mbits.table_id),
+			.id = 0,
+			.reserved = 0,
 			.dummy = 0,
-			.domain = !!mbits.transfer,
-			.direction = 0,
+			.is_fdb = !!mbits.transfer,
+			.is_egress = 0,
 		}
 	};
 	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index ecd3fb250c..cb2803d080 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -565,8 +565,9 @@ struct mlx5_flow_tbl_data_entry {
 	uint32_t is_egress:1; /**< Egress table. */
 	uint32_t is_transfer:1; /**< Transfer table. */
 	uint32_t dummy:1; /**<  DR table. */
-	uint32_t reserve:27; /**< Reserved to future using. */
-	uint32_t table_id; /**< Table ID. */
+	uint32_t id:22; /**< Table ID. */
+	uint32_t reserve:5; /**< Reserved to future using. */
+	uint32_t level; /**< Table level. */
 };
 
 /* Sub rdma-core actions list. */
@@ -700,6 +701,7 @@ struct mlx5_flow_handle {
 /** Device flow structure only for DV flow creation. */
 struct mlx5_flow_dv_workspace {
 	uint32_t group; /**< The group index. */
+	uint32_t table_id; /**< Flow table identifier. */
 	uint8_t transfer; /**< 1 if the flow is E-Switch flow. */
 	int actions_n; /**< number of actions. */
 	void *actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS]; /**< Action list. */
@@ -1060,6 +1062,7 @@ struct mlx5_flow_split_info {
 	uint32_t flow_idx; /**< This memory pool index to the flow. */
 	uint32_t prefix_mark; /**< Prefix subflow mark flag. */
 	uint64_t prefix_layers; /**< Prefix subflow layers. */
+	uint32_t table_id; /**< Flow table identifier. */
 };
 
 typedef int (*mlx5_flow_validate_t)(struct rte_eth_dev *dev,
@@ -1411,9 +1414,10 @@ int flow_dv_tbl_match_cb(struct mlx5_hlist *list,
 void flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			   struct mlx5_hlist_entry *entry);
 struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
-		uint32_t table_id, uint8_t egress, uint8_t transfer,
+		uint32_t table_level, uint8_t egress, uint8_t transfer,
 		bool external, const struct mlx5_flow_tunnel *tunnel,
-		uint32_t group_id, uint8_t dummy, struct rte_flow_error *error);
+		uint32_t group_id, uint8_t dummy,
+		uint32_t table_id, struct rte_flow_error *error);
 
 struct mlx5_hlist_entry *flow_dv_tag_create_cb(struct mlx5_hlist *list,
 					       uint64_t key, void *cb_ctx);
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 8f2ea71357..d8ea440668 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -9357,20 +9357,21 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	tbl_data->group_id = tt_prm->group_id;
 	tbl_data->external = !!tt_prm->external;
 	tbl_data->tunnel_offload = is_tunnel_offload_active(dev);
-	tbl_data->is_egress = !!key.direction;
-	tbl_data->is_transfer = !!key.domain;
+	tbl_data->is_egress = !!key.is_egress;
+	tbl_data->is_transfer = !!key.is_fdb;
 	tbl_data->dummy = !!key.dummy;
-	tbl_data->table_id = key.table_id;
+	tbl_data->level = key.level;
+	tbl_data->id = key.id;
 	tbl = &tbl_data->tbl;
 	if (key.dummy)
 		return &tbl_data->entry;
-	if (key.domain)
+	if (key.is_fdb)
 		domain = sh->fdb_domain;
-	else if (key.direction)
+	else if (key.is_egress)
 		domain = sh->tx_domain;
 	else
 		domain = sh->rx_domain;
-	ret = mlx5_flow_os_create_flow_tbl(domain, key.table_id, &tbl->obj);
+	ret = mlx5_flow_os_create_flow_tbl(domain, key.level, &tbl->obj);
 	if (ret) {
 		rte_flow_error_set(error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -9378,7 +9379,7 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
 		return NULL;
 	}
-	if (key.table_id) {
+	if (key.level != 0) {
 		ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
 					(tbl->obj, &tbl_data->jump.action);
 		if (ret) {
@@ -9391,9 +9392,9 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 			return NULL;
 		}
 	}
-	MKSTR(matcher_name, "%s_%s_%u_matcher_cache",
-	      key.domain ? "FDB" : "NIC", key.direction ? "egress" : "ingress",
-	      key.table_id);
+	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_cache",
+	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
+	      key.level, key.id);
 	mlx5_cache_list_init(&tbl_data->matchers, matcher_name, 0, sh,
 			     flow_dv_matcher_create_cb,
 			     flow_dv_matcher_match_cb,
@@ -9410,10 +9411,11 @@ flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
 		container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
 	union mlx5_flow_tbl_key key = { .v64 = key64 };
 
-	return tbl_data->table_id != key.table_id ||
+	return tbl_data->level != key.level ||
+	       tbl_data->id != key.id ||
 	       tbl_data->dummy != key.dummy ||
-	       tbl_data->is_transfer != key.domain ||
-	       tbl_data->is_egress != key.direction;
+	       tbl_data->is_transfer != !!key.is_fdb ||
+	       tbl_data->is_egress != !!key.is_egress;
 }
 
 /**
@@ -9421,14 +9423,16 @@ flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] table_id
- *   Table id to use.
+ * @param[in] table_level
+ *   Table level to use.
  * @param[in] egress
  *   Direction of the table.
  * @param[in] transfer
  *   E-Switch or NIC flow.
  * @param[in] dummy
  *   Dummy entry for dv API.
+ * @param[in] table_id
+ *   Table id to use.
  * @param[out] error
  *   pointer to error structure.
  *
@@ -9437,20 +9441,23 @@ flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
  */
 struct mlx5_flow_tbl_resource *
 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
-			 uint32_t table_id, uint8_t egress,
+			 uint32_t table_level, uint8_t egress,
 			 uint8_t transfer,
 			 bool external,
 			 const struct mlx5_flow_tunnel *tunnel,
 			 uint32_t group_id, uint8_t dummy,
+			 uint32_t table_id,
 			 struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	union mlx5_flow_tbl_key table_key = {
 		{
-			.table_id = table_id,
-			.dummy = dummy,
-			.domain = !!transfer,
-			.direction = !!egress,
+			.level = table_level,
+			.id = table_id,
+			.reserved = 0,
+			.dummy = !!dummy,
+			.is_fdb = !!transfer,
+			.is_egress = !!egress,
 		}
 	};
 	struct mlx5_flow_tbl_tunnel_prm tt_prm = {
@@ -9473,8 +9480,10 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 				   "cannot get table");
 		return NULL;
 	}
-	DRV_LOG(DEBUG, "Table_id %u tunnel %u group %u registered.",
-		table_id, tunnel ? tunnel->tunnel_id : 0, group_id);
+	DRV_LOG(DEBUG, "table_level %u table_id %u "
+		"tunnel %u group %u registered.",
+		table_level, table_id,
+		tunnel ? tunnel->tunnel_id : 0, group_id);
 	tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
 	return &tbl_data->tbl;
 }
@@ -9501,7 +9510,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 					tbl_data->tunnel->tunnel_id : 0,
 			.group = tbl_data->group_id
 		};
-		uint32_t table_id = tbl_data->table_id;
+		uint32_t table_level = tbl_data->level;
 
 		tunnel_grp_hash = tbl_data->tunnel ?
 					tbl_data->tunnel->groups :
@@ -9510,8 +9519,9 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 		if (he)
 			mlx5_hlist_unregister(tunnel_grp_hash, he);
 		DRV_LOG(DEBUG,
-			"Table_id %u tunnel %u group %u released.",
-			table_id,
+			"table_level %u id %u tunnel %u group %u released.",
+			table_level,
+			tbl_data->id,
 			tbl_data->tunnel ?
 			tbl_data->tunnel->tunnel_id : 0,
 			tbl_data->group_id);
@@ -9639,10 +9649,10 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 	 * tunnel offload API requires this registration for cases when
 	 * tunnel match rule was inserted before tunnel set rule.
 	 */
-	tbl = flow_dv_tbl_resource_get(dev, key->table_id,
-				       key->direction, key->domain,
+	tbl = flow_dv_tbl_resource_get(dev, key->level,
+				       key->is_egress, key->is_fdb,
 				       dev_flow->external, tunnel,
-				       group_id, 0, error);
+				       group_id, 0, key->id, error);
 	if (!tbl)
 		return -rte_errno;	/* No need to refill the error info */
 	tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
@@ -10121,7 +10131,7 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
 		is_egress = 1;
 	tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
 					is_egress, is_transfer,
-					true, NULL, 0, 0, error);
+					true, NULL, 0, 0, 0, error);
 	if (!tbl) {
 		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -11474,7 +11484,7 @@ flow_dv_translate(struct rte_eth_dev *dev,
 						       attr->transfer,
 						       !!dev_flow->external,
 						       tunnel, jump_group, 0,
-						       error);
+						       0, error);
 			if (!tbl)
 				return rte_flow_error_set
 						(error, errno,
@@ -12016,9 +12026,10 @@ flow_dv_translate(struct rte_eth_dev *dev,
 	matcher.priority = mlx5_get_matcher_priority(dev, attr,
 					matcher.priority);
 	/* reserved field no needs to be set to 0 here. */
-	tbl_key.domain = attr->transfer;
-	tbl_key.direction = attr->egress;
-	tbl_key.table_id = dev_flow->dv.group;
+	tbl_key.is_fdb = attr->transfer;
+	tbl_key.is_egress = attr->egress;
+	tbl_key.level = dev_flow->dv.group;
+	tbl_key.id = dev_flow->dv.table_id;
 	if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow,
 				     tunnel, attr->group, error))
 		return -rte_errno;
@@ -13622,7 +13633,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
 	/* Create the meter table with METER level. */
 	dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
 					    egress, transfer, false, NULL, 0,
-					    0, &error);
+					    0, 0, &error);
 	if (!dtb->tbl) {
 		DRV_LOG(ERR, "Failed to create meter policer table.");
 		return -1;
@@ -13631,7 +13642,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
 	dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
 					    MLX5_FLOW_TABLE_LEVEL_SUFFIX,
 					    egress, transfer, false, NULL, 0,
-					    0, &error);
+					    0, 0, &error);
 	if (!dtb->sfx_tbl) {
 		DRV_LOG(ERR, "Failed to create meter suffix table.");
 		return -1;
@@ -14154,7 +14165,8 @@ mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev)
 	void *flow = NULL;
 	int ret = -1;
 
-	tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL, 0, 0, NULL);
+	tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL,
+					0, 0, 0, NULL);
 	if (!tbl)
 		goto err;
 	dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
-- 
2.27.0