[PATCH v2 10/14] net/mlx5: add flow jump action

[Suanming Mou <suanmingm@nvidia.com>]

Jump action connects different level of flow tables and allows packet
handling in the chain of flows.

A new action construct data struct is also added in this commit to help
to handle not only the dynamic jump action but also for the other generic
dynamic actions. The actions with empty mask configuration means dynamic
action, and the dedicated action will be created with the flow action
configuration during flow creation. In that dynamic action case, the action
will be appended to the table template's action list during table creation.
When creating the flows, traverse the action list and pick the dynamic
action configuration details from flow actions as the action construct data
struct describes, then create the dedicated dynamic actions.

This commit adds the jump action and the generic dynamic action construct
mechanism.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
---
 drivers/net/mlx5/mlx5.h         |   1 +
 drivers/net/mlx5/mlx5_flow.h    |  25 ++-
 drivers/net/mlx5/mlx5_flow_hw.c | 270 +++++++++++++++++++++++++++++---
 3 files changed, 275 insertions(+), 21 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 17aa4b683b..bb0d07d870 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1524,6 +1524,7 @@ struct mlx5_priv {
 	/* HW steering global drop action. */
 	struct mlx5dr_action *hw_drop[MLX5_HW_ACTION_FLAG_MAX]
 				     [MLX5DR_TABLE_TYPE_MAX];
+	struct mlx5_indexed_pool *acts_ipool; /* Action data indexed pool. */
 #endif
 };
 
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index a0cc4028fa..675a5bfee5 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1020,10 +1020,25 @@ struct rte_flow {
 /* HWS flow struct. */
 struct rte_flow_hw {
 	uint32_t idx; /* Flow index from indexed pool. */
+	uint32_t fate_type; /* Fate action type. */
+	union {
+		/* Jump action. */
+		struct mlx5_hw_jump_action *jump;
+	};
 	struct rte_flow_template_table *table; /* The table flow allcated from. */
 	struct mlx5dr_rule rule; /* HWS layer data struct. */
 } __rte_packed;
 
+/* rte flow action translate to DR action struct. */
+struct mlx5_action_construct_data {
+	LIST_ENTRY(mlx5_action_construct_data) next;
+	/* Ensure the action types are matched. */
+	int type;
+	uint32_t idx;  /* Data index. */
+	uint16_t action_src; /* rte_flow_action src offset. */
+	uint16_t action_dst; /* mlx5dr_rule_action dst offset. */
+};
+
 /* Flow item template struct. */
 struct rte_flow_pattern_template {
 	LIST_ENTRY(rte_flow_pattern_template) next;
@@ -1051,9 +1066,17 @@ struct mlx5_hw_jump_action {
 	struct mlx5dr_action *hws_action;
 };
 
+/* The maximum actions support in the flow. */
+#define MLX5_HW_MAX_ACTS 16
+
 /* DR action set struct. */
 struct mlx5_hw_actions {
-	struct mlx5dr_action *drop; /* Drop action. */
+	/* Dynamic action list. */
+	LIST_HEAD(act_list, mlx5_action_construct_data) act_list;
+	struct mlx5_hw_jump_action *jump; /* Jump action. */
+	uint32_t acts_num:4; /* Total action number. */
+	/* Translated DR action array from action template. */
+	struct mlx5dr_rule_action rule_acts[MLX5_HW_MAX_ACTS];
 };
 
 /* mlx5 action template struct. */
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index ed14eacce2..f320d0db8c 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -36,18 +36,158 @@ static uint32_t mlx5_hw_act_flag[MLX5_HW_ACTION_FLAG_MAX]
 	},
 };
 
+/**
+ * Register destination table DR jump action.
+ *
+ * @param[in] dev
+ *   Pointer to the rte_eth_dev structure.
+ * @param[in] table_attr
+ *   Pointer to the flow attributes.
+ * @param[in] dest_group
+ *   The destination group ID.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *    Table on success, NULL otherwise and rte_errno is set.
+ */
+static struct mlx5_hw_jump_action *
+flow_hw_jump_action_register(struct rte_eth_dev *dev,
+			     const struct rte_flow_attr *attr,
+			     uint32_t dest_group,
+			     struct rte_flow_error *error)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct rte_flow_attr jattr = *attr;
+	struct mlx5_flow_group *grp;
+	struct mlx5_flow_cb_ctx ctx = {
+		.dev = dev,
+		.error = error,
+		.data = &jattr,
+	};
+	struct mlx5_list_entry *ge;
+
+	jattr.group = dest_group;
+	ge = mlx5_hlist_register(priv->sh->flow_tbls, dest_group, &ctx);
+	if (!ge)
+		return NULL;
+	grp = container_of(ge, struct mlx5_flow_group, entry);
+	return &grp->jump;
+}
+
+/**
+ * Release jump action.
+ *
+ * @param[in] dev
+ *   Pointer to the rte_eth_dev structure.
+ * @param[in] jump
+ *   Pointer to the jump action.
+ */
+
+static void
+flow_hw_jump_release(struct rte_eth_dev *dev, struct mlx5_hw_jump_action *jump)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_group *grp;
+
+	grp = container_of
+		(jump, struct mlx5_flow_group, jump);
+	mlx5_hlist_unregister(priv->sh->flow_tbls, &grp->entry);
+}
+
 /**
  * Destroy DR actions created by action template.
  *
  * For DR actions created during table creation's action translate.
  * Need to destroy the DR action when destroying the table.
  *
+ * @param[in] dev
+ *   Pointer to the rte_eth_dev structure.
  * @param[in] acts
  *   Pointer to the template HW steering DR actions.
  */
 static void
-__flow_hw_action_template_destroy(struct mlx5_hw_actions *acts __rte_unused)
+__flow_hw_action_template_destroy(struct rte_eth_dev *dev,
+				 struct mlx5_hw_actions *acts)
 {
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	if (acts->jump) {
+		struct mlx5_flow_group *grp;
+
+		grp = container_of
+			(acts->jump, struct mlx5_flow_group, jump);
+		mlx5_hlist_unregister(priv->sh->flow_tbls, &grp->entry);
+		acts->jump = NULL;
+	}
+}
+
+/**
+ * Append dynamic action to the dynamic action list.
+ *
+ * @param[in] priv
+ *   Pointer to the port private data structure.
+ * @param[in] acts
+ *   Pointer to the template HW steering DR actions.
+ * @param[in] type
+ *   Action type.
+ * @param[in] action_src
+ *   Offset of source rte flow action.
+ * @param[in] action_dst
+ *   Offset of destination DR action.
+ *
+ * @return
+ *    0 on success, negative value otherwise and rte_errno is set.
+ */
+static __rte_always_inline struct mlx5_action_construct_data *
+__flow_hw_act_data_alloc(struct mlx5_priv *priv,
+			 enum rte_flow_action_type type,
+			 uint16_t action_src,
+			 uint16_t action_dst)
+{
+	struct mlx5_action_construct_data *act_data;
+	uint32_t idx = 0;
+
+	act_data = mlx5_ipool_zmalloc(priv->acts_ipool, &idx);
+	if (!act_data)
+		return NULL;
+	act_data->idx = idx;
+	act_data->type = type;
+	act_data->action_src = action_src;
+	act_data->action_dst = action_dst;
+	return act_data;
+}
+
+/**
+ * Append dynamic action to the dynamic action list.
+ *
+ * @param[in] priv
+ *   Pointer to the port private data structure.
+ * @param[in] acts
+ *   Pointer to the template HW steering DR actions.
+ * @param[in] type
+ *   Action type.
+ * @param[in] action_src
+ *   Offset of source rte flow action.
+ * @param[in] action_dst
+ *   Offset of destination DR action.
+ *
+ * @return
+ *    0 on success, negative value otherwise and rte_errno is set.
+ */
+static __rte_always_inline int
+__flow_hw_act_data_general_append(struct mlx5_priv *priv,
+				  struct mlx5_hw_actions *acts,
+				  enum rte_flow_action_type type,
+				  uint16_t action_src,
+				  uint16_t action_dst)
+{	struct mlx5_action_construct_data *act_data;
+
+	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
+	if (!act_data)
+		return -1;
+	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
+	return 0;
 }
 
 /**
@@ -80,14 +220,16 @@ flow_hw_actions_translate(struct rte_eth_dev *dev,
 			  const struct rte_flow_template_table_attr *table_attr,
 			  struct mlx5_hw_actions *acts,
 			  struct rte_flow_actions_template *at,
-			  struct rte_flow_error *error __rte_unused)
+			  struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr *attr = &table_attr->flow_attr;
 	struct rte_flow_action *actions = at->actions;
+	struct rte_flow_action *action_start = actions;
 	struct rte_flow_action *masks = at->masks;
 	bool actions_end = false;
-	uint32_t type;
+	uint32_t type, i;
+	int err;
 
 	if (attr->transfer)
 		type = MLX5DR_TABLE_TYPE_FDB;
@@ -95,14 +237,34 @@ flow_hw_actions_translate(struct rte_eth_dev *dev,
 		type = MLX5DR_TABLE_TYPE_NIC_TX;
 	else
 		type = MLX5DR_TABLE_TYPE_NIC_RX;
-	for (; !actions_end; actions++, masks++) {
+	for (i = 0; !actions_end; actions++, masks++) {
 		switch (actions->type) {
 		case RTE_FLOW_ACTION_TYPE_INDIRECT:
 			break;
 		case RTE_FLOW_ACTION_TYPE_VOID:
 			break;
 		case RTE_FLOW_ACTION_TYPE_DROP:
-			acts->drop = priv->hw_drop[!!attr->group][type];
+			acts->rule_acts[i++].action =
+				priv->hw_drop[!!attr->group][type];
+			break;
+		case RTE_FLOW_ACTION_TYPE_JUMP:
+			if (masks->conf) {
+				uint32_t jump_group =
+					((const struct rte_flow_action_jump *)
+					actions->conf)->group;
+				acts->jump = flow_hw_jump_action_register
+						(dev, attr, jump_group, error);
+				if (!acts->jump)
+					goto err;
+				acts->rule_acts[i].action = (!!attr->group) ?
+						acts->jump->hws_action :
+						acts->jump->root_action;
+			} else if (__flow_hw_act_data_general_append
+					(priv, acts, actions->type,
+					 actions - action_start, i)){
+				goto err;
+			}
+			i++;
 			break;
 		case RTE_FLOW_ACTION_TYPE_END:
 			actions_end = true;
@@ -111,7 +273,14 @@ flow_hw_actions_translate(struct rte_eth_dev *dev,
 			break;
 		}
 	}
+	acts->acts_num = i;
 	return 0;
+err:
+	err = rte_errno;
+	__flow_hw_action_template_destroy(dev, acts);
+	return rte_flow_error_set(error, err,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				  "fail to create rte table");
 }
 
 /**
@@ -120,6 +289,10 @@ flow_hw_actions_translate(struct rte_eth_dev *dev,
  * For action template contains dynamic actions, these actions need to
  * be updated according to the rte_flow action during flow creation.
  *
+ * @param[in] dev
+ *   Pointer to the rte_eth_dev structure.
+ * @param[in] job
+ *   Pointer to job descriptor.
  * @param[in] hw_acts
  *   Pointer to translated actions from template.
  * @param[in] actions
@@ -133,31 +306,63 @@ flow_hw_actions_translate(struct rte_eth_dev *dev,
  *    0 on success, negative value otherwise and rte_errno is set.
  */
 static __rte_always_inline int
-flow_hw_actions_construct(struct mlx5_hw_actions *hw_acts,
+flow_hw_actions_construct(struct rte_eth_dev *dev,
+			  struct mlx5_hw_q_job *job,
+			  struct mlx5_hw_actions *hw_acts,
 			  const struct rte_flow_action actions[],
 			  struct mlx5dr_rule_action *rule_acts,
 			  uint32_t *acts_num)
 {
-	bool actions_end = false;
-	uint32_t i;
+	struct rte_flow_template_table *table = job->flow->table;
+	struct mlx5_action_construct_data *act_data;
+	const struct rte_flow_action *action;
+	struct rte_flow_attr attr = {
+			.ingress = 1,
+	};
 
-	for (i = 0; !actions_end || (i >= MLX5_HW_MAX_ACTS); actions++) {
-		switch (actions->type) {
+	memcpy(rule_acts, hw_acts->rule_acts,
+	       sizeof(*rule_acts) * hw_acts->acts_num);
+	*acts_num = hw_acts->acts_num;
+	if (LIST_EMPTY(&hw_acts->act_list))
+		return 0;
+	attr.group = table->grp->group_id;
+	if (table->type == MLX5DR_TABLE_TYPE_FDB) {
+		attr.transfer = 1;
+		attr.ingress = 1;
+	} else if (table->type == MLX5DR_TABLE_TYPE_NIC_TX) {
+		attr.egress = 1;
+		attr.ingress = 0;
+	} else {
+		attr.ingress = 1;
+	}
+	LIST_FOREACH(act_data, &hw_acts->act_list, next) {
+		uint32_t jump_group;
+		struct mlx5_hw_jump_action *jump;
+
+		action = &actions[act_data->action_src];
+		MLX5_ASSERT(action->type == RTE_FLOW_ACTION_TYPE_INDIRECT ||
+			    (int)action->type == act_data->type);
+		switch (action->type) {
 		case RTE_FLOW_ACTION_TYPE_INDIRECT:
 			break;
 		case RTE_FLOW_ACTION_TYPE_VOID:
 			break;
-		case RTE_FLOW_ACTION_TYPE_DROP:
-			rule_acts[i++].action = hw_acts->drop;
-			break;
-		case RTE_FLOW_ACTION_TYPE_END:
-			actions_end = true;
+		case RTE_FLOW_ACTION_TYPE_JUMP:
+			jump_group = ((const struct rte_flow_action_jump *)
+						action->conf)->group;
+			jump = flow_hw_jump_action_register
+				(dev, &attr, jump_group, NULL);
+			if (!jump)
+				return -1;
+			rule_acts[act_data->action_dst].action =
+			(!!attr.group) ? jump->hws_action : jump->root_action;
+			job->flow->jump = jump;
+			job->flow->fate_type = MLX5_FLOW_FATE_JUMP;
 			break;
 		default:
 			break;
 		}
 	}
-	*acts_num = i;
 	return 0;
 }
 
@@ -239,7 +444,8 @@ flow_hw_async_flow_create(struct rte_eth_dev *dev,
 	rule_attr.user_data = job;
 	hw_acts = &table->ats[action_template_index].acts;
 	/* Construct the flow action array based on the input actions.*/
-	flow_hw_actions_construct(hw_acts, actions, rule_acts, &acts_num);
+	flow_hw_actions_construct(dev, job, hw_acts, actions,
+				  rule_acts, &acts_num);
 	ret = mlx5dr_rule_create(table->matcher,
 				 pattern_template_index, items,
 				 rule_acts, acts_num,
@@ -356,8 +562,11 @@ flow_hw_pull(struct rte_eth_dev *dev,
 		job = (struct mlx5_hw_q_job *)res[i].user_data;
 		/* Restore user data. */
 		res[i].user_data = job->user_data;
-		if (job->type == MLX5_HW_Q_JOB_TYPE_DESTROY)
+		if (job->type == MLX5_HW_Q_JOB_TYPE_DESTROY) {
+			if (job->flow->fate_type == MLX5_FLOW_FATE_JUMP)
+				flow_hw_jump_release(dev, job->flow->jump);
 			mlx5_ipool_free(job->flow->table->flow, job->flow->idx);
+		}
 		priv->hw_q[queue].job[priv->hw_q[queue].job_idx++] = job;
 	}
 	return ret;
@@ -642,6 +851,7 @@ flow_hw_table_create(struct rte_eth_dev *dev,
 			rte_errno = EINVAL;
 			goto at_error;
 		}
+		LIST_INIT(&tbl->ats[i].acts.act_list);
 		err = flow_hw_actions_translate(dev, attr,
 						&tbl->ats[i].acts,
 						action_templates[i], error);
@@ -657,7 +867,7 @@ flow_hw_table_create(struct rte_eth_dev *dev,
 	return tbl;
 at_error:
 	while (i--) {
-		__flow_hw_action_template_destroy(&tbl->ats[i].acts);
+		__flow_hw_action_template_destroy(dev, &tbl->ats[i].acts);
 		__atomic_sub_fetch(&action_templates[i]->refcnt,
 				   1, __ATOMIC_RELAXED);
 	}
@@ -716,7 +926,7 @@ flow_hw_table_destroy(struct rte_eth_dev *dev,
 		__atomic_sub_fetch(&table->its[i]->refcnt,
 				   1, __ATOMIC_RELAXED);
 	for (i = 0; i < table->nb_action_templates; i++) {
-		__flow_hw_action_template_destroy(&table->ats[i].acts);
+		__flow_hw_action_template_destroy(dev, &table->ats[i].acts);
 		__atomic_sub_fetch(&table->ats[i].action_template->refcnt,
 				   1, __ATOMIC_RELAXED);
 	}
@@ -1167,6 +1377,15 @@ flow_hw_configure(struct rte_eth_dev *dev,
 	struct mlx5_hw_q *hw_q;
 	struct mlx5_hw_q_job *job = NULL;
 	uint32_t mem_size, i, j;
+	struct mlx5_indexed_pool_config cfg = {
+		.size = sizeof(struct rte_flow_hw),
+		.trunk_size = 4096,
+		.need_lock = 1,
+		.release_mem_en = !!priv->sh->config.reclaim_mode,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.type = "mlx5_hw_action_construct_data",
+	};
 
 	if (!port_attr || !nb_queue || !queue_attr) {
 		rte_errno = EINVAL;
@@ -1185,6 +1404,9 @@ flow_hw_configure(struct rte_eth_dev *dev,
 		}
 		flow_hw_resource_release(dev);
 	}
+	priv->acts_ipool = mlx5_ipool_create(&cfg);
+	if (!priv->acts_ipool)
+		goto err;
 	/* Allocate the queue job descriptor LIFO. */
 	mem_size = sizeof(priv->hw_q[0]) * nb_queue;
 	for (i = 0; i < nb_queue; i++) {
@@ -1252,6 +1474,10 @@ flow_hw_configure(struct rte_eth_dev *dev,
 		claim_zero(mlx5dr_context_close(dr_ctx));
 	mlx5_free(priv->hw_q);
 	priv->hw_q = NULL;
+	if (priv->acts_ipool) {
+		mlx5_ipool_destroy(priv->acts_ipool);
+		priv->acts_ipool = NULL;
+	}
 	return rte_flow_error_set(error, rte_errno,
 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				  "fail to configure port");
@@ -1293,6 +1519,10 @@ flow_hw_resource_release(struct rte_eth_dev *dev)
 			mlx5dr_action_destroy(priv->hw_drop[i][j]);
 		}
 	}
+	if (priv->acts_ipool) {
+		mlx5_ipool_destroy(priv->acts_ipool);
+		priv->acts_ipool = NULL;
+	}
 	mlx5_free(priv->hw_q);
 	priv->hw_q = NULL;
 	claim_zero(mlx5dr_context_close(priv->dr_ctx));
-- 
2.25.1