[PATCH v3 14/14] net/mlx5: add header reformat action

[Suanming Mou <suanmingm@nvidia.com>]

HW steering header reformat action can work under bulk mode. In
this case, when create the table, bulk size of header reformat
actions will be allocated in low level. Afterwards, when create
flow, just simply specify the action index in the bulk and the
encapsulation data to the action will be enough.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
---
 drivers/net/mlx5/mlx5.h         |   1 +
 drivers/net/mlx5/mlx5_flow.h    |  21 +++
 drivers/net/mlx5/mlx5_flow_dv.c |   4 +-
 drivers/net/mlx5/mlx5_flow_hw.c | 228 +++++++++++++++++++++++++++++++-
 4 files changed, 251 insertions(+), 3 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index a0f9fbe365..bd69aa2334 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -342,6 +342,7 @@ struct mlx5_hw_q_job {
 	uint32_t type; /* Job type. */
 	struct rte_flow_hw *flow; /* Flow attached to the job. */
 	void *user_data; /* Job user data. */
+	uint8_t *encap_data; /* Encap data. */
 };
 
 /* HW steering job descriptor LIFO pool. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 4c224bbf52..09f0d7a75d 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1040,6 +1040,14 @@ struct mlx5_action_construct_data {
 	uint16_t action_src; /* rte_flow_action src offset. */
 	uint16_t action_dst; /* mlx5dr_rule_action dst offset. */
 	union {
+		struct {
+			/* encap src(item) offset. */
+			uint16_t src;
+			/* encap dst data offset. */
+			uint16_t dst;
+			/* encap data len. */
+			uint16_t len;
+		} encap;
 		struct {
 			uint64_t types; /* RSS hash types. */
 			uint32_t level; /* RSS level. */
@@ -1076,6 +1084,13 @@ struct mlx5_hw_jump_action {
 	struct mlx5dr_action *hws_action;
 };
 
+/* Encap decap action struct. */
+struct mlx5_hw_encap_decap_action {
+	struct mlx5dr_action *action; /* Action object. */
+	size_t data_size; /* Action metadata size. */
+	uint8_t data[]; /* Action data. */
+};
+
 /* The maximum actions support in the flow. */
 #define MLX5_HW_MAX_ACTS 16
 
@@ -1085,6 +1100,9 @@ struct mlx5_hw_actions {
 	LIST_HEAD(act_list, mlx5_action_construct_data) act_list;
 	struct mlx5_hw_jump_action *jump; /* Jump action. */
 	struct mlx5_hrxq *tir; /* TIR action. */
+	/* Encap/Decap action. */
+	struct mlx5_hw_encap_decap_action *encap_decap;
+	uint16_t encap_decap_pos; /* Encap/Decap action position. */
 	uint32_t acts_num:4; /* Total action number. */
 	uint32_t mark:1; /* Indicate the mark action. */
 	/* Translated DR action array from action template. */
@@ -2027,4 +2045,7 @@ int flow_dv_action_query(struct rte_eth_dev *dev,
 			 const struct rte_flow_action_handle *handle,
 			 void *data,
 			 struct rte_flow_error *error);
+size_t flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type);
+int flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
+			   size_t *size, struct rte_flow_error *error);
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 5f85100324..7a012f7bb9 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -4034,7 +4034,7 @@ flow_dv_push_vlan_action_resource_register
  * @return
  *   sizeof struct item_type, 0 if void or irrelevant.
  */
-static size_t
+size_t
 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
 {
 	size_t retval;
@@ -4100,7 +4100,7 @@ flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-static int
+int
 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
 			   size_t *size, struct rte_flow_error *error)
 {
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index 95df6e5190..cc70b85369 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -332,6 +332,50 @@ __flow_hw_act_data_general_append(struct mlx5_priv *priv,
 	return 0;
 }
 
+/**
+ * Append dynamic encap action to the dynamic action list.
+ *
+ * @param[in] priv
+ *   Pointer to the port private data structure.
+ * @param[in] acts
+ *   Pointer to the template HW steering DR actions.
+ * @param[in] type
+ *   Action type.
+ * @param[in] action_src
+ *   Offset of source rte flow action.
+ * @param[in] action_dst
+ *   Offset of destination DR action.
+ * @param[in] encap_src
+ *   Offset of source encap raw data.
+ * @param[in] encap_dst
+ *   Offset of destination encap raw data.
+ * @param[in] len
+ *   Length of the data to be updated.
+ *
+ * @return
+ *    0 on success, negative value otherwise and rte_errno is set.
+ */
+static __rte_always_inline int
+__flow_hw_act_data_encap_append(struct mlx5_priv *priv,
+				struct mlx5_hw_actions *acts,
+				enum rte_flow_action_type type,
+				uint16_t action_src,
+				uint16_t action_dst,
+				uint16_t encap_src,
+				uint16_t encap_dst,
+				uint16_t len)
+{	struct mlx5_action_construct_data *act_data;
+
+	act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
+	if (!act_data)
+		return -1;
+	act_data->encap.src = encap_src;
+	act_data->encap.dst = encap_dst;
+	act_data->encap.len = len;
+	LIST_INSERT_HEAD(&acts->act_list, act_data, next);
+	return 0;
+}
+
 /**
  * Append shared RSS action to the dynamic action list.
  *
@@ -422,6 +466,53 @@ flow_hw_shared_action_translate(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/**
+ * Translate encap items to encapsulation list.
+ *
+ * @param[in] dev
+ *   Pointer to the rte_eth_dev data structure.
+ * @param[in] acts
+ *   Pointer to the template HW steering DR actions.
+ * @param[in] type
+ *   Action type.
+ * @param[in] action_src
+ *   Offset of source rte flow action.
+ * @param[in] action_dst
+ *   Offset of destination DR action.
+ * @param[in] items
+ *   Encap item pattern.
+ * @param[in] items_m
+ *   Encap item mask indicates which part are constant and dynamic.
+ *
+ * @return
+ *    0 on success, negative value otherwise and rte_errno is set.
+ */
+static __rte_always_inline int
+flow_hw_encap_item_translate(struct rte_eth_dev *dev,
+			     struct mlx5_hw_actions *acts,
+			     enum rte_flow_action_type type,
+			     uint16_t action_src,
+			     uint16_t action_dst,
+			     const struct rte_flow_item *items,
+			     const struct rte_flow_item *items_m)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	size_t len, total_len = 0;
+	uint32_t i = 0;
+
+	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++, items_m++, i++) {
+		len = flow_dv_get_item_hdr_len(items->type);
+		if ((!items_m->spec ||
+		    memcmp(items_m->spec, items->spec, len)) &&
+		    __flow_hw_act_data_encap_append(priv, acts, type,
+						    action_src, action_dst, i,
+						    total_len, len))
+			return -1;
+		total_len += len;
+	}
+	return 0;
+}
+
 /**
  * Translate rte_flow actions to DR action.
  *
@@ -459,6 +550,12 @@ flow_hw_actions_translate(struct rte_eth_dev *dev,
 	struct rte_flow_action *actions = at->actions;
 	struct rte_flow_action *action_start = actions;
 	struct rte_flow_action *masks = at->masks;
+	enum mlx5dr_action_reformat_type refmt_type = 0;
+	const struct rte_flow_action_raw_encap *raw_encap_data;
+	const struct rte_flow_item *enc_item = NULL, *enc_item_m = NULL;
+	uint16_t reformat_pos = MLX5_HW_MAX_ACTS, reformat_src = 0;
+	uint8_t *encap_data = NULL;
+	size_t data_size = 0;
 	bool actions_end = false;
 	uint32_t type, i;
 	int err;
@@ -560,6 +657,56 @@ flow_hw_actions_translate(struct rte_eth_dev *dev,
 			}
 			i++;
 			break;
+		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
+			MLX5_ASSERT(reformat_pos == MLX5_HW_MAX_ACTS);
+			enc_item = ((const struct rte_flow_action_vxlan_encap *)
+				   actions->conf)->definition;
+			enc_item_m =
+				((const struct rte_flow_action_vxlan_encap *)
+				 masks->conf)->definition;
+			reformat_pos = i++;
+			reformat_src = actions - action_start;
+			refmt_type = MLX5DR_ACTION_REFORMAT_TYPE_L2_TO_TNL_L2;
+			break;
+		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
+			MLX5_ASSERT(reformat_pos == MLX5_HW_MAX_ACTS);
+			enc_item = ((const struct rte_flow_action_nvgre_encap *)
+				   actions->conf)->definition;
+			enc_item_m =
+				((const struct rte_flow_action_nvgre_encap *)
+				actions->conf)->definition;
+			reformat_pos = i++;
+			reformat_src = actions - action_start;
+			refmt_type = MLX5DR_ACTION_REFORMAT_TYPE_L2_TO_TNL_L2;
+			break;
+		case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
+		case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
+			MLX5_ASSERT(reformat_pos == MLX5_HW_MAX_ACTS);
+			reformat_pos = i++;
+			refmt_type = MLX5DR_ACTION_REFORMAT_TYPE_TNL_L2_TO_L2;
+			break;
+		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
+			raw_encap_data =
+				(const struct rte_flow_action_raw_encap *)
+				 actions->conf;
+			encap_data = raw_encap_data->data;
+			data_size = raw_encap_data->size;
+			if (reformat_pos != MLX5_HW_MAX_ACTS) {
+				refmt_type = data_size <
+				MLX5_ENCAPSULATION_DECISION_SIZE ?
+				MLX5DR_ACTION_REFORMAT_TYPE_TNL_L3_TO_L2 :
+				MLX5DR_ACTION_REFORMAT_TYPE_L2_TO_TNL_L3;
+			} else {
+				reformat_pos = i++;
+				refmt_type =
+				MLX5DR_ACTION_REFORMAT_TYPE_L2_TO_TNL_L2;
+			}
+			reformat_src = actions - action_start;
+			break;
+		case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
+			reformat_pos = i++;
+			refmt_type = MLX5DR_ACTION_REFORMAT_TYPE_TNL_L2_TO_L2;
+			break;
 		case RTE_FLOW_ACTION_TYPE_END:
 			actions_end = true;
 			break;
@@ -567,6 +714,45 @@ flow_hw_actions_translate(struct rte_eth_dev *dev,
 			break;
 		}
 	}
+	if (reformat_pos != MLX5_HW_MAX_ACTS) {
+		uint8_t buf[MLX5_ENCAP_MAX_LEN];
+
+		if (enc_item) {
+			MLX5_ASSERT(!encap_data);
+			if (flow_dv_convert_encap_data
+				(enc_item, buf, &data_size, error) ||
+			    flow_hw_encap_item_translate
+				(dev, acts, (action_start + reformat_src)->type,
+				 reformat_src, reformat_pos,
+				 enc_item, enc_item_m))
+				goto err;
+			encap_data = buf;
+		} else if (encap_data && __flow_hw_act_data_encap_append
+				(priv, acts,
+				 (action_start + reformat_src)->type,
+				 reformat_src, reformat_pos, 0, 0, data_size)) {
+			goto err;
+		}
+		acts->encap_decap = mlx5_malloc(MLX5_MEM_ZERO,
+				    sizeof(*acts->encap_decap) + data_size,
+				    0, SOCKET_ID_ANY);
+		if (!acts->encap_decap)
+			goto err;
+		if (data_size) {
+			acts->encap_decap->data_size = data_size;
+			memcpy(acts->encap_decap->data, encap_data, data_size);
+		}
+		acts->encap_decap->action = mlx5dr_action_create_reformat
+				(priv->dr_ctx, refmt_type,
+				 data_size, encap_data,
+				 rte_log2_u32(table_attr->nb_flows),
+				 mlx5_hw_act_flag[!!attr->group][type]);
+		if (!acts->encap_decap->action)
+			goto err;
+		acts->rule_acts[reformat_pos].action =
+						acts->encap_decap->action;
+		acts->encap_decap_pos = reformat_pos;
+	}
 	acts->acts_num = i;
 	return 0;
 err:
@@ -722,6 +908,9 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,
 	struct rte_flow_template_table *table = job->flow->table;
 	struct mlx5_action_construct_data *act_data;
 	const struct rte_flow_action *action;
+	const struct rte_flow_action_raw_encap *raw_encap_data;
+	const struct rte_flow_item *enc_item = NULL;
+	uint8_t *buf = job->encap_data;
 	struct rte_flow_attr attr = {
 			.ingress = 1,
 	};
@@ -743,6 +932,9 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,
 	} else {
 		attr.ingress = 1;
 	}
+	if (hw_acts->encap_decap && hw_acts->encap_decap->data_size)
+		memcpy(buf, hw_acts->encap_decap->data,
+		       hw_acts->encap_decap->data_size);
 	LIST_FOREACH(act_data, &hw_acts->act_list, next) {
 		uint32_t jump_group;
 		uint32_t tag;
@@ -798,10 +990,38 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,
 				 &rule_acts[act_data->action_dst]))
 				return -1;
 			break;
+		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
+			enc_item = ((const struct rte_flow_action_vxlan_encap *)
+				   action->conf)->definition;
+			rte_memcpy((void *)&buf[act_data->encap.dst],
+				   enc_item[act_data->encap.src].spec,
+				   act_data->encap.len);
+			break;
+		case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
+			enc_item = ((const struct rte_flow_action_nvgre_encap *)
+				   action->conf)->definition;
+			rte_memcpy((void *)&buf[act_data->encap.dst],
+				   enc_item[act_data->encap.src].spec,
+				   act_data->encap.len);
+			break;
+		case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
+			raw_encap_data =
+				(const struct rte_flow_action_raw_encap *)
+				 action->conf;
+			rte_memcpy((void *)&buf[act_data->encap.dst],
+				   raw_encap_data->data, act_data->encap.len);
+			MLX5_ASSERT(raw_encap_data->size ==
+				    act_data->encap.len);
+			break;
 		default:
 			break;
 		}
 	}
+	if (hw_acts->encap_decap) {
+		rule_acts[hw_acts->encap_decap_pos].reformat.offset =
+				job->flow->idx - 1;
+		rule_acts[hw_acts->encap_decap_pos].reformat.data = buf;
+	}
 	return 0;
 }
 
@@ -1863,6 +2083,7 @@ flow_hw_configure(struct rte_eth_dev *dev,
 			goto err;
 		}
 		mem_size += (sizeof(struct mlx5_hw_q_job *) +
+			    sizeof(uint8_t) * MLX5_ENCAP_MAX_LEN +
 			    sizeof(struct mlx5_hw_q_job)) *
 			    queue_attr[0]->size;
 	}
@@ -1873,6 +2094,8 @@ flow_hw_configure(struct rte_eth_dev *dev,
 		goto err;
 	}
 	for (i = 0; i < nb_queue; i++) {
+		uint8_t *encap = NULL;
+
 		priv->hw_q[i].job_idx = queue_attr[i]->size;
 		priv->hw_q[i].size = queue_attr[i]->size;
 		if (i == 0)
@@ -1883,8 +2106,11 @@ flow_hw_configure(struct rte_eth_dev *dev,
 					    &job[queue_attr[i - 1]->size];
 		job = (struct mlx5_hw_q_job *)
 		      &priv->hw_q[i].job[queue_attr[i]->size];
-		for (j = 0; j < queue_attr[i]->size; j++)
+		encap = (uint8_t *)&job[queue_attr[i]->size];
+		for (j = 0; j < queue_attr[i]->size; j++) {
+			job[j].encap_data = &encap[j * MLX5_ENCAP_MAX_LEN];
 			priv->hw_q[i].job[j] = &job[j];
+		}
 	}
 	dr_ctx_attr.pd = priv->sh->cdev->pd;
 	dr_ctx_attr.queues = nb_queue;
-- 
2.25.1