[dpdk-dev] [PATCH 14/20] net/mlx5: extend flow mark support

[Viacheslav Ovsiienko <viacheslavo@mellanox.com>]

Flow MARK item is newly supported along with MARK action. MARK
action and item are supported on both Rx and Tx. It works on the
metadata reg_c[] only if extensive flow metadata register is
supported. Without the support, MARK action behaves same as
before - valid only on Rx and no MARK item is valid.

FLAG action is also modified accordingly. FLAG action is
supported on both Rx and Tx via reg_c[] if extensive flow
metadata register is supported.

However, the new MARK/FLAG item and action are currently
disabled until register copy on loopback is supported by
forthcoming patches.

The actual index of engaged metadata reg_c[] register to
support FLAG/MARK actions depends on dv_xmeta_en devarg value.

For extensive metadata mode 1 the reg_c[1] is used and
transitive MARK data width is 24. For extensive metadata mode 2
the reg_c[0] is used and transitive MARK data width might be
restricted to 0 or 16 bits, depending on kernel usage of reg_c[0].
The actual supported width can be discovered by series of trials
with rte_flow_validate().

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
Acked-by: Matan Azrad <matan@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.h    |   5 +-
 drivers/net/mlx5/mlx5_flow_dv.c | 383 ++++++++++++++++++++++++++++++++++++++--
 2 files changed, 370 insertions(+), 18 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 9371e11..d6209ff 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -102,6 +102,7 @@ enum mlx5_feature_name {
 #define MLX5_FLOW_ITEM_METADATA (1u << 16)
 #define MLX5_FLOW_ITEM_PORT_ID (1u << 17)
 #define MLX5_FLOW_ITEM_TAG (1u << 18)
+#define MLX5_FLOW_ITEM_MARK (1u << 19)
 
 /* Pattern MISC bits. */
 #define MLX5_FLOW_LAYER_ICMP (1u << 19)
@@ -194,6 +195,7 @@ enum mlx5_feature_name {
 #define MLX5_FLOW_ACTION_INC_TCP_ACK (1u << 30)
 #define MLX5_FLOW_ACTION_DEC_TCP_ACK (1u << 31)
 #define MLX5_FLOW_ACTION_SET_TAG (1ull << 32)
+#define MLX5_FLOW_ACTION_MARK_EXT (1ull << 33)
 
 #define MLX5_FLOW_FATE_ACTIONS \
 	(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE | \
@@ -228,7 +230,8 @@ enum mlx5_feature_name {
 				      MLX5_FLOW_ACTION_INC_TCP_ACK | \
 				      MLX5_FLOW_ACTION_DEC_TCP_ACK | \
 				      MLX5_FLOW_ACTION_OF_SET_VLAN_VID | \
-				      MLX5_FLOW_ACTION_SET_TAG)
+				      MLX5_FLOW_ACTION_SET_TAG | \
+				      MLX5_FLOW_ACTION_MARK_EXT)
 
 #define MLX5_FLOW_VLAN_ACTIONS (MLX5_FLOW_ACTION_OF_POP_VLAN | \
 				MLX5_FLOW_ACTION_OF_PUSH_VLAN)
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 08e78b0..5714a6d 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -993,6 +993,125 @@ struct field_modify_info modify_tcp[] = {
 }
 
 /**
+ * Convert MARK action to DV specification. This routine is used
+ * in extensive metadata only and requires metadata register to be
+ * handled. In legacy mode hardware tag resource is engaged.
+ *
+ * @param[in] dev
+ *   Pointer to the rte_eth_dev structure.
+ * @param[in] conf
+ *   Pointer to MARK action specification.
+ * @param[in,out] resource
+ *   Pointer to the modify-header resource.
+ * @param[out] error
+ *   Pointer to the error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_convert_action_mark(struct rte_eth_dev *dev,
+			    const struct rte_flow_action_mark *conf,
+			    struct mlx5_flow_dv_modify_hdr_resource *resource,
+			    struct rte_flow_error *error)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
+					   priv->sh->dv_mark_mask);
+	rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
+	struct rte_flow_item item = {
+		.spec = &data,
+		.mask = &mask,
+	};
+	struct field_modify_info reg_c_x[] = {
+		{4, 0, 0}, /* dynamic instead of MLX5_MODI_META_REG_C_1. */
+		{0, 0, 0},
+	};
+	enum modify_reg reg;
+
+	if (!mask)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  NULL, "zero mark action mask");
+	reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
+	if (reg < 0)
+		return reg;
+	assert(reg > 0);
+	reg_c_x[0].id = reg_to_field[reg];
+	return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
+					     MLX5_MODIFICATION_TYPE_SET, error);
+}
+
+/**
+ * Validate MARK item.
+ *
+ * @param[in] dev
+ *   Pointer to the rte_eth_dev structure.
+ * @param[in] item
+ *   Item specification.
+ * @param[in] attr
+ *   Attributes of flow that includes this item.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_validate_item_mark(struct rte_eth_dev *dev,
+			   const struct rte_flow_item *item,
+			   const struct rte_flow_attr *attr __rte_unused,
+			   struct rte_flow_error *error)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_config *config = &priv->config;
+	const struct rte_flow_item_mark *spec = item->spec;
+	const struct rte_flow_item_mark *mask = item->mask;
+	const struct rte_flow_item_mark nic_mask = {
+		.id = priv->sh->dv_mark_mask,
+	};
+	int ret;
+
+	if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
+					  "extended metadata feature"
+					  " isn't enabled");
+	if (!mlx5_flow_ext_mreg_supported(dev))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
+					  "extended metadata register"
+					  " isn't supported");
+	if (!nic_mask.id)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM, item,
+					  "extended metadata register"
+					  " isn't available");
+	ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
+	if (ret < 0)
+		return ret;
+	if (!spec)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+					  item->spec,
+					  "data cannot be empty");
+	if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &spec->id,
+					  "mark id exceeds the limit");
+	if (!mask)
+		mask = &nic_mask;
+	ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
+					(const uint8_t *)&nic_mask,
+					sizeof(struct rte_flow_item_mark),
+					error);
+	if (ret < 0)
+		return ret;
+	return 0;
+}
+
+/**
  * Validate META item.
  *
  * @param[in] dev
@@ -1465,6 +1584,139 @@ struct field_modify_info modify_tcp[] = {
 	return 0;
 }
 
+/*
+ * Validate the FLAG action.
+ *
+ * @param[in] dev
+ *   Pointer to the rte_eth_dev structure.
+ * @param[in] action_flags
+ *   Holds the actions detected until now.
+ * @param[in] attr
+ *   Pointer to flow attributes
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_validate_action_flag(struct rte_eth_dev *dev,
+			     uint64_t action_flags,
+			     const struct rte_flow_attr *attr,
+			     struct rte_flow_error *error)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_config *config = &priv->config;
+	int ret;
+
+	/* Fall back if no extended metadata register support. */
+	if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
+		return mlx5_flow_validate_action_flag(action_flags, attr,
+						      error);
+	/* Extensive metadata mode requires registers. */
+	if (!mlx5_flow_ext_mreg_supported(dev))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "no metadata registers "
+					  "to support flag action");
+	if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "extended metadata register"
+					  " isn't available");
+	ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
+	if (ret < 0)
+		return ret;
+	assert(ret > 0);
+	if (action_flags & MLX5_FLOW_ACTION_DROP)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't drop and flag in same flow");
+	if (action_flags & MLX5_FLOW_ACTION_MARK)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't mark and flag in same flow");
+	if (action_flags & MLX5_FLOW_ACTION_FLAG)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't have 2 flag"
+					  " actions in same flow");
+	return 0;
+}
+
+/**
+ * Validate MARK action.
+ *
+ * @param[in] dev
+ *   Pointer to the rte_eth_dev structure.
+ * @param[in] action
+ *   Pointer to action.
+ * @param[in] action_flags
+ *   Holds the actions detected until now.
+ * @param[in] attr
+ *   Pointer to flow attributes
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_validate_action_mark(struct rte_eth_dev *dev,
+			     const struct rte_flow_action *action,
+			     uint64_t action_flags,
+			     const struct rte_flow_attr *attr,
+			     struct rte_flow_error *error)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_config *config = &priv->config;
+	const struct rte_flow_action_mark *mark = action->conf;
+	int ret;
+
+	/* Fall back if no extended metadata register support. */
+	if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
+		return mlx5_flow_validate_action_mark(action, action_flags,
+						      attr, error);
+	/* Extensive metadata mode requires registers. */
+	if (!mlx5_flow_ext_mreg_supported(dev))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "no metadata registers "
+					  "to support mark action");
+	if (!priv->sh->dv_mark_mask)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "extended metadata register"
+					  " isn't available");
+	ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
+	if (ret < 0)
+		return ret;
+	assert(ret > 0);
+	if (!mark)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, action,
+					  "configuration cannot be null");
+	if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &mark->id,
+					  "mark id exceeds the limit");
+	if (action_flags & MLX5_FLOW_ACTION_DROP)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't drop and mark in same flow");
+	if (action_flags & MLX5_FLOW_ACTION_FLAG)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't flag and mark in same flow");
+	if (action_flags & MLX5_FLOW_ACTION_MARK)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					  "can't have 2 mark actions in same"
+					  " flow");
+	return 0;
+}
+
 /**
  * Validate SET_TAG action.
  *
@@ -3732,6 +3984,8 @@ struct field_modify_info modify_tcp[] = {
 			.dst_port = RTE_BE16(UINT16_MAX),
 		}
 	};
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_config *dev_conf = &priv->config;
 
 	if (items == NULL)
 		return -1;
@@ -3888,6 +4142,14 @@ struct field_modify_info modify_tcp[] = {
 				return ret;
 			last_item = MLX5_FLOW_LAYER_MPLS;
 			break;
+
+		case RTE_FLOW_ITEM_TYPE_MARK:
+			ret = flow_dv_validate_item_mark(dev, items, attr,
+							 error);
+			if (ret < 0)
+				return ret;
+			last_item = MLX5_FLOW_ITEM_MARK;
+			break;
 		case RTE_FLOW_ITEM_TYPE_META:
 			ret = flow_dv_validate_item_meta(dev, items, attr,
 							 error);
@@ -3949,21 +4211,39 @@ struct field_modify_info modify_tcp[] = {
 			++actions_n;
 			break;
 		case RTE_FLOW_ACTION_TYPE_FLAG:
-			ret = mlx5_flow_validate_action_flag(action_flags,
-							     attr, error);
+			ret = flow_dv_validate_action_flag(dev, action_flags,
+							   attr, error);
 			if (ret < 0)
 				return ret;
-			action_flags |= MLX5_FLOW_ACTION_FLAG;
-			++actions_n;
+			if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
+				/* Count all modify-header actions as one. */
+				if (!(action_flags &
+				      MLX5_FLOW_MODIFY_HDR_ACTIONS))
+					++actions_n;
+				action_flags |= MLX5_FLOW_ACTION_FLAG |
+						MLX5_FLOW_ACTION_MARK_EXT;
+			} else {
+				action_flags |= MLX5_FLOW_ACTION_FLAG;
+				++actions_n;
+			}
 			break;
 		case RTE_FLOW_ACTION_TYPE_MARK:
-			ret = mlx5_flow_validate_action_mark(actions,
-							     action_flags,
-							     attr, error);
+			ret = flow_dv_validate_action_mark(dev, actions,
+							   action_flags,
+							   attr, error);
 			if (ret < 0)
 				return ret;
-			action_flags |= MLX5_FLOW_ACTION_MARK;
-			++actions_n;
+			if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
+				/* Count all modify-header actions as one. */
+				if (!(action_flags &
+				      MLX5_FLOW_MODIFY_HDR_ACTIONS))
+					++actions_n;
+				action_flags |= MLX5_FLOW_ACTION_MARK |
+						MLX5_FLOW_ACTION_MARK_EXT;
+			} else {
+				action_flags |= MLX5_FLOW_ACTION_MARK;
+				++actions_n;
+			}
 			break;
 		case RTE_FLOW_ACTION_TYPE_SET_TAG:
 			ret = flow_dv_validate_action_set_tag(dev, actions,
@@ -4234,12 +4514,14 @@ struct field_modify_info modify_tcp[] = {
 					  " actions in the same rule");
 	/* Eswitch has few restrictions on using items and actions */
 	if (attr->transfer) {
-		if (action_flags & MLX5_FLOW_ACTION_FLAG)
+		if (!mlx5_flow_ext_mreg_supported(dev) &&
+		    action_flags & MLX5_FLOW_ACTION_FLAG)
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
 						  NULL,
 						  "unsupported action FLAG");
-		if (action_flags & MLX5_FLOW_ACTION_MARK)
+		if (!mlx5_flow_ext_mreg_supported(dev) &&
+		    action_flags & MLX5_FLOW_ACTION_MARK)
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
 						  NULL,
@@ -5202,6 +5484,44 @@ struct field_modify_info modify_tcp[] = {
 }
 
 /**
+ * Add MARK item to matcher
+ *
+ * @param[in] dev
+ *   The device to configure through.
+ * @param[in, out] matcher
+ *   Flow matcher.
+ * @param[in, out] key
+ *   Flow matcher value.
+ * @param[in] item
+ *   Flow pattern to translate.
+ */
+static void
+flow_dv_translate_item_mark(struct rte_eth_dev *dev,
+			    void *matcher, void *key,
+			    const struct rte_flow_item *item)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	const struct rte_flow_item_mark *mark;
+	uint32_t value;
+	uint32_t mask;
+
+	mark = item->mask ? (const void *)item->mask :
+			    &rte_flow_item_mark_mask;
+	mask = mark->id & priv->sh->dv_mark_mask;
+	mark = (const void *)item->spec;
+	assert(mark);
+	value = mark->id & priv->sh->dv_mark_mask & mask;
+	if (mask) {
+		enum modify_reg reg;
+
+		/* Get the metadata register index for the mark. */
+		reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
+		assert(reg > 0);
+		flow_dv_match_meta_reg(matcher, key, reg, value, mask);
+	}
+}
+
+/**
  * Add META item to matcher
  *
  * @param[in, out] matcher
@@ -5210,8 +5530,6 @@ struct field_modify_info modify_tcp[] = {
  *   Flow matcher value.
  * @param[in] item
  *   Flow pattern to translate.
- * @param[in] inner
- *   Item is inner pattern.
  */
 static void
 flow_dv_translate_item_meta(void *matcher, void *key,
@@ -5882,6 +6200,7 @@ struct field_modify_info modify_tcp[] = {
 		    struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_config *dev_conf = &priv->config;
 	struct rte_flow *flow = dev_flow->flow;
 	uint64_t item_flags = 0;
 	uint64_t last_item = 0;
@@ -5916,7 +6235,7 @@ struct field_modify_info modify_tcp[] = {
 	if (attr->transfer)
 		mhdr_res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
 	if (priority == MLX5_FLOW_PRIO_RSVD)
-		priority = priv->config.flow_prio - 1;
+		priority = dev_conf->flow_prio - 1;
 	for (; !actions_end ; actions++) {
 		const struct rte_flow_action_queue *queue;
 		const struct rte_flow_action_rss *rss;
@@ -5947,6 +6266,19 @@ struct field_modify_info modify_tcp[] = {
 			action_flags |= MLX5_FLOW_ACTION_PORT_ID;
 			break;
 		case RTE_FLOW_ACTION_TYPE_FLAG:
+			action_flags |= MLX5_FLOW_ACTION_FLAG;
+			if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
+				struct rte_flow_action_mark mark = {
+					.id = MLX5_FLOW_MARK_DEFAULT,
+				};
+
+				if (flow_dv_convert_action_mark(dev, &mark,
+								&mhdr_res,
+								error))
+					return -rte_errno;
+				action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
+				break;
+			}
 			tag_resource.tag =
 				mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
 			if (!dev_flow->dv.tag_resource)
@@ -5955,9 +6287,22 @@ struct field_modify_info modify_tcp[] = {
 					return errno;
 			dev_flow->dv.actions[actions_n++] =
 				dev_flow->dv.tag_resource->action;
-			action_flags |= MLX5_FLOW_ACTION_FLAG;
 			break;
 		case RTE_FLOW_ACTION_TYPE_MARK:
+			action_flags |= MLX5_FLOW_ACTION_MARK;
+			if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
+				const struct rte_flow_action_mark *mark =
+					(const struct rte_flow_action_mark *)
+						actions->conf;
+
+				if (flow_dv_convert_action_mark(dev, mark,
+								&mhdr_res,
+								error))
+					return -rte_errno;
+				action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
+				break;
+			}
+			/* Legacy (non-extensive) MARK action. */
 			tag_resource.tag = mlx5_flow_mark_set
 			      (((const struct rte_flow_action_mark *)
 			       (actions->conf))->id);
@@ -5967,7 +6312,6 @@ struct field_modify_info modify_tcp[] = {
 					return errno;
 			dev_flow->dv.actions[actions_n++] =
 				dev_flow->dv.tag_resource->action;
-			action_flags |= MLX5_FLOW_ACTION_MARK;
 			break;
 		case RTE_FLOW_ACTION_TYPE_SET_TAG:
 			if (flow_dv_convert_action_set_tag
@@ -6004,7 +6348,7 @@ struct field_modify_info modify_tcp[] = {
 			action_flags |= MLX5_FLOW_ACTION_RSS;
 			break;
 		case RTE_FLOW_ACTION_TYPE_COUNT:
-			if (!priv->config.devx) {
+			if (!dev_conf->devx) {
 				rte_errno = ENOTSUP;
 				goto cnt_err;
 			}
@@ -6417,6 +6761,11 @@ struct field_modify_info modify_tcp[] = {
 						    items, last_item, tunnel);
 			last_item = MLX5_FLOW_LAYER_MPLS;
 			break;
+		case RTE_FLOW_ITEM_TYPE_MARK:
+			flow_dv_translate_item_mark(dev, match_mask,
+						    match_value, items);
+			last_item = MLX5_FLOW_ITEM_MARK;
+			break;
 		case RTE_FLOW_ITEM_TYPE_META:
 			flow_dv_translate_item_meta(match_mask, match_value,
 						    items);
-- 
1.8.3.1