[dpdk-dev] [RFC] net/mlx5: add meter hierarchy support

[Shun Hao <shunh@nvidia.com>]

Currently, using multiple meters in one flow is not supported. Users
must give every meter a different group, and use multiple flows to
jump from one group to another, so as to apply multiple meters on the
traffic path.

It's not good for performance cause the jumps between every two meters
are waste, so there's requirement that to have a better way for meter
hierarchy.

To support it, now meter action is supported in meter policy, which
means after one meter, in its policy, the packet can go to next
meter directly, without any extra jump action. This makes all meters
in the hierarchy be chained together directly. Users just need to
use the first meter in their flow, and all meters in the hierarchy
will be working.

There's no API changes, just add support of meter action in policy
creation and related validation.

Validation during policy creation:
1. The target meter must exist and must be termination policy meter.
2. No more fate actions accepted besides the meter action.
3. All meters in the hierarchy should not be a loop.

Policy creation:
1. If the final fate action of the meter chain is not RSS, just
create the policy table and policy flow, which has the destination
of the direct next meter and its policy table.
2. If the final fate action of the meter chain is RSS, hold on for
the policy table creation, until the first RTE flow uses this meter.
This is just similar as the creation and usage of a normal RSS policy
meter.

Limitation:
1. The max meter number in one hierarchy is 8.
2. Currently only meters with termination policy are allowed in meter
hierarchy.

Signed-off-by: Shun Hao <shunh@nvidia.com>
---
 doc/guides/nics/mlx5.rst        | 18 +++++++++++++++++
 drivers/net/mlx5/mlx5.h         |  5 +++++
 drivers/net/mlx5/mlx5_flow.h    |  1 +
 drivers/net/mlx5/mlx5_flow_dv.c | 35 +++++++++++++++++++++++++++++++++
 4 files changed, 59 insertions(+)

diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 83299646dd..7296860dd1 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -1909,3 +1909,21 @@ all flows with assistance of external tools.
    .. code-block:: console
 
        mlx_steering_dump.py -f <output_file> -flowptr <flow_ptr>
+
+How to use meter hierarchy
+-----------------
+
+This section demonstrates how to use meter hierarchy. A meter M and a meter N
+can be chained and used together in one flow.
+
+1. Use the later meter N to create a new policy X:
+
+        add port meter policy 0 X g_actions meter mtr_id N / end ...
+
+2. Use the new policy X to create the former meter M:
+
+        create port meter 0 M 1 X yes 0xffff 1 0
+
+3. Apply meter M in flow:
+
+        flow create 0 ingress transfer pattern eth / end actions meter mtr_id M / end
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 32b2817bf2..96741b9948 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -682,6 +682,8 @@ struct mlx5_meter_policy_action_container {
 
 /* Flow meter policy parameter structure. */
 struct mlx5_flow_meter_policy {
+	uint32_t next_mtr_id;
+	/* The next meter id if in meter hierarchy. */
 	uint32_t is_rss:1;
 	/* Is RSS policy table. */
 	uint32_t ingress:1;
@@ -692,6 +694,8 @@ struct mlx5_flow_meter_policy {
 	/* Rule applies to transfer domain. */
 	uint32_t is_queue:1;
 	/* Is queue action in policy table. */
+	uint32_t is_meter:1;
+	/* Is meter action in policy table. */
 	rte_spinlock_t sl;
 	uint32_t ref_cnt;
 	/* Use count. */
@@ -710,6 +714,7 @@ struct mlx5_flow_meter_policy {
 #define MLX5_MTR_SUB_POLICY_NUM_SHIFT  3
 #define MLX5_MTR_SUB_POLICY_NUM_MASK  0x7
 #define MLX5_MTRS_DEFAULT_RULE_PRIORITY 0xFFFF
+#define MLX5_MTR_CHAIN_MAX_NUM 8
 
 /* Flow meter default policy parameter structure.
  * Policy index 0 is reserved by default policy table.
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 2f2aa962f9..3190d9c32d 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -454,6 +454,7 @@ enum mlx5_flow_fate_type {
 	MLX5_FLOW_FATE_DROP,
 	MLX5_FLOW_FATE_DEFAULT_MISS,
 	MLX5_FLOW_FATE_SHARED_RSS,
+	MLX5_FLOW_FATE_MTR,
 	MLX5_FLOW_FATE_MAX,
 };
 
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index c50649a107..7a20e4ee3b 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -14977,6 +14977,9 @@ __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
 				action_flags |= MLX5_FLOW_ACTION_JUMP;
 				break;
 			}
+			case RTE_FLOW_ACTION_TYPE_METER:
+				/* Get next meter's action. */
+				break;
 			default:
 				return -rte_mtr_error_set(error, ENOTSUP,
 					  RTE_MTR_ERROR_TYPE_METER_POLICY,
@@ -15578,6 +15581,9 @@ __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
 				hrxq->action;
 				acts[i].actions_n++;
 				break;
+			case MLX5_FLOW_FATE_MTR:
+				/* Set dv_actions with next meter's action. */
+				break;
 			default:
 				/*Queue action do nothing*/
 				break;
@@ -16420,11 +16426,14 @@ flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
 	struct mlx5_dev_config *dev_conf = &priv->config;
 	const struct rte_flow_action *act;
 	uint64_t action_flags = 0;
+	uint64_t fate_flags = 0;
 	int actions_n;
 	int i, ret;
 	struct rte_flow_error flow_err;
 	uint8_t domain_color[RTE_COLORS] = {0};
 	uint8_t def_domain = MLX5_MTR_ALL_DOMAIN_BIT;
+	struct mlx5_flow_meter_info *fm;
+	const struct rte_flow_action_meter *mtr;
 
 	if (!priv->config.dv_esw_en)
 		def_domain &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
@@ -16602,12 +16611,38 @@ flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
 				++actions_n;
 				action_flags |= MLX5_FLOW_ACTION_JUMP;
 				break;
+			case RTE_FLOW_ACTION_TYPE_METER:
+				mtr = act->conf;
+				fm = mlx5_flow_meter_find(priv, mtr->mtr_id,
+							  NULL);
+				if (!fm)
+					return -rte_mtr_error_set(error,
+						EINVAL,
+						RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
+						"Meter not found");
+				if (fm->def_policy)
+					return -rte_mtr_error_set(error,
+						EINVAL,
+						RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
+						"Default policy meter "
+						"not supported.");
+				++actions_n;
+				action_flags |=
+				MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
+				break;
 			default:
 				return -rte_mtr_error_set(error, ENOTSUP,
 					RTE_MTR_ERROR_TYPE_METER_POLICY,
 					NULL,
 					"Doesn't support optional action");
 			}
+			fate_flags = action_flags &
+			    ~(MLX5_FLOW_ACTION_MARK | MLX5_FLOW_ACTION_SET_TAG);
+			if (fate_flags && (fate_flags & (fate_flags - 1)))
+				return -rte_mtr_error_set(error, EINVAL,
+					  RTE_MTR_ERROR_TYPE_METER_POLICY,
+					  NULL, "Multiple fate actions "
+					  "not supported.");
 		}
 		/* Yellow is not supported, just skip. */
 		if (i == RTE_COLOR_YELLOW)
-- 
2.20.0