[PATCH 20.11 1/5] net/mlx5: fix RSS expansion for inner tunnel VLAN

[PATCH 20.11 1/5] net/mlx5: fix RSS expansion for inner tunnel VLAN

[Lior Margalit]

[ upstream commit 3f02c7ff6815286dfe81efbf2db196bdafc752b3 ]

The RSS expansion algorithm is using a graph to find the possible
expansion paths. The VLAN item in the flow pattern requires special
treatment, because it should not be added implicitly by the expansion
algorithm.  If the flow pattern ends with ETH item, the pattern will be
expanded with IPv4 and IPv6.
For example:
testpmd> flow create ... eth / end actions rss / end
ETH END
ETH IPV4 END
ETH IPV6 END
If a VLAN item follows the ETH item in the flow pattern, the pattern
will be expanded with IPv4 and IPv6 following the VLAN item.
For example:
testpmd> flow create ... eth / vlan / end actions rss level 1 / end
ETH VLAN END
ETH VLAN IPV4 END
ETH VLAN IPV6 END

The case of inner tunnel VLAN item was not taken care of so the flow
pattern did not expand with IPv6 and IPv4 as expected.
Example with inner VLAN:
testpmd> flow create ... / vxlan / eth / vlan / end actions rss level 2
/ end
The current result of the expansion alg:
ETH IPV6 UDP VXLAN ETH VLAN END
The expected result of the expansion alg:
ETH IPV6 UDP VXLAN ETH VLAN END
ETH IPV6 UDP VXLAN ETH VLAN IPV4 END
ETH IPV6 UDP VXLAN ETH VLAN IPV6 END

The fix is to introduce a new flag to set on a graph expansion node
to apply the 'explicit' behavior, meaning the node is not added to
the expanded pattern, if it is not found in the flow pattern, but the
expansion alg can go deeper to its next nodes.

Fixes: c7870bfe09dc ("ethdev: move RSS expansion code to mlx5 driver")
Cc: stable@dpdk.org

Signed-off-by: Lior Margalit <lmargalit@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_flow.c | 88 ++++++++++++++++++------------------
 1 file changed, 44 insertions(+), 44 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 3c35b87964..2b916899e8 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -100,10 +100,25 @@ struct mlx5_flow_expand_node {
 	 * RSS types bit-field associated with this node
 	 * (see ETH_RSS_* definitions).
 	 */
-	uint8_t optional;
-	/**< optional expand field. Default 0 to expand, 1 not go deeper. */
+	uint64_t node_flags;
+	/**<
+	 *  Bit-fields that define how the node is used in the expansion.
+	 * (see MLX5_EXPANSION_NODE_* definitions).
+	 */
 };
 
+/* Optional expand field. The expansion alg will not go deeper. */
+#define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0)
+
+/* The node is not added implicitly as expansion to the flow pattern.
+ * If the node type does not match the flow pattern item type, the
+ * expansion alg will go deeper to its next items.
+ * In the current implementation, the list of next nodes indexes can
+ * have up to one node with this flag set and it has to be the last
+ * node index (before the list terminator).
+ */
+#define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1)
+
 /** Object returned by mlx5_flow_expand_rss(). */
 struct mlx5_flow_expand_rss {
 	uint32_t entries;
@@ -331,10 +346,17 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 			continue;
 		}
 		last_item = item;
-		for (i = 0; node->next && node->next[i]; ++i) {
+		i = 0;
+		while (node->next && node->next[i]) {
 			next = &graph[node->next[i]];
 			if (next->type == item->type)
 				break;
+			if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) {
+				node = next;
+				i = 0;
+			} else {
+				++i;
+			}
 		}
 		if (next)
 			node = next;
@@ -393,6 +415,16 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 	}
 	memset(flow_items, 0, sizeof(flow_items));
 	next_node = node->next;
+	while (next_node) {
+		/*
+		 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT
+		 * flag set, because they were not found in the flow pattern.
+		 */
+		node = &graph[*next_node];
+		if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT))
+			break;
+		next_node = node->next;
+	}
 	stack[stack_pos] = next_node;
 	node = next_node ? &graph[*next_node] : NULL;
 	while (node) {
@@ -427,7 +459,8 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 			addr = (void *)(((uintptr_t)addr) + n);
 		}
 		/* Go deeper. */
-		if (!node->optional && node->next) {
+		if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) &&
+				node->next) {
 			next_node = node->next;
 			if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
 				rte_errno = E2BIG;
@@ -452,10 +485,7 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 enum mlx5_expansion {
 	MLX5_EXPANSION_ROOT,
 	MLX5_EXPANSION_ROOT_OUTER,
-	MLX5_EXPANSION_ROOT_ETH_VLAN,
-	MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
 	MLX5_EXPANSION_OUTER_ETH,
-	MLX5_EXPANSION_OUTER_ETH_VLAN,
 	MLX5_EXPANSION_OUTER_VLAN,
 	MLX5_EXPANSION_OUTER_IPV4,
 	MLX5_EXPANSION_OUTER_IPV4_UDP,
@@ -470,7 +500,6 @@ enum mlx5_expansion {
 	MLX5_EXPANSION_GRE_KEY,
 	MLX5_EXPANSION_MPLS,
 	MLX5_EXPANSION_ETH,
-	MLX5_EXPANSION_ETH_VLAN,
 	MLX5_EXPANSION_VLAN,
 	MLX5_EXPANSION_IPV4,
 	MLX5_EXPANSION_IPV4_UDP,
@@ -497,22 +526,7 @@ static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 						  MLX5_EXPANSION_OUTER_IPV6),
 		.type = RTE_FLOW_ITEM_TYPE_END,
 	},
-	[MLX5_EXPANSION_ROOT_ETH_VLAN] = {
-		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
-		.type = RTE_FLOW_ITEM_TYPE_END,
-	},
-	[MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
-		.next = MLX5_FLOW_EXPAND_RSS_NEXT
-						(MLX5_EXPANSION_OUTER_ETH_VLAN),
-		.type = RTE_FLOW_ITEM_TYPE_END,
-	},
 	[MLX5_EXPANSION_OUTER_ETH] = {
-		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
-						  MLX5_EXPANSION_OUTER_IPV6),
-		.type = RTE_FLOW_ITEM_TYPE_ETH,
-		.rss_types = 0,
-	},
-	[MLX5_EXPANSION_OUTER_ETH_VLAN] = {
 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
 		.type = RTE_FLOW_ITEM_TYPE_ETH,
 		.rss_types = 0,
@@ -521,6 +535,7 @@ static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
 						  MLX5_EXPANSION_OUTER_IPV6),
 		.type = RTE_FLOW_ITEM_TYPE_VLAN,
+		.node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
 	},
 	[MLX5_EXPANSION_OUTER_IPV4] = {
 		.next = MLX5_FLOW_EXPAND_RSS_NEXT
@@ -597,7 +612,7 @@ static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 						  MLX5_EXPANSION_IPV6,
 						  MLX5_EXPANSION_MPLS),
 		.type = RTE_FLOW_ITEM_TYPE_GRE_KEY,
-		.optional = 1,
+		.node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
 	},
 	[MLX5_EXPANSION_NVGRE] = {
 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
@@ -608,13 +623,9 @@ static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 						  MLX5_EXPANSION_IPV6,
 						  MLX5_EXPANSION_ETH),
 		.type = RTE_FLOW_ITEM_TYPE_MPLS,
+		.node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
 	},
 	[MLX5_EXPANSION_ETH] = {
-		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
-						  MLX5_EXPANSION_IPV6),
-		.type = RTE_FLOW_ITEM_TYPE_ETH,
-	},
-	[MLX5_EXPANSION_ETH_VLAN] = {
 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
 		.type = RTE_FLOW_ITEM_TYPE_ETH,
 	},
@@ -622,6 +633,7 @@ static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 						  MLX5_EXPANSION_IPV6),
 		.type = RTE_FLOW_ITEM_TYPE_VLAN,
+		.node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
 	},
 	[MLX5_EXPANSION_IPV4] = {
 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
@@ -3458,20 +3470,8 @@ flow_get_shared_rss_action(struct rte_eth_dev *dev,
 }
 
 static unsigned int
-find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
+find_graph_root(uint32_t rss_level)
 {
-	const struct rte_flow_item *item;
-	unsigned int has_vlan = 0;
-
-	for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
-		if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
-			has_vlan = 1;
-			break;
-		}
-	}
-	if (has_vlan)
-		return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
-				       MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
 	return rss_level < 2 ? MLX5_EXPANSION_ROOT :
 			       MLX5_EXPANSION_ROOT_OUTER;
 }
@@ -5361,7 +5361,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 	int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
 	union {
 		struct mlx5_flow_expand_rss buf;
-		uint8_t buffer[2048];
+		uint8_t buffer[4096];
 	} expand_buffer;
 	union {
 		struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
@@ -5451,7 +5451,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 	if (rss && rss->types) {
 		unsigned int graph_root;
 
-		graph_root = find_graph_root(items, rss->level);
+		graph_root = find_graph_root(rss->level);
 		ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
 					   items, rss->types,
 					   mlx5_support_expansion, graph_root);
-- 
2.25.1

[PATCH 20.11 2/5] net/mlx5: fix RSS expansion for explicit graph node

[Lior Margalit]

[ upstream commit 69d268b4fff3791c3fea772a43abca1660982005 ]

The RSS expansion algorithm is using a graph to find the possible
expansion paths. A graph node with the 'explicit' flag will be skipped,
if it is not found in the flow pattern.

The current implementation misses the case where the node with the
explicit flag is in the middle of the expanded path.
For example:
testpmd> flow create 0 ingress pattern eth / ipv6 / udp / vxlan / end
actions rss level 2 types tcp end / end
The VLAN node has the explicit flag, so it is currently included in the
expanded flow:
ETH IPV6 UDP VXLAN END
ETH IPV6 UDP VXLAN ETH VLAN IPV4 TCP END
ETH IPV6 UDP VXLAN ETH VLAN IPV6 TCP END

The fix is to skip the nodes with the explicit flag while iterating over
the possible expansion paths. Using the above example, the flows will be:
ETH IPV6 UDP VXLAN END
ETH IPV6 UDP VXLAN ETH IPV4 TCP END
ETH IPV6 UDP VXLAN ETH IPV6 TCP END

Fixes: 3f02c7ff6815 ("net/mlx5: fix RSS expansion for inner tunnel VLAN")
Cc: stable@dpdk.org

Signed-off-by: Lior Margalit <lmargalit@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_flow.c | 44 ++++++++++++++++++++++++------------
 1 file changed, 29 insertions(+), 15 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 2b916899e8..cacccd6cc8 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -282,6 +282,26 @@ mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item)
 	return ret;
 }
 
+static const int *
+mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[],
+		const int *next_node)
+{
+	const struct mlx5_flow_expand_node *node = NULL;
+	const int *next = next_node;
+
+	while (next && *next) {
+		/*
+		 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT
+		 * flag set, because they were not found in the flow pattern.
+		 */
+		node = &graph[*next];
+		if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT))
+			break;
+		next = node->next;
+	}
+	return next;
+}
+
 #define MLX5_RSS_EXP_ELT_N 16
 
 /**
@@ -414,17 +434,8 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 		}
 	}
 	memset(flow_items, 0, sizeof(flow_items));
-	next_node = node->next;
-	while (next_node) {
-		/*
-		 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT
-		 * flag set, because they were not found in the flow pattern.
-		 */
-		node = &graph[*next_node];
-		if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT))
-			break;
-		next_node = node->next;
-	}
+	next_node = mlx5_flow_expand_rss_skip_explicit(graph,
+			node->next);
 	stack[stack_pos] = next_node;
 	node = next_node ? &graph[*next_node] : NULL;
 	while (node) {
@@ -461,7 +472,8 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 		/* Go deeper. */
 		if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) &&
 				node->next) {
-			next_node = node->next;
+			next_node = mlx5_flow_expand_rss_skip_explicit(graph,
+					node->next);
 			if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
 				rte_errno = E2BIG;
 				return -rte_errno;
@@ -469,15 +481,17 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 			stack[stack_pos] = next_node;
 		} else if (*(next_node + 1)) {
 			/* Follow up with the next possibility. */
-			++next_node;
+			next_node = mlx5_flow_expand_rss_skip_explicit(graph,
+					++next_node);
 		} else {
 			/* Move to the next path. */
 			if (stack_pos)
 				next_node = stack[--stack_pos];
-			next_node++;
+			next_node = mlx5_flow_expand_rss_skip_explicit(graph,
+					++next_node);
 			stack[stack_pos] = next_node;
 		}
-		node = *next_node ? &graph[*next_node] : NULL;
+		node = next_node && *next_node ? &graph[*next_node] : NULL;
 	};
 	return lsize;
 }
-- 
2.25.1

[PATCH 20.11 3/5] net/mlx5: fix RSS expansion traversal over next nodes

[Lior Margalit]

[ upstream commit aa52e5f0f9e6fe4acf3cd54ce7e392944f68d6c4 ]

The RSS expansion is based on DFS algorithm to traverse over the possible
expansion paths.

The current implementation breaks out, if it reaches the terminator of
the "next nodes" array, instead of going backwards to try the next path.
For example:
testpmd> flow create 0 ingress pattern eth / ipv6 / udp / vxlan / end
actions rss level 2 types tcp end / end
The paths found are:
ETH IPV6 UDP VXLAN END
ETH IPV6 UDP VXLAN ETH IPV4 TCP END
ETH IPV6 UDP VXLAN ETH IPV6 TCP END
The traversal stopped after getting to the terminator of the next nodes
of the ETH node. It missed the rest of the nodes in the next nodes array
of the VXLAN node.

The fix is to go backwards when reaching the terminator of the current
level and find if there is a "next node" to start traversing a new path.
Using the above example, the flows will be:
ETH IPV6 UDP VXLAN END
ETH IPV6 UDP VXLAN ETH IPV4 TCP END
ETH IPV6 UDP VXLAN ETH IPV6 TCP END
ETH IPV6 UDP VXLAN IPV4 TCP END
ETH IPV6 UDP VXLAN IPV6 TCP END
The traversal will find additional paths, because it traverses through
all the next nodes array of the VXLAN node.

Fixes: 4ed05fcd441b ("ethdev: add flow API to expand RSS flows")
Cc: stable@dpdk.org

Signed-off-by: Lior Margalit <lmargalit@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_flow.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index cacccd6cc8..5cebd5ed6f 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -483,12 +483,22 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 			/* Follow up with the next possibility. */
 			next_node = mlx5_flow_expand_rss_skip_explicit(graph,
 					++next_node);
+		} else if (!stack_pos) {
+			/*
+			 * Completing the traverse over the different paths.
+			 * The next_node is advanced to the terminator.
+			 */
+			++next_node;
 		} else {
 			/* Move to the next path. */
-			if (stack_pos)
+			while (stack_pos) {
 				next_node = stack[--stack_pos];
+				next_node++;
+				if (*next_node)
+					break;
+			}
 			next_node = mlx5_flow_expand_rss_skip_explicit(graph,
-					++next_node);
+					next_node);
 			stack[stack_pos] = next_node;
 		}
 		node = next_node && *next_node ? &graph[*next_node] : NULL;
-- 
2.25.1

[PATCH 20.11 4/5] net/mlx5: fix RSS expansion for L2/L3 VXLAN

[Lior Margalit]

[ upstream commit 0c3fa683964ead2597a542e8dccb87b23343a15f ]

The RSS expansion algorithm is using a graph to find the possible
expansion paths. The current implementation does not differentiate
between standard (L2) VXLAN and L3 VXLAN. As result the flow is expanded
with all possible paths.
For example:
testpmd> flow create... / vxlan / end actions rss level 2 / end
It is currently expanded to the following paths:
ETH IPV4 UDP VXLAN END
ETH IPV4 UDP VXLAN ETH IPV4 END
ETH IPV4 UDP VXLAN ETH IPV6 END
ETH IPV4 UDP VXLAN IPV4 END
ETH IPV4 UDP VXLAN IPV6 END

The fix is to adjust the expansion according to the outer UDP destination
port. In case flow pattern defines a match on the standard udp port, 4789,
or does not define a match on the destination port, which also implies
setting the standard one, the expansion for the above example will be:
ETH IPV4 UDP VXLAN END
ETH IPV4 UDP VXLAN ETH IPV4 END
ETH IPV4 UDP VXLAN ETH IPV6 END
Otherwise, the expansion will be:
ETH IPV4 UDP VXLAN END
ETH IPV4 UDP VXLAN IPV4 END
ETH IPV4 UDP VXLAN IPV6 END

Fixes: f4f06e361516 ("net/mlx5: add flow VXLAN item")
Cc: stable@dpdk.org

Signed-off-by: Lior Margalit <lmargalit@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_flow.c | 71 +++++++++++++++++++++++++++++++++---
 1 file changed, 66 insertions(+), 5 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 5cebd5ed6f..befad22cbb 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -132,6 +132,12 @@ struct mlx5_flow_expand_rss {
 static void
 mlx5_dbg__print_pattern(const struct rte_flow_item *item);
 
+static const struct mlx5_flow_expand_node *
+mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern,
+		unsigned int item_idx,
+		const struct mlx5_flow_expand_node graph[],
+		const struct mlx5_flow_expand_node *node);
+
 static bool
 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item)
 {
@@ -341,7 +347,7 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 	const int *stack[MLX5_RSS_EXP_ELT_N];
 	int stack_pos = 0;
 	struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N];
-	unsigned int i;
+	unsigned int i, item_idx, last_expand_item_idx = 0;
 	size_t lsize;
 	size_t user_pattern_size = 0;
 	void *addr = NULL;
@@ -349,7 +355,7 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 	struct rte_flow_item missed_item;
 	int missed = 0;
 	int elt = 0;
-	const struct rte_flow_item *last_item = NULL;
+	const struct rte_flow_item *last_expand_item = NULL;
 
 	memset(&missed_item, 0, sizeof(missed_item));
 	lsize = offsetof(struct mlx5_flow_expand_rss, entry) +
@@ -360,12 +366,15 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 	buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N];
 	buf->entries = 0;
 	addr = buf->entry[0].pattern;
-	for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
+	for (item = pattern, item_idx = 0;
+			item->type != RTE_FLOW_ITEM_TYPE_END;
+			item++, item_idx++) {
 		if (!mlx5_flow_is_rss_expandable_item(item)) {
 			user_pattern_size += sizeof(*item);
 			continue;
 		}
-		last_item = item;
+		last_expand_item = item;
+		last_expand_item_idx = item_idx;
 		i = 0;
 		while (node->next && node->next[i]) {
 			next = &graph[node->next[i]];
@@ -397,7 +406,7 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 	 * Check if the last valid item has spec set, need complete pattern,
 	 * and the pattern can be used for expansion.
 	 */
-	missed_item.type = mlx5_flow_expand_rss_item_complete(last_item);
+	missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item);
 	if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) {
 		/* Item type END indicates expansion is not required. */
 		return lsize;
@@ -432,6 +441,9 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 			addr = (void *)(((uintptr_t)addr) +
 					elt * sizeof(*item));
 		}
+	} else if (last_expand_item != NULL) {
+		node = mlx5_flow_expand_rss_adjust_node(pattern,
+				last_expand_item_idx, graph, node);
 	}
 	memset(flow_items, 0, sizeof(flow_items));
 	next_node = mlx5_flow_expand_rss_skip_explicit(graph,
@@ -518,6 +530,8 @@ enum mlx5_expansion {
 	MLX5_EXPANSION_OUTER_IPV6_UDP,
 	MLX5_EXPANSION_OUTER_IPV6_TCP,
 	MLX5_EXPANSION_VXLAN,
+	MLX5_EXPANSION_STD_VXLAN,
+	MLX5_EXPANSION_L3_VXLAN,
 	MLX5_EXPANSION_VXLAN_GPE,
 	MLX5_EXPANSION_GRE,
 	MLX5_EXPANSION_NVGRE,
@@ -617,6 +631,15 @@ static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
 						  MLX5_EXPANSION_IPV6),
 		.type = RTE_FLOW_ITEM_TYPE_VXLAN,
 	},
+	[MLX5_EXPANSION_STD_VXLAN] = {
+			.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
+					.type = RTE_FLOW_ITEM_TYPE_VXLAN,
+	},
+	[MLX5_EXPANSION_L3_VXLAN] = {
+			.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
+					MLX5_EXPANSION_IPV6),
+					.type = RTE_FLOW_ITEM_TYPE_VXLAN,
+	},
 	[MLX5_EXPANSION_VXLAN_GPE] = {
 		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
 						  MLX5_EXPANSION_IPV4,
@@ -8103,3 +8126,41 @@ mlx5_dbg__print_pattern(const struct rte_flow_item *item)
 	}
 	printf("END\n");
 }
+
+static int
+mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item)
+{
+	const struct rte_flow_item_udp *spec = udp_item->spec;
+	const struct rte_flow_item_udp *mask = udp_item->mask;
+	uint16_t udp_dport = 0;
+
+	if (spec != NULL) {
+		if (!mask)
+			mask = &rte_flow_item_udp_mask;
+		udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port &
+				mask->hdr.dst_port);
+	}
+	return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN);
+}
+
+static const struct mlx5_flow_expand_node *
+mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern,
+		unsigned int item_idx,
+		const struct mlx5_flow_expand_node graph[],
+		const struct mlx5_flow_expand_node *node)
+{
+	const struct rte_flow_item *item = pattern + item_idx, *prev_item;
+	switch (item->type) {
+	case RTE_FLOW_ITEM_TYPE_VXLAN:
+		MLX5_ASSERT(item_idx > 0);
+		prev_item = pattern + item_idx - 1;
+		MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP);
+		if (mlx5_flow_is_std_vxlan_port(prev_item))
+			return &graph[MLX5_EXPANSION_STD_VXLAN];
+		else
+			return &graph[MLX5_EXPANSION_L3_VXLAN];
+		break;
+	default:
+		return node;
+	}
+}
-- 
2.25.1

[PATCH 20.11 5/5] net/mlx5: fix RSS expansion with EtherType

[Lior Margalit]

[ upstream commit a451287102fec4eecdb2938760a57334bccc204f ]

The RSS expansion algorithm is using a graph to find the possible
expansion paths. A graph node with the 'explicit' flag will be skipped,
if it is not found in the flow pattern.
The current implementation misses a check for the explicit flag when
expanding the pattern according to ETH item with EtherType.
For example:
testpmd> flow create 0 ingress pattern eth / ipv6 / udp / vxlan / eth
type is 2048 / end actions rss level 2 types udp end / end
The "eth type is 2048" item in the pattern may be expanded to "ETH IPv4".
The ETH node in the expansion graph is followed by VLAN node marked as
explicit. The fix is to skip the VLAN node and continue the expansion
with its next nodes, IPv4 and IPv6.
The expansion paths for the above example will be:
ETH IPV6 UDP VXLAN ETH END
ETH IPV6 UDP VXLAN ETH IPV4 UDP END

Fixes: 69d268b4fff3 ("net/mlx5: fix RSS expansion for explicit graph node")
Cc: stable@dpdk.org

Signed-off-by: Lior Margalit <lmargalit@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_flow.c | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index befad22cbb..85d755b5df 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -414,13 +414,20 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
 	if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) {
 		next = NULL;
 		missed = 1;
-		for (i = 0; node->next && node->next[i]; ++i) {
+		i = 0;
+		while (node->next && node->next[i]) {
 			next = &graph[node->next[i]];
 			if (next->type == missed_item.type) {
 				flow_items[0].type = missed_item.type;
 				flow_items[1].type = RTE_FLOW_ITEM_TYPE_END;
 				break;
 			}
+			if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) {
+				node = next;
+				i = 0;
+			} else {
+				++i;
+			}
 			next = NULL;
 		}
 	}
@@ -8150,17 +8157,21 @@ mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern,
 		const struct mlx5_flow_expand_node *node)
 {
 	const struct rte_flow_item *item = pattern + item_idx, *prev_item;
-	switch (item->type) {
-	case RTE_FLOW_ITEM_TYPE_VXLAN:
+
+	if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN &&
+			node != NULL &&
+			node->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
+		/*
+		 * The expansion node is VXLAN and it is also the last
+		 * expandable item in the pattern, so need to continue
+		 * expansion of the inner tunnel.
+		 */
 		MLX5_ASSERT(item_idx > 0);
 		prev_item = pattern + item_idx - 1;
 		MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP);
 		if (mlx5_flow_is_std_vxlan_port(prev_item))
 			return &graph[MLX5_EXPANSION_STD_VXLAN];
-		else
-			return &graph[MLX5_EXPANSION_L3_VXLAN];
-		break;
-	default:
-		return node;
+		return &graph[MLX5_EXPANSION_L3_VXLAN];
 	}
+	return node;
 }
-- 
2.25.1

Re: [PATCH 20.11 1/5] net/mlx5: fix RSS expansion for inner tunnel VLAN

[Xueming(Steven) Li]

Series applied to 20.11.4 list, thanks!

On Tue, 2021-11-30 at 14:17 +0200, Lior Margalit wrote:
> [ upstream commit 3f02c7ff6815286dfe81efbf2db196bdafc752b3 ]
> 
> The RSS expansion algorithm is using a graph to find the possible
> expansion paths. The VLAN item in the flow pattern requires special
> treatment, because it should not be added implicitly by the expansion
> algorithm.  If the flow pattern ends with ETH item, the pattern will be
> expanded with IPv4 and IPv6.
> For example:
> testpmd> flow create ... eth / end actions rss / end
> ETH END
> ETH IPV4 END
> ETH IPV6 END
> If a VLAN item follows the ETH item in the flow pattern, the pattern
> will be expanded with IPv4 and IPv6 following the VLAN item.
> For example:
> testpmd> flow create ... eth / vlan / end actions rss level 1 / end
> ETH VLAN END
> ETH VLAN IPV4 END
> ETH VLAN IPV6 END
> 
> The case of inner tunnel VLAN item was not taken care of so the flow
> pattern did not expand with IPv6 and IPv4 as expected.
> Example with inner VLAN:
> testpmd> flow create ... / vxlan / eth / vlan / end actions rss level 2
> / end
> The current result of the expansion alg:
> ETH IPV6 UDP VXLAN ETH VLAN END
> The expected result of the expansion alg:
> ETH IPV6 UDP VXLAN ETH VLAN END
> ETH IPV6 UDP VXLAN ETH VLAN IPV4 END
> ETH IPV6 UDP VXLAN ETH VLAN IPV6 END
> 
> The fix is to introduce a new flag to set on a graph expansion node
> to apply the 'explicit' behavior, meaning the node is not added to
> the expanded pattern, if it is not found in the flow pattern, but the
> expansion alg can go deeper to its next nodes.
> 
> Fixes: c7870bfe09dc ("ethdev: move RSS expansion code to mlx5 driver")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Lior Margalit <lmargalit@nvidia.com>
> Acked-by: Matan Azrad <matan@nvidia.com>
> ---
>  drivers/net/mlx5/mlx5_flow.c | 88 ++++++++++++++++++------------------
>  1 file changed, 44 insertions(+), 44 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index 3c35b87964..2b916899e8 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -100,10 +100,25 @@ struct mlx5_flow_expand_node {
>  	 * RSS types bit-field associated with this node
>  	 * (see ETH_RSS_* definitions).
>  	 */
> -	uint8_t optional;
> -	/**< optional expand field. Default 0 to expand, 1 not go deeper. */
> +	uint64_t node_flags;
> +	/**<
> +	 *  Bit-fields that define how the node is used in the expansion.
> +	 * (see MLX5_EXPANSION_NODE_* definitions).
> +	 */
>  };
>  
> +/* Optional expand field. The expansion alg will not go deeper. */
> +#define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0)
> +
> +/* The node is not added implicitly as expansion to the flow pattern.
> + * If the node type does not match the flow pattern item type, the
> + * expansion alg will go deeper to its next items.
> + * In the current implementation, the list of next nodes indexes can
> + * have up to one node with this flag set and it has to be the last
> + * node index (before the list terminator).
> + */
> +#define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1)
> +
>  /** Object returned by mlx5_flow_expand_rss(). */
>  struct mlx5_flow_expand_rss {
>  	uint32_t entries;
> @@ -331,10 +346,17 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
>  			continue;
>  		}
>  		last_item = item;
> -		for (i = 0; node->next && node->next[i]; ++i) {
> +		i = 0;
> +		while (node->next && node->next[i]) {
>  			next = &graph[node->next[i]];
>  			if (next->type == item->type)
>  				break;
> +			if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) {
> +				node = next;
> +				i = 0;
> +			} else {
> +				++i;
> +			}
>  		}
>  		if (next)
>  			node = next;
> @@ -393,6 +415,16 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
>  	}
>  	memset(flow_items, 0, sizeof(flow_items));
>  	next_node = node->next;
> +	while (next_node) {
> +		/*
> +		 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT
> +		 * flag set, because they were not found in the flow pattern.
> +		 */
> +		node = &graph[*next_node];
> +		if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT))
> +			break;
> +		next_node = node->next;
> +	}
>  	stack[stack_pos] = next_node;
>  	node = next_node ? &graph[*next_node] : NULL;
>  	while (node) {
> @@ -427,7 +459,8 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
>  			addr = (void *)(((uintptr_t)addr) + n);
>  		}
>  		/* Go deeper. */
> -		if (!node->optional && node->next) {
> +		if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) &&
> +				node->next) {
>  			next_node = node->next;
>  			if (stack_pos++ == MLX5_RSS_EXP_ELT_N) {
>  				rte_errno = E2BIG;
> @@ -452,10 +485,7 @@ mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size,
>  enum mlx5_expansion {
>  	MLX5_EXPANSION_ROOT,
>  	MLX5_EXPANSION_ROOT_OUTER,
> -	MLX5_EXPANSION_ROOT_ETH_VLAN,
> -	MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN,
>  	MLX5_EXPANSION_OUTER_ETH,
> -	MLX5_EXPANSION_OUTER_ETH_VLAN,
>  	MLX5_EXPANSION_OUTER_VLAN,
>  	MLX5_EXPANSION_OUTER_IPV4,
>  	MLX5_EXPANSION_OUTER_IPV4_UDP,
> @@ -470,7 +500,6 @@ enum mlx5_expansion {
>  	MLX5_EXPANSION_GRE_KEY,
>  	MLX5_EXPANSION_MPLS,
>  	MLX5_EXPANSION_ETH,
> -	MLX5_EXPANSION_ETH_VLAN,
>  	MLX5_EXPANSION_VLAN,
>  	MLX5_EXPANSION_IPV4,
>  	MLX5_EXPANSION_IPV4_UDP,
> @@ -497,22 +526,7 @@ static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
>  						  MLX5_EXPANSION_OUTER_IPV6),
>  		.type = RTE_FLOW_ITEM_TYPE_END,
>  	},
> -	[MLX5_EXPANSION_ROOT_ETH_VLAN] = {
> -		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH_VLAN),
> -		.type = RTE_FLOW_ITEM_TYPE_END,
> -	},
> -	[MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN] = {
> -		.next = MLX5_FLOW_EXPAND_RSS_NEXT
> -						(MLX5_EXPANSION_OUTER_ETH_VLAN),
> -		.type = RTE_FLOW_ITEM_TYPE_END,
> -	},
>  	[MLX5_EXPANSION_OUTER_ETH] = {
> -		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
> -						  MLX5_EXPANSION_OUTER_IPV6),
> -		.type = RTE_FLOW_ITEM_TYPE_ETH,
> -		.rss_types = 0,
> -	},
> -	[MLX5_EXPANSION_OUTER_ETH_VLAN] = {
>  		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN),
>  		.type = RTE_FLOW_ITEM_TYPE_ETH,
>  		.rss_types = 0,
> @@ -521,6 +535,7 @@ static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
>  		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
>  						  MLX5_EXPANSION_OUTER_IPV6),
>  		.type = RTE_FLOW_ITEM_TYPE_VLAN,
> +		.node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
>  	},
>  	[MLX5_EXPANSION_OUTER_IPV4] = {
>  		.next = MLX5_FLOW_EXPAND_RSS_NEXT
> @@ -597,7 +612,7 @@ static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
>  						  MLX5_EXPANSION_IPV6,
>  						  MLX5_EXPANSION_MPLS),
>  		.type = RTE_FLOW_ITEM_TYPE_GRE_KEY,
> -		.optional = 1,
> +		.node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
>  	},
>  	[MLX5_EXPANSION_NVGRE] = {
>  		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
> @@ -608,13 +623,9 @@ static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
>  						  MLX5_EXPANSION_IPV6,
>  						  MLX5_EXPANSION_ETH),
>  		.type = RTE_FLOW_ITEM_TYPE_MPLS,
> +		.node_flags = MLX5_EXPANSION_NODE_OPTIONAL,
>  	},
>  	[MLX5_EXPANSION_ETH] = {
> -		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
> -						  MLX5_EXPANSION_IPV6),
> -		.type = RTE_FLOW_ITEM_TYPE_ETH,
> -	},
> -	[MLX5_EXPANSION_ETH_VLAN] = {
>  		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN),
>  		.type = RTE_FLOW_ITEM_TYPE_ETH,
>  	},
> @@ -622,6 +633,7 @@ static const struct mlx5_flow_expand_node mlx5_support_expansion[] = {
>  		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
>  						  MLX5_EXPANSION_IPV6),
>  		.type = RTE_FLOW_ITEM_TYPE_VLAN,
> +		.node_flags = MLX5_EXPANSION_NODE_EXPLICIT,
>  	},
>  	[MLX5_EXPANSION_IPV4] = {
>  		.next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
> @@ -3458,20 +3470,8 @@ flow_get_shared_rss_action(struct rte_eth_dev *dev,
>  }
>  
>  static unsigned int
> -find_graph_root(const struct rte_flow_item pattern[], uint32_t rss_level)
> +find_graph_root(uint32_t rss_level)
>  {
> -	const struct rte_flow_item *item;
> -	unsigned int has_vlan = 0;
> -
> -	for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
> -		if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
> -			has_vlan = 1;
> -			break;
> -		}
> -	}
> -	if (has_vlan)
> -		return rss_level < 2 ? MLX5_EXPANSION_ROOT_ETH_VLAN :
> -				       MLX5_EXPANSION_ROOT_OUTER_ETH_VLAN;
>  	return rss_level < 2 ? MLX5_EXPANSION_ROOT :
>  			       MLX5_EXPANSION_ROOT_OUTER;
>  }
> @@ -5361,7 +5361,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
>  	int shared_actions_n = MLX5_MAX_SHARED_ACTIONS;
>  	union {
>  		struct mlx5_flow_expand_rss buf;
> -		uint8_t buffer[2048];
> +		uint8_t buffer[4096];
>  	} expand_buffer;
>  	union {
>  		struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS];
> @@ -5451,7 +5451,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
>  	if (rss && rss->types) {
>  		unsigned int graph_root;
>  
> -		graph_root = find_graph_root(items, rss->level);
> +		graph_root = find_graph_root(rss->level);
>  		ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
>  					   items, rss->types,
>  					   mlx5_support_expansion, graph_root);