[dpdk-dev] [DPDK 18.08 v1 00/12] net/mlx5: flow rework

[dpdk-dev] [DPDK 18.08 v1 00/12] net/mlx5: flow rework

[Nelio Laranjeiro]

First version of for the flow engine rework of MLX5 to prepare the
introduction for the ENCAP/DECAP and PUSH/POP actions done via TC
flower/filter.

This first series depends on [1] and is a work in progress, recent work added
on Tunnel RSS are still absent as well as the tunnel support.  Those will be
added in further versions.

Expected for the next 18.08 release:

- same level of features,
- TC flow support for port redirection,
- TC filter support for ENCAP/DECAP and PUSH/POP.

[1] https://dpdk.org/dev/patchwork/patch/40462/

Nelio Laranjeiro (12):
  net/mlx5: remove flow support
  net/mlx5: handle drop queues are regular queues
  net/mlx5: support flow Ethernet item among with drop action
  net/mlx5: add flow queue action
  net/mlx5: add flow stop/start
  net/mlx5: add flow VLAN item
  net/mlx5: add flow IPv4 item
  net/mlx5: add flow IPv6 item
  net/mlx5: add flow UDP item
  net/mlx5: add flow TCP item
  net/mlx5: add mark/flag flow action
  net/mlx5: add RSS flow action

 drivers/net/mlx5/mlx5.c      |    9 -
 drivers/net/mlx5/mlx5.h      |    3 +-
 drivers/net/mlx5/mlx5_flow.c | 3615 +++++++++++-----------------------
 drivers/net/mlx5/mlx5_rxq.c  |  221 +++
 drivers/net/mlx5/mlx5_rxtx.h |    6 +
 5 files changed, 1388 insertions(+), 2466 deletions(-)

-- 
2.17.0

[dpdk-dev] [DPDK 18.08 v1 01/12] net/mlx5: remove flow support

[Nelio Laranjeiro]

This start a series to re-work the flow engine in mlx5 to easily support
flow conversion to Verbs or TC.  This is necessary to handle both regular
flows and representors flows.

As the full file needs to be clean-up to re-write all items/actions
processing, this patch starts to disable the regular code and only let the
PMD to start in isolated mode.

After this patch flow API will not be usable.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 2958 +---------------------------------
 1 file changed, 80 insertions(+), 2878 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index ec6d00f21..a45cb06e1 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -31,2274 +31,49 @@
 #include "mlx5_prm.h"
 #include "mlx5_glue.h"
 
-/* Flow priority for control plane flows. */
-#define MLX5_CTRL_FLOW_PRIORITY 1
-
-/* Internet Protocol versions. */
-#define MLX5_IPV4 4
-#define MLX5_IPV6 6
-#define MLX5_GRE 47
-
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-struct ibv_flow_spec_counter_action {
-	int dummy;
-};
-#endif
-
-/* Dev ops structure defined in mlx5.c */
-extern const struct eth_dev_ops mlx5_dev_ops;
-extern const struct eth_dev_ops mlx5_dev_ops_isolate;
-
-/** Structure give to the conversion functions. */
-struct mlx5_flow_data {
-	struct rte_eth_dev *dev; /** Ethernet device. */
-	struct mlx5_flow_parse *parser; /** Parser context. */
-	struct rte_flow_error *error; /** Error context. */
-};
-
-static int
-mlx5_flow_create_eth(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vlan(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_ipv4(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_ipv6(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_udp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_tcp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vxlan(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vxlan_gpe(const struct rte_flow_item *item,
-			   const void *default_mask,
-			   struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_gre(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-struct mlx5_flow_parse;
-
-static void
-mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
-		      unsigned int size);
-
-static int
-mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
-
-static int
-mlx5_flow_create_count(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser);
-
-/* Hash RX queue types. */
-enum hash_rxq_type {
-	HASH_RXQ_TCPV4,
-	HASH_RXQ_UDPV4,
-	HASH_RXQ_IPV4,
-	HASH_RXQ_TCPV6,
-	HASH_RXQ_UDPV6,
-	HASH_RXQ_IPV6,
-	HASH_RXQ_ETH,
-	HASH_RXQ_TUNNEL,
-};
-
-/* Initialization data for hash RX queue. */
-struct hash_rxq_init {
-	uint64_t hash_fields; /* Fields that participate in the hash. */
-	uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
-	unsigned int flow_priority; /* Flow priority to use. */
-	unsigned int ip_version; /* Internet protocol. */
-};
-
-/* Initialization data for hash RX queues. */
-const struct hash_rxq_init hash_rxq_init[] = {
-	[HASH_RXQ_TCPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4 |
-				IBV_RX_HASH_SRC_PORT_TCP |
-				IBV_RX_HASH_DST_PORT_TCP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV4,
-	},
-	[HASH_RXQ_UDPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4 |
-				IBV_RX_HASH_SRC_PORT_UDP |
-				IBV_RX_HASH_DST_PORT_UDP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV4,
-	},
-	[HASH_RXQ_IPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4),
-		.dpdk_rss_hf = (ETH_RSS_IPV4 |
-				ETH_RSS_FRAG_IPV4),
-		.flow_priority = 1,
-		.ip_version = MLX5_IPV4,
-	},
-	[HASH_RXQ_TCPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6 |
-				IBV_RX_HASH_SRC_PORT_TCP |
-				IBV_RX_HASH_DST_PORT_TCP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV6,
-	},
-	[HASH_RXQ_UDPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6 |
-				IBV_RX_HASH_SRC_PORT_UDP |
-				IBV_RX_HASH_DST_PORT_UDP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV6,
-	},
-	[HASH_RXQ_IPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6),
-		.dpdk_rss_hf = (ETH_RSS_IPV6 |
-				ETH_RSS_FRAG_IPV6),
-		.flow_priority = 1,
-		.ip_version = MLX5_IPV6,
-	},
-	[HASH_RXQ_ETH] = {
-		.hash_fields = 0,
-		.dpdk_rss_hf = 0,
-		.flow_priority = 2,
-	},
-};
-
-/* Number of entries in hash_rxq_init[]. */
-const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
-
-/** Structure for holding counter stats. */
-struct mlx5_flow_counter_stats {
-	uint64_t hits; /**< Number of packets matched by the rule. */
-	uint64_t bytes; /**< Number of bytes matched by the rule. */
-};
-
-/** Structure for Drop queue. */
-struct mlx5_hrxq_drop {
-	struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
-	struct ibv_qp *qp; /**< Verbs queue pair. */
-	struct ibv_wq *wq; /**< Verbs work queue. */
-	struct ibv_cq *cq; /**< Verbs completion queue. */
-};
-
-/* Flows structures. */
-struct mlx5_flow {
-	uint64_t hash_fields; /**< Fields that participate in the hash. */
-	struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
-	struct ibv_flow *ibv_flow; /**< Verbs flow. */
-	struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
-};
-
-/* Drop flows structures. */
-struct mlx5_flow_drop {
-	struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
-	struct ibv_flow *ibv_flow; /**< Verbs flow. */
-};
-
-struct rte_flow {
-	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
-	uint32_t mark:1; /**< Set if the flow is marked. */
-	uint32_t drop:1; /**< Drop queue. */
-	struct rte_flow_action_rss rss_conf; /**< RSS configuration */
-	uint16_t (*queues)[]; /**< Queues indexes to use. */
-	uint8_t rss_key[40]; /**< copy of the RSS key. */
-	uint32_t tunnel; /**< Tunnel type of RTE_PTYPE_TUNNEL_XXX. */
-	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
-	struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
-	struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
-	/**< Flow with Rx queue. */
-};
-
-/** Static initializer for items. */
-#define ITEMS(...) \
-	(const enum rte_flow_item_type []){ \
-		__VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
-	}
-
-#define IS_TUNNEL(type) ( \
-	(type) == RTE_FLOW_ITEM_TYPE_VXLAN || \
-	(type) == RTE_FLOW_ITEM_TYPE_VXLAN_GPE || \
-	(type) == RTE_FLOW_ITEM_TYPE_GRE)
-
-const uint32_t flow_ptype[] = {
-	[RTE_FLOW_ITEM_TYPE_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN,
-	[RTE_FLOW_ITEM_TYPE_VXLAN_GPE] = RTE_PTYPE_TUNNEL_VXLAN_GPE,
-	[RTE_FLOW_ITEM_TYPE_GRE] = RTE_PTYPE_TUNNEL_GRE,
-};
-
-#define PTYPE_IDX(t) ((RTE_PTYPE_TUNNEL_MASK & (t)) >> 12)
-
-const uint32_t ptype_ext[] = {
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] = RTE_PTYPE_TUNNEL_VXLAN |
-					      RTE_PTYPE_L4_UDP,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN_GPE)]	= RTE_PTYPE_TUNNEL_VXLAN_GPE |
-						  RTE_PTYPE_L4_UDP,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_GRE)] = RTE_PTYPE_TUNNEL_GRE,
-};
-
-/** Structure to generate a simple graph of layers supported by the NIC. */
-struct mlx5_flow_items {
-	/** List of possible actions for these items. */
-	const enum rte_flow_action_type *const actions;
-	/** Bit-masks corresponding to the possibilities for the item. */
-	const void *mask;
-	/**
-	 * Default bit-masks to use when item->mask is not provided. When
-	 * \default_mask is also NULL, the full supported bit-mask (\mask) is
-	 * used instead.
-	 */
-	const void *default_mask;
-	/** Bit-masks size in bytes. */
-	const unsigned int mask_sz;
-	/**
-	 * Conversion function from rte_flow to NIC specific flow.
-	 *
-	 * @param item
-	 *   rte_flow item to convert.
-	 * @param default_mask
-	 *   Default bit-masks to use when item->mask is not provided.
-	 * @param data
-	 *   Internal structure to store the conversion.
-	 *
-	 * @return
-	 *   0 on success, a negative errno value otherwise and rte_errno is
-	 *   set.
-	 */
-	int (*convert)(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data);
-	/** Size in bytes of the destination structure. */
-	const unsigned int dst_sz;
-	/** List of possible following items.  */
-	const enum rte_flow_item_type *const items;
-};
-
-/** Valid action for this PMD. */
-static const enum rte_flow_action_type valid_actions[] = {
-	RTE_FLOW_ACTION_TYPE_DROP,
-	RTE_FLOW_ACTION_TYPE_QUEUE,
-	RTE_FLOW_ACTION_TYPE_MARK,
-	RTE_FLOW_ACTION_TYPE_FLAG,
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	RTE_FLOW_ACTION_TYPE_COUNT,
-#endif
-	RTE_FLOW_ACTION_TYPE_END,
-};
-
-/** Graph of supported items and associated actions. */
-static const struct mlx5_flow_items mlx5_flow_items[] = {
-	[RTE_FLOW_ITEM_TYPE_END] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_VXLAN,
-			       RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
-			       RTE_FLOW_ITEM_TYPE_GRE),
-	},
-	[RTE_FLOW_ITEM_TYPE_ETH] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_eth){
-			.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
-			.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
-			.type = -1,
-		},
-		.default_mask = &rte_flow_item_eth_mask,
-		.mask_sz = sizeof(struct rte_flow_item_eth),
-		.convert = mlx5_flow_create_eth,
-		.dst_sz = sizeof(struct ibv_flow_spec_eth),
-	},
-	[RTE_FLOW_ITEM_TYPE_VLAN] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vlan){
-			.tci = -1,
-			.inner_type = -1,
-		},
-		.default_mask = &rte_flow_item_vlan_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vlan),
-		.convert = mlx5_flow_create_vlan,
-		.dst_sz = 0,
-	},
-	[RTE_FLOW_ITEM_TYPE_IPV4] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
-			       RTE_FLOW_ITEM_TYPE_TCP,
-			       RTE_FLOW_ITEM_TYPE_GRE),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_ipv4){
-			.hdr = {
-				.src_addr = -1,
-				.dst_addr = -1,
-				.type_of_service = -1,
-				.next_proto_id = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_ipv4_mask,
-		.mask_sz = sizeof(struct rte_flow_item_ipv4),
-		.convert = mlx5_flow_create_ipv4,
-		.dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
-	},
-	[RTE_FLOW_ITEM_TYPE_IPV6] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
-			       RTE_FLOW_ITEM_TYPE_TCP,
-			       RTE_FLOW_ITEM_TYPE_GRE),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_ipv6){
-			.hdr = {
-				.src_addr = {
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-				},
-				.dst_addr = {
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-				},
-				.vtc_flow = -1,
-				.proto = -1,
-				.hop_limits = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_ipv6_mask,
-		.mask_sz = sizeof(struct rte_flow_item_ipv6),
-		.convert = mlx5_flow_create_ipv6,
-		.dst_sz = sizeof(struct ibv_flow_spec_ipv6),
-	},
-	[RTE_FLOW_ITEM_TYPE_UDP] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN,
-			       RTE_FLOW_ITEM_TYPE_VXLAN_GPE),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_udp){
-			.hdr = {
-				.src_port = -1,
-				.dst_port = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_udp_mask,
-		.mask_sz = sizeof(struct rte_flow_item_udp),
-		.convert = mlx5_flow_create_udp,
-		.dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
-	},
-	[RTE_FLOW_ITEM_TYPE_TCP] = {
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_tcp){
-			.hdr = {
-				.src_port = -1,
-				.dst_port = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_tcp_mask,
-		.mask_sz = sizeof(struct rte_flow_item_tcp),
-		.convert = mlx5_flow_create_tcp,
-		.dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
-	},
-	[RTE_FLOW_ITEM_TYPE_GRE] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_gre){
-			.protocol = -1,
-		},
-		.default_mask = &rte_flow_item_gre_mask,
-		.mask_sz = sizeof(struct rte_flow_item_gre),
-		.convert = mlx5_flow_create_gre,
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
-	},
-	[RTE_FLOW_ITEM_TYPE_VXLAN] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4, /* For L3 VXLAN. */
-			       RTE_FLOW_ITEM_TYPE_IPV6), /* For L3 VXLAN. */
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vxlan){
-			.vni = "\xff\xff\xff",
-		},
-		.default_mask = &rte_flow_item_vxlan_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vxlan),
-		.convert = mlx5_flow_create_vxlan,
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
-	},
-	[RTE_FLOW_ITEM_TYPE_VXLAN_GPE] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vxlan_gpe){
-			.vni = "\xff\xff\xff",
-		},
-		.default_mask = &rte_flow_item_vxlan_gpe_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vxlan_gpe),
-		.convert = mlx5_flow_create_vxlan_gpe,
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
-	},
-};
-
-/** Structure to pass to the conversion function. */
-struct mlx5_flow_parse {
-	uint32_t inner; /**< Verbs value, set once tunnel is encountered. */
-	uint32_t create:1;
-	/**< Whether resources should remain after a validate. */
-	uint32_t drop:1; /**< Target is a drop queue. */
-	uint32_t mark:1; /**< Mark is present in the flow. */
-	uint32_t count:1; /**< Count is present in the flow. */
-	uint32_t mark_id; /**< Mark identifier. */
-	struct rte_flow_action_rss rss_conf; /**< RSS configuration */
-	uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
-	uint8_t rss_key[40]; /**< copy of the RSS key. */
-	enum hash_rxq_type layer; /**< Last pattern layer detected. */
-	enum hash_rxq_type out_layer; /**< Last outer pattern layer detected. */
-	uint32_t tunnel; /**< Tunnel type of RTE_PTYPE_TUNNEL_XXX. */
-	struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
-	struct {
-		struct ibv_flow_attr *ibv_attr;
-		/**< Pointer to Verbs attributes. */
-		unsigned int offset;
-		/**< Current position or total size of the attribute. */
-		uint64_t hash_fields; /**< Verbs hash fields. */
-	} queue[RTE_DIM(hash_rxq_init)];
-};
-
-static const struct rte_flow_ops mlx5_flow_ops = {
-	.validate = mlx5_flow_validate,
-	.create = mlx5_flow_create,
-	.destroy = mlx5_flow_destroy,
-	.flush = mlx5_flow_flush,
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	.query = mlx5_flow_query,
-#else
-	.query = NULL,
-#endif
-	.isolate = mlx5_flow_isolate,
-};
-
-/* Convert FDIR request to Generic flow. */
-struct mlx5_fdir {
-	struct rte_flow_attr attr;
-	struct rte_flow_action actions[2];
-	struct rte_flow_item items[4];
-	struct rte_flow_item_eth l2;
-	struct rte_flow_item_eth l2_mask;
-	union {
-		struct rte_flow_item_ipv4 ipv4;
-		struct rte_flow_item_ipv6 ipv6;
-	} l3;
-	union {
-		struct rte_flow_item_ipv4 ipv4;
-		struct rte_flow_item_ipv6 ipv6;
-	} l3_mask;
-	union {
-		struct rte_flow_item_udp udp;
-		struct rte_flow_item_tcp tcp;
-	} l4;
-	union {
-		struct rte_flow_item_udp udp;
-		struct rte_flow_item_tcp tcp;
-	} l4_mask;
-	struct rte_flow_action_queue queue;
-};
-
-/* Verbs specification header. */
-struct ibv_spec_header {
-	enum ibv_flow_spec_type type;
-	uint16_t size;
-};
-
-/**
- * Check item is fully supported by the NIC matching capability.
- *
- * @param item[in]
- *   Item specification.
- * @param mask[in]
- *   Bit-masks covering supported fields to compare with spec, last and mask in
- *   \item.
- * @param size
- *   Bit-Mask size in bytes.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_item_validate(const struct rte_flow_item *item,
-			const uint8_t *mask, unsigned int size)
-{
-	unsigned int i;
-	const uint8_t *spec = item->spec;
-	const uint8_t *last = item->last;
-	const uint8_t *m = item->mask ? item->mask : mask;
-
-	if (!spec && (item->mask || last))
-		goto error;
-	if (!spec)
-		return 0;
-	/*
-	 * Single-pass check to make sure that:
-	 * - item->mask is supported, no bits are set outside mask.
-	 * - Both masked item->spec and item->last are equal (no range
-	 *   supported).
-	 */
-	for (i = 0; i < size; i++) {
-		if (!m[i])
-			continue;
-		if ((m[i] | mask[i]) != mask[i])
-			goto error;
-		if (last && ((spec[i] & m[i]) != (last[i] & m[i])))
-			goto error;
-	}
-	return 0;
-error:
-	rte_errno = ENOTSUP;
-	return -rte_errno;
-}
-
-/**
- * Extract attribute to the parser.
- *
- * @param[in] attr
- *   Flow rule attributes.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_attributes(const struct rte_flow_attr *attr,
-			     struct rte_flow_error *error)
-{
-	if (attr->group) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
-				   NULL,
-				   "groups are not supported");
-		return -rte_errno;
-	}
-	if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
-				   NULL,
-				   "priorities are not supported");
-		return -rte_errno;
-	}
-	if (attr->egress) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
-				   NULL,
-				   "egress is not supported");
-		return -rte_errno;
-	}
-	if (attr->transfer) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
-				   NULL,
-				   "transfer is not supported");
-		return -rte_errno;
-	}
-	if (!attr->ingress) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
-				   NULL,
-				   "only ingress is supported");
-		return -rte_errno;
-	}
-	return 0;
-}
-
-/**
- * Extract actions request to the parser.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[in] actions
- *   Associated actions (list terminated by the END action).
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_actions(struct rte_eth_dev *dev,
-			  const struct rte_flow_action actions[],
-			  struct rte_flow_error *error,
-			  struct mlx5_flow_parse *parser)
-{
-	enum { FATE = 1, MARK = 2, COUNT = 4, };
-	uint32_t overlap = 0;
-	struct priv *priv = dev->data->dev_private;
-
-	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
-		if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
-			continue;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			parser->drop = 1;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
-			const struct rte_flow_action_queue *queue =
-				(const struct rte_flow_action_queue *)
-				actions->conf;
-
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			if (!queue || (queue->index > (priv->rxqs_n - 1)))
-				goto exit_action_not_supported;
-			parser->queues[0] = queue->index;
-			parser->rss_conf = (struct rte_flow_action_rss){
-				.queue_num = 1,
-				.queue = parser->queues,
-			};
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
-			const struct rte_flow_action_rss *rss =
-				(const struct rte_flow_action_rss *)
-				actions->conf;
-			const uint8_t *rss_key;
-			uint32_t rss_key_len;
-			uint16_t n;
-
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			if (rss->func &&
-			    rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "the only supported RSS hash"
-						   " function is Toeplitz");
-				return -rte_errno;
-			}
-#ifndef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-			if (parser->rss_conf.level > 1) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "a nonzero RSS encapsulation"
-						   " level is not supported");
-				return -rte_errno;
-			}
-#endif
-			if (parser->rss_conf.level > 2) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "RSS encapsulation level"
-						   " > 1 is not supported");
-				return -rte_errno;
-			}
-			if (rss->types & MLX5_RSS_HF_MASK) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "unsupported RSS type"
-						   " requested");
-				return -rte_errno;
-			}
-			if (rss->key_len) {
-				rss_key_len = rss->key_len;
-				rss_key = rss->key;
-			} else {
-				rss_key_len = rss_hash_default_key_len;
-				rss_key = rss_hash_default_key;
-			}
-			if (rss_key_len != RTE_DIM(parser->rss_key)) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "RSS hash key must be"
-						   " exactly 40 bytes long");
-				return -rte_errno;
-			}
-			if (!rss->queue_num) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "no valid queues");
-				return -rte_errno;
-			}
-			if (rss->queue_num > RTE_DIM(parser->queues)) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "too many queues for RSS"
-						   " context");
-				return -rte_errno;
-			}
-			for (n = 0; n < rss->queue_num; ++n) {
-				if (rss->queue[n] >= priv->rxqs_n) {
-					rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "queue id > number of"
-						   " queues");
-					return -rte_errno;
-				}
-			}
-			parser->rss_conf = (struct rte_flow_action_rss){
-				.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
-				.level = rss->level,
-				.types = rss->types,
-				.key_len = rss_key_len,
-				.queue_num = rss->queue_num,
-				.key = memcpy(parser->rss_key, rss_key,
-					      sizeof(*rss_key) * rss_key_len),
-				.queue = memcpy(parser->queues, rss->queue,
-						sizeof(*rss->queue) *
-						rss->queue_num),
-			};
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
-			const struct rte_flow_action_mark *mark =
-				(const struct rte_flow_action_mark *)
-				actions->conf;
-
-			if (overlap & MARK)
-				goto exit_action_overlap;
-			overlap |= MARK;
-			if (!mark) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "mark must be defined");
-				return -rte_errno;
-			} else if (mark->id >= MLX5_FLOW_MARK_MAX) {
-				rte_flow_error_set(error, ENOTSUP,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "mark must be between 0"
-						   " and 16777199");
-				return -rte_errno;
-			}
-			parser->mark = 1;
-			parser->mark_id = mark->id;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
-			if (overlap & MARK)
-				goto exit_action_overlap;
-			overlap |= MARK;
-			parser->mark = 1;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
-			   priv->config.flow_counter_en) {
-			if (overlap & COUNT)
-				goto exit_action_overlap;
-			overlap |= COUNT;
-			parser->count = 1;
-		} else {
-			goto exit_action_not_supported;
-		}
-	}
-	/* When fate is unknown, drop traffic. */
-	if (!(overlap & FATE))
-		parser->drop = 1;
-	if (parser->drop && parser->mark)
-		parser->mark = 0;
-	if (!parser->rss_conf.queue_num && !parser->drop) {
-		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "no valid action");
-		return -rte_errno;
-	}
-	return 0;
-exit_action_not_supported:
-	rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
-			   actions, "action not supported");
-	return -rte_errno;
-exit_action_overlap:
-	rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
-			   actions, "overlapping actions are not supported");
-	return -rte_errno;
-}
-
-/**
- * Validate items.
- *
- * @param[in] items
- *   Pattern specification (list terminated by the END pattern item).
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_items_validate(struct rte_eth_dev *dev,
-				 const struct rte_flow_item items[],
-				 struct rte_flow_error *error,
-				 struct mlx5_flow_parse *parser)
-{
-	struct priv *priv = dev->data->dev_private;
-	const struct mlx5_flow_items *cur_item = mlx5_flow_items;
-	unsigned int i;
-	int ret = 0;
-
-	/* Initialise the offsets to start after verbs attribute. */
-	for (i = 0; i != hash_rxq_init_n; ++i)
-		parser->queue[i].offset = sizeof(struct ibv_flow_attr);
-	for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
-		const struct mlx5_flow_items *token = NULL;
-		unsigned int n;
-
-		if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
-			continue;
-		for (i = 0;
-		     cur_item->items &&
-		     cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
-		     ++i) {
-			if (cur_item->items[i] == items->type) {
-				token = &mlx5_flow_items[items->type];
-				break;
-			}
-		}
-		if (!token) {
-			ret = -ENOTSUP;
-			goto exit_item_not_supported;
-		}
-		cur_item = token;
-		ret = mlx5_flow_item_validate(items,
-					      (const uint8_t *)cur_item->mask,
-					      cur_item->mask_sz);
-		if (ret)
-			goto exit_item_not_supported;
-		if (IS_TUNNEL(items->type)) {
-			if (parser->tunnel) {
-				rte_flow_error_set(error, ENOTSUP,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   items,
-						   "Cannot recognize multiple"
-						   " tunnel encapsulations.");
-				return -rte_errno;
-			}
-			if (!priv->config.tunnel_en &&
-			    parser->rss_conf.level > 1) {
-				rte_flow_error_set(error, ENOTSUP,
-					RTE_FLOW_ERROR_TYPE_ITEM,
-					items,
-					"RSS on tunnel is not supported");
-				return -rte_errno;
-			}
-			parser->inner = IBV_FLOW_SPEC_INNER;
-			parser->tunnel = flow_ptype[items->type];
-		}
-		if (parser->drop) {
-			parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
-		} else {
-			for (n = 0; n != hash_rxq_init_n; ++n)
-				parser->queue[n].offset += cur_item->dst_sz;
-		}
-	}
-	if (parser->drop) {
-		parser->queue[HASH_RXQ_ETH].offset +=
-			sizeof(struct ibv_flow_spec_action_drop);
-	}
-	if (parser->mark) {
-		for (i = 0; i != hash_rxq_init_n; ++i)
-			parser->queue[i].offset +=
-				sizeof(struct ibv_flow_spec_action_tag);
-	}
-	if (parser->count) {
-		unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
-
-		for (i = 0; i != hash_rxq_init_n; ++i)
-			parser->queue[i].offset += size;
-	}
-	return 0;
-exit_item_not_supported:
-	return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
-				  items, "item not supported");
-}
-
-/**
- * Allocate memory space to store verbs flow attributes.
- *
- * @param[in] size
- *   Amount of byte to allocate.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   A verbs flow attribute on success, NULL otherwise and rte_errno is set.
- */
-static struct ibv_flow_attr *
-mlx5_flow_convert_allocate(unsigned int size, struct rte_flow_error *error)
-{
-	struct ibv_flow_attr *ibv_attr;
-
-	ibv_attr = rte_calloc(__func__, 1, size, 0);
-	if (!ibv_attr) {
-		rte_flow_error_set(error, ENOMEM,
-				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				   NULL,
-				   "cannot allocate verbs spec attributes");
-		return NULL;
-	}
-	return ibv_attr;
-}
-
-/**
- * Make inner packet matching with an higher priority from the non Inner
- * matching.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[in, out] parser
- *   Internal parser structure.
- * @param attr
- *   User flow attribute.
- */
-static void
-mlx5_flow_update_priority(struct rte_eth_dev *dev,
-			  struct mlx5_flow_parse *parser,
-			  const struct rte_flow_attr *attr)
-{
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-	uint16_t priority;
-
-	/*			8 priorities	>= 16 priorities
-	 * Control flow:	4-7		8-15
-	 * User normal flow:	1-3		4-7
-	 * User tunnel flow:	0-2		0-3
-	 */
-	priority = attr->priority * MLX5_VERBS_FLOW_PRIO_8;
-	if (priv->config.max_verbs_prio == MLX5_VERBS_FLOW_PRIO_8)
-		priority /= 2;
-	/*
-	 * Lower non-tunnel flow Verbs priority 1 if only support 8 Verbs
-	 * priorities, lower 4 otherwise.
-	 */
-	if (!parser->inner) {
-		if (priv->config.max_verbs_prio == MLX5_VERBS_FLOW_PRIO_8)
-			priority += 1;
-		else
-			priority += MLX5_VERBS_FLOW_PRIO_8 / 2;
-	}
-	if (parser->drop) {
-		parser->queue[HASH_RXQ_ETH].ibv_attr->priority = priority +
-				hash_rxq_init[HASH_RXQ_ETH].flow_priority;
-		return;
-	}
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		parser->queue[i].ibv_attr->priority = priority +
-				hash_rxq_init[i].flow_priority;
-	}
-}
-
-/**
- * Finalise verbs flow attributes.
- *
- * @param[in, out] parser
- *   Internal parser structure.
- */
-static void
-mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser)
-{
-	unsigned int i;
-	uint32_t inner = parser->inner;
-
-	/* Don't create extra flows for outer RSS. */
-	if (parser->tunnel && parser->rss_conf.level < 2)
-		return;
-	/*
-	 * Fill missing layers in verbs specifications, or compute the correct
-	 * offset to allocate the memory space for the attributes and
-	 * specifications.
-	 */
-	for (i = 0; i != hash_rxq_init_n - 1; ++i) {
-		union {
-			struct ibv_flow_spec_ipv4_ext ipv4;
-			struct ibv_flow_spec_ipv6 ipv6;
-			struct ibv_flow_spec_tcp_udp udp_tcp;
-			struct ibv_flow_spec_eth eth;
-		} specs;
-		void *dst;
-		uint16_t size;
-
-		if (i == parser->layer)
-			continue;
-		if (parser->layer == HASH_RXQ_ETH ||
-		    parser->layer == HASH_RXQ_TUNNEL) {
-			if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
-				size = sizeof(struct ibv_flow_spec_ipv4_ext);
-				specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
-					.type = inner | IBV_FLOW_SPEC_IPV4_EXT,
-					.size = size,
-				};
-			} else {
-				size = sizeof(struct ibv_flow_spec_ipv6);
-				specs.ipv6 = (struct ibv_flow_spec_ipv6){
-					.type = inner | IBV_FLOW_SPEC_IPV6,
-					.size = size,
-				};
-			}
-			if (parser->queue[i].ibv_attr) {
-				dst = (void *)((uintptr_t)
-					       parser->queue[i].ibv_attr +
-					       parser->queue[i].offset);
-				memcpy(dst, &specs, size);
-				++parser->queue[i].ibv_attr->num_of_specs;
-			}
-			parser->queue[i].offset += size;
-		}
-		if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
-		    (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
-			size = sizeof(struct ibv_flow_spec_tcp_udp);
-			specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
-				.type = inner | ((i == HASH_RXQ_UDPV4 ||
-					  i == HASH_RXQ_UDPV6) ?
-					 IBV_FLOW_SPEC_UDP :
-					 IBV_FLOW_SPEC_TCP),
-				.size = size,
-			};
-			if (parser->queue[i].ibv_attr) {
-				dst = (void *)((uintptr_t)
-					       parser->queue[i].ibv_attr +
-					       parser->queue[i].offset);
-				memcpy(dst, &specs, size);
-				++parser->queue[i].ibv_attr->num_of_specs;
-			}
-			parser->queue[i].offset += size;
-		}
-	}
-}
-
-/**
- * Update flows according to pattern and RSS hash fields.
- *
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_rss(struct mlx5_flow_parse *parser)
-{
-	unsigned int i;
-	enum hash_rxq_type start;
-	enum hash_rxq_type layer;
-	int outer = parser->tunnel && parser->rss_conf.level < 2;
-	uint64_t rss = parser->rss_conf.types;
-
-	/* Default to outer RSS. */
-	if (!parser->rss_conf.level)
-		parser->rss_conf.level = 1;
-	layer = outer ? parser->out_layer : parser->layer;
-	if (layer == HASH_RXQ_TUNNEL)
-		layer = HASH_RXQ_ETH;
-	if (outer) {
-		/* Only one hash type for outer RSS. */
-		if (rss && layer == HASH_RXQ_ETH) {
-			start = HASH_RXQ_TCPV4;
-		} else if (rss && layer != HASH_RXQ_ETH &&
-			   !(rss & hash_rxq_init[layer].dpdk_rss_hf)) {
-			/* If RSS not match L4 pattern, try L3 RSS. */
-			if (layer < HASH_RXQ_IPV4)
-				layer = HASH_RXQ_IPV4;
-			else if (layer > HASH_RXQ_IPV4 && layer < HASH_RXQ_IPV6)
-				layer = HASH_RXQ_IPV6;
-			start = layer;
-		} else {
-			start = layer;
-		}
-		/* Scan first valid hash type. */
-		for (i = start; rss && i <= layer; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-			if (hash_rxq_init[i].dpdk_rss_hf & rss)
-				break;
-		}
-		if (rss && i <= layer)
-			parser->queue[layer].hash_fields =
-					hash_rxq_init[i].hash_fields;
-		/* Trim unused hash types. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (parser->queue[i].ibv_attr && i != layer) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			}
-		}
-	} else {
-		/* Expand for inner or normal RSS. */
-		if (rss && (layer == HASH_RXQ_ETH || layer == HASH_RXQ_IPV4))
-			start = HASH_RXQ_TCPV4;
-		else if (rss && layer == HASH_RXQ_IPV6)
-			start = HASH_RXQ_TCPV6;
-		else
-			start = layer;
-		/* For L4 pattern, try L3 RSS if no L4 RSS. */
-		/* Trim unused hash types. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-			if (i < start || i > layer) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-				continue;
-			}
-			if (!rss)
-				continue;
-			if (hash_rxq_init[i].dpdk_rss_hf & rss) {
-				parser->queue[i].hash_fields =
-						hash_rxq_init[i].hash_fields;
-			} else if (i != layer) {
-				/* Remove unused RSS expansion. */
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			} else if (layer < HASH_RXQ_IPV4 &&
-				   (hash_rxq_init[HASH_RXQ_IPV4].dpdk_rss_hf &
-				    rss)) {
-				/* Allow IPv4 RSS on L4 pattern. */
-				parser->queue[i].hash_fields =
-					hash_rxq_init[HASH_RXQ_IPV4]
-						.hash_fields;
-			} else if (i > HASH_RXQ_IPV4 && i < HASH_RXQ_IPV6 &&
-				   (hash_rxq_init[HASH_RXQ_IPV6].dpdk_rss_hf &
-				    rss)) {
-				/* Allow IPv4 RSS on L4 pattern. */
-				parser->queue[i].hash_fields =
-					hash_rxq_init[HASH_RXQ_IPV6]
-						.hash_fields;
-			}
-		}
-	}
-	return 0;
-}
-
-/**
- * Validate and convert a flow supported by the NIC.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[in] attr
- *   Flow rule attributes.
- * @param[in] pattern
- *   Pattern specification (list terminated by the END pattern item).
- * @param[in] actions
- *   Associated actions (list terminated by the END action).
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert(struct rte_eth_dev *dev,
-		  const struct rte_flow_attr *attr,
-		  const struct rte_flow_item items[],
-		  const struct rte_flow_action actions[],
-		  struct rte_flow_error *error,
-		  struct mlx5_flow_parse *parser)
-{
-	const struct mlx5_flow_items *cur_item = mlx5_flow_items;
-	unsigned int i;
-	int ret;
-
-	/* First step. Validate the attributes, items and actions. */
-	*parser = (struct mlx5_flow_parse){
-		.create = parser->create,
-		.layer = HASH_RXQ_ETH,
-		.mark_id = MLX5_FLOW_MARK_DEFAULT,
-	};
-	ret = mlx5_flow_convert_attributes(attr, error);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert_actions(dev, actions, error, parser);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert_items_validate(dev, items, error, parser);
-	if (ret)
-		return ret;
-	mlx5_flow_convert_finalise(parser);
-	/*
-	 * Second step.
-	 * Allocate the memory space to store verbs specifications.
-	 */
-	if (parser->drop) {
-		unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
-
-		parser->queue[HASH_RXQ_ETH].ibv_attr =
-			mlx5_flow_convert_allocate(offset, error);
-		if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
-			goto exit_enomem;
-		parser->queue[HASH_RXQ_ETH].offset =
-			sizeof(struct ibv_flow_attr);
-	} else {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			unsigned int offset;
-
-			offset = parser->queue[i].offset;
-			parser->queue[i].ibv_attr =
-				mlx5_flow_convert_allocate(offset, error);
-			if (!parser->queue[i].ibv_attr)
-				goto exit_enomem;
-			parser->queue[i].offset = sizeof(struct ibv_flow_attr);
-		}
-	}
-	/* Third step. Conversion parse, fill the specifications. */
-	parser->inner = 0;
-	parser->tunnel = 0;
-	parser->layer = HASH_RXQ_ETH;
-	for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
-		struct mlx5_flow_data data = {
-			.dev = dev,
-			.parser = parser,
-			.error = error,
-		};
-
-		if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
-			continue;
-		cur_item = &mlx5_flow_items[items->type];
-		ret = cur_item->convert(items,
-					(cur_item->default_mask ?
-					 cur_item->default_mask :
-					 cur_item->mask),
-					 &data);
-		if (ret)
-			goto exit_free;
-	}
-	if (!parser->drop) {
-		/* RSS check, remove unused hash types. */
-		ret = mlx5_flow_convert_rss(parser);
-		if (ret)
-			goto exit_free;
-		/* Complete missing specification. */
-		mlx5_flow_convert_finalise(parser);
-	}
-	mlx5_flow_update_priority(dev, parser, attr);
-	if (parser->mark)
-		mlx5_flow_create_flag_mark(parser, parser->mark_id);
-	if (parser->count && parser->create) {
-		mlx5_flow_create_count(dev, parser);
-		if (!parser->cs)
-			goto exit_count_error;
-	}
-exit_free:
-	/* Only verification is expected, all resources should be released. */
-	if (!parser->create) {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (parser->queue[i].ibv_attr) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			}
-		}
-	}
-	return ret;
-exit_enomem:
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser->queue[i].ibv_attr) {
-			rte_free(parser->queue[i].ibv_attr);
-			parser->queue[i].ibv_attr = NULL;
-		}
-	}
-	rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL, "cannot allocate verbs spec attributes");
-	return -rte_errno;
-exit_count_error:
-	rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL, "cannot create counter");
-	return -rte_errno;
-}
-
-/**
- * Copy the specification created into the flow.
- *
- * @param parser
- *   Internal parser structure.
- * @param src
- *   Create specification.
- * @param size
- *   Size in bytes of the specification to copy.
- */
-static void
-mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
-		      unsigned int size)
-{
-	unsigned int i;
-	void *dst;
-
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset);
-		memcpy(dst, src, size);
-		++parser->queue[i].ibv_attr->num_of_specs;
-		parser->queue[i].offset += size;
-	}
-}
-
-/**
- * Convert Ethernet item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_eth(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_eth *spec = item->spec;
-	const struct rte_flow_item_eth *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
-	struct ibv_flow_spec_eth eth = {
-		.type = parser->inner | IBV_FLOW_SPEC_ETH,
-		.size = eth_size,
-	};
-
-	parser->layer = HASH_RXQ_ETH;
-	if (spec) {
-		unsigned int i;
-
-		if (!mask)
-			mask = default_mask;
-		memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
-		memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
-		eth.val.ether_type = spec->type;
-		memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
-		memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
-		eth.mask.ether_type = mask->type;
-		/* Remove unwanted bits from values. */
-		for (i = 0; i < ETHER_ADDR_LEN; ++i) {
-			eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
-			eth.val.src_mac[i] &= eth.mask.src_mac[i];
-		}
-		eth.val.ether_type &= eth.mask.ether_type;
-	}
-	mlx5_flow_create_copy(parser, &eth, eth_size);
-	return 0;
-}
-
-/**
- * Convert VLAN item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_vlan(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_vlan *spec = item->spec;
-	const struct rte_flow_item_vlan *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	struct ibv_flow_spec_eth *eth;
-	const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
-	const char *msg = "VLAN cannot be empty";
-
-	if (spec) {
-		unsigned int i;
-		if (!mask)
-			mask = default_mask;
-
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-
-			eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				       parser->queue[i].offset - eth_size);
-			eth->val.vlan_tag = spec->tci;
-			eth->mask.vlan_tag = mask->tci;
-			eth->val.vlan_tag &= eth->mask.vlan_tag;
-			/*
-			 * From verbs perspective an empty VLAN is equivalent
-			 * to a packet without VLAN layer.
-			 */
-			if (!eth->mask.vlan_tag)
-				goto error;
-			/* Outer TPID cannot be matched. */
-			if (eth->mask.ether_type) {
-				msg = "VLAN TPID matching is not supported";
-				goto error;
-			}
-			eth->val.ether_type = spec->inner_type;
-			eth->mask.ether_type = mask->inner_type;
-			eth->val.ether_type &= eth->mask.ether_type;
-		}
-		return 0;
-	}
-error:
-	return rte_flow_error_set(data->error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
-				  item, msg);
-}
-
-/**
- * Convert IPv4 item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_ipv4(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-	struct priv *priv = data->dev->data->dev_private;
-	const struct rte_flow_item_ipv4 *spec = item->spec;
-	const struct rte_flow_item_ipv4 *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
-	struct ibv_flow_spec_ipv4_ext ipv4 = {
-		.type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
-		.size = ipv4_size,
-	};
-
-	if (parser->layer == HASH_RXQ_TUNNEL &&
-	    parser->tunnel == ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] &&
-	    !priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	parser->layer = HASH_RXQ_IPV4;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
-			.src_ip = spec->hdr.src_addr,
-			.dst_ip = spec->hdr.dst_addr,
-			.proto = spec->hdr.next_proto_id,
-			.tos = spec->hdr.type_of_service,
-		};
-		ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
-			.src_ip = mask->hdr.src_addr,
-			.dst_ip = mask->hdr.dst_addr,
-			.proto = mask->hdr.next_proto_id,
-			.tos = mask->hdr.type_of_service,
-		};
-		/* Remove unwanted bits from values. */
-		ipv4.val.src_ip &= ipv4.mask.src_ip;
-		ipv4.val.dst_ip &= ipv4.mask.dst_ip;
-		ipv4.val.proto &= ipv4.mask.proto;
-		ipv4.val.tos &= ipv4.mask.tos;
-	}
-	mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
-	return 0;
-}
-
-/**
- * Convert IPv6 item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_ipv6(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-	struct priv *priv = data->dev->data->dev_private;
-	const struct rte_flow_item_ipv6 *spec = item->spec;
-	const struct rte_flow_item_ipv6 *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
-	struct ibv_flow_spec_ipv6 ipv6 = {
-		.type = parser->inner | IBV_FLOW_SPEC_IPV6,
-		.size = ipv6_size,
-	};
-
-	if (parser->layer == HASH_RXQ_TUNNEL &&
-	    parser->tunnel == ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] &&
-	    !priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	parser->layer = HASH_RXQ_IPV6;
-	if (spec) {
-		unsigned int i;
-		uint32_t vtc_flow_val;
-		uint32_t vtc_flow_mask;
-
-		if (!mask)
-			mask = default_mask;
-		memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
-		       RTE_DIM(ipv6.val.src_ip));
-		memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
-		       RTE_DIM(ipv6.val.dst_ip));
-		memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
-		       RTE_DIM(ipv6.mask.src_ip));
-		memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
-		       RTE_DIM(ipv6.mask.dst_ip));
-		vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
-		vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
-		ipv6.val.flow_label =
-			rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
-					 IPV6_HDR_FL_SHIFT);
-		ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
-					 IPV6_HDR_TC_SHIFT;
-		ipv6.val.next_hdr = spec->hdr.proto;
-		ipv6.val.hop_limit = spec->hdr.hop_limits;
-		ipv6.mask.flow_label =
-			rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
-					 IPV6_HDR_FL_SHIFT);
-		ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
-					  IPV6_HDR_TC_SHIFT;
-		ipv6.mask.next_hdr = mask->hdr.proto;
-		ipv6.mask.hop_limit = mask->hdr.hop_limits;
-		/* Remove unwanted bits from values. */
-		for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
-			ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
-			ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
-		}
-		ipv6.val.flow_label &= ipv6.mask.flow_label;
-		ipv6.val.traffic_class &= ipv6.mask.traffic_class;
-		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
-		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
-	}
-	mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
-	return 0;
-}
-
-/**
- * Convert UDP item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_udp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_udp *spec = item->spec;
-	const struct rte_flow_item_udp *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
-	struct ibv_flow_spec_tcp_udp udp = {
-		.type = parser->inner | IBV_FLOW_SPEC_UDP,
-		.size = udp_size,
-	};
-
-	if (parser->layer == HASH_RXQ_IPV4)
-		parser->layer = HASH_RXQ_UDPV4;
-	else
-		parser->layer = HASH_RXQ_UDPV6;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		udp.val.dst_port = spec->hdr.dst_port;
-		udp.val.src_port = spec->hdr.src_port;
-		udp.mask.dst_port = mask->hdr.dst_port;
-		udp.mask.src_port = mask->hdr.src_port;
-		/* Remove unwanted bits from values. */
-		udp.val.src_port &= udp.mask.src_port;
-		udp.val.dst_port &= udp.mask.dst_port;
-	}
-	mlx5_flow_create_copy(parser, &udp, udp_size);
-	return 0;
-}
-
-/**
- * Convert TCP item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_tcp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_tcp *spec = item->spec;
-	const struct rte_flow_item_tcp *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
-	struct ibv_flow_spec_tcp_udp tcp = {
-		.type = parser->inner | IBV_FLOW_SPEC_TCP,
-		.size = tcp_size,
-	};
-
-	if (parser->layer == HASH_RXQ_IPV4)
-		parser->layer = HASH_RXQ_TCPV4;
-	else
-		parser->layer = HASH_RXQ_TCPV6;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		tcp.val.dst_port = spec->hdr.dst_port;
-		tcp.val.src_port = spec->hdr.src_port;
-		tcp.mask.dst_port = mask->hdr.dst_port;
-		tcp.mask.src_port = mask->hdr.src_port;
-		/* Remove unwanted bits from values. */
-		tcp.val.src_port &= tcp.mask.src_port;
-		tcp.val.dst_port &= tcp.mask.dst_port;
-	}
-	mlx5_flow_create_copy(parser, &tcp, tcp_size);
-	return 0;
-}
-
-/**
- * Convert VXLAN item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_vxlan(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_vxlan *spec = item->spec;
-	const struct rte_flow_item_vxlan *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel vxlan = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
-	union vni {
-		uint32_t vlan_id;
-		uint8_t vni[4];
-	} id;
-
-	id.vni[0] = 0;
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
-	/* Default VXLAN to outer RSS. */
-	if (!parser->rss_conf.level)
-		parser->rss_conf.level = 1;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		memcpy(&id.vni[1], spec->vni, 3);
-		vxlan.val.tunnel_id = id.vlan_id;
-		memcpy(&id.vni[1], mask->vni, 3);
-		vxlan.mask.tunnel_id = id.vlan_id;
-		/* Remove unwanted bits from values. */
-		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
-	}
-	/*
-	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
-	 * layer is defined in the Verbs specification it is interpreted as
-	 * wildcard and all packets will match this rule, if it follows a full
-	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
-	 * before will also match this rule.
-	 * To avoid such situation, VNI 0 is currently refused.
-	 */
-	/* Only allow tunnel w/o tunnel id pattern after proper outer spec. */
-	if (parser->out_layer == HASH_RXQ_ETH && !vxlan.val.tunnel_id)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "VxLAN vni cannot be 0");
-	mlx5_flow_create_copy(parser, &vxlan, size);
-	return 0;
-}
-
-/**
- * Convert VXLAN-GPE item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_vxlan_gpe(const struct rte_flow_item *item,
-			   const void *default_mask,
-			   struct mlx5_flow_data *data)
-{
-	struct priv *priv = data->dev->data->dev_private;
-	const struct rte_flow_item_vxlan_gpe *spec = item->spec;
-	const struct rte_flow_item_vxlan_gpe *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel vxlan = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
-	union vni {
-		uint32_t vlan_id;
-		uint8_t vni[4];
-	} id;
-
-	if (!priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	id.vni[0] = 0;
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN_GPE)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
-	/* Default VXLAN-GPE to outer RSS. */
-	if (!parser->rss_conf.level)
-		parser->rss_conf.level = 1;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		memcpy(&id.vni[1], spec->vni, 3);
-		vxlan.val.tunnel_id = id.vlan_id;
-		memcpy(&id.vni[1], mask->vni, 3);
-		vxlan.mask.tunnel_id = id.vlan_id;
-		if (spec->protocol)
-			return rte_flow_error_set(data->error, EINVAL,
-						  RTE_FLOW_ERROR_TYPE_ITEM,
-						  item,
-						  "VxLAN-GPE protocol not"
-						  " supported");
-		/* Remove unwanted bits from values. */
-		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
-	}
-	/*
-	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
-	 * layer is defined in the Verbs specification it is interpreted as
-	 * wildcard and all packets will match this rule, if it follows a full
-	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
-	 * before will also match this rule.
-	 * To avoid such situation, VNI 0 is currently refused.
-	 */
-	/* Only allow tunnel w/o tunnel id pattern after proper outer spec. */
-	if (parser->out_layer == HASH_RXQ_ETH && !vxlan.val.tunnel_id)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "VxLAN-GPE vni cannot be 0");
-	mlx5_flow_create_copy(parser, &vxlan, size);
-	return 0;
-}
-
-/**
- * Convert GRE item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_gre(const struct rte_flow_item *item __rte_unused,
-		     const void *default_mask __rte_unused,
-		     struct mlx5_flow_data *data)
-{
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel tunnel = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
-	struct ibv_flow_spec_ipv4_ext *ipv4;
-	struct ibv_flow_spec_ipv6 *ipv6;
-	unsigned int i;
-
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_GRE)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
-	/* Default GRE to inner RSS. */
-	if (!parser->rss_conf.level)
-		parser->rss_conf.level = 2;
-	/* Update encapsulation IP layer protocol. */
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		if (parser->out_layer == HASH_RXQ_IPV4) {
-			ipv4 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset -
-				sizeof(struct ibv_flow_spec_ipv4_ext));
-			if (ipv4->mask.proto && ipv4->val.proto != MLX5_GRE)
-				break;
-			ipv4->val.proto = MLX5_GRE;
-			ipv4->mask.proto = 0xff;
-		} else if (parser->out_layer == HASH_RXQ_IPV6) {
-			ipv6 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset -
-				sizeof(struct ibv_flow_spec_ipv6));
-			if (ipv6->mask.next_hdr &&
-			    ipv6->val.next_hdr != MLX5_GRE)
-				break;
-			ipv6->val.next_hdr = MLX5_GRE;
-			ipv6->mask.next_hdr = 0xff;
-		}
-	}
-	if (i != hash_rxq_init_n)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "IP protocol of GRE must be 47");
-	mlx5_flow_create_copy(parser, &tunnel, size);
-	return 0;
-}
-
-/**
- * Convert mark/flag action to Verbs specification.
- *
- * @param parser
- *   Internal parser structure.
- * @param mark_id
- *   Mark identifier.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
-{
-	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
-	struct ibv_flow_spec_action_tag tag = {
-		.type = IBV_FLOW_SPEC_ACTION_TAG,
-		.size = size,
-		.tag_id = mlx5_flow_mark_set(mark_id),
-	};
-
-	assert(parser->mark);
-	mlx5_flow_create_copy(parser, &tag, size);
-	return 0;
-}
-
-/**
- * Convert count action to Verbs specification.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Pointer to MLX5 flow parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused,
-		       struct mlx5_flow_parse *parser __rte_unused)
-{
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	struct priv *priv = dev->data->dev_private;
-	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
-	struct ibv_counter_set_init_attr init_attr = {0};
-	struct ibv_flow_spec_counter_action counter = {
-		.type = IBV_FLOW_SPEC_ACTION_COUNT,
-		.size = size,
-		.counter_set_handle = 0,
-	};
-
-	init_attr.counter_set_id = 0;
-	parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr);
-	if (!parser->cs) {
-		rte_errno = EINVAL;
-		return -rte_errno;
-	}
-	counter.counter_set_handle = parser->cs->handle;
-	mlx5_flow_create_copy(parser, &counter, size);
-#endif
-	return 0;
-}
-
-/**
- * Complete flow rule creation with a drop queue.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev,
-				   struct mlx5_flow_parse *parser,
-				   struct rte_flow *flow,
-				   struct rte_flow_error *error)
-{
-	struct priv *priv = dev->data->dev_private;
-	struct ibv_flow_spec_action_drop *drop;
-	unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
-
-	assert(priv->pd);
-	assert(priv->ctx);
-	flow->drop = 1;
-	drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr +
-			parser->queue[HASH_RXQ_ETH].offset);
-	*drop = (struct ibv_flow_spec_action_drop){
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-			.size = size,
-	};
-	++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs;
-	parser->queue[HASH_RXQ_ETH].offset += size;
-	flow->frxq[HASH_RXQ_ETH].ibv_attr =
-		parser->queue[HASH_RXQ_ETH].ibv_attr;
-	if (parser->count)
-		flow->cs = parser->cs;
-	if (!dev->data->dev_started)
-		return 0;
-	parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
-	flow->frxq[HASH_RXQ_ETH].ibv_flow =
-		mlx5_glue->create_flow(priv->flow_drop_queue->qp,
-				       flow->frxq[HASH_RXQ_ETH].ibv_attr);
-	if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-		rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "flow rule creation failure");
-		goto error;
-	}
-	return 0;
-error:
-	assert(flow);
-	if (flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-		claim_zero(mlx5_glue->destroy_flow
-			   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-		flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
-	}
-	if (flow->frxq[HASH_RXQ_ETH].ibv_attr) {
-		rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
-		flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL;
-	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-		parser->cs = NULL;
-	}
-	return -rte_errno;
-}
-
-/**
- * Create hash Rx queues when RSS is enabled.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev,
-				  struct mlx5_flow_parse *parser,
-				  struct rte_flow *flow,
-				  struct rte_flow_error *error)
-{
-	unsigned int i;
-
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
-		parser->queue[i].ibv_attr = NULL;
-		flow->frxq[i].hash_fields = parser->queue[i].hash_fields;
-		if (!dev->data->dev_started)
-			continue;
-		flow->frxq[i].hrxq =
-			mlx5_hrxq_get(dev,
-				      parser->rss_conf.key,
-				      parser->rss_conf.key_len,
-				      flow->frxq[i].hash_fields,
-				      parser->rss_conf.queue,
-				      parser->rss_conf.queue_num,
-				      parser->tunnel,
-				      parser->rss_conf.level);
-		if (flow->frxq[i].hrxq)
-			continue;
-		flow->frxq[i].hrxq =
-			mlx5_hrxq_new(dev,
-				      parser->rss_conf.key,
-				      parser->rss_conf.key_len,
-				      flow->frxq[i].hash_fields,
-				      parser->rss_conf.queue,
-				      parser->rss_conf.queue_num,
-				      parser->tunnel,
-				      parser->rss_conf.level);
-		if (!flow->frxq[i].hrxq) {
-			return rte_flow_error_set(error, ENOMEM,
-						  RTE_FLOW_ERROR_TYPE_HANDLE,
-						  NULL,
-						  "cannot create hash rxq");
-		}
-	}
-	return 0;
-}
-
-/**
- * RXQ update after flow rule creation.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Pointer to the flow rule.
- */
-static void
-mlx5_flow_create_update_rxqs(struct rte_eth_dev *dev, struct rte_flow *flow)
-{
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-	unsigned int j;
-
-	if (!dev->data->dev_started)
-		return;
-	for (i = 0; i != flow->rss_conf.queue_num; ++i) {
-		struct mlx5_rxq_data *rxq_data = (*priv->rxqs)
-						 [(*flow->queues)[i]];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
-		uint8_t tunnel = PTYPE_IDX(flow->tunnel);
-
-		rxq_data->mark |= flow->mark;
-		if (!tunnel)
-			continue;
-		rxq_ctrl->tunnel_types[tunnel] += 1;
-		/* Clear tunnel type if more than one tunnel types set. */
-		for (j = 0; j != RTE_DIM(rxq_ctrl->tunnel_types); ++j) {
-			if (j == tunnel)
-				continue;
-			if (rxq_ctrl->tunnel_types[j] > 0) {
-				rxq_data->tunnel = 0;
-				break;
-			}
-		}
-		if (j == RTE_DIM(rxq_ctrl->tunnel_types))
-			rxq_data->tunnel = flow->tunnel;
-	}
-}
-
-/**
- * Dump flow hash RX queue detail.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Pointer to the rte_flow.
- * @param hrxq_idx
- *   Hash RX queue index.
- */
-static void
-mlx5_flow_dump(struct rte_eth_dev *dev __rte_unused,
-	       struct rte_flow *flow __rte_unused,
-	       unsigned int hrxq_idx __rte_unused)
-{
-#ifndef NDEBUG
-	uintptr_t spec_ptr;
-	uint16_t j;
-	char buf[256];
-	uint8_t off;
+/* Dev ops structure defined in mlx5.c */
+extern const struct eth_dev_ops mlx5_dev_ops;
+extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
-	spec_ptr = (uintptr_t)(flow->frxq[hrxq_idx].ibv_attr + 1);
-	for (j = 0, off = 0; j < flow->frxq[hrxq_idx].ibv_attr->num_of_specs;
-	     j++) {
-		struct ibv_flow_spec *spec = (void *)spec_ptr;
-		off += sprintf(buf + off, " %x(%hu)", spec->hdr.type,
-			       spec->hdr.size);
-		spec_ptr += spec->hdr.size;
-	}
-	DRV_LOG(DEBUG,
-		"port %u Verbs flow %p type %u: hrxq:%p qp:%p ind:%p,"
-		" hash:%" PRIx64 "/%u specs:%hhu(%hu), priority:%hu, type:%d,"
-		" flags:%x, comp_mask:%x specs:%s",
-		dev->data->port_id, (void *)flow, hrxq_idx,
-		(void *)flow->frxq[hrxq_idx].hrxq,
-		(void *)flow->frxq[hrxq_idx].hrxq->qp,
-		(void *)flow->frxq[hrxq_idx].hrxq->ind_table,
-		flow->frxq[hrxq_idx].hash_fields |
-		(flow->tunnel &&
-		 flow->rss_conf.level > 1 ? (uint32_t)IBV_RX_HASH_INNER : 0),
-		flow->rss_conf.queue_num,
-		flow->frxq[hrxq_idx].ibv_attr->num_of_specs,
-		flow->frxq[hrxq_idx].ibv_attr->size,
-		flow->frxq[hrxq_idx].ibv_attr->priority,
-		flow->frxq[hrxq_idx].ibv_attr->type,
-		flow->frxq[hrxq_idx].ibv_attr->flags,
-		flow->frxq[hrxq_idx].ibv_attr->comp_mask,
-		buf);
-#endif
-}
+struct rte_flow {
+	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
+};
 
-/**
- * Complete flow rule creation.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_action_queue(struct rte_eth_dev *dev,
-			      struct mlx5_flow_parse *parser,
-			      struct rte_flow *flow,
-			      struct rte_flow_error *error)
-{
-	struct priv *priv __rte_unused = dev->data->dev_private;
-	int ret;
-	unsigned int i;
-	unsigned int flows_n = 0;
+static const struct rte_flow_ops mlx5_flow_ops = {
+	.isolate = mlx5_flow_isolate,
+};
 
-	assert(priv->pd);
-	assert(priv->ctx);
-	assert(!parser->drop);
-	ret = mlx5_flow_create_action_queue_rss(dev, parser, flow, error);
-	if (ret)
-		goto error;
-	if (parser->count)
-		flow->cs = parser->cs;
-	if (!dev->data->dev_started)
-		return 0;
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!flow->frxq[i].hrxq)
-			continue;
-		flow->frxq[i].ibv_flow =
-			mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
-					       flow->frxq[i].ibv_attr);
-		mlx5_flow_dump(dev, flow, i);
-		if (!flow->frxq[i].ibv_flow) {
-			rte_flow_error_set(error, ENOMEM,
-					   RTE_FLOW_ERROR_TYPE_HANDLE,
-					   NULL, "flow rule creation failure");
-			goto error;
-		}
-		++flows_n;
-	}
-	if (!flows_n) {
-		rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "internal error in flow creation");
-		goto error;
-	}
-	mlx5_flow_create_update_rxqs(dev, flow);
-	return 0;
-error:
-	ret = rte_errno; /* Save rte_errno before cleanup. */
-	assert(flow);
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (flow->frxq[i].ibv_flow) {
-			struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
+/* Convert FDIR request to Generic flow. */
+struct mlx5_fdir {
+	struct rte_flow_attr attr;
+	struct rte_flow_action actions[2];
+	struct rte_flow_item items[4];
+	struct rte_flow_item_eth l2;
+	struct rte_flow_item_eth l2_mask;
+	union {
+		struct rte_flow_item_ipv4 ipv4;
+		struct rte_flow_item_ipv6 ipv6;
+	} l3;
+	union {
+		struct rte_flow_item_ipv4 ipv4;
+		struct rte_flow_item_ipv6 ipv6;
+	} l3_mask;
+	union {
+		struct rte_flow_item_udp udp;
+		struct rte_flow_item_tcp tcp;
+	} l4;
+	union {
+		struct rte_flow_item_udp udp;
+		struct rte_flow_item_tcp tcp;
+	} l4_mask;
+	struct rte_flow_action_queue queue;
+};
 
-			claim_zero(mlx5_glue->destroy_flow(ibv_flow));
-		}
-		if (flow->frxq[i].hrxq)
-			mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
-		if (flow->frxq[i].ibv_attr)
-			rte_free(flow->frxq[i].ibv_attr);
-	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-		parser->cs = NULL;
-	}
-	rte_errno = ret; /* Restore rte_errno. */
-	return -rte_errno;
-}
+/* Verbs specification header. */
+struct ibv_spec_header {
+	enum ibv_flow_spec_type type;
+	uint16_t size;
+};
 
 /**
  * Convert a flow.
@@ -2320,69 +95,17 @@ mlx5_flow_create_action_queue(struct rte_eth_dev *dev,
  *   A flow on success, NULL otherwise and rte_errno is set.
  */
 static struct rte_flow *
-mlx5_flow_list_create(struct rte_eth_dev *dev,
-		      struct mlx5_flows *list,
-		      const struct rte_flow_attr *attr,
-		      const struct rte_flow_item items[],
-		      const struct rte_flow_action actions[],
+mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
+		      struct mlx5_flows *list __rte_unused,
+		      const struct rte_flow_attr *attr __rte_unused,
+		      const struct rte_flow_item items[] __rte_unused,
+		      const struct rte_flow_action actions[] __rte_unused,
 		      struct rte_flow_error *error)
 {
-	struct mlx5_flow_parse parser = { .create = 1, };
-	struct rte_flow *flow = NULL;
-	unsigned int i;
-	int ret;
-
-	ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser);
-	if (ret)
-		goto exit;
-	flow = rte_calloc(__func__, 1,
-			  sizeof(*flow) +
-			  parser.rss_conf.queue_num * sizeof(uint16_t),
-			  0);
-	if (!flow) {
-		rte_flow_error_set(error, ENOMEM,
-				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				   NULL,
-				   "cannot allocate flow memory");
-		return NULL;
-	}
-	/* Copy configuration. */
-	flow->queues = (uint16_t (*)[])(flow + 1);
-	flow->tunnel = parser.tunnel;
-	flow->rss_conf = (struct rte_flow_action_rss){
-		.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
-		.level = 0,
-		.types = parser.rss_conf.types,
-		.key_len = parser.rss_conf.key_len,
-		.queue_num = parser.rss_conf.queue_num,
-		.key = memcpy(flow->rss_key, parser.rss_conf.key,
-			      sizeof(*parser.rss_conf.key) *
-			      parser.rss_conf.key_len),
-		.queue = memcpy(flow->queues, parser.rss_conf.queue,
-				sizeof(*parser.rss_conf.queue) *
-				parser.rss_conf.queue_num),
-	};
-	flow->mark = parser.mark;
-	/* finalise the flow. */
-	if (parser.drop)
-		ret = mlx5_flow_create_action_queue_drop(dev, &parser, flow,
-							 error);
-	else
-		ret = mlx5_flow_create_action_queue(dev, &parser, flow, error);
-	if (ret)
-		goto exit;
-	TAILQ_INSERT_TAIL(list, flow, next);
-	DRV_LOG(DEBUG, "port %u flow created %p", dev->data->port_id,
-		(void *)flow);
-	return flow;
-exit:
-	DRV_LOG(ERR, "port %u flow creation error: %s", dev->data->port_id,
-		error->message);
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser.queue[i].ibv_attr)
-			rte_free(parser.queue[i].ibv_attr);
-	}
-	rte_free(flow);
+	rte_flow_error_set(error, ENOTSUP,
+			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			   NULL,
+			   "action not supported");
 	return NULL;
 }
 
@@ -2393,15 +116,16 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
  * @see rte_flow_ops
  */
 int
-mlx5_flow_validate(struct rte_eth_dev *dev,
-		   const struct rte_flow_attr *attr,
-		   const struct rte_flow_item items[],
-		   const struct rte_flow_action actions[],
+mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
+		   const struct rte_flow_attr *attr __rte_unused,
+		   const struct rte_flow_item items[] __rte_unused,
+		   const struct rte_flow_action actions[] __rte_unused,
 		   struct rte_flow_error *error)
 {
-	struct mlx5_flow_parse parser = { .create = 0, };
-
-	return mlx5_flow_convert(dev, attr, items, actions, error, &parser);
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "action not supported");
 }
 
 /**
@@ -2411,16 +135,17 @@ mlx5_flow_validate(struct rte_eth_dev *dev,
  * @see rte_flow_ops
  */
 struct rte_flow *
-mlx5_flow_create(struct rte_eth_dev *dev,
-		 const struct rte_flow_attr *attr,
-		 const struct rte_flow_item items[],
-		 const struct rte_flow_action actions[],
+mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
+		 const struct rte_flow_attr *attr __rte_unused,
+		 const struct rte_flow_item items[] __rte_unused,
+		 const struct rte_flow_action actions[] __rte_unused,
 		 struct rte_flow_error *error)
 {
-	struct priv *priv = dev->data->dev_private;
-
-	return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions,
-				     error);
+	rte_flow_error_set(error, ENOTSUP,
+			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			   NULL,
+			   "action not supported");
+	return NULL;
 }
 
 /**
@@ -2434,99 +159,10 @@ mlx5_flow_create(struct rte_eth_dev *dev,
  *   Flow to destroy.
  */
 static void
-mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
-		       struct rte_flow *flow)
+mlx5_flow_list_destroy(struct rte_eth_dev *dev __rte_unused,
+		       struct mlx5_flows *list __rte_unused,
+		       struct rte_flow *flow __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-
-	if (flow->drop || !dev->data->dev_started)
-		goto free;
-	for (i = 0; flow->tunnel && i != flow->rss_conf.queue_num; ++i) {
-		/* Update queue tunnel type. */
-		struct mlx5_rxq_data *rxq_data = (*priv->rxqs)
-						 [(*flow->queues)[i]];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
-		uint8_t tunnel = PTYPE_IDX(flow->tunnel);
-
-		assert(rxq_ctrl->tunnel_types[tunnel] > 0);
-		rxq_ctrl->tunnel_types[tunnel] -= 1;
-		if (!rxq_ctrl->tunnel_types[tunnel]) {
-			/* Update tunnel type. */
-			uint8_t j;
-			uint8_t types = 0;
-			uint8_t last;
-
-			for (j = 0; j < RTE_DIM(rxq_ctrl->tunnel_types); j++)
-				if (rxq_ctrl->tunnel_types[j]) {
-					types += 1;
-					last = j;
-				}
-			/* Keep same if more than one tunnel types left. */
-			if (types == 1)
-				rxq_data->tunnel = ptype_ext[last];
-			else if (types == 0)
-				/* No tunnel type left. */
-				rxq_data->tunnel = 0;
-		}
-	}
-	for (i = 0; flow->mark && i != flow->rss_conf.queue_num; ++i) {
-		struct rte_flow *tmp;
-		int mark = 0;
-
-		/*
-		 * To remove the mark from the queue, the queue must not be
-		 * present in any other marked flow (RSS or not).
-		 */
-		TAILQ_FOREACH(tmp, list, next) {
-			unsigned int j;
-			uint16_t *tqs = NULL;
-			uint16_t tq_n = 0;
-
-			if (!tmp->mark)
-				continue;
-			for (j = 0; j != hash_rxq_init_n; ++j) {
-				if (!tmp->frxq[j].hrxq)
-					continue;
-				tqs = tmp->frxq[j].hrxq->ind_table->queues;
-				tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
-			}
-			if (!tq_n)
-				continue;
-			for (j = 0; (j != tq_n) && !mark; j++)
-				if (tqs[j] == (*flow->queues)[i])
-					mark = 1;
-		}
-		(*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
-	}
-free:
-	if (flow->drop) {
-		if (flow->frxq[HASH_RXQ_ETH].ibv_flow)
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-		rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
-	} else {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			struct mlx5_flow *frxq = &flow->frxq[i];
-
-			if (frxq->ibv_flow)
-				claim_zero(mlx5_glue->destroy_flow
-					   (frxq->ibv_flow));
-			if (frxq->hrxq)
-				mlx5_hrxq_release(dev, frxq->hrxq);
-			if (frxq->ibv_attr)
-				rte_free(frxq->ibv_attr);
-		}
-	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-	}
-	TAILQ_REMOVE(list, flow, next);
-	DRV_LOG(DEBUG, "port %u flow destroyed %p", dev->data->port_id,
-		(void *)flow);
-	rte_free(flow);
 }
 
 /**
@@ -2558,97 +194,9 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_flow_create_drop_queue(struct rte_eth_dev *dev)
+mlx5_flow_create_drop_queue(struct rte_eth_dev *dev __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_hrxq_drop *fdq = NULL;
-
-	assert(priv->pd);
-	assert(priv->ctx);
-	fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
-	if (!fdq) {
-		DRV_LOG(WARNING,
-			"port %u cannot allocate memory for drop queue",
-			dev->data->port_id);
-		rte_errno = ENOMEM;
-		return -rte_errno;
-	}
-	fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
-	if (!fdq->cq) {
-		DRV_LOG(WARNING, "port %u cannot allocate CQ for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->wq = mlx5_glue->create_wq
-		(priv->ctx,
-		 &(struct ibv_wq_init_attr){
-			.wq_type = IBV_WQT_RQ,
-			.max_wr = 1,
-			.max_sge = 1,
-			.pd = priv->pd,
-			.cq = fdq->cq,
-		 });
-	if (!fdq->wq) {
-		DRV_LOG(WARNING, "port %u cannot allocate WQ for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->ind_table = mlx5_glue->create_rwq_ind_table
-		(priv->ctx,
-		 &(struct ibv_rwq_ind_table_init_attr){
-			.log_ind_tbl_size = 0,
-			.ind_tbl = &fdq->wq,
-			.comp_mask = 0,
-		 });
-	if (!fdq->ind_table) {
-		DRV_LOG(WARNING,
-			"port %u cannot allocate indirection table for drop"
-			" queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->qp = mlx5_glue->create_qp_ex
-		(priv->ctx,
-		 &(struct ibv_qp_init_attr_ex){
-			.qp_type = IBV_QPT_RAW_PACKET,
-			.comp_mask =
-				IBV_QP_INIT_ATTR_PD |
-				IBV_QP_INIT_ATTR_IND_TABLE |
-				IBV_QP_INIT_ATTR_RX_HASH,
-			.rx_hash_conf = (struct ibv_rx_hash_conf){
-				.rx_hash_function =
-					IBV_RX_HASH_FUNC_TOEPLITZ,
-				.rx_hash_key_len = rss_hash_default_key_len,
-				.rx_hash_key = rss_hash_default_key,
-				.rx_hash_fields_mask = 0,
-				},
-			.rwq_ind_tbl = fdq->ind_table,
-			.pd = priv->pd
-		 });
-	if (!fdq->qp) {
-		DRV_LOG(WARNING, "port %u cannot allocate QP for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	priv->flow_drop_queue = fdq;
 	return 0;
-error:
-	if (fdq->qp)
-		claim_zero(mlx5_glue->destroy_qp(fdq->qp));
-	if (fdq->ind_table)
-		claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
-	if (fdq->wq)
-		claim_zero(mlx5_glue->destroy_wq(fdq->wq));
-	if (fdq->cq)
-		claim_zero(mlx5_glue->destroy_cq(fdq->cq));
-	if (fdq)
-		rte_free(fdq);
-	priv->flow_drop_queue = NULL;
-	return -rte_errno;
 }
 
 /**
@@ -2658,23 +206,8 @@ mlx5_flow_create_drop_queue(struct rte_eth_dev *dev)
  *   Pointer to Ethernet device.
  */
 void
-mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
+mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
-
-	if (!fdq)
-		return;
-	if (fdq->qp)
-		claim_zero(mlx5_glue->destroy_qp(fdq->qp));
-	if (fdq->ind_table)
-		claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
-	if (fdq->wq)
-		claim_zero(mlx5_glue->destroy_wq(fdq->wq));
-	if (fdq->cq)
-		claim_zero(mlx5_glue->destroy_cq(fdq->cq));
-	rte_free(fdq);
-	priv->flow_drop_queue = NULL;
 }
 
 /**
@@ -2686,70 +219,9 @@ mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
  *   Pointer to a TAILQ flow list.
  */
 void
-mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
+mlx5_flow_stop(struct rte_eth_dev *dev __rte_unused,
+	       struct mlx5_flows *list __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct rte_flow *flow;
-	unsigned int i;
-
-	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
-		struct mlx5_ind_table_ibv *ind_tbl = NULL;
-
-		if (flow->drop) {
-			if (!flow->frxq[HASH_RXQ_ETH].ibv_flow)
-				continue;
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-			flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
-			DRV_LOG(DEBUG, "port %u flow %p removed",
-				dev->data->port_id, (void *)flow);
-			/* Next flow. */
-			continue;
-		}
-		/* Verify the flow has not already been cleaned. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_flow)
-				continue;
-			/*
-			 * Indirection table may be necessary to remove the
-			 * flags in the Rx queues.
-			 * This helps to speed-up the process by avoiding
-			 * another loop.
-			 */
-			ind_tbl = flow->frxq[i].hrxq->ind_table;
-			break;
-		}
-		if (i == hash_rxq_init_n)
-			return;
-		if (flow->mark) {
-			assert(ind_tbl);
-			for (i = 0; i != ind_tbl->queues_n; ++i)
-				(*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
-		}
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_flow)
-				continue;
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[i].ibv_flow));
-			flow->frxq[i].ibv_flow = NULL;
-			mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
-			flow->frxq[i].hrxq = NULL;
-		}
-		DRV_LOG(DEBUG, "port %u flow %p removed", dev->data->port_id,
-			(void *)flow);
-	}
-	/* Cleanup Rx queue tunnel info. */
-	for (i = 0; i != priv->rxqs_n; ++i) {
-		struct mlx5_rxq_data *q = (*priv->rxqs)[i];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(q, struct mlx5_rxq_ctrl, rxq);
-
-		if (!q)
-			continue;
-		memset((void *)rxq_ctrl->tunnel_types, 0,
-		       sizeof(rxq_ctrl->tunnel_types));
-		q->tunnel = 0;
-	}
 }
 
 /**
@@ -2764,76 +236,9 @@ mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
+mlx5_flow_start(struct rte_eth_dev *dev __rte_unused,
+		struct mlx5_flows *list __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct rte_flow *flow;
-
-	TAILQ_FOREACH(flow, list, next) {
-		unsigned int i;
-
-		if (flow->drop) {
-			flow->frxq[HASH_RXQ_ETH].ibv_flow =
-				mlx5_glue->create_flow
-				(priv->flow_drop_queue->qp,
-				 flow->frxq[HASH_RXQ_ETH].ibv_attr);
-			if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p cannot be applied",
-					dev->data->port_id, (void *)flow);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-			DRV_LOG(DEBUG, "port %u flow %p applied",
-				dev->data->port_id, (void *)flow);
-			/* Next flow. */
-			continue;
-		}
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_attr)
-				continue;
-			flow->frxq[i].hrxq =
-				mlx5_hrxq_get(dev, flow->rss_conf.key,
-					      flow->rss_conf.key_len,
-					      flow->frxq[i].hash_fields,
-					      flow->rss_conf.queue,
-					      flow->rss_conf.queue_num,
-					      flow->tunnel,
-					      flow->rss_conf.level);
-			if (flow->frxq[i].hrxq)
-				goto flow_create;
-			flow->frxq[i].hrxq =
-				mlx5_hrxq_new(dev, flow->rss_conf.key,
-					      flow->rss_conf.key_len,
-					      flow->frxq[i].hash_fields,
-					      flow->rss_conf.queue,
-					      flow->rss_conf.queue_num,
-					      flow->tunnel,
-					      flow->rss_conf.level);
-			if (!flow->frxq[i].hrxq) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p cannot create hash"
-					" rxq",
-					dev->data->port_id, (void *)flow);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-flow_create:
-			mlx5_flow_dump(dev, flow, i);
-			flow->frxq[i].ibv_flow =
-				mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
-						       flow->frxq[i].ibv_attr);
-			if (!flow->frxq[i].ibv_flow) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p type %u cannot be"
-					" applied",
-					dev->data->port_id, (void *)flow, i);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-		}
-		mlx5_flow_create_update_rxqs(dev, flow);
-	}
 	return 0;
 }
 
@@ -2887,7 +292,6 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
-		.priority = MLX5_CTRL_FLOW_PRIORITY,
 	};
 	struct rte_flow_item items[] = {
 		{
@@ -2997,83 +401,6 @@ mlx5_flow_flush(struct rte_eth_dev *dev,
 	return 0;
 }
 
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-/**
- * Query flow counter.
- *
- * @param cs
- *   the counter set.
- * @param counter_value
- *   returned data from the counter.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_query_count(struct ibv_counter_set *cs,
-		      struct mlx5_flow_counter_stats *counter_stats,
-		      struct rte_flow_query_count *query_count,
-		      struct rte_flow_error *error)
-{
-	uint64_t counters[2];
-	struct ibv_query_counter_set_attr query_cs_attr = {
-		.cs = cs,
-		.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
-	};
-	struct ibv_counter_set_data query_out = {
-		.out = counters,
-		.outlen = 2 * sizeof(uint64_t),
-	};
-	int err = mlx5_glue->query_counter_set(&query_cs_attr, &query_out);
-
-	if (err)
-		return rte_flow_error_set(error, err,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "cannot read counter");
-	query_count->hits_set = 1;
-	query_count->bytes_set = 1;
-	query_count->hits = counters[0] - counter_stats->hits;
-	query_count->bytes = counters[1] - counter_stats->bytes;
-	if (query_count->reset) {
-		counter_stats->hits = counters[0];
-		counter_stats->bytes = counters[1];
-	}
-	return 0;
-}
-
-/**
- * Query a flows.
- *
- * @see rte_flow_query()
- * @see rte_flow_ops
- */
-int
-mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
-		struct rte_flow *flow,
-		const struct rte_flow_action *action __rte_unused,
-		void *data,
-		struct rte_flow_error *error)
-{
-	if (flow->cs) {
-		int ret;
-
-		ret = mlx5_flow_query_count(flow->cs,
-					    &flow->counter_stats,
-					    (struct rte_flow_query_count *)data,
-					    error);
-		if (ret)
-			return ret;
-	} else {
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "no counter found for flow");
-	}
-	return 0;
-}
-#endif
-
 /**
  * Isolated mode.
  *
@@ -3313,18 +640,11 @@ mlx5_fdir_filter_add(struct rte_eth_dev *dev,
 			.type = 0,
 		},
 	};
-	struct mlx5_flow_parse parser = {
-		.layer = HASH_RXQ_ETH,
-	};
 	struct rte_flow_error error;
 	struct rte_flow *flow;
 	int ret;
 
 	ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
-				attributes.actions, &error, &parser);
 	if (ret)
 		return ret;
 	flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr,
@@ -3350,91 +670,11 @@ mlx5_fdir_filter_add(struct rte_eth_dev *dev,
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx5_fdir_filter_delete(struct rte_eth_dev *dev,
-			const struct rte_eth_fdir_filter *fdir_filter)
+mlx5_fdir_filter_delete(struct rte_eth_dev *dev __rte_unused,
+			const struct rte_eth_fdir_filter *fdir_filter
+			__rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_fdir attributes = {
-		.attr.group = 0,
-	};
-	struct mlx5_flow_parse parser = {
-		.create = 1,
-		.layer = HASH_RXQ_ETH,
-	};
-	struct rte_flow_error error;
-	struct rte_flow *flow;
-	unsigned int i;
-	int ret;
-
-	ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
-				attributes.actions, &error, &parser);
-	if (ret)
-		goto exit;
-	/*
-	 * Special case for drop action which is only set in the
-	 * specifications when the flow is created.  In this situation the
-	 * drop specification is missing.
-	 */
-	if (parser.drop) {
-		struct ibv_flow_spec_action_drop *drop;
-
-		drop = (void *)((uintptr_t)parser.queue[parser.layer].ibv_attr +
-				parser.queue[parser.layer].offset);
-		*drop = (struct ibv_flow_spec_action_drop){
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-			.size = sizeof(struct ibv_flow_spec_action_drop),
-		};
-		parser.queue[parser.layer].ibv_attr->num_of_specs++;
-	}
-	TAILQ_FOREACH(flow, &priv->flows, next) {
-		struct ibv_flow_attr *attr;
-		struct ibv_spec_header *attr_h;
-		void *spec;
-		struct ibv_flow_attr *flow_attr;
-		struct ibv_spec_header *flow_h;
-		void *flow_spec;
-		unsigned int specs_n;
-
-		attr = parser.queue[parser.layer].ibv_attr;
-		flow_attr = flow->frxq[parser.layer].ibv_attr;
-		/* Compare first the attributes. */
-		if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
-			continue;
-		if (attr->num_of_specs == 0)
-			continue;
-		spec = (void *)((uintptr_t)attr +
-				sizeof(struct ibv_flow_attr));
-		flow_spec = (void *)((uintptr_t)flow_attr +
-				     sizeof(struct ibv_flow_attr));
-		specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
-		for (i = 0; i != specs_n; ++i) {
-			attr_h = spec;
-			flow_h = flow_spec;
-			if (memcmp(spec, flow_spec,
-				   RTE_MIN(attr_h->size, flow_h->size)))
-				goto wrong_flow;
-			spec = (void *)((uintptr_t)spec + attr_h->size);
-			flow_spec = (void *)((uintptr_t)flow_spec +
-					     flow_h->size);
-		}
-		/* At this point, the flow match. */
-		break;
-wrong_flow:
-		/* The flow does not match. */
-		continue;
-	}
-	ret = rte_errno; /* Save rte_errno before cleanup. */
-	if (flow)
-		mlx5_flow_list_destroy(dev, &priv->flows, flow);
-exit:
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser.queue[i].ibv_attr)
-			rte_free(parser.queue[i].ibv_attr);
-	}
-	rte_errno = ret; /* Restore rte_errno. */
+	rte_errno = ENOTSUP;
 	return -rte_errno;
 }
 
@@ -3601,45 +841,7 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
  *   number of supported Verbs flow priority.
  */
 unsigned int
-mlx5_get_max_verbs_prio(struct rte_eth_dev *dev)
+mlx5_get_max_verbs_prio(struct rte_eth_dev *dev __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	unsigned int verb_priorities = MLX5_VERBS_FLOW_PRIO_8;
-	struct {
-		struct ibv_flow_attr attr;
-		struct ibv_flow_spec_eth eth;
-		struct ibv_flow_spec_action_drop drop;
-	} flow_attr = {
-		.attr = {
-			.num_of_specs = 2,
-		},
-		.eth = {
-			.type = IBV_FLOW_SPEC_ETH,
-			.size = sizeof(struct ibv_flow_spec_eth),
-		},
-		.drop = {
-			.size = sizeof(struct ibv_flow_spec_action_drop),
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-		},
-	};
-	struct ibv_flow *flow;
-
-	do {
-		flow_attr.attr.priority = verb_priorities - 1;
-		flow = mlx5_glue->create_flow(priv->flow_drop_queue->qp,
-					      &flow_attr.attr);
-		if (flow) {
-			claim_zero(mlx5_glue->destroy_flow(flow));
-			/* Try more priorities. */
-			verb_priorities *= 2;
-		} else {
-			/* Failed, restore last right number. */
-			verb_priorities /= 2;
-			break;
-		}
-	} while (1);
-	DRV_LOG(DEBUG, "port %u Verbs flow priorities: %d,"
-		" user flow priorities: %d",
-		dev->data->port_id, verb_priorities, MLX5_CTRL_FLOW_PRIORITY);
-	return verb_priorities;
+	return 8;
 }
-- 
2.17.0

[dpdk-dev] [DPDK 18.08 v1 02/12] net/mlx5: handle drop queues are regular queues

[Nelio Laranjeiro]

Drop queues are essentially used in flows due to Verbs API, the
information if the fate of the flow is a drop or not is already present
in the flow.  Due to this, drop queues can be fully mapped on regular
queues.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5.c      |   9 --
 drivers/net/mlx5/mlx5.h      |   3 +-
 drivers/net/mlx5/mlx5_flow.c |  26 -----
 drivers/net/mlx5/mlx5_rxq.c  | 221 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_rxtx.h |   6 +
 5 files changed, 229 insertions(+), 36 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index f455ea8ea..8188a67e3 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -252,7 +252,6 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 		priv->txqs_n = 0;
 		priv->txqs = NULL;
 	}
-	mlx5_flow_delete_drop_queue(dev);
 	mlx5_mprq_free_mp(dev);
 	mlx5_mr_release(dev);
 	if (priv->pd != NULL) {
@@ -1161,14 +1160,6 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 		mlx5_link_update(eth_dev, 0);
 		/* Store device configuration on private structure. */
 		priv->config = config;
-		/* Create drop queue. */
-		err = mlx5_flow_create_drop_queue(eth_dev);
-		if (err) {
-			DRV_LOG(ERR, "port %u drop queue allocation failed: %s",
-				eth_dev->data->port_id, strerror(rte_errno));
-			err = rte_errno;
-			goto port_error;
-		}
 		/* Supported Verbs flow priority number detection. */
 		if (verb_priorities == 0)
 			verb_priorities = mlx5_get_max_verbs_prio(eth_dev);
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index c4c962b92..69f4f7eb6 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -169,7 +169,7 @@ struct priv {
 	struct rte_intr_handle intr_handle; /* Interrupt handler. */
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
-	struct mlx5_hrxq_drop *flow_drop_queue; /* Flow drop queue. */
+	struct mlx5_hrxq *flow_drop_queue; /* Flow drop queue. */
 	struct mlx5_flows flows; /* RTE Flow rules. */
 	struct mlx5_flows ctrl_flows; /* Control flow rules. */
 	struct {
@@ -294,6 +294,7 @@ int mlx5_traffic_restart(struct rte_eth_dev *dev);
 
 /* mlx5_flow.c */
 
+void mlx5_flow_print(struct rte_flow *flow);
 unsigned int mlx5_get_max_verbs_prio(struct rte_eth_dev *dev);
 int mlx5_flow_validate(struct rte_eth_dev *dev,
 		       const struct rte_flow_attr *attr,
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index a45cb06e1..6497e99c1 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -184,32 +184,6 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
 	}
 }
 
-/**
- * Create drop queue.
- *
- * @param dev
- *   Pointer to Ethernet device.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-int
-mlx5_flow_create_drop_queue(struct rte_eth_dev *dev __rte_unused)
-{
-	return 0;
-}
-
-/**
- * Delete drop queue.
- *
- * @param dev
- *   Pointer to Ethernet device.
- */
-void
-mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev __rte_unused)
-{
-}
-
 /**
  * Remove all flows.
  *
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 171ce9e30..7ac9bd000 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1967,3 +1967,224 @@ mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev)
 	}
 	return ret;
 }
+
+/**
+ * Create a drop Rx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_rxq_ibv *
+mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct ibv_cq *cq;
+	struct ibv_wq *wq = NULL;
+	struct mlx5_rxq_ibv *rxq;
+
+	cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
+	if (!cq) {
+		DEBUG("port %u cannot allocate CQ for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	wq = mlx5_glue->create_wq(priv->ctx,
+		 &(struct ibv_wq_init_attr){
+			.wq_type = IBV_WQT_RQ,
+			.max_wr = 1,
+			.max_sge = 1,
+			.pd = priv->pd,
+			.cq = cq,
+		 });
+	if (!wq) {
+		DEBUG("port %u cannot allocate WQ for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	rxq = rte_calloc(__func__, 1, sizeof(*rxq), 0);
+	if (!rxq) {
+		DEBUG("port %u cannot allocate drop Rx queue memory",
+		      dev->data->port_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	rxq->cq = cq;
+	rxq->wq = wq;
+	return rxq;
+error:
+	if (wq)
+		claim_zero(mlx5_glue->destroy_wq(wq));
+	if (cq)
+		claim_zero(mlx5_glue->destroy_cq(cq));
+	return NULL;
+}
+
+/**
+ * Release a drop Rx queue Verbs object.
+ *
+ * @param rxq
+ *   Pointer to the drop Verbs Rx queue.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+void
+mlx5_rxq_ibv_drop_release(struct mlx5_rxq_ibv *rxq)
+{
+	if (rxq->wq)
+		claim_zero(mlx5_glue->destroy_wq(rxq->wq));
+	if (rxq->cq)
+		claim_zero(mlx5_glue->destroy_cq(rxq->cq));
+	rte_free(rxq);
+}
+
+/**
+ * Create a drop indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_ind_table_ibv *
+mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	struct mlx5_rxq_ibv *rxq;
+	struct mlx5_ind_table_ibv tmpl;
+
+	rxq = mlx5_rxq_ibv_drop_new(dev);
+	if (!rxq)
+		return NULL;
+	tmpl.ind_table = mlx5_glue->create_rwq_ind_table
+		(priv->ctx,
+		 &(struct ibv_rwq_ind_table_init_attr){
+			.log_ind_tbl_size = 0,
+			.ind_tbl = &rxq->wq,
+			.comp_mask = 0,
+		 });
+	if (!tmpl.ind_table) {
+		DEBUG("port %u cannot allocate indirection table for drop"
+		      " queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	ind_tbl = rte_calloc(__func__, 1, sizeof(*ind_tbl), 0);
+	if (!ind_tbl) {
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	ind_tbl->ind_table = tmpl.ind_table;
+	return ind_tbl;
+error:
+	mlx5_rxq_ibv_drop_release(rxq);
+	return NULL;
+}
+
+/**
+ * Release a drop indirection table.
+ *
+ * @param ind_tbl
+ *   Indirection table to release.
+ */
+void
+mlx5_ind_table_ibv_drop_release(struct mlx5_ind_table_ibv *ind_tbl)
+{
+	claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl->ind_table));
+	rte_free(ind_tbl);
+}
+
+/**
+ * Create a drop Rx Hash queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_hrxq *
+mlx5_hrxq_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	struct ibv_qp *qp;
+	struct mlx5_hrxq *hrxq;
+
+	if (priv->flow_drop_queue) {
+		rte_atomic32_inc(&priv->flow_drop_queue->refcnt);
+		return priv->flow_drop_queue;
+	}
+	ind_tbl = mlx5_ind_table_ibv_drop_new(dev);
+	if (!ind_tbl)
+		return NULL;
+	qp = mlx5_glue->create_qp_ex(priv->ctx,
+		 &(struct ibv_qp_init_attr_ex){
+			.qp_type = IBV_QPT_RAW_PACKET,
+			.comp_mask =
+				IBV_QP_INIT_ATTR_PD |
+				IBV_QP_INIT_ATTR_IND_TABLE |
+				IBV_QP_INIT_ATTR_RX_HASH,
+			.rx_hash_conf = (struct ibv_rx_hash_conf){
+				.rx_hash_function =
+					IBV_RX_HASH_FUNC_TOEPLITZ,
+				.rx_hash_key_len = rss_hash_default_key_len,
+				.rx_hash_key = rss_hash_default_key,
+				.rx_hash_fields_mask = 0,
+				},
+			.rwq_ind_tbl = ind_tbl->ind_table,
+			.pd = priv->pd
+		 });
+	if (!qp) {
+		DEBUG("port %u cannot allocate QP for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	hrxq = rte_calloc(__func__, 1, sizeof(*hrxq), 0);
+	if (!hrxq) {
+		DRV_LOG(WARNING,
+			"port %u cannot allocate memory for drop queue",
+			dev->data->port_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	hrxq->ind_table = ind_tbl;
+	hrxq->qp = qp;
+	priv->flow_drop_queue = hrxq;
+	rte_atomic32_set(&hrxq->refcnt, 1);
+	return hrxq;
+error:
+	if (ind_tbl)
+		mlx5_ind_table_ibv_drop_release(ind_tbl);
+	return NULL;
+}
+
+/**
+ * Release a drop hash Rx queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param hrxq
+ *   Pointer to Hash Rx queue to release.
+ */
+void
+mlx5_hrxq_drop_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
+		claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
+		mlx5_ind_table_ibv_drop_release(hrxq->ind_table);
+		rte_free(hrxq);
+		priv->flow_drop_queue = NULL;
+	}
+}
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index a6017f0dd..3f7a206c1 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -245,6 +245,8 @@ struct mlx5_rxq_ibv *mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
 struct mlx5_rxq_ibv *mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
 int mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv);
 int mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv);
+struct mlx5_rxq_ibv *mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev);
+void mlx5_rxq_ibv_drop_release(struct mlx5_rxq_ibv *rxq);
 int mlx5_rxq_ibv_verify(struct rte_eth_dev *dev);
 struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx,
 				   uint16_t desc, unsigned int socket,
@@ -265,6 +267,8 @@ struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_get(struct rte_eth_dev *dev,
 int mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
 			       struct mlx5_ind_table_ibv *ind_tbl);
 int mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev);
+void mlx5_ind_table_ibv_drop_release(struct mlx5_ind_table_ibv *ind_tbl);
 struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev,
 				const uint8_t *rss_key, uint32_t rss_key_len,
 				uint64_t hash_fields,
@@ -277,6 +281,8 @@ struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev,
 				uint32_t tunnel, uint32_t rss_level);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
 int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev);
+void mlx5_hrxq_drop_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq);
 uint64_t mlx5_get_rx_port_offloads(void);
 uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
 
-- 
2.17.0

[dpdk-dev] [DPDK 18.08 v1 03/12] net/mlx5: support flow Ethernet item among with drop action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 676 ++++++++++++++++++++++++++++++++---
 1 file changed, 634 insertions(+), 42 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 6497e99c1..85dc5edaf 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -35,11 +35,62 @@
 extern const struct eth_dev_ops mlx5_dev_ops;
 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
+/* Pattern Layer bits. */
+#define MLX5_FLOW_LAYER_L2 (1u << 0)
+#define MLX5_FLOW_LAYER_L3_IPV4 (1u << 1)
+#define MLX5_FLOW_LAYER_L3_IPV6 (1u << 2)
+#define MLX5_FLOW_LAYER_L4_UDP (1u << 3)
+#define MLX5_FLOW_LAYER_L4_TCP (1u << 4)
+#define MLX5_FLOW_LAYER_VLAN (1u << 5)
+/* Masks. */
+#define MLX5_FLOW_LAYER_L3 \
+	(MLX5_FLOW_LAYER_L3_IPV4 | MLX5_FLOW_LAYER_L3_IPV6)
+#define MLX5_FLOW_LAYER_L4 \
+	(MLX5_FLOW_LAYER_L4_UDP | MLX5_FLOW_LAYER_L4_TCP)
+#define MLX5_FLOW_LAYER_OUTER \
+	(MLX5_FLOW_LAYER_L2 | MLX5_FLOW_LAYER_L3 | \
+	 MLX5_FLOW_LAYER_L4)
+
+/* Action fate on the packet. */
+#define MLX5_FLOW_FATE_DROP (1u << 0)
+
+/* Verbs flow priority per layer level. */
+#define MLX5_FLOW_PRIO_L4 0
+#define MLX5_FLOW_PRIO_L3 1
+#define MLX5_FLOW_PRIO_L2 2
+
+/* Control flow priority. */
+#define MLX5_FLOW_CTRL 0xffffffff
+#define MLX5_FLOW_CTRL_PRIO_OFFSET (MLX5_FLOW_PRIO_L2 + 1)
+
+/** Handles information leading to a drop fate. */
+struct mlx5_flow_verbs {
+	unsigned int size; /**< Size of the attribute. */
+	uint32_t layers;
+	/**< Bit-fields of present layers see MLX5_FLOW_ITEMS_*. */
+	uint32_t fate;
+	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
+	struct {
+		struct ibv_flow_attr *attr;
+		/**< Pointer to the Specification buffer. */
+		uint8_t *specs; /**< Pointer to the specifications. */
+	};
+	struct ibv_flow *flow; /**< Verbs flow pointer. */
+	struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */
+};
+
+/* Flow structure. */
 struct rte_flow {
 	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
+	struct rte_flow_attr attributes; /**< User flow attribute. */
+	struct mlx5_flow_verbs verbs; /* Verbs drop flow. */
 };
 
 static const struct rte_flow_ops mlx5_flow_ops = {
+	.validate = mlx5_flow_validate,
+	.create = mlx5_flow_create,
+	.destroy = mlx5_flow_destroy,
+	.flush = mlx5_flow_flush,
 	.isolate = mlx5_flow_isolate,
 };
 
@@ -76,12 +127,374 @@ struct ibv_spec_header {
 };
 
 /**
- * Convert a flow.
+ * Flow debug purpose function only used when CONFIG_RTE_LIBRTE_MLX5_DEBUG=y
+ *
+ * @param flow
+ *   Pointer to the flow structure to display.
+ */
+void
+mlx5_flow_print(struct rte_flow *flow __rte_unused)
+{
+#ifndef NDEBUG
+	fprintf(stdout, "---------8<------------\n");
+	fprintf(stdout, "%s: flow information\n", MLX5_DRIVER_NAME);
+	fprintf(stdout, " attributes: group %u priority %u ingress %d egress %d"
+		" transfer %d\n", flow->attributes.group,
+		flow->attributes.priority,
+		flow->attributes.ingress,
+		flow->attributes.egress,
+		flow->attributes.transfer);
+	fprintf(stdout, " layers: %s/%s/%s\n",
+		flow->verbs.layers & MLX5_FLOW_LAYER_L2 ? "l2" : "-",
+		flow->verbs.layers & MLX5_FLOW_LAYER_L3 ? "l3" : "-",
+		flow->verbs.layers & MLX5_FLOW_LAYER_L4 ? "l4" : "-");
+	if (flow->verbs.fate & MLX5_FLOW_FATE_DROP)
+		fprintf(stdout, " fate: drop queue\n");
+	if (flow->verbs.attr) {
+		struct ibv_spec_header *hdr =
+			(struct ibv_spec_header *)flow->verbs.specs;
+		const int n = flow->verbs.attr->num_of_specs;
+		int i;
+
+		fprintf(stdout, " Verbs attributes: priority %u specs_n %u\n",
+			flow->verbs.attr->priority,
+			flow->verbs.attr->num_of_specs);
+		for (i = 0; i != n; ++i) {
+			rte_hexdump(stdout, " ", hdr, hdr->size);
+			hdr = (struct ibv_spec_header *)
+				((uint8_t *)hdr + hdr->size);
+		}
+	}
+	fprintf(stdout, "--------->8------------\n");
+#endif
+}
+
+/**
+ * Validate Attributes provided by the user.
+ *
+ * @param attr
+ *   Pointer to flow attributes
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param error
+ *   Pointer to error structure.
+ */
+static int
+mlx5_flow_attributes(const struct rte_flow_attr *attr,
+		     struct rte_flow *flow,
+		     struct rte_flow_error *error)
+{
+	if (attr->group)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+					  NULL,
+					  "groups are not supported");
+	if (attr->priority && attr->priority != MLX5_FLOW_CTRL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+					  NULL,
+					  "priorities are not supported");
+	if (attr->egress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+					  NULL,
+					  "egress is not supported");
+	if (attr->transfer)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
+					  NULL,
+					  "transfer is not supported");
+	if (!attr->ingress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+					  NULL,
+					  "only ingress is supported");
+	flow->attributes = *attr;
+	return 0;
+}
+
+/**
+ * Check support for a given item.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param default_mask[in]
+ *   Bit-masks covering supported fields to compare with spec, last and mask in
+ *   \item.
+ * @param nic_mask[in]
+ *   Bit-masks covering supported fields by the NIC to compare with user mask.
+ * @param size
+ *   Bit-Mask size in bytes.
+ * @param error[out]
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_validate(const struct rte_flow_item *item,
+			const uint8_t *default_mask,
+			const uint8_t *nic_mask,
+			unsigned int size,
+			struct rte_flow_error *error)
+{
+	const uint8_t *mask = item->mask ? item->mask : default_mask;
+	unsigned int i;
+
+	assert(nic_mask);
+	for (i = 0; i < size; ++i)
+		if ((nic_mask[i] | mask[i]) != nic_mask[i])
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "mask enables non supported"
+						  " bits");
+	if (!item->spec && (item->mask || item->last))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "mask/last without a spec is not"
+					  " supported");
+	if (item->spec && item->last) {
+		uint8_t spec[size];
+		uint8_t last[size];
+		unsigned int i;
+		int ret;
+
+		for (i = 0; i < size; ++i) {
+			spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
+			last[i] = ((const uint8_t *)item->last)[i] & mask[i];
+		}
+		ret = memcmp(spec, last, size);
+		if (ret != 0)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "spec and last different"
+						  " is not supported");
+	}
+	return 0;
+}
+
+/**
+ * Add a verbs specification.
+ *
+ * @param flow
+ *   Pointer to flow structure.
+ * @param src
+ *   Create specification.
+ * @param size
+ *   Size in bytes of the specification to copy.
+ */
+static void
+mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
+{
+	if (flow->verbs.specs) {
+		void *dst;
+
+		dst = (void *)(flow->verbs.specs + flow->verbs.size);
+		memcpy(dst, src, size);
+		++flow->verbs.attr->num_of_specs;
+	}
+	flow->verbs.size += size;
+}
+
+/**
+ * Validate Ethernet layer and possibly create the Verbs specification.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
+		   struct rte_flow_error *error)
+{
+	const struct rte_flow_item_eth *spec = item->spec;
+	const struct rte_flow_item_eth *mask = item->mask;
+	const struct rte_flow_item_eth nic_mask = {
+		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.type = RTE_BE16(0xffff),
+	};
+	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	struct ibv_flow_spec_eth eth = {
+		.type = IBV_FLOW_SPEC_ETH,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->verbs.layers & MLX5_FLOW_LAYER_L2)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L2 layers already configured");
+	if (!mask)
+		mask = &rte_flow_item_eth_mask;
+	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+				      (const uint8_t *)&nic_mask,
+				      sizeof(struct rte_flow_item_eth),
+				      error);
+	if (ret)
+		return ret;
+	if (spec) {
+		unsigned int i;
+
+		memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
+		memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
+		eth.val.ether_type = spec->type;
+		memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
+		memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
+		eth.mask.ether_type = mask->type;
+		/* Remove unwanted bits from values. */
+		for (i = 0; i < ETHER_ADDR_LEN; ++i) {
+			eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
+			eth.val.src_mac[i] &= eth.mask.src_mac[i];
+		}
+		eth.val.ether_type &= eth.mask.ether_type;
+	}
+	mlx5_flow_spec_verbs_add(flow, &eth, size);
+	flow->verbs.layers |= MLX5_FLOW_LAYER_L2;
+	return 0;
+}
+
+/**
+ * Validate items provided by the user.
+ *
+ * @param items
+ *   Pointer to flow items array.
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_items(const struct rte_flow_item items[],
+		struct rte_flow *flow __rte_unused,
+		struct rte_flow_error *error)
+{
+	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
+		int ret = 0;
+
+		switch (items->type) {
+		case RTE_FLOW_ITEM_TYPE_VOID:
+			break;
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			ret = mlx5_flow_item_eth(items, flow, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  items,
+						  "item not supported");
+		}
+		if (ret < 0)
+			return ret;
+	}
+	if (!flow->verbs.layers) {
+		const struct rte_flow_item item = {
+			.type = RTE_FLOW_ITEM_TYPE_ETH,
+		};
+
+		return mlx5_flow_item_eth(&item, flow, error);
+	}
+	return 0;
+}
+
+/**
+ * Validate action drop provided by the user.
+ *
+ * @param actions
+ *   Pointer to flow actions array.
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param error
+ *   Pointer to error structure.
+ */
+static int
+mlx5_flow_action_drop(const struct rte_flow_action *actions,
+		      struct rte_flow *flow,
+		      struct rte_flow_error *error)
+{
+	unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
+	struct ibv_flow_spec_action_drop drop = {
+			.type = IBV_FLOW_SPEC_ACTION_DROP,
+			.size = size,
+	};
+
+	if (flow->verbs.fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "multiple fate actions are not"
+					  " supported");
+	mlx5_flow_spec_verbs_add(flow, &drop, size);
+	flow->verbs.fate |= MLX5_FLOW_FATE_DROP;
+	return 0;
+}
+
+/**
+ * Validate actions provided by the user.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param actions
+ *   Pointer to flow actions array.
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param error
+ *   Pointer to error structure.
+ */
+static int
+mlx5_flow_actions(struct rte_eth_dev *dev __rte_unused,
+		  const struct rte_flow_action actions[],
+		  struct rte_flow *flow __rte_unused,
+		  struct rte_flow_error *error)
+{
+	int ret;
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_DROP:
+			ret = mlx5_flow_action_drop(actions, flow, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+		if (ret < 0)
+			return ret;
+	}
+	if (!flow->verbs.fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "no fate action found");
+	return 0;
+}
+
+/**
+ * Validate the rule and return a flow structure filled accordingly.
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to a TAILQ flow list.
+ * @param flow
+ *   Pointer an flow structure.
+ * @param flow_size
+ *   Size of the allocated space to store the flow information.
  * @param[in] attr
  *   Flow rule attributes.
  * @param[in] pattern
@@ -92,21 +505,42 @@ struct ibv_spec_header {
  *   Perform verbose error reporting if not NULL.
  *
  * @return
- *   A flow on success, NULL otherwise and rte_errno is set.
+ *   the amount of bytes necessary to create the flow, a negative errno value
+ *   otherwise and rte_errno is set.
  */
-static struct rte_flow *
-mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
-		      struct mlx5_flows *list __rte_unused,
-		      const struct rte_flow_attr *attr __rte_unused,
-		      const struct rte_flow_item items[] __rte_unused,
-		      const struct rte_flow_action actions[] __rte_unused,
-		      struct rte_flow_error *error)
+static int
+mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
+		size_t flow_size,
+		const struct rte_flow_attr *attr,
+		const struct rte_flow_item items[],
+		const struct rte_flow_action actions[],
+		struct rte_flow_error *error)
 {
-	rte_flow_error_set(error, ENOTSUP,
-			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL,
-			   "action not supported");
-	return NULL;
+	struct rte_flow vflow = { .verbs.layers = 0, };
+	size_t size;
+	int ret;
+
+	/* Make a first virtual parse. */
+	ret = mlx5_flow_actions(dev, actions, &vflow, error);
+	if (ret)
+		return ret;
+	ret = mlx5_flow_items(items, &vflow, error);
+	if (ret)
+		return ret;
+	/* Size of the verbs specification is now known. */
+	size = sizeof(vflow) + sizeof(struct ibv_flow_attr) + vflow.verbs.size;
+	if (size <= flow_size) {
+		ret = mlx5_flow_attributes(attr, flow, error);
+		if (ret)
+			return ret;
+		ret = mlx5_flow_items(items, flow, error);
+		if (ret)
+			return ret;
+		ret = mlx5_flow_actions(dev, actions, flow, error);
+		if (ret)
+			return ret;
+	}
+	return size;
 }
 
 /**
@@ -116,16 +550,175 @@ mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
  * @see rte_flow_ops
  */
 int
-mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
-		   const struct rte_flow_attr *attr __rte_unused,
-		   const struct rte_flow_item items[] __rte_unused,
-		   const struct rte_flow_action actions[] __rte_unused,
+mlx5_flow_validate(struct rte_eth_dev *dev,
+		   const struct rte_flow_attr *attr,
+		   const struct rte_flow_item items[],
+		   const struct rte_flow_action actions[],
 		   struct rte_flow_error *error)
 {
-	return rte_flow_error_set(error, ENOTSUP,
-				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				  NULL,
-				  "action not supported");
+	int ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+
+	if (ret < 0)
+		return ret;
+	return 0;
+}
+
+/**
+ * Remove the flow.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param flow
+ *   Pointer to flow structure.
+ */
+static void
+mlx5_flow_fate_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	if (flow->verbs.fate & MLX5_FLOW_FATE_DROP) {
+		if (flow->verbs.flow) {
+			claim_zero(mlx5_glue->destroy_flow(flow->verbs.flow));
+			flow->verbs.flow = NULL;
+		}
+	}
+	if (flow->verbs.hrxq) {
+		mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
+		flow->verbs.hrxq = NULL;
+	}
+}
+
+/**
+ * Update the verbs specification according to the pattern it matches.
+ *
+ * @param layers
+ *   Bit-fields of present layers see MLX5_FLOW_ITEMS_*.
+ * @param attr[in, out]
+ *   Pointer to Verbs attribute to update.
+ * @param control[in]
+ *   The specification is used for default PMD flows.
+ */
+static void
+mlx5_flow_verbs_priority(uint32_t layers, struct ibv_flow_attr *attr,
+			 const uint32_t control)
+{
+	if (layers & MLX5_FLOW_LAYER_L4)
+		attr->priority = MLX5_FLOW_PRIO_L4;
+	else if (layers & MLX5_FLOW_LAYER_L3)
+		attr->priority = MLX5_FLOW_PRIO_L3;
+	else
+		attr->priority = MLX5_FLOW_PRIO_L2;
+	if (control)
+		attr->priority += MLX5_FLOW_CTRL_PRIO_OFFSET;
+}
+
+/**
+ * Apply the flow.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param flow
+ *   Pointer to flow structure.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_fate_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
+	       struct rte_flow_error *error)
+{
+	if (flow->verbs.fate & MLX5_FLOW_FATE_DROP)
+		flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
+		if (!flow->verbs.hrxq)
+			return rte_flow_error_set
+				(error, errno,
+				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				 NULL,
+				 "cannot allocate Drop queue");
+	else
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "no fate found");
+	mlx5_flow_verbs_priority(flow->verbs.layers, flow->verbs.attr,
+				 flow->attributes.priority == MLX5_FLOW_CTRL);
+	flow->verbs.flow =
+		mlx5_glue->create_flow(flow->verbs.hrxq->qp, flow->verbs.attr);
+	if (!flow->verbs.flow) {
+		if (flow->verbs.fate & MLX5_FLOW_FATE_DROP)
+			mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
+		flow->verbs.hrxq = NULL;
+		return rte_flow_error_set(error, errno,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "kernel module refuses to create"
+					  " flow");
+	}
+	return 0;
+}
+
+/**
+ * Create a flow.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param list
+ *   Pointer to a TAILQ flow list.
+ * @param[in] attr
+ *   Flow rule attributes.
+ * @param[in] pattern
+ *   Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ *   Associated actions (list terminated by the END action).
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   A flow on success, NULL otherwise and rte_errno is set.
+ */
+static struct rte_flow *
+mlx5_flow_list_create(struct rte_eth_dev *dev,
+		      struct mlx5_flows *list,
+		      const struct rte_flow_attr *attr,
+		      const struct rte_flow_item items[],
+		      const struct rte_flow_action actions[],
+		      struct rte_flow_error *error)
+{
+	struct rte_flow *flow;
+	size_t size;
+	int ret;
+
+	ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+	if (ret < 0)
+		return NULL;
+	size = ret;
+	flow = rte_zmalloc(__func__, size, 0);
+	if (!flow) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "cannot allocate memory");
+		return NULL;
+	}
+	flow->verbs.attr = (struct ibv_flow_attr *)(flow + 1);
+	flow->verbs.specs = (uint8_t *)(flow->verbs.attr + 1);
+	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
+	if (ret < 0)
+		goto error;
+	assert((size_t)ret == size);
+	if (!dev->data->dev_started)
+		return flow;
+	ret = mlx5_flow_fate_apply(dev, flow, error);
+	if (ret < 0)
+		goto error;
+	TAILQ_INSERT_TAIL(list, flow, next);
+	return flow;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	mlx5_flow_fate_remove(dev, flow);
+	rte_free(flow);
+	rte_errno = ret; /* Restore rte_errno. */
+	return NULL;
 }
 
 /**
@@ -135,17 +728,15 @@ mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
  * @see rte_flow_ops
  */
 struct rte_flow *
-mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
-		 const struct rte_flow_attr *attr __rte_unused,
-		 const struct rte_flow_item items[] __rte_unused,
-		 const struct rte_flow_action actions[] __rte_unused,
+mlx5_flow_create(struct rte_eth_dev *dev,
+		 const struct rte_flow_attr *attr,
+		 const struct rte_flow_item items[],
+		 const struct rte_flow_action actions[],
 		 struct rte_flow_error *error)
 {
-	rte_flow_error_set(error, ENOTSUP,
-			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL,
-			   "action not supported");
-	return NULL;
+	return mlx5_flow_list_create
+		(dev, &((struct priv *)dev->data->dev_private)->flows,
+		 attr, items, actions, error);
 }
 
 /**
@@ -159,10 +750,12 @@ mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
  *   Flow to destroy.
  */
 static void
-mlx5_flow_list_destroy(struct rte_eth_dev *dev __rte_unused,
-		       struct mlx5_flows *list __rte_unused,
-		       struct rte_flow *flow __rte_unused)
+mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
+		       struct rte_flow *flow)
 {
+	mlx5_flow_fate_remove(dev, flow);
+	TAILQ_REMOVE(list, flow, next);
+	rte_free(flow);
 }
 
 /**
@@ -266,6 +859,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
+		.priority = MLX5_FLOW_CTRL,
 	};
 	struct rte_flow_item items[] = {
 		{
@@ -369,9 +963,8 @@ int
 mlx5_flow_flush(struct rte_eth_dev *dev,
 		struct rte_flow_error *error __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->flows);
+	mlx5_flow_list_flush(dev,
+			     &((struct priv *)dev->data->dev_private)->flows);
 	return 0;
 }
 
@@ -684,9 +1277,8 @@ mlx5_fdir_filter_update(struct rte_eth_dev *dev,
 static void
 mlx5_fdir_filter_flush(struct rte_eth_dev *dev)
 {
-	struct priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->flows);
+	mlx5_flow_list_flush(dev,
+			     &((struct priv *)dev->data->dev_private)->flows);
 }
 
 /**
-- 
2.17.0

[dpdk-dev] [DPDK 18.08 v1 04/12] net/mlx5: add flow queue action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 83 +++++++++++++++++++++++++++++++++---
 1 file changed, 77 insertions(+), 6 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 85dc5edaf..3e16f67d6 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -53,6 +53,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
 /* Action fate on the packet. */
 #define MLX5_FLOW_FATE_DROP (1u << 0)
+#define MLX5_FLOW_FATE_QUEUE (1u << 1)
 
 /* Verbs flow priority per layer level. */
 #define MLX5_FLOW_PRIO_L4 0
@@ -83,7 +84,8 @@ struct mlx5_flow_verbs {
 struct rte_flow {
 	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
 	struct rte_flow_attr attributes; /**< User flow attribute. */
-	struct mlx5_flow_verbs verbs; /* Verbs drop flow. */
+	struct mlx5_flow_verbs verbs; /* Verbs flow. */
+	uint16_t queue; /**< Destination queue to redirect traffic to. */
 };
 
 static const struct rte_flow_ops mlx5_flow_ops = {
@@ -150,6 +152,8 @@ mlx5_flow_print(struct rte_flow *flow __rte_unused)
 		flow->verbs.layers & MLX5_FLOW_LAYER_L4 ? "l4" : "-");
 	if (flow->verbs.fate & MLX5_FLOW_FATE_DROP)
 		fprintf(stdout, " fate: drop queue\n");
+	else if (flow->verbs.fate & MLX5_FLOW_FATE_QUEUE)
+		fprintf(stdout, " fate: target queue %u\n", flow->queue);
 	if (flow->verbs.attr) {
 		struct ibv_spec_header *hdr =
 			(struct ibv_spec_header *)flow->verbs.specs;
@@ -442,6 +446,48 @@ mlx5_flow_action_drop(const struct rte_flow_action *actions,
 	return 0;
 }
 
+/**
+ * Validate action queue provided by the user.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param actions
+ *   Pointer to flow actions array.
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param error
+ *   Pointer to error structure.
+ */
+static int
+mlx5_flow_action_queue(struct rte_eth_dev *dev,
+		       const struct rte_flow_action *actions,
+		       struct rte_flow *flow,
+		       struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_action_queue *queue = actions->conf;
+
+	if (flow->verbs.fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "multiple fate actions are not"
+					  " supported");
+	if (queue->index >= priv->rxqs_n)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &queue->index,
+					  "queue index out of range");
+	if (!(*priv->rxqs)[queue->index])
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &queue->index,
+					  "queue is not configured");
+	flow->queue = queue->index;
+	flow->verbs.fate |= MLX5_FLOW_FATE_QUEUE;
+	return 0;
+}
+
 /**
  * Validate actions provided by the user.
  *
@@ -455,9 +501,9 @@ mlx5_flow_action_drop(const struct rte_flow_action *actions,
  *   Pointer to error structure.
  */
 static int
-mlx5_flow_actions(struct rte_eth_dev *dev __rte_unused,
+mlx5_flow_actions(struct rte_eth_dev *dev,
 		  const struct rte_flow_action actions[],
-		  struct rte_flow *flow __rte_unused,
+		  struct rte_flow *flow,
 		  struct rte_flow_error *error)
 {
 	int ret;
@@ -469,6 +515,9 @@ mlx5_flow_actions(struct rte_eth_dev *dev __rte_unused,
 		case RTE_FLOW_ACTION_TYPE_DROP:
 			ret = mlx5_flow_action_drop(actions, flow, error);
 			break;
+		case RTE_FLOW_ACTION_TYPE_QUEUE:
+			ret = mlx5_flow_action_queue(dev, actions, flow, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -581,7 +630,10 @@ mlx5_flow_fate_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
 		}
 	}
 	if (flow->verbs.hrxq) {
-		mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
+		if (flow->verbs.fate & MLX5_FLOW_FATE_DROP)
+			mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
+		else if (flow->verbs.fate & MLX5_FLOW_FATE_QUEUE)
+			mlx5_hrxq_release(dev, flow->verbs.hrxq);
 		flow->verbs.hrxq = NULL;
 	}
 }
@@ -627,7 +679,7 @@ static int
 mlx5_flow_fate_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 	       struct rte_flow_error *error)
 {
-	if (flow->verbs.fate & MLX5_FLOW_FATE_DROP)
+	if (flow->verbs.fate & MLX5_FLOW_FATE_DROP) {
 		flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
 		if (!flow->verbs.hrxq)
 			return rte_flow_error_set
@@ -635,11 +687,28 @@ mlx5_flow_fate_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				 NULL,
 				 "cannot allocate Drop queue");
-	else
+	} else if (flow->verbs.fate & MLX5_FLOW_FATE_QUEUE) {
+		struct mlx5_hrxq *hrxq;
+
+		hrxq = mlx5_hrxq_get(dev, rss_hash_default_key,
+				     rss_hash_default_key_len, 0,
+				     &flow->queue, 1, 0, 0);
+		if (!hrxq)
+			hrxq = mlx5_hrxq_new(dev, rss_hash_default_key,
+					     rss_hash_default_key_len, 0,
+					     &flow->queue, 1, 0, 0);
+		if (!hrxq)
+			return rte_flow_error_set(error, rte_errno,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"cannot create flow");
+		flow->verbs.hrxq = hrxq;
+	} else {
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					  NULL,
 					  "no fate found");
+	}
 	mlx5_flow_verbs_priority(flow->verbs.layers, flow->verbs.attr,
 				 flow->attributes.priority == MLX5_FLOW_CTRL);
 	flow->verbs.flow =
@@ -647,6 +716,8 @@ mlx5_flow_fate_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 	if (!flow->verbs.flow) {
 		if (flow->verbs.fate & MLX5_FLOW_FATE_DROP)
 			mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
+		else
+			mlx5_hrxq_release(dev, flow->verbs.hrxq);
 		flow->verbs.hrxq = NULL;
 		return rte_flow_error_set(error, errno,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-- 
2.17.0

[dpdk-dev] [DPDK 18.08 v1 05/12] net/mlx5: add flow stop/start

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 3e16f67d6..8c6309188 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -857,9 +857,12 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
  *   Pointer to a TAILQ flow list.
  */
 void
-mlx5_flow_stop(struct rte_eth_dev *dev __rte_unused,
-	       struct mlx5_flows *list __rte_unused)
+mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
 {
+	struct rte_flow *flow;
+
+	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
+		mlx5_flow_fate_remove(dev, flow);
 }
 
 /**
@@ -874,10 +877,23 @@ mlx5_flow_stop(struct rte_eth_dev *dev __rte_unused,
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_flow_start(struct rte_eth_dev *dev __rte_unused,
-		struct mlx5_flows *list __rte_unused)
+mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
 {
+	struct rte_flow *flow;
+	struct rte_flow_error error;
+	int ret = 0;
+
+	TAILQ_FOREACH(flow, list, next) {
+		ret = mlx5_flow_fate_apply(dev, flow, &error);
+		if (ret < 0)
+			goto error;
+	}
 	return 0;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	mlx5_flow_stop(dev, list);
+	rte_errno = ret; /* Restore rte_errno. */
+	return -rte_errno;
 }
 
 /**
-- 
2.17.0

[dpdk-dev] [DPDK 18.08 v1 06/12] net/mlx5: add flow VLAN item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 114 +++++++++++++++++++++++++++++++++++
 1 file changed, 114 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 8c6309188..ed27914aa 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -368,6 +368,117 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 	return 0;
 }
 
+/**
+ * Update the VLAN tag in the Ethernet spec.
+ *
+ * @param attr[in, out]
+ *   Pointer to Verbs attributes structure.
+ * @param eth[in]
+ *   Verbs structure containing the VLAN information to copy.
+ */
+static void
+mlx5_flow_item_vlan_update(struct ibv_flow_attr *attr,
+			   struct ibv_flow_spec_eth *eth)
+{
+	unsigned int i;
+	enum ibv_flow_spec_type search = IBV_FLOW_SPEC_ETH;
+	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
+		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
+
+	for (i = 0; i != attr->num_of_specs; ++i) {
+		if (hdr->type == search) {
+			struct ibv_flow_spec_eth *e =
+				(struct ibv_flow_spec_eth *)hdr;
+
+			e->val.vlan_tag = eth->val.vlan_tag;
+			e->mask.vlan_tag = eth->mask.vlan_tag;
+			break;
+		}
+		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
+	}
+}
+
+/**
+ * Validate VLAN layer and possibly create/modify the Verbs specification.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
+		    struct rte_flow_error *error)
+{
+	const struct rte_flow_item_vlan *spec = item->spec;
+	const struct rte_flow_item_vlan *mask = item->mask;
+	const struct rte_flow_item_vlan nic_mask = {
+		.tci = RTE_BE16(0x0fff),
+	};
+	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	struct ibv_flow_spec_eth eth = {
+		.type = IBV_FLOW_SPEC_ETH,
+		.size = size,
+	};
+	int ret;
+	const uint32_t lm = (MLX5_FLOW_LAYER_L3 | MLX5_FLOW_LAYER_L4);
+	const uint32_t vlanm = MLX5_FLOW_LAYER_VLAN;
+	const uint32_t l2m = MLX5_FLOW_LAYER_L2;
+
+	if (flow->verbs.layers & vlanm)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L2 layers already configured");
+	else if ((flow->verbs.layers & lm) != 0)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L2 layer cannot follow L3/L4 layer");
+	if (!mask)
+		mask = &rte_flow_item_vlan_mask;
+	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+				      (const uint8_t *)&nic_mask,
+				      sizeof(struct rte_flow_item_vlan), error);
+	if (ret)
+		return ret;
+	if (spec) {
+		eth.val.vlan_tag = spec->tci;
+		eth.mask.vlan_tag = mask->tci;
+		eth.val.vlan_tag &= eth.mask.vlan_tag;
+		eth.val.ether_type = spec->inner_type;
+		eth.mask.ether_type = mask->inner_type;
+		eth.val.ether_type &= eth.mask.ether_type;
+	}
+	/*
+	 * From verbs perspective an empty VLAN is equivalent
+	 * to a packet without VLAN layer.
+	 */
+	if (!eth.mask.vlan_tag)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+					  item->spec,
+					  "VLAN cannot be empty");
+	/* Outer TPID cannot be matched. */
+	if (eth.mask.ether_type)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+					  item->spec,
+					  "VLAN TPID matching is not"
+					  " supported");
+	if (!(flow->verbs.layers & l2m))
+		mlx5_flow_spec_verbs_add(flow, &eth, size);
+	else
+		mlx5_flow_item_vlan_update(flow->verbs.attr, &eth);
+	flow->verbs.layers |= MLX5_FLOW_LAYER_L2 | MLX5_FLOW_LAYER_VLAN;
+	return 0;
+}
+
 /**
  * Validate items provided by the user.
  *
@@ -395,6 +506,9 @@ mlx5_flow_items(const struct rte_flow_item items[],
 		case RTE_FLOW_ITEM_TYPE_ETH:
 			ret = mlx5_flow_item_eth(items, flow, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			ret = mlx5_flow_item_vlan(items, flow, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.17.0

[dpdk-dev] [DPDK 18.08 v1 07/12] net/mlx5: add flow IPv4 item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 78 ++++++++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index ed27914aa..003c4aadd 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -479,6 +479,81 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 	return 0;
 }
 
+/**
+ * Validate IPv4 layer and possibly create the Verbs specification.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
+		    struct rte_flow_error *error)
+{
+	const struct rte_flow_item_ipv4 *spec = item->spec;
+	const struct rte_flow_item_ipv4 *mask = item->mask;
+	const struct rte_flow_item_ipv4 nic_mask = {
+		.hdr = {
+			.src_addr = RTE_BE32(0xffffffff),
+			.dst_addr = RTE_BE32(0xffffffff),
+			.type_of_service = 0xff,
+			.next_proto_id = 0xff,
+		},
+	};
+	unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
+	struct ibv_flow_spec_ipv4_ext ipv4 = {
+		.type = IBV_FLOW_SPEC_IPV4_EXT,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->verbs.layers & MLX5_FLOW_LAYER_L3)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "multiple L3 layers not supported");
+	else if (flow->verbs.layers & MLX5_FLOW_LAYER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 cannot follow an L4 layer.");
+	if (!mask)
+		mask = &rte_flow_item_ipv4_mask;
+	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+				      (const uint8_t *)&nic_mask,
+				      sizeof(struct rte_flow_item_ipv4), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
+			.src_ip = spec->hdr.src_addr,
+			.dst_ip = spec->hdr.dst_addr,
+			.proto = spec->hdr.next_proto_id,
+			.tos = spec->hdr.type_of_service,
+		};
+		ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
+			.src_ip = mask->hdr.src_addr,
+			.dst_ip = mask->hdr.dst_addr,
+			.proto = mask->hdr.next_proto_id,
+			.tos = mask->hdr.type_of_service,
+		};
+		/* Remove unwanted bits from values. */
+		ipv4.val.src_ip &= ipv4.mask.src_ip;
+		ipv4.val.dst_ip &= ipv4.mask.dst_ip;
+		ipv4.val.proto &= ipv4.mask.proto;
+		ipv4.val.tos &= ipv4.mask.tos;
+	}
+	mlx5_flow_spec_verbs_add(flow, &ipv4, size);
+	flow->verbs.layers |= MLX5_FLOW_LAYER_L3_IPV4;
+	return 0;
+}
+
 /**
  * Validate items provided by the user.
  *
@@ -509,6 +584,9 @@ mlx5_flow_items(const struct rte_flow_item items[],
 		case RTE_FLOW_ITEM_TYPE_VLAN:
 			ret = mlx5_flow_item_vlan(items, flow, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			ret = mlx5_flow_item_ipv4(items, flow, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.17.0

[dpdk-dev] [DPDK 18.08 v1 08/12] net/mlx5: add flow IPv6 item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 103 +++++++++++++++++++++++++++++++++++
 1 file changed, 103 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 003c4aadd..6f3e50452 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -554,6 +554,106 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 	return 0;
 }
 
+/**
+ * Validate IPv6 layer and possibly create the Verbs specification.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
+		    struct rte_flow_error *error)
+{
+	const struct rte_flow_item_ipv6 *spec = item->spec;
+	const struct rte_flow_item_ipv6 *mask = item->mask;
+	const struct rte_flow_item_ipv6 nic_mask = {
+		.hdr = {
+			.src_addr =
+				"\xff\xff\xff\xff\xff\xff\xff\xff"
+				"\xff\xff\xff\xff\xff\xff\xff\xff",
+			.dst_addr =
+				"\xff\xff\xff\xff\xff\xff\xff\xff"
+				"\xff\xff\xff\xff\xff\xff\xff\xff",
+			.vtc_flow = RTE_BE32(0xffffffff),
+			.proto = 0xff,
+			.hop_limits = 0xff,
+		},
+	};
+	unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
+	struct ibv_flow_spec_ipv6 ipv6 = {
+		.type = IBV_FLOW_SPEC_IPV6,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->verbs.layers & MLX5_FLOW_LAYER_L3)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "multiple L3 layers not supported");
+	else if (flow->verbs.layers & MLX5_FLOW_LAYER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 cannot follow an L4 layer.");
+	if (!mask)
+		mask = &rte_flow_item_ipv6_mask;
+	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+				      (const uint8_t *)&nic_mask,
+				      sizeof(struct rte_flow_item_ipv6), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		unsigned int i;
+		uint32_t vtc_flow_val;
+		uint32_t vtc_flow_mask;
+
+		memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
+		       RTE_DIM(ipv6.val.src_ip));
+		memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
+		       RTE_DIM(ipv6.val.dst_ip));
+		memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
+		       RTE_DIM(ipv6.mask.src_ip));
+		memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
+		       RTE_DIM(ipv6.mask.dst_ip));
+		vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
+		vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
+		ipv6.val.flow_label =
+			rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
+					 IPV6_HDR_FL_SHIFT);
+		ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
+					 IPV6_HDR_TC_SHIFT;
+		ipv6.val.next_hdr = spec->hdr.proto;
+		ipv6.val.hop_limit = spec->hdr.hop_limits;
+		ipv6.mask.flow_label =
+			rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
+					 IPV6_HDR_FL_SHIFT);
+		ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
+					  IPV6_HDR_TC_SHIFT;
+		ipv6.mask.next_hdr = mask->hdr.proto;
+		ipv6.mask.hop_limit = mask->hdr.hop_limits;
+		/* Remove unwanted bits from values. */
+		for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
+			ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
+			ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
+		}
+		ipv6.val.flow_label &= ipv6.mask.flow_label;
+		ipv6.val.traffic_class &= ipv6.mask.traffic_class;
+		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
+		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
+	}
+	mlx5_flow_spec_verbs_add(flow, &ipv6, size);
+	flow->verbs.layers |= MLX5_FLOW_LAYER_L3_IPV6;
+	return 0;
+}
+
 /**
  * Validate items provided by the user.
  *
@@ -587,6 +687,9 @@ mlx5_flow_items(const struct rte_flow_item items[],
 		case RTE_FLOW_ITEM_TYPE_IPV4:
 			ret = mlx5_flow_item_ipv4(items, flow, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			ret = mlx5_flow_item_ipv6(items, flow, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.17.0

[dpdk-dev] [DPDK 18.08 v1 09/12] net/mlx5: add flow UDP item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 55 ++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 6f3e50452..66ebe6d36 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -654,6 +654,58 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 	return 0;
 }
 
+/**
+ * Validate UDP layer and possibly create the Verbs specification.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
+		   struct rte_flow_error *error)
+{
+	const struct rte_flow_item_udp *spec = item->spec;
+	const struct rte_flow_item_udp *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
+	struct ibv_flow_spec_tcp_udp udp = {
+		.type = IBV_FLOW_SPEC_UDP,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->verbs.layers & MLX5_FLOW_LAYER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L4 layer is already present");
+	if (!mask)
+		mask = &rte_flow_item_udp_mask;
+	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+				      (const uint8_t *)&rte_flow_item_udp_mask,
+				      sizeof(struct rte_flow_item_udp), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		udp.val.dst_port = spec->hdr.dst_port;
+		udp.val.src_port = spec->hdr.src_port;
+		udp.mask.dst_port = mask->hdr.dst_port;
+		udp.mask.src_port = mask->hdr.src_port;
+		/* Remove unwanted bits from values. */
+		udp.val.src_port &= udp.mask.src_port;
+		udp.val.dst_port &= udp.mask.dst_port;
+	}
+	mlx5_flow_spec_verbs_add(flow, &udp, size);
+	flow->verbs.layers |= MLX5_FLOW_LAYER_L4_UDP;
+	return 0;
+}
+
 /**
  * Validate items provided by the user.
  *
@@ -690,6 +742,9 @@ mlx5_flow_items(const struct rte_flow_item items[],
 		case RTE_FLOW_ITEM_TYPE_IPV6:
 			ret = mlx5_flow_item_ipv6(items, flow, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			ret = mlx5_flow_item_udp(items, flow, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.17.0

[dpdk-dev] [DPDK 18.08 v1 10/12] net/mlx5: add flow TCP item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 55 ++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 66ebe6d36..ce1b4e94b 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -706,6 +706,58 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 	return 0;
 }
 
+/**
+ * Validate TCP layer and possibly create the Verbs specification.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
+		   struct rte_flow_error *error)
+{
+	const struct rte_flow_item_tcp *spec = item->spec;
+	const struct rte_flow_item_tcp *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
+	struct ibv_flow_spec_tcp_udp tcp = {
+		.type = IBV_FLOW_SPEC_TCP,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->verbs.layers & MLX5_FLOW_LAYER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L4 layer is already present");
+	if (!mask)
+		mask = &rte_flow_item_tcp_mask;
+	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+				      (const uint8_t *)&rte_flow_item_tcp_mask,
+				      sizeof(struct rte_flow_item_tcp), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		tcp.val.dst_port = spec->hdr.dst_port;
+		tcp.val.src_port = spec->hdr.src_port;
+		tcp.mask.dst_port = mask->hdr.dst_port;
+		tcp.mask.src_port = mask->hdr.src_port;
+		/* Remove unwanted bits from values. */
+		tcp.val.src_port &= tcp.mask.src_port;
+		tcp.val.dst_port &= tcp.mask.dst_port;
+	}
+	mlx5_flow_spec_verbs_add(flow, &tcp, size);
+	flow->verbs.layers |= MLX5_FLOW_LAYER_L4_TCP;
+	return 0;
+}
+
 /**
  * Validate items provided by the user.
  *
@@ -745,6 +797,9 @@ mlx5_flow_items(const struct rte_flow_item items[],
 		case RTE_FLOW_ITEM_TYPE_UDP:
 			ret = mlx5_flow_item_udp(items, flow, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			ret = mlx5_flow_item_tcp(items, flow, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.17.0

[dpdk-dev] [DPDK 18.08 v1 11/12] net/mlx5: add mark/flag flow action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 156 +++++++++++++++++++++++++++++++++++
 1 file changed, 156 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index ce1b4e94b..4ef0a3fee 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -55,6 +55,10 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_FLOW_FATE_DROP (1u << 0)
 #define MLX5_FLOW_FATE_QUEUE (1u << 1)
 
+/* Modify a packet. */
+#define MLX5_FLOW_MOD_FLAG (1u << 0)
+#define MLX5_FLOW_MOD_MARK (1u << 1)
+
 /* Verbs flow priority per layer level. */
 #define MLX5_FLOW_PRIO_L4 0
 #define MLX5_FLOW_PRIO_L3 1
@@ -69,6 +73,8 @@ struct mlx5_flow_verbs {
 	unsigned int size; /**< Size of the attribute. */
 	uint32_t layers;
 	/**< Bit-fields of present layers see MLX5_FLOW_ITEMS_*. */
+	uint32_t modifier;
+	/**< Bit-fields of present modifier see MLX5_FLOW_MOD_*. */
 	uint32_t fate;
 	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
 	struct {
@@ -893,6 +899,107 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/**
+ * Validate action flag provided by the user.
+ *
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_action_flag(struct rte_flow *flow)
+{
+	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
+	struct ibv_flow_spec_action_tag tag = {
+		.type = IBV_FLOW_SPEC_ACTION_TAG,
+		.size = size,
+		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
+	};
+
+	if (flow->verbs.modifier & MLX5_FLOW_MOD_MARK)
+		return 0;
+	mlx5_flow_spec_verbs_add(flow, &tag, size);
+	flow->verbs.modifier |= MLX5_FLOW_MOD_FLAG;
+	return 0;
+}
+
+/**
+ * Update verbs specification to modify the flag to mark.
+ *
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param mark_id
+ *   Mark identifier to replace the flag.
+ */
+static void
+mlx5_flow_action_mark_fate_queue(struct rte_flow *flow, uint32_t mark_id)
+{
+	int i;
+
+	/* Update Verbs specification. */
+	for (i = 0; i != flow->verbs.attr->num_of_specs; ++i) {
+		struct ibv_spec_header *hdr =
+			(struct ibv_spec_header *)flow->verbs.attr;
+
+		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
+			struct ibv_flow_spec_action_tag *t =
+				(struct ibv_flow_spec_action_tag *)hdr;
+
+			t->tag_id = mlx5_flow_mark_set(mark_id);
+		}
+	}
+}
+
+/**
+ * Validate action mark provided by the user.
+ *
+ * @param actions
+ *   Pointer to flow actions array.
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_action_mark(const struct rte_flow_action *actions,
+		      struct rte_flow *flow,
+		      struct rte_flow_error *error)
+{
+	const struct rte_flow_action_mark *mark = actions->conf;
+	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
+	struct ibv_flow_spec_action_tag tag = {
+		.type = IBV_FLOW_SPEC_ACTION_TAG,
+		.size = size,
+	};
+
+	if (!mark)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "configuration cannot be null");
+	if (mark->id >= MLX5_FLOW_MARK_MAX)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &mark->id,
+					  "mark must be between 0 and"
+					  " 16777199");
+	if (flow->verbs.modifier & MLX5_FLOW_MOD_FLAG) {
+		mlx5_flow_action_mark_fate_queue(flow, mark->id);
+	} else {
+		tag.tag_id = mlx5_flow_mark_set(mark->id);
+		mlx5_flow_spec_verbs_add(flow, &tag, size);
+	}
+	flow->verbs.modifier |= MLX5_FLOW_MOD_MARK;
+	return 0;
+}
+
 /**
  * Validate actions provided by the user.
  *
@@ -917,6 +1024,18 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 		switch (actions->type) {
 		case RTE_FLOW_ACTION_TYPE_VOID:
 			break;
+		case RTE_FLOW_ACTION_TYPE_FLAG:
+			if (flow->verbs.modifier & MLX5_FLOW_MOD_MARK)
+				return rte_flow_error_set
+					(error, ENOTSUP,
+					 RTE_FLOW_ERROR_TYPE_ACTION,
+					 actions,
+					 "Flag after Mark is not supported");
+			ret = mlx5_flow_action_flag(flow);
+			break;
+		case RTE_FLOW_ACTION_TYPE_MARK:
+			ret = mlx5_flow_action_mark(actions, flow, error);
+			break;
 		case RTE_FLOW_ACTION_TYPE_DROP:
 			ret = mlx5_flow_action_drop(actions, flow, error);
 			break;
@@ -997,6 +1116,42 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Enable/Disable Mark flag in Rx queues.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param flow
+ *   Pointer to flow structure.
+ * @param enable
+ *   0 to disabled, positive value otherwise.
+ */
+static void
+mlx5_flow_rxq_mark(struct rte_eth_dev *dev, struct rte_flow *flow, int enable)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_data *rxq_data;
+	const uint32_t mask = MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK;
+	struct rte_flow *tmp;
+	uint32_t mark = !!enable;
+
+	if (!(flow->verbs.modifier & mask))
+		return;
+	rxq_data = (*priv->rxqs)[flow->queue];
+	/**
+	 * Mark/Flag bit can only be disabled when there is no other
+	 * flow applied using the same queue has a MARK/FLOW action
+	 * configured.
+	 */
+	TAILQ_FOREACH(tmp, &priv->flows, next) {
+		if (tmp == flow)
+			continue;
+		if (tmp->queue == flow->queue)
+			mark |= !!(tmp->verbs.modifier & mask);
+	}
+	rxq_data->mark = mark;
+}
+
 /**
  * Validate a flow supported by the NIC.
  *
@@ -1187,6 +1342,7 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
 	ret = mlx5_flow_fate_apply(dev, flow, error);
 	if (ret < 0)
 		goto error;
+	mlx5_flow_rxq_mark(dev, flow, 1);
 	TAILQ_INSERT_TAIL(list, flow, next);
 	return flow;
 error:
-- 
2.17.0

[dpdk-dev] [DPDK 18.08 v1 12/12] net/mlx5: add RSS flow action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 681 +++++++++++++++++++++++++----------
 1 file changed, 484 insertions(+), 197 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 4ef0a3fee..27354615f 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -54,6 +54,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* Action fate on the packet. */
 #define MLX5_FLOW_FATE_DROP (1u << 0)
 #define MLX5_FLOW_FATE_QUEUE (1u << 1)
+#define MLX5_FLOW_FATE_RSS (1u << 2)
 
 /* Modify a packet. */
 #define MLX5_FLOW_MOD_FLAG (1u << 0)
@@ -68,15 +69,40 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_FLOW_CTRL 0xffffffff
 #define MLX5_FLOW_CTRL_PRIO_OFFSET (MLX5_FLOW_PRIO_L2 + 1)
 
+#define MLX5_RSS_EXP_SUPP(...) \
+	((const enum rte_flow_item_type []) \
+	 { __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END })
+
+/** Supported expansion of items. */
+static const enum rte_flow_item_type *mlx5_supported_expansion[] = {
+	MLX5_RSS_EXP_SUPP(RTE_FLOW_ITEM_TYPE_ETH),
+	MLX5_RSS_EXP_SUPP(RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV4),
+	MLX5_RSS_EXP_SUPP(RTE_FLOW_ITEM_TYPE_ETH,
+			  RTE_FLOW_ITEM_TYPE_IPV4,
+			  RTE_FLOW_ITEM_TYPE_UDP),
+	MLX5_RSS_EXP_SUPP(RTE_FLOW_ITEM_TYPE_ETH,
+			  RTE_FLOW_ITEM_TYPE_IPV4,
+			  RTE_FLOW_ITEM_TYPE_TCP),
+	MLX5_RSS_EXP_SUPP(RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_IPV6),
+	MLX5_RSS_EXP_SUPP(RTE_FLOW_ITEM_TYPE_ETH,
+			  RTE_FLOW_ITEM_TYPE_IPV6,
+			  RTE_FLOW_ITEM_TYPE_UDP),
+	MLX5_RSS_EXP_SUPP(RTE_FLOW_ITEM_TYPE_ETH,
+			  RTE_FLOW_ITEM_TYPE_IPV6,
+			  RTE_FLOW_ITEM_TYPE_TCP),
+};
+
 /** Handles information leading to a drop fate. */
 struct mlx5_flow_verbs {
-	unsigned int size; /**< Size of the attribute. */
+	LIST_ENTRY(mlx5_flow_verbs) next;
+	/**< Pointer to the next Verbs flow structure. */
 	uint32_t layers;
 	/**< Bit-fields of present layers see MLX5_FLOW_ITEMS_*. */
 	uint32_t modifier;
 	/**< Bit-fields of present modifier see MLX5_FLOW_MOD_*. */
 	uint32_t fate;
 	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
+	unsigned int size; /**< Size of the attribute. */
 	struct {
 		struct ibv_flow_attr *attr;
 		/**< Pointer to the Specification buffer. */
@@ -84,14 +110,19 @@ struct mlx5_flow_verbs {
 	};
 	struct ibv_flow *flow; /**< Verbs flow pointer. */
 	struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */
+	uint64_t hash_fields; /**< Verbs hash Rx queue hash fields. */
 };
 
 /* Flow structure. */
 struct rte_flow {
 	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
 	struct rte_flow_attr attributes; /**< User flow attribute. */
-	struct mlx5_flow_verbs verbs; /* Verbs flow. */
-	uint16_t queue; /**< Destination queue to redirect traffic to. */
+	LIST_HEAD(verbs, mlx5_flow_verbs) verbs; /**< Verbs flows list. */
+	struct mlx5_flow_verbs *cur_verbs;
+	/**< Current Verbs flow structure being filled. */
+	struct rte_flow_action_rss rss;/**< RSS context. */
+	uint8_t key[40]; /**< RSS hash key. */
+	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
 };
 
 static const struct rte_flow_ops mlx5_flow_ops = {
@@ -144,6 +175,8 @@ void
 mlx5_flow_print(struct rte_flow *flow __rte_unused)
 {
 #ifndef NDEBUG
+	struct mlx5_flow_verbs *verbs = LIST_FIRST(&flow->verbs);
+
 	fprintf(stdout, "---------8<------------\n");
 	fprintf(stdout, "%s: flow information\n", MLX5_DRIVER_NAME);
 	fprintf(stdout, " attributes: group %u priority %u ingress %d egress %d"
@@ -152,27 +185,36 @@ mlx5_flow_print(struct rte_flow *flow __rte_unused)
 		flow->attributes.ingress,
 		flow->attributes.egress,
 		flow->attributes.transfer);
-	fprintf(stdout, " layers: %s/%s/%s\n",
-		flow->verbs.layers & MLX5_FLOW_LAYER_L2 ? "l2" : "-",
-		flow->verbs.layers & MLX5_FLOW_LAYER_L3 ? "l3" : "-",
-		flow->verbs.layers & MLX5_FLOW_LAYER_L4 ? "l4" : "-");
-	if (flow->verbs.fate & MLX5_FLOW_FATE_DROP)
+	if (verbs->fate & MLX5_FLOW_FATE_DROP) {
 		fprintf(stdout, " fate: drop queue\n");
-	else if (flow->verbs.fate & MLX5_FLOW_FATE_QUEUE)
-		fprintf(stdout, " fate: target queue %u\n", flow->queue);
-	if (flow->verbs.attr) {
-		struct ibv_spec_header *hdr =
-			(struct ibv_spec_header *)flow->verbs.specs;
-		const int n = flow->verbs.attr->num_of_specs;
-		int i;
-
-		fprintf(stdout, " Verbs attributes: priority %u specs_n %u\n",
-			flow->verbs.attr->priority,
-			flow->verbs.attr->num_of_specs);
-		for (i = 0; i != n; ++i) {
-			rte_hexdump(stdout, " ", hdr, hdr->size);
-			hdr = (struct ibv_spec_header *)
-				((uint8_t *)hdr + hdr->size);
+	} else {
+		uint16_t i;
+
+		fprintf(stdout, " fate: target queues");
+		for (i = 0; i != flow->rss.queue_num; ++i)
+			fprintf(stdout, " %u", (*flow->queue)[i]);
+		fprintf(stdout, "\n");
+	}
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		fprintf(stdout, " layers: %s/%s/%s\n",
+			verbs->layers & MLX5_FLOW_LAYER_L2 ? "l2" : "-",
+			verbs->layers & MLX5_FLOW_LAYER_L3 ? "l3" : "-",
+			verbs->layers & MLX5_FLOW_LAYER_L4 ? "l4" : "-");
+		if (verbs->attr) {
+			struct ibv_spec_header *hdr =
+				(struct ibv_spec_header *)verbs->specs;
+			const int n = verbs->attr->num_of_specs;
+			int i;
+
+			fprintf(stdout, " Verbs attributes: priority %u"
+				" specs_n %u\n",
+				verbs->attr->priority,
+				verbs->attr->num_of_specs);
+			for (i = 0; i != n; ++i) {
+				rte_hexdump(stdout, " ", hdr, hdr->size);
+				hdr = (struct ibv_spec_header *)
+					((uint8_t *)hdr + hdr->size);
+			}
 		}
 	}
 	fprintf(stdout, "--------->8------------\n");
@@ -289,24 +331,25 @@ mlx5_flow_item_validate(const struct rte_flow_item *item,
 /**
  * Add a verbs specification.
  *
- * @param flow
- *   Pointer to flow structure.
+ * @param verbs
+ *   Pointer to mlx5_flow_verbs structure.
  * @param src
  *   Create specification.
  * @param size
  *   Size in bytes of the specification to copy.
  */
 static void
-mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
+mlx5_flow_spec_verbs_add(struct mlx5_flow_verbs *verbs, void *src,
+			 unsigned int size)
 {
-	if (flow->verbs.specs) {
+	if (verbs->specs) {
 		void *dst;
 
-		dst = (void *)(flow->verbs.specs + flow->verbs.size);
+		dst = (void *)(verbs->specs + verbs->size);
 		memcpy(dst, src, size);
-		++flow->verbs.attr->num_of_specs;
+		++verbs->attr->num_of_specs;
 	}
-	flow->verbs.size += size;
+	verbs->size += size;
 }
 
 /**
@@ -334,13 +377,14 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 		.type = RTE_BE16(0xffff),
 	};
 	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 	struct ibv_flow_spec_eth eth = {
 		.type = IBV_FLOW_SPEC_ETH,
 		.size = size,
 	};
 	int ret;
 
-	if (flow->verbs.layers & MLX5_FLOW_LAYER_L2)
+	if (verbs->layers & MLX5_FLOW_LAYER_L2)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -369,8 +413,8 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 		}
 		eth.val.ether_type &= eth.mask.ether_type;
 	}
-	mlx5_flow_spec_verbs_add(flow, &eth, size);
-	flow->verbs.layers |= MLX5_FLOW_LAYER_L2;
+	mlx5_flow_spec_verbs_add(verbs, &eth, size);
+	verbs->layers |= MLX5_FLOW_LAYER_L2;
 	return 0;
 }
 
@@ -426,6 +470,7 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 	const struct rte_flow_item_vlan nic_mask = {
 		.tci = RTE_BE16(0x0fff),
 	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 	struct ibv_flow_spec_eth eth = {
 		.type = IBV_FLOW_SPEC_ETH,
@@ -436,12 +481,12 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 	const uint32_t vlanm = MLX5_FLOW_LAYER_VLAN;
 	const uint32_t l2m = MLX5_FLOW_LAYER_L2;
 
-	if (flow->verbs.layers & vlanm)
+	if (verbs->layers & vlanm)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
 					  "L2 layers already configured");
-	else if ((flow->verbs.layers & lm) != 0)
+	else if ((verbs->layers & lm) != 0)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -477,11 +522,11 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 					  item->spec,
 					  "VLAN TPID matching is not"
 					  " supported");
-	if (!(flow->verbs.layers & l2m))
-		mlx5_flow_spec_verbs_add(flow, &eth, size);
+	if (!(verbs->layers & l2m))
+		mlx5_flow_spec_verbs_add(verbs, &eth, size);
 	else
-		mlx5_flow_item_vlan_update(flow->verbs.attr, &eth);
-	flow->verbs.layers |= MLX5_FLOW_LAYER_L2 | MLX5_FLOW_LAYER_VLAN;
+		mlx5_flow_item_vlan_update(verbs->attr, &eth);
+	verbs->layers |= MLX5_FLOW_LAYER_L2 | MLX5_FLOW_LAYER_VLAN;
 	return 0;
 }
 
@@ -512,6 +557,7 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 			.next_proto_id = 0xff,
 		},
 	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 	unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 	struct ibv_flow_spec_ipv4_ext ipv4 = {
 		.type = IBV_FLOW_SPEC_IPV4_EXT,
@@ -519,12 +565,12 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 	};
 	int ret;
 
-	if (flow->verbs.layers & MLX5_FLOW_LAYER_L3)
+	if (verbs->layers & MLX5_FLOW_LAYER_L3)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
 					  "multiple L3 layers not supported");
-	else if (flow->verbs.layers & MLX5_FLOW_LAYER_L4)
+	else if (verbs->layers & MLX5_FLOW_LAYER_L4)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -555,8 +601,8 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 		ipv4.val.proto &= ipv4.mask.proto;
 		ipv4.val.tos &= ipv4.mask.tos;
 	}
-	mlx5_flow_spec_verbs_add(flow, &ipv4, size);
-	flow->verbs.layers |= MLX5_FLOW_LAYER_L3_IPV4;
+	mlx5_flow_spec_verbs_add(verbs, &ipv4, size);
+	verbs->layers |= MLX5_FLOW_LAYER_L3_IPV4;
 	return 0;
 }
 
@@ -592,6 +638,7 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 			.hop_limits = 0xff,
 		},
 	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 	unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 	struct ibv_flow_spec_ipv6 ipv6 = {
 		.type = IBV_FLOW_SPEC_IPV6,
@@ -599,12 +646,12 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 	};
 	int ret;
 
-	if (flow->verbs.layers & MLX5_FLOW_LAYER_L3)
+	if (verbs->layers & MLX5_FLOW_LAYER_L3)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
 					  "multiple L3 layers not supported");
-	else if (flow->verbs.layers & MLX5_FLOW_LAYER_L4)
+	else if (verbs->layers & MLX5_FLOW_LAYER_L4)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -655,8 +702,8 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
 	}
-	mlx5_flow_spec_verbs_add(flow, &ipv6, size);
-	flow->verbs.layers |= MLX5_FLOW_LAYER_L3_IPV6;
+	mlx5_flow_spec_verbs_add(verbs, &ipv6, size);
+	verbs->layers |= MLX5_FLOW_LAYER_L3_IPV6;
 	return 0;
 }
 
@@ -679,6 +726,7 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 {
 	const struct rte_flow_item_udp *spec = item->spec;
 	const struct rte_flow_item_udp *mask = item->mask;
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 	struct ibv_flow_spec_tcp_udp udp = {
 		.type = IBV_FLOW_SPEC_UDP,
@@ -686,7 +734,7 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 	};
 	int ret;
 
-	if (flow->verbs.layers & MLX5_FLOW_LAYER_L4)
+	if (verbs->layers & MLX5_FLOW_LAYER_L4)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -707,8 +755,8 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 		udp.val.src_port &= udp.mask.src_port;
 		udp.val.dst_port &= udp.mask.dst_port;
 	}
-	mlx5_flow_spec_verbs_add(flow, &udp, size);
-	flow->verbs.layers |= MLX5_FLOW_LAYER_L4_UDP;
+	mlx5_flow_spec_verbs_add(verbs, &udp, size);
+	verbs->layers |= MLX5_FLOW_LAYER_L4_UDP;
 	return 0;
 }
 
@@ -731,6 +779,7 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 {
 	const struct rte_flow_item_tcp *spec = item->spec;
 	const struct rte_flow_item_tcp *mask = item->mask;
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 	struct ibv_flow_spec_tcp_udp tcp = {
 		.type = IBV_FLOW_SPEC_TCP,
@@ -738,7 +787,7 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 	};
 	int ret;
 
-	if (flow->verbs.layers & MLX5_FLOW_LAYER_L4)
+	if (verbs->layers & MLX5_FLOW_LAYER_L4)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -759,8 +808,8 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 		tcp.val.src_port &= tcp.mask.src_port;
 		tcp.val.dst_port &= tcp.mask.dst_port;
 	}
-	mlx5_flow_spec_verbs_add(flow, &tcp, size);
-	flow->verbs.layers |= MLX5_FLOW_LAYER_L4_TCP;
+	mlx5_flow_spec_verbs_add(verbs, &tcp, size);
+	verbs->layers |= MLX5_FLOW_LAYER_L4_TCP;
 	return 0;
 }
 
@@ -782,6 +831,8 @@ mlx5_flow_items(const struct rte_flow_item items[],
 		struct rte_flow *flow __rte_unused,
 		struct rte_flow_error *error)
 {
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+
 	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
 		int ret = 0;
 
@@ -815,7 +866,7 @@ mlx5_flow_items(const struct rte_flow_item items[],
 		if (ret < 0)
 			return ret;
 	}
-	if (!flow->verbs.layers) {
+	if (!verbs->layers) {
 		const struct rte_flow_item item = {
 			.type = RTE_FLOW_ITEM_TYPE_ETH,
 		};
@@ -845,15 +896,16 @@ mlx5_flow_action_drop(const struct rte_flow_action *actions,
 			.type = IBV_FLOW_SPEC_ACTION_DROP,
 			.size = size,
 	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 
-	if (flow->verbs.fate)
+	if (verbs->fate)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ACTION,
 					  actions,
 					  "multiple fate actions are not"
 					  " supported");
-	mlx5_flow_spec_verbs_add(flow, &drop, size);
-	flow->verbs.fate |= MLX5_FLOW_FATE_DROP;
+	mlx5_flow_spec_verbs_add(verbs, &drop, size);
+	verbs->fate |= MLX5_FLOW_FATE_DROP;
 	return 0;
 }
 
@@ -877,8 +929,9 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
 {
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_action_queue *queue = actions->conf;
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 
-	if (flow->verbs.fate)
+	if (verbs->fate)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ACTION,
 					  actions,
@@ -894,8 +947,144 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 					  &queue->index,
 					  "queue is not configured");
-	flow->queue = queue->index;
-	flow->verbs.fate |= MLX5_FLOW_FATE_QUEUE;
+	if (flow->queue)
+		(*flow->queue)[0] = queue->index;
+	flow->rss.queue_num = 1;
+	verbs->fate |= MLX5_FLOW_FATE_QUEUE;
+	return 0;
+}
+
+/**
+ * Store the Verbs hash field according to the layer and types.
+ *
+ * @param verbs
+ *   Pointer to a struct mlx5_flow_verb.
+ * @param types
+ *   RSS types for this flow (see ETH_RSS_*).
+ */
+static void
+mlx5_flow_action_rss_hash_filed(struct mlx5_flow_verbs *verbs, uint32_t types)
+{
+	if ((types & ETH_RSS_NONFRAG_IPV4_TCP) &&
+	    (verbs->layers & MLX5_FLOW_LAYER_L4_TCP))
+		verbs->hash_fields = IBV_RX_HASH_SRC_IPV4 |
+			IBV_RX_HASH_DST_IPV4 |
+			IBV_RX_HASH_SRC_PORT_TCP |
+			IBV_RX_HASH_DST_PORT_TCP;
+	else if ((types & ETH_RSS_NONFRAG_IPV4_UDP) &&
+		 (verbs->layers & MLX5_FLOW_LAYER_L4_UDP))
+		verbs->hash_fields = IBV_RX_HASH_SRC_IPV4 |
+			IBV_RX_HASH_DST_IPV4 |
+			IBV_RX_HASH_SRC_PORT_UDP |
+			IBV_RX_HASH_DST_PORT_UDP;
+	else if ((types & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4)) &&
+		 (verbs->layers & MLX5_FLOW_LAYER_L3_IPV4))
+		verbs->hash_fields = IBV_RX_HASH_SRC_IPV4 |
+			IBV_RX_HASH_DST_IPV4;
+	else if ((types & ETH_RSS_NONFRAG_IPV6_TCP) &&
+		 (verbs->layers & MLX5_FLOW_LAYER_L4_TCP))
+		verbs->hash_fields = IBV_RX_HASH_SRC_IPV6 |
+			IBV_RX_HASH_DST_IPV6 |
+			IBV_RX_HASH_SRC_PORT_TCP |
+			IBV_RX_HASH_DST_PORT_TCP;
+	else if ((types & ETH_RSS_NONFRAG_IPV6_UDP) &&
+		 (verbs->layers & MLX5_FLOW_LAYER_L3_IPV6))
+		verbs->hash_fields = IBV_RX_HASH_SRC_IPV6 |
+			IBV_RX_HASH_DST_IPV6 |
+			IBV_RX_HASH_SRC_PORT_UDP |
+			IBV_RX_HASH_DST_PORT_UDP;
+	else if ((types & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6)) &&
+		 (verbs->layers & MLX5_FLOW_LAYER_L3_IPV6))
+		verbs->hash_fields = IBV_RX_HASH_SRC_IPV6 |
+			IBV_RX_HASH_DST_IPV6;
+	else
+		verbs->hash_fields = 0;
+}
+
+/**
+ * Validate action queue provided by the user.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param actions
+ *   Pointer to flow actions array.
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param error
+ *   Pointer to error structure.
+ */
+static int
+mlx5_flow_action_rss(struct rte_eth_dev *dev,
+		     const struct rte_flow_action *actions,
+		     struct rte_flow *flow,
+		     struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_action_rss *rss = actions->conf;
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+	unsigned int i;
+
+	/*
+	 * If verbs list is not empty, this action has already been parsed
+	 * with another items list.
+	 */
+	if (!LIST_EMPTY(&flow->verbs)) {
+		verbs->fate |= MLX5_FLOW_FATE_RSS;
+		return 0;
+	}
+	if (verbs->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "multiple fate actions are not"
+					  " supported");
+	if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
+	    rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->func,
+					  "RSS hash function not supported");
+	if (rss->level > 1)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->level,
+					  "tunnel RSS is not supported");
+	if (rss->key_len < rss_hash_default_key_len)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->key_len,
+					  "RSS hash key too small");
+	if (rss->key_len > rss_hash_default_key_len)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->key_len,
+					  "RSS hash key too large");
+	if (rss->queue_num > priv->config.ind_table_max_size)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->queue_num,
+					  "number of queues too large");
+	if (rss->types & MLX5_RSS_HF_MASK)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->types,
+					  "some RSS protocols are not"
+					  " supported");
+	for (i = 0; i != rss->queue_num; ++i) {
+		if (!(*priv->rxqs)[rss->queue[i]])
+			return rte_flow_error_set
+				(error, EINVAL,
+				 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+				 &rss->queue[i],
+				 "queue is not configured");
+	}
+	if (flow->queue)
+		memcpy((*flow->queue), rss->queue,
+		       rss->queue_num * sizeof(uint16_t));
+	flow->rss.queue_num = rss->queue_num;
+	memcpy(flow->key, rss->key, rss_hash_default_key_len);
+	flow->rss.types = rss->types;
+	verbs->fate |= MLX5_FLOW_FATE_RSS;
 	return 0;
 }
 
@@ -919,31 +1108,32 @@ mlx5_flow_action_flag(struct rte_flow *flow)
 		.size = size,
 		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
 	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 
-	if (flow->verbs.modifier & MLX5_FLOW_MOD_MARK)
+	if (verbs->modifier & MLX5_FLOW_MOD_MARK)
 		return 0;
-	mlx5_flow_spec_verbs_add(flow, &tag, size);
-	flow->verbs.modifier |= MLX5_FLOW_MOD_FLAG;
+	mlx5_flow_spec_verbs_add(verbs, &tag, size);
+	verbs->modifier |= MLX5_FLOW_MOD_FLAG;
 	return 0;
 }
 
 /**
  * Update verbs specification to modify the flag to mark.
  *
- * @param flow
- *   Pointer to the rte_flow structure.
+ * @param verbs
+ *   Pointer to the mlx5_flow_verbs structure.
  * @param mark_id
  *   Mark identifier to replace the flag.
  */
 static void
-mlx5_flow_action_mark_fate_queue(struct rte_flow *flow, uint32_t mark_id)
+mlx5_flow_verbs_mark_update(struct mlx5_flow_verbs *verbs, uint32_t mark_id)
 {
 	int i;
 
 	/* Update Verbs specification. */
-	for (i = 0; i != flow->verbs.attr->num_of_specs; ++i) {
+	for (i = 0; i != verbs->attr->num_of_specs; ++i) {
 		struct ibv_spec_header *hdr =
-			(struct ibv_spec_header *)flow->verbs.attr;
+			(struct ibv_spec_header *)verbs->attr;
 
 		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
 			struct ibv_flow_spec_action_tag *t =
@@ -978,6 +1168,7 @@ mlx5_flow_action_mark(const struct rte_flow_action *actions,
 		.type = IBV_FLOW_SPEC_ACTION_TAG,
 		.size = size,
 	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 
 	if (!mark)
 		return rte_flow_error_set(error, EINVAL,
@@ -990,13 +1181,13 @@ mlx5_flow_action_mark(const struct rte_flow_action *actions,
 					  &mark->id,
 					  "mark must be between 0 and"
 					  " 16777199");
-	if (flow->verbs.modifier & MLX5_FLOW_MOD_FLAG) {
-		mlx5_flow_action_mark_fate_queue(flow, mark->id);
+	if (verbs->modifier & MLX5_FLOW_MOD_FLAG) {
+		mlx5_flow_verbs_mark_update(verbs, mark->id);
 	} else {
 		tag.tag_id = mlx5_flow_mark_set(mark->id);
-		mlx5_flow_spec_verbs_add(flow, &tag, size);
+		mlx5_flow_spec_verbs_add(verbs, &tag, size);
 	}
-	flow->verbs.modifier |= MLX5_FLOW_MOD_MARK;
+	verbs->modifier |= MLX5_FLOW_MOD_MARK;
 	return 0;
 }
 
@@ -1019,13 +1210,14 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 		  struct rte_flow_error *error)
 {
 	int ret;
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 
 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 		switch (actions->type) {
 		case RTE_FLOW_ACTION_TYPE_VOID:
 			break;
 		case RTE_FLOW_ACTION_TYPE_FLAG:
-			if (flow->verbs.modifier & MLX5_FLOW_MOD_MARK)
+			if (verbs->modifier & MLX5_FLOW_MOD_MARK)
 				return rte_flow_error_set
 					(error, ENOTSUP,
 					 RTE_FLOW_ERROR_TYPE_ACTION,
@@ -1042,6 +1234,9 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 		case RTE_FLOW_ACTION_TYPE_QUEUE:
 			ret = mlx5_flow_action_queue(dev, actions, flow, error);
 			break;
+		case RTE_FLOW_ACTION_TYPE_RSS:
+			ret = mlx5_flow_action_rss(dev, actions, flow, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -1051,7 +1246,7 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 		if (ret < 0)
 			return ret;
 	}
-	if (!flow->verbs.fate)
+	if (!verbs->fate)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					  NULL,
@@ -1059,6 +1254,30 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/**
+ * Update the verbs specification according to the pattern it matches.
+ *
+ * @param layers
+ *   Bit-fields of present layers see MLX5_FLOW_ITEMS_*.
+ * @param attr[in, out]
+ *   Pointer to Verbs attribute to update.
+ * @param control[in]
+ *   The specification is used for default PMD flows.
+ */
+static void
+mlx5_flow_verbs_priority(uint32_t layers, struct ibv_flow_attr *attr,
+			 const uint32_t control)
+{
+	if (layers & MLX5_FLOW_LAYER_L4)
+		attr->priority = MLX5_FLOW_PRIO_L4;
+	else if (layers & MLX5_FLOW_LAYER_L3)
+		attr->priority = MLX5_FLOW_PRIO_L3;
+	else
+		attr->priority = MLX5_FLOW_PRIO_L2;
+	if (control)
+		attr->priority += MLX5_FLOW_CTRL_PRIO_OFFSET;
+}
+
 /**
  * Validate the rule and return a flow structure filled accordingly.
  *
@@ -1089,31 +1308,89 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 		const struct rte_flow_action actions[],
 		struct rte_flow_error *error)
 {
-	struct rte_flow vflow = { .verbs.layers = 0, };
-	size_t size;
+	struct mlx5_flow_verbs vverbs = { .layers = 0, };
+	struct rte_flow vflow = { .cur_verbs = &vverbs, };
+	struct rte_flow_expand_rss *buf;
 	int ret;
+	size_t size;
+	uint32_t i;
 
 	/* Make a first virtual parse. */
 	ret = mlx5_flow_actions(dev, actions, &vflow, error);
 	if (ret)
 		return ret;
-	ret = mlx5_flow_items(items, &vflow, error);
+	ret = rte_flow_expand_rss(NULL, 0, items, vflow.rss.types,
+				  mlx5_supported_expansion);
+	assert(ret > 0);
+	buf = rte_calloc(__func__, 1, ret, 0);
+	if (!buf) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "not enough memory to expand the RSS flow");
+		goto error;
+	}
+	ret = rte_flow_expand_rss(buf, ret, items, vflow.rss.types,
+				  mlx5_supported_expansion);
+	assert(ret > 0);
+	/* Create the flow. */
+	size = sizeof(vflow) + vflow.rss.queue_num * sizeof(uint16_t);
+	size = RTE_ALIGN_CEIL(size, sizeof(void *));
+	if (size >= flow_size)
+		flow = &vflow;
+	else
+		flow->queue = (void *)(flow + 1);
+	LIST_INIT(&flow->verbs);
+	ret = mlx5_flow_attributes(attr, flow, error);
 	if (ret)
 		return ret;
-	/* Size of the verbs specification is now known. */
-	size = sizeof(vflow) + sizeof(struct ibv_flow_attr) + vflow.verbs.size;
-	if (size <= flow_size) {
-		ret = mlx5_flow_attributes(attr, flow, error);
-		if (ret)
-			return ret;
-		ret = mlx5_flow_items(items, flow, error);
+	for (i = 0; i != buf->entries; ++i) {
+		size_t off = size;
+
+		memset(&vverbs, 0, sizeof(vverbs));
+		flow->cur_verbs = &vverbs;
+		ret = mlx5_flow_items
+			((const struct rte_flow_item *)buf->patterns[i],
+			 flow, error);
 		if (ret)
-			return ret;
+			goto error;
 		ret = mlx5_flow_actions(dev, actions, flow, error);
 		if (ret)
-			return ret;
+			goto error;
+		/* Size of the verbs specification is now known. */
+		size += sizeof(struct ibv_flow_attr) + sizeof(vverbs) +
+			vverbs.size;
+		if (size <= flow_size) {
+			flow->cur_verbs = (void *)((uintptr_t)flow + off);
+			flow->cur_verbs->attr = (void *)(flow->cur_verbs + 1);
+			flow->cur_verbs->specs =
+				(void *)(flow->cur_verbs->attr + 1);
+			ret = mlx5_flow_items
+				((const struct rte_flow_item *)buf->patterns[i],
+				 flow, error);
+			if (ret)
+				goto error;
+			ret = mlx5_flow_actions(dev, actions, flow, error);
+			if (ret)
+				goto error;
+			/*
+			 * Note: This verbs priority adjustment could be done
+			 * using the priority reported by the expansion.
+			 */
+			mlx5_flow_verbs_priority(flow->cur_verbs->layers,
+						 flow->cur_verbs->attr,
+						 flow->attributes.priority ==
+						 MLX5_FLOW_CTRL);
+			mlx5_flow_action_rss_hash_filed(flow->cur_verbs,
+							flow->rss.types);
+			LIST_INSERT_HEAD(&flow->verbs, flow->cur_verbs, next);
+		}
 	}
+	rte_free(buf);
 	return size;
+error:
+	rte_free(buf);
+	return ret;
 }
 
 /**
@@ -1131,25 +1408,43 @@ mlx5_flow_rxq_mark(struct rte_eth_dev *dev, struct rte_flow *flow, int enable)
 {
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_rxq_data *rxq_data;
+	uint32_t i;
 	const uint32_t mask = MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK;
-	struct rte_flow *tmp;
-	uint32_t mark = !!enable;
+	struct mlx5_flow_verbs *verbs;
 
-	if (!(flow->verbs.modifier & mask))
+	verbs = LIST_FIRST(&flow->verbs);
+	if (!(verbs->modifier & mask))
 		return;
-	rxq_data = (*priv->rxqs)[flow->queue];
-	/**
-	 * Mark/Flag bit can only be disabled when there is no other
-	 * flow applied using the same queue has a MARK/FLOW action
-	 * configured.
-	 */
-	TAILQ_FOREACH(tmp, &priv->flows, next) {
-		if (tmp == flow)
-			continue;
-		if (tmp->queue == flow->queue)
-			mark |= !!(tmp->verbs.modifier & mask);
+	for (i = 0; i != flow->rss.queue_num; ++i) {
+		int mark = 0;
+		unsigned int qidx = (*flow->queue)[i];
+		struct rte_flow *tmp;
+		rxq_data = (*priv->rxqs)[qidx];
+
+		if (enable) {
+			rxq_data->mark = !!enable;
+			break;
+		}
+		/**
+		 * Mark/Flag bit can only be disabled when there is no other
+		 * flow applied using the same queue has a MARK/FLOW action
+		 * configured.
+		 */
+		TAILQ_FOREACH(tmp, &priv->flows, next) {
+			uint32_t j;
+			struct mlx5_flow_verbs *vtmp;
+
+			if (tmp == flow)
+				continue;
+			vtmp = LIST_FIRST(&tmp->verbs);
+			for (j = 0; j != tmp->rss.queue_num; ++j)
+				if ((*tmp->queue)[i] == qidx) {
+					mark |= !!(vtmp->modifier & mask);
+					break;
+				}
+		}
+		rxq_data->mark = mark;
 	}
-	rxq_data->mark = mark;
 }
 
 /**
@@ -1183,43 +1478,21 @@ mlx5_flow_validate(struct rte_eth_dev *dev,
 static void
 mlx5_flow_fate_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
-	if (flow->verbs.fate & MLX5_FLOW_FATE_DROP) {
-		if (flow->verbs.flow) {
-			claim_zero(mlx5_glue->destroy_flow(flow->verbs.flow));
-			flow->verbs.flow = NULL;
+	struct mlx5_flow_verbs *verbs;
+
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (verbs->flow) {
+			claim_zero(mlx5_glue->destroy_flow(verbs->flow));
+			verbs->flow = NULL;
+		}
+		if (verbs->hrxq) {
+			if (verbs->fate & MLX5_FLOW_FATE_DROP)
+				mlx5_hrxq_drop_release(dev, verbs->hrxq);
+			else
+				mlx5_hrxq_release(dev, verbs->hrxq);
+			verbs->hrxq = NULL;
 		}
 	}
-	if (flow->verbs.hrxq) {
-		if (flow->verbs.fate & MLX5_FLOW_FATE_DROP)
-			mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
-		else if (flow->verbs.fate & MLX5_FLOW_FATE_QUEUE)
-			mlx5_hrxq_release(dev, flow->verbs.hrxq);
-		flow->verbs.hrxq = NULL;
-	}
-}
-
-/**
- * Update the verbs specification according to the pattern it matches.
- *
- * @param layers
- *   Bit-fields of present layers see MLX5_FLOW_ITEMS_*.
- * @param attr[in, out]
- *   Pointer to Verbs attribute to update.
- * @param control[in]
- *   The specification is used for default PMD flows.
- */
-static void
-mlx5_flow_verbs_priority(uint32_t layers, struct ibv_flow_attr *attr,
-			 const uint32_t control)
-{
-	if (layers & MLX5_FLOW_LAYER_L4)
-		attr->priority = MLX5_FLOW_PRIO_L4;
-	else if (layers & MLX5_FLOW_LAYER_L3)
-		attr->priority = MLX5_FLOW_PRIO_L3;
-	else
-		attr->priority = MLX5_FLOW_PRIO_L2;
-	if (control)
-		attr->priority += MLX5_FLOW_CTRL_PRIO_OFFSET;
 }
 
 /**
@@ -1239,53 +1512,66 @@ static int
 mlx5_flow_fate_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 	       struct rte_flow_error *error)
 {
-	if (flow->verbs.fate & MLX5_FLOW_FATE_DROP) {
-		flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
-		if (!flow->verbs.hrxq)
-			return rte_flow_error_set
-				(error, errno,
-				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				 NULL,
-				 "cannot allocate Drop queue");
-	} else if (flow->verbs.fate & MLX5_FLOW_FATE_QUEUE) {
-		struct mlx5_hrxq *hrxq;
-
-		hrxq = mlx5_hrxq_get(dev, rss_hash_default_key,
-				     rss_hash_default_key_len, 0,
-				     &flow->queue, 1, 0, 0);
-		if (!hrxq)
-			hrxq = mlx5_hrxq_new(dev, rss_hash_default_key,
-					     rss_hash_default_key_len, 0,
-					     &flow->queue, 1, 0, 0);
-		if (!hrxq)
-			return rte_flow_error_set(error, rte_errno,
-					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					NULL,
-					"cannot create flow");
-		flow->verbs.hrxq = hrxq;
-	} else {
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "no fate found");
+	struct mlx5_flow_verbs *verbs;
+	int err;
+
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (verbs->fate & MLX5_FLOW_FATE_DROP) {
+			verbs->hrxq = mlx5_hrxq_drop_new(dev);
+			if (!verbs->hrxq) {
+				rte_flow_error_set
+					(error, errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					 NULL,
+					 "cannot get drop hash queue");
+				goto error;
+			}
+		} else {
+			struct mlx5_hrxq *hrxq;
+
+			hrxq = mlx5_hrxq_get(dev, flow->key,
+					     rss_hash_default_key_len,
+					     verbs->hash_fields,
+					     (*flow->queue),
+					     flow->rss.queue_num, 0, 0);
+			if (!hrxq)
+				hrxq = mlx5_hrxq_new(dev, flow->key,
+						     rss_hash_default_key_len,
+						     verbs->hash_fields,
+						     (*flow->queue),
+						     flow->rss.queue_num, 0, 0);
+			if (!hrxq) {
+				rte_flow_error_set
+					(error, rte_errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					 NULL,
+					 "cannot get hash queue");
+				goto error;
+			}
+			verbs->hrxq = hrxq;
+		}
+		verbs->flow =
+			mlx5_glue->create_flow(verbs->hrxq->qp, verbs->attr);
+		if (!verbs->flow) {
+			rte_flow_error_set(error, errno,
+					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					   NULL,
+					   "hardware refuses to create flow");
+			goto error;
+		}
 	}
-	mlx5_flow_verbs_priority(flow->verbs.layers, flow->verbs.attr,
-				 flow->attributes.priority == MLX5_FLOW_CTRL);
-	flow->verbs.flow =
-		mlx5_glue->create_flow(flow->verbs.hrxq->qp, flow->verbs.attr);
-	if (!flow->verbs.flow) {
-		if (flow->verbs.fate & MLX5_FLOW_FATE_DROP)
-			mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
+	return 0;
+error:
+	err = rte_errno; /* Save rte_errno before cleanup. */
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (verbs->fate & MLX5_FLOW_FATE_DROP)
+			mlx5_hrxq_drop_release(dev, verbs->hrxq);
 		else
-			mlx5_hrxq_release(dev, flow->verbs.hrxq);
-		flow->verbs.hrxq = NULL;
-		return rte_flow_error_set(error, errno,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "kernel module refuses to create"
-					  " flow");
+			mlx5_hrxq_release(dev, verbs->hrxq);
+		verbs->hrxq = NULL;
 	}
-	return 0;
+	rte_errno = err; /* Restore rte_errno. */
+	return -rte_errno;
 }
 
 /**
@@ -1315,42 +1601,43 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
 		      const struct rte_flow_action actions[],
 		      struct rte_flow_error *error)
 {
-	struct rte_flow *flow;
-	size_t size;
+	struct rte_flow *flow = NULL;
+	size_t size = 0;
 	int ret;
 
-	ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
 	if (ret < 0)
 		return NULL;
 	size = ret;
-	flow = rte_zmalloc(__func__, size, 0);
+	flow = rte_calloc(__func__, 1, size, 0);
 	if (!flow) {
 		rte_flow_error_set(error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL,
-				   "cannot allocate memory");
+				   "not enough memory to create flow");
 		return NULL;
 	}
-	flow->verbs.attr = (struct ibv_flow_attr *)(flow + 1);
-	flow->verbs.specs = (uint8_t *)(flow->verbs.attr + 1);
 	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
-	if (ret < 0)
-		goto error;
+	if (ret < 0) {
+		rte_free(flow);
+		return NULL;
+	}
 	assert((size_t)ret == size);
 	if (!dev->data->dev_started)
 		return flow;
 	ret = mlx5_flow_fate_apply(dev, flow, error);
-	if (ret < 0)
-		goto error;
+	if (ret < 0) {
+		ret = rte_errno; /* Save rte_errno before cleanup. */
+		if (flow) {
+			mlx5_flow_fate_remove(dev, flow);
+			rte_free(flow);
+		}
+		rte_errno = ret; /* Restore rte_errno. */
+		return NULL;
+	}
 	mlx5_flow_rxq_mark(dev, flow, 1);
 	TAILQ_INSERT_TAIL(list, flow, next);
 	return flow;
-error:
-	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_fate_remove(dev, flow);
-	rte_free(flow);
-	rte_errno = ret; /* Restore rte_errno. */
-	return NULL;
 }
 
 /**
-- 
2.17.0

Re: [dpdk-dev] [DPDK 18.08 v1 00/12] net/mlx5: flow rework

[Wiles, Keith]

The subject does not have [PATCH ] so it will be missed by patchwork, right?

> On May 28, 2018, at 6:21 AM, Nelio Laranjeiro <nelio.laranjeiro@6wind.com> wrote:
> 
> First version of for the flow engine rework of MLX5 to prepare the
> introduction for the ENCAP/DECAP and PUSH/POP actions done via TC
> flower/filter.
> 
> This first series depends on [1] and is a work in progress, recent work added
> on Tunnel RSS are still absent as well as the tunnel support.  Those will be
> added in further versions.
> 
> Expected for the next 18.08 release:
> 
> - same level of features,
> - TC flow support for port redirection,
> - TC filter support for ENCAP/DECAP and PUSH/POP.
> 
> [1] https://dpdk.org/dev/patchwork/patch/40462/
> 
> Nelio Laranjeiro (12):
>  net/mlx5: remove flow support
>  net/mlx5: handle drop queues are regular queues
>  net/mlx5: support flow Ethernet item among with drop action
>  net/mlx5: add flow queue action
>  net/mlx5: add flow stop/start
>  net/mlx5: add flow VLAN item
>  net/mlx5: add flow IPv4 item
>  net/mlx5: add flow IPv6 item
>  net/mlx5: add flow UDP item
>  net/mlx5: add flow TCP item
>  net/mlx5: add mark/flag flow action
>  net/mlx5: add RSS flow action
> 
> drivers/net/mlx5/mlx5.c      |    9 -
> drivers/net/mlx5/mlx5.h      |    3 +-
> drivers/net/mlx5/mlx5_flow.c | 3615 +++++++++++-----------------------
> drivers/net/mlx5/mlx5_rxq.c  |  221 +++
> drivers/net/mlx5/mlx5_rxtx.h |    6 +
> 5 files changed, 1388 insertions(+), 2466 deletions(-)
> 
> -- 
> 2.17.0
> 

Regards,
Keith

Re: [dpdk-dev] [DPDK 18.08 v1 00/12] net/mlx5: flow rework

[Ferruh Yigit]

On 5/28/2018 2:32 PM, Wiles, Keith wrote:
> The subject does not have [PATCH ] so it will be missed by patchwork, right?

Patchwork already have them [1], it seems patchwork doesn't take patch
subject-prefix into account.

[1]
Only I already mark the set as deferred, that is why it may not been seen in new
patches list.
https://dpdk.org/dev/patchwork/patch/40466/

> 
>> On May 28, 2018, at 6:21 AM, Nelio Laranjeiro <nelio.laranjeiro@6wind.com> wrote:
>>
>> First version of for the flow engine rework of MLX5 to prepare the
>> introduction for the ENCAP/DECAP and PUSH/POP actions done via TC
>> flower/filter.
>>
>> This first series depends on [1] and is a work in progress, recent work added
>> on Tunnel RSS are still absent as well as the tunnel support.  Those will be
>> added in further versions.
>>
>> Expected for the next 18.08 release:
>>
>> - same level of features,
>> - TC flow support for port redirection,
>> - TC filter support for ENCAP/DECAP and PUSH/POP.
>>
>> [1] https://dpdk.org/dev/patchwork/patch/40462/
>>
>> Nelio Laranjeiro (12):
>>  net/mlx5: remove flow support
>>  net/mlx5: handle drop queues are regular queues
>>  net/mlx5: support flow Ethernet item among with drop action
>>  net/mlx5: add flow queue action
>>  net/mlx5: add flow stop/start
>>  net/mlx5: add flow VLAN item
>>  net/mlx5: add flow IPv4 item
>>  net/mlx5: add flow IPv6 item
>>  net/mlx5: add flow UDP item
>>  net/mlx5: add flow TCP item
>>  net/mlx5: add mark/flag flow action
>>  net/mlx5: add RSS flow action
>>
>> drivers/net/mlx5/mlx5.c      |    9 -
>> drivers/net/mlx5/mlx5.h      |    3 +-
>> drivers/net/mlx5/mlx5_flow.c | 3615 +++++++++++-----------------------
>> drivers/net/mlx5/mlx5_rxq.c  |  221 +++
>> drivers/net/mlx5/mlx5_rxtx.h |    6 +
>> 5 files changed, 1388 insertions(+), 2466 deletions(-)
>>
>> -- 
>> 2.17.0
>>
> 
> Regards,
> Keith
> 

Re: [dpdk-dev] [DPDK 18.08 v1 00/12] net/mlx5: flow rework

[Nélio Laranjeiro]

Hi Keith,

On Mon, May 28, 2018 at 01:32:34PM +0000, Wiles, Keith wrote:
> The subject does not have [PATCH ] so it will be missed by patchwork, right?

I see them on the patchwork [1], it seems it uses anything else to
determine if it is a patch or not.

> > On May 28, 2018, at 6:21 AM, Nelio Laranjeiro <nelio.laranjeiro@6wind.com> wrote:
> > 
> > First version of for the flow engine rework of MLX5 to prepare the
> > introduction for the ENCAP/DECAP and PUSH/POP actions done via TC
> > flower/filter.
> > 
> > This first series depends on [1] and is a work in progress, recent work added
> > on Tunnel RSS are still absent as well as the tunnel support.  Those will be
> > added in further versions.
> > 
> > Expected for the next 18.08 release:
> > 
> > - same level of features,
> > - TC flow support for port redirection,
> > - TC filter support for ENCAP/DECAP and PUSH/POP.
> > 
> > [1] https://dpdk.org/dev/patchwork/patch/40462/
> > 
> > Nelio Laranjeiro (12):
> >  net/mlx5: remove flow support
> >  net/mlx5: handle drop queues are regular queues
> >  net/mlx5: support flow Ethernet item among with drop action
> >  net/mlx5: add flow queue action
> >  net/mlx5: add flow stop/start
> >  net/mlx5: add flow VLAN item
> >  net/mlx5: add flow IPv4 item
> >  net/mlx5: add flow IPv6 item
> >  net/mlx5: add flow UDP item
> >  net/mlx5: add flow TCP item
> >  net/mlx5: add mark/flag flow action
> >  net/mlx5: add RSS flow action
> > 
> > drivers/net/mlx5/mlx5.c      |    9 -
> > drivers/net/mlx5/mlx5.h      |    3 +-
> > drivers/net/mlx5/mlx5_flow.c | 3615 +++++++++++-----------------------
> > drivers/net/mlx5/mlx5_rxq.c  |  221 +++
> > drivers/net/mlx5/mlx5_rxtx.h |    6 +
> > 5 files changed, 1388 insertions(+), 2466 deletions(-)
> > 
> > -- 
> > 2.17.0
> > 
> 
> Regards,
> Keith

Thanks,

[1] https://dpdk.org/dev/patchwork/project/dpdk/list/?submitter=243

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH v2 00/20] net/mlx5: flow rework

[Nelio Laranjeiro]

Re-work flow engine to support port redirection actions through TC.

This first series depends on [1] which is implemented in commit 
"net/mlx5: support inner RSS computation" and on [2].
Next series will bring the port redirection as announced[3].

[1] https://patches.dpdk.org/project/dpdk/list/?series=267
[2] https://mails.dpdk.org/archives/dev/2018-June/104192.html
[3] https://mails.dpdk.org/archives/dev/2018-May/103043.html

Nelio Laranjeiro (20):
  net/mlx5: remove flow support
  net/mlx5: handle drop queues are regular queues
  net/mlx5: replace verbs priorities by flow
  net/mlx5: support flow Ethernet item among with drop action
  net/mlx5: add flow queue action
  net/mlx5: add flow stop/start
  net/mlx5: add flow VLAN item
  net/mlx5: add flow IPv4 item
  net/mlx5: add flow IPv6 item
  net/mlx5: add flow UDP item
  net/mlx5: add flow TCP item
  net/mlx5: add mark/flag flow action
  net/mlx5: add RSS flow action
  net/mlx5: remove useless arguments in hrxq API
  net/mlx5: support inner RSS computation
  net/mlx5: add flow VXLAN item
  net/mlx5: add flow VXLAN-GPE item
  net/mlx5: add flow GRE item
  net/mlx5: add flow MPLS item
  net/mlx5: add count flow action

 drivers/net/mlx5/mlx5.c      |   22 +-
 drivers/net/mlx5/mlx5.h      |   18 +-
 drivers/net/mlx5/mlx5_flow.c | 4738 ++++++++++++++++------------------
 drivers/net/mlx5/mlx5_rxq.c  |  271 +-
 drivers/net/mlx5/mlx5_rxtx.h |   17 +-
 5 files changed, 2552 insertions(+), 2514 deletions(-)

-- 
2.18.0

[dpdk-dev] [PATCH v2 01/20] net/mlx5: remove flow support

[Nelio Laranjeiro]

This start a series to re-work the flow engine in mlx5 to easily support
flow conversion to Verbs or TC.  This is necessary to handle both regular
flows and representors flows.

As the full file needs to be clean-up to re-write all items/actions
processing, this patch starts to disable the regular code and only let the
PMD to start in isolated mode.

After this patch flow API will not be usable.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 3095 +---------------------------------
 1 file changed, 80 insertions(+), 3015 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 45207d70e..a45cb06e1 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -31,2406 +31,49 @@
 #include "mlx5_prm.h"
 #include "mlx5_glue.h"
 
-/* Flow priority for control plane flows. */
-#define MLX5_CTRL_FLOW_PRIORITY 1
-
-/* Internet Protocol versions. */
-#define MLX5_IPV4 4
-#define MLX5_IPV6 6
-#define MLX5_GRE 47
-
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-struct ibv_flow_spec_counter_action {
-	int dummy;
-};
-#endif
-
-/* Dev ops structure defined in mlx5.c */
-extern const struct eth_dev_ops mlx5_dev_ops;
-extern const struct eth_dev_ops mlx5_dev_ops_isolate;
-
-/** Structure give to the conversion functions. */
-struct mlx5_flow_data {
-	struct rte_eth_dev *dev; /** Ethernet device. */
-	struct mlx5_flow_parse *parser; /** Parser context. */
-	struct rte_flow_error *error; /** Error context. */
-};
-
-static int
-mlx5_flow_create_eth(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vlan(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_ipv4(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_ipv6(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_udp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_tcp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vxlan(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vxlan_gpe(const struct rte_flow_item *item,
-			   const void *default_mask,
-			   struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_gre(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_mpls(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-struct mlx5_flow_parse;
-
-static void
-mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
-		      unsigned int size);
-
-static int
-mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
-
-static int
-mlx5_flow_create_count(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser);
-
-/* Hash RX queue types. */
-enum hash_rxq_type {
-	HASH_RXQ_TCPV4,
-	HASH_RXQ_UDPV4,
-	HASH_RXQ_IPV4,
-	HASH_RXQ_TCPV6,
-	HASH_RXQ_UDPV6,
-	HASH_RXQ_IPV6,
-	HASH_RXQ_ETH,
-	HASH_RXQ_TUNNEL,
-};
-
-/* Initialization data for hash RX queue. */
-struct hash_rxq_init {
-	uint64_t hash_fields; /* Fields that participate in the hash. */
-	uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
-	unsigned int flow_priority; /* Flow priority to use. */
-	unsigned int ip_version; /* Internet protocol. */
-};
-
-/* Initialization data for hash RX queues. */
-const struct hash_rxq_init hash_rxq_init[] = {
-	[HASH_RXQ_TCPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4 |
-				IBV_RX_HASH_SRC_PORT_TCP |
-				IBV_RX_HASH_DST_PORT_TCP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV4,
-	},
-	[HASH_RXQ_UDPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4 |
-				IBV_RX_HASH_SRC_PORT_UDP |
-				IBV_RX_HASH_DST_PORT_UDP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV4,
-	},
-	[HASH_RXQ_IPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4),
-		.dpdk_rss_hf = (ETH_RSS_IPV4 |
-				ETH_RSS_FRAG_IPV4),
-		.flow_priority = 1,
-		.ip_version = MLX5_IPV4,
-	},
-	[HASH_RXQ_TCPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6 |
-				IBV_RX_HASH_SRC_PORT_TCP |
-				IBV_RX_HASH_DST_PORT_TCP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV6,
-	},
-	[HASH_RXQ_UDPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6 |
-				IBV_RX_HASH_SRC_PORT_UDP |
-				IBV_RX_HASH_DST_PORT_UDP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV6,
-	},
-	[HASH_RXQ_IPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6),
-		.dpdk_rss_hf = (ETH_RSS_IPV6 |
-				ETH_RSS_FRAG_IPV6),
-		.flow_priority = 1,
-		.ip_version = MLX5_IPV6,
-	},
-	[HASH_RXQ_ETH] = {
-		.hash_fields = 0,
-		.dpdk_rss_hf = 0,
-		.flow_priority = 2,
-	},
-};
-
-/* Number of entries in hash_rxq_init[]. */
-const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
-
-/** Structure for holding counter stats. */
-struct mlx5_flow_counter_stats {
-	uint64_t hits; /**< Number of packets matched by the rule. */
-	uint64_t bytes; /**< Number of bytes matched by the rule. */
-};
-
-/** Structure for Drop queue. */
-struct mlx5_hrxq_drop {
-	struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
-	struct ibv_qp *qp; /**< Verbs queue pair. */
-	struct ibv_wq *wq; /**< Verbs work queue. */
-	struct ibv_cq *cq; /**< Verbs completion queue. */
-};
-
-/* Flows structures. */
-struct mlx5_flow {
-	uint64_t hash_fields; /**< Fields that participate in the hash. */
-	struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
-	struct ibv_flow *ibv_flow; /**< Verbs flow. */
-	struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
-};
-
-/* Drop flows structures. */
-struct mlx5_flow_drop {
-	struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
-	struct ibv_flow *ibv_flow; /**< Verbs flow. */
-};
-
-struct rte_flow {
-	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
-	uint32_t mark:1; /**< Set if the flow is marked. */
-	uint32_t drop:1; /**< Drop queue. */
-	struct rte_flow_action_rss rss_conf; /**< RSS configuration */
-	uint16_t (*queues)[]; /**< Queues indexes to use. */
-	uint8_t rss_key[40]; /**< copy of the RSS key. */
-	uint32_t tunnel; /**< Tunnel type of RTE_PTYPE_TUNNEL_XXX. */
-	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
-	struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
-	struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
-	/**< Flow with Rx queue. */
-};
-
-/** Static initializer for items. */
-#define ITEMS(...) \
-	(const enum rte_flow_item_type []){ \
-		__VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
-	}
-
-#define IS_TUNNEL(type) ( \
-	(type) == RTE_FLOW_ITEM_TYPE_VXLAN || \
-	(type) == RTE_FLOW_ITEM_TYPE_VXLAN_GPE || \
-	(type) == RTE_FLOW_ITEM_TYPE_GRE || \
-	(type) == RTE_FLOW_ITEM_TYPE_MPLS)
-
-const uint32_t flow_ptype[] = {
-	[RTE_FLOW_ITEM_TYPE_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN,
-	[RTE_FLOW_ITEM_TYPE_VXLAN_GPE] = RTE_PTYPE_TUNNEL_VXLAN_GPE,
-	[RTE_FLOW_ITEM_TYPE_GRE] = RTE_PTYPE_TUNNEL_GRE,
-	[RTE_FLOW_ITEM_TYPE_MPLS] = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
-};
-
-#define PTYPE_IDX(t) ((RTE_PTYPE_TUNNEL_MASK & (t)) >> 12)
-
-const uint32_t ptype_ext[] = {
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] = RTE_PTYPE_TUNNEL_VXLAN |
-					      RTE_PTYPE_L4_UDP,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN_GPE)]	= RTE_PTYPE_TUNNEL_VXLAN_GPE |
-						  RTE_PTYPE_L4_UDP,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_GRE)] = RTE_PTYPE_TUNNEL_GRE,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_GRE)] =
-		RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_UDP)] =
-		RTE_PTYPE_TUNNEL_MPLS_IN_GRE | RTE_PTYPE_L4_UDP,
-};
-
-/** Structure to generate a simple graph of layers supported by the NIC. */
-struct mlx5_flow_items {
-	/** List of possible actions for these items. */
-	const enum rte_flow_action_type *const actions;
-	/** Bit-masks corresponding to the possibilities for the item. */
-	const void *mask;
-	/**
-	 * Default bit-masks to use when item->mask is not provided. When
-	 * \default_mask is also NULL, the full supported bit-mask (\mask) is
-	 * used instead.
-	 */
-	const void *default_mask;
-	/** Bit-masks size in bytes. */
-	const unsigned int mask_sz;
-	/**
-	 * Conversion function from rte_flow to NIC specific flow.
-	 *
-	 * @param item
-	 *   rte_flow item to convert.
-	 * @param default_mask
-	 *   Default bit-masks to use when item->mask is not provided.
-	 * @param data
-	 *   Internal structure to store the conversion.
-	 *
-	 * @return
-	 *   0 on success, a negative errno value otherwise and rte_errno is
-	 *   set.
-	 */
-	int (*convert)(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data);
-	/** Size in bytes of the destination structure. */
-	const unsigned int dst_sz;
-	/** List of possible following items.  */
-	const enum rte_flow_item_type *const items;
-};
-
-/** Valid action for this PMD. */
-static const enum rte_flow_action_type valid_actions[] = {
-	RTE_FLOW_ACTION_TYPE_DROP,
-	RTE_FLOW_ACTION_TYPE_QUEUE,
-	RTE_FLOW_ACTION_TYPE_MARK,
-	RTE_FLOW_ACTION_TYPE_FLAG,
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	RTE_FLOW_ACTION_TYPE_COUNT,
-#endif
-	RTE_FLOW_ACTION_TYPE_END,
-};
-
-/** Graph of supported items and associated actions. */
-static const struct mlx5_flow_items mlx5_flow_items[] = {
-	[RTE_FLOW_ITEM_TYPE_END] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_VXLAN,
-			       RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
-			       RTE_FLOW_ITEM_TYPE_GRE),
-	},
-	[RTE_FLOW_ITEM_TYPE_ETH] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_eth){
-			.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
-			.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
-			.type = -1,
-		},
-		.default_mask = &rte_flow_item_eth_mask,
-		.mask_sz = sizeof(struct rte_flow_item_eth),
-		.convert = mlx5_flow_create_eth,
-		.dst_sz = sizeof(struct ibv_flow_spec_eth),
-	},
-	[RTE_FLOW_ITEM_TYPE_VLAN] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vlan){
-			.tci = -1,
-			.inner_type = -1,
-		},
-		.default_mask = &rte_flow_item_vlan_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vlan),
-		.convert = mlx5_flow_create_vlan,
-		.dst_sz = 0,
-	},
-	[RTE_FLOW_ITEM_TYPE_IPV4] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
-			       RTE_FLOW_ITEM_TYPE_TCP,
-			       RTE_FLOW_ITEM_TYPE_GRE),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_ipv4){
-			.hdr = {
-				.src_addr = -1,
-				.dst_addr = -1,
-				.type_of_service = -1,
-				.next_proto_id = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_ipv4_mask,
-		.mask_sz = sizeof(struct rte_flow_item_ipv4),
-		.convert = mlx5_flow_create_ipv4,
-		.dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
-	},
-	[RTE_FLOW_ITEM_TYPE_IPV6] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
-			       RTE_FLOW_ITEM_TYPE_TCP,
-			       RTE_FLOW_ITEM_TYPE_GRE),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_ipv6){
-			.hdr = {
-				.src_addr = {
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-				},
-				.dst_addr = {
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-				},
-				.vtc_flow = -1,
-				.proto = -1,
-				.hop_limits = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_ipv6_mask,
-		.mask_sz = sizeof(struct rte_flow_item_ipv6),
-		.convert = mlx5_flow_create_ipv6,
-		.dst_sz = sizeof(struct ibv_flow_spec_ipv6),
-	},
-	[RTE_FLOW_ITEM_TYPE_UDP] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN,
-			       RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
-			       RTE_FLOW_ITEM_TYPE_MPLS),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_udp){
-			.hdr = {
-				.src_port = -1,
-				.dst_port = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_udp_mask,
-		.mask_sz = sizeof(struct rte_flow_item_udp),
-		.convert = mlx5_flow_create_udp,
-		.dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
-	},
-	[RTE_FLOW_ITEM_TYPE_TCP] = {
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_tcp){
-			.hdr = {
-				.src_port = -1,
-				.dst_port = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_tcp_mask,
-		.mask_sz = sizeof(struct rte_flow_item_tcp),
-		.convert = mlx5_flow_create_tcp,
-		.dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
-	},
-	[RTE_FLOW_ITEM_TYPE_GRE] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6,
-			       RTE_FLOW_ITEM_TYPE_MPLS),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_gre){
-			.protocol = -1,
-		},
-		.default_mask = &rte_flow_item_gre_mask,
-		.mask_sz = sizeof(struct rte_flow_item_gre),
-		.convert = mlx5_flow_create_gre,
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-		.dst_sz = sizeof(struct ibv_flow_spec_gre),
-#else
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
-#endif
-	},
-	[RTE_FLOW_ITEM_TYPE_MPLS] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_mpls){
-			.label_tc_s = "\xff\xff\xf0",
-		},
-		.default_mask = &rte_flow_item_mpls_mask,
-		.mask_sz = sizeof(struct rte_flow_item_mpls),
-		.convert = mlx5_flow_create_mpls,
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-		.dst_sz = sizeof(struct ibv_flow_spec_mpls),
-#endif
-	},
-	[RTE_FLOW_ITEM_TYPE_VXLAN] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4, /* For L3 VXLAN. */
-			       RTE_FLOW_ITEM_TYPE_IPV6), /* For L3 VXLAN. */
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vxlan){
-			.vni = "\xff\xff\xff",
-		},
-		.default_mask = &rte_flow_item_vxlan_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vxlan),
-		.convert = mlx5_flow_create_vxlan,
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
-	},
-	[RTE_FLOW_ITEM_TYPE_VXLAN_GPE] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vxlan_gpe){
-			.vni = "\xff\xff\xff",
-		},
-		.default_mask = &rte_flow_item_vxlan_gpe_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vxlan_gpe),
-		.convert = mlx5_flow_create_vxlan_gpe,
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
-	},
-};
-
-/** Structure to pass to the conversion function. */
-struct mlx5_flow_parse {
-	uint32_t inner; /**< Verbs value, set once tunnel is encountered. */
-	uint32_t create:1;
-	/**< Whether resources should remain after a validate. */
-	uint32_t drop:1; /**< Target is a drop queue. */
-	uint32_t mark:1; /**< Mark is present in the flow. */
-	uint32_t count:1; /**< Count is present in the flow. */
-	uint32_t mark_id; /**< Mark identifier. */
-	struct rte_flow_action_rss rss_conf; /**< RSS configuration */
-	uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
-	uint8_t rss_key[40]; /**< copy of the RSS key. */
-	enum hash_rxq_type layer; /**< Last pattern layer detected. */
-	enum hash_rxq_type out_layer; /**< Last outer pattern layer detected. */
-	uint32_t tunnel; /**< Tunnel type of RTE_PTYPE_TUNNEL_XXX. */
-	struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
-	struct {
-		struct ibv_flow_attr *ibv_attr;
-		/**< Pointer to Verbs attributes. */
-		unsigned int offset;
-		/**< Current position or total size of the attribute. */
-		uint64_t hash_fields; /**< Verbs hash fields. */
-	} queue[RTE_DIM(hash_rxq_init)];
-};
-
-static const struct rte_flow_ops mlx5_flow_ops = {
-	.validate = mlx5_flow_validate,
-	.create = mlx5_flow_create,
-	.destroy = mlx5_flow_destroy,
-	.flush = mlx5_flow_flush,
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	.query = mlx5_flow_query,
-#else
-	.query = NULL,
-#endif
-	.isolate = mlx5_flow_isolate,
-};
-
-/* Convert FDIR request to Generic flow. */
-struct mlx5_fdir {
-	struct rte_flow_attr attr;
-	struct rte_flow_action actions[2];
-	struct rte_flow_item items[4];
-	struct rte_flow_item_eth l2;
-	struct rte_flow_item_eth l2_mask;
-	union {
-		struct rte_flow_item_ipv4 ipv4;
-		struct rte_flow_item_ipv6 ipv6;
-	} l3;
-	union {
-		struct rte_flow_item_ipv4 ipv4;
-		struct rte_flow_item_ipv6 ipv6;
-	} l3_mask;
-	union {
-		struct rte_flow_item_udp udp;
-		struct rte_flow_item_tcp tcp;
-	} l4;
-	union {
-		struct rte_flow_item_udp udp;
-		struct rte_flow_item_tcp tcp;
-	} l4_mask;
-	struct rte_flow_action_queue queue;
-};
-
-/* Verbs specification header. */
-struct ibv_spec_header {
-	enum ibv_flow_spec_type type;
-	uint16_t size;
-};
-
-/**
- * Check item is fully supported by the NIC matching capability.
- *
- * @param item[in]
- *   Item specification.
- * @param mask[in]
- *   Bit-masks covering supported fields to compare with spec, last and mask in
- *   \item.
- * @param size
- *   Bit-Mask size in bytes.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_item_validate(const struct rte_flow_item *item,
-			const uint8_t *mask, unsigned int size)
-{
-	unsigned int i;
-	const uint8_t *spec = item->spec;
-	const uint8_t *last = item->last;
-	const uint8_t *m = item->mask ? item->mask : mask;
-
-	if (!spec && (item->mask || last))
-		goto error;
-	if (!spec)
-		return 0;
-	/*
-	 * Single-pass check to make sure that:
-	 * - item->mask is supported, no bits are set outside mask.
-	 * - Both masked item->spec and item->last are equal (no range
-	 *   supported).
-	 */
-	for (i = 0; i < size; i++) {
-		if (!m[i])
-			continue;
-		if ((m[i] | mask[i]) != mask[i])
-			goto error;
-		if (last && ((spec[i] & m[i]) != (last[i] & m[i])))
-			goto error;
-	}
-	return 0;
-error:
-	rte_errno = ENOTSUP;
-	return -rte_errno;
-}
-
-/**
- * Extract attribute to the parser.
- *
- * @param[in] attr
- *   Flow rule attributes.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_attributes(const struct rte_flow_attr *attr,
-			     struct rte_flow_error *error)
-{
-	if (attr->group) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
-				   NULL,
-				   "groups are not supported");
-		return -rte_errno;
-	}
-	if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
-				   NULL,
-				   "priorities are not supported");
-		return -rte_errno;
-	}
-	if (attr->egress) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
-				   NULL,
-				   "egress is not supported");
-		return -rte_errno;
-	}
-	if (attr->transfer) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
-				   NULL,
-				   "transfer is not supported");
-		return -rte_errno;
-	}
-	if (!attr->ingress) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
-				   NULL,
-				   "only ingress is supported");
-		return -rte_errno;
-	}
-	return 0;
-}
-
-/**
- * Extract actions request to the parser.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[in] actions
- *   Associated actions (list terminated by the END action).
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_actions(struct rte_eth_dev *dev,
-			  const struct rte_flow_action actions[],
-			  struct rte_flow_error *error,
-			  struct mlx5_flow_parse *parser)
-{
-	enum { FATE = 1, MARK = 2, COUNT = 4, };
-	uint32_t overlap = 0;
-	struct priv *priv = dev->data->dev_private;
-
-	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
-		if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
-			continue;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			parser->drop = 1;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
-			const struct rte_flow_action_queue *queue =
-				(const struct rte_flow_action_queue *)
-				actions->conf;
-
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			if (!queue || (queue->index > (priv->rxqs_n - 1)))
-				goto exit_action_not_supported;
-			parser->queues[0] = queue->index;
-			parser->rss_conf = (struct rte_flow_action_rss){
-				.queue_num = 1,
-				.queue = parser->queues,
-			};
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
-			const struct rte_flow_action_rss *rss =
-				(const struct rte_flow_action_rss *)
-				actions->conf;
-			const uint8_t *rss_key;
-			uint32_t rss_key_len;
-			uint16_t n;
-
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			if (rss->func &&
-			    rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "the only supported RSS hash"
-						   " function is Toeplitz");
-				return -rte_errno;
-			}
-#ifndef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-			if (parser->rss_conf.level > 1) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "a nonzero RSS encapsulation"
-						   " level is not supported");
-				return -rte_errno;
-			}
-#endif
-			if (parser->rss_conf.level > 2) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "RSS encapsulation level"
-						   " > 1 is not supported");
-				return -rte_errno;
-			}
-			if (rss->types & MLX5_RSS_HF_MASK) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "unsupported RSS type"
-						   " requested");
-				return -rte_errno;
-			}
-			if (rss->key_len) {
-				rss_key_len = rss->key_len;
-				rss_key = rss->key;
-			} else {
-				rss_key_len = rss_hash_default_key_len;
-				rss_key = rss_hash_default_key;
-			}
-			if (rss_key_len != RTE_DIM(parser->rss_key)) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "RSS hash key must be"
-						   " exactly 40 bytes long");
-				return -rte_errno;
-			}
-			if (!rss->queue_num) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "no valid queues");
-				return -rte_errno;
-			}
-			if (rss->queue_num > RTE_DIM(parser->queues)) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "too many queues for RSS"
-						   " context");
-				return -rte_errno;
-			}
-			for (n = 0; n < rss->queue_num; ++n) {
-				if (rss->queue[n] >= priv->rxqs_n) {
-					rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "queue id > number of"
-						   " queues");
-					return -rte_errno;
-				}
-			}
-			parser->rss_conf = (struct rte_flow_action_rss){
-				.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
-				.level = rss->level ? rss->level : 1,
-				.types = rss->types,
-				.key_len = rss_key_len,
-				.queue_num = rss->queue_num,
-				.key = memcpy(parser->rss_key, rss_key,
-					      sizeof(*rss_key) * rss_key_len),
-				.queue = memcpy(parser->queues, rss->queue,
-						sizeof(*rss->queue) *
-						rss->queue_num),
-			};
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
-			const struct rte_flow_action_mark *mark =
-				(const struct rte_flow_action_mark *)
-				actions->conf;
-
-			if (overlap & MARK)
-				goto exit_action_overlap;
-			overlap |= MARK;
-			if (!mark) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "mark must be defined");
-				return -rte_errno;
-			} else if (mark->id >= MLX5_FLOW_MARK_MAX) {
-				rte_flow_error_set(error, ENOTSUP,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "mark must be between 0"
-						   " and 16777199");
-				return -rte_errno;
-			}
-			parser->mark = 1;
-			parser->mark_id = mark->id;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
-			if (overlap & MARK)
-				goto exit_action_overlap;
-			overlap |= MARK;
-			parser->mark = 1;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
-			   priv->config.flow_counter_en) {
-			if (overlap & COUNT)
-				goto exit_action_overlap;
-			overlap |= COUNT;
-			parser->count = 1;
-		} else {
-			goto exit_action_not_supported;
-		}
-	}
-	/* When fate is unknown, drop traffic. */
-	if (!(overlap & FATE))
-		parser->drop = 1;
-	if (parser->drop && parser->mark)
-		parser->mark = 0;
-	if (!parser->rss_conf.queue_num && !parser->drop) {
-		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "no valid action");
-		return -rte_errno;
-	}
-	return 0;
-exit_action_not_supported:
-	rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
-			   actions, "action not supported");
-	return -rte_errno;
-exit_action_overlap:
-	rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
-			   actions, "overlapping actions are not supported");
-	return -rte_errno;
-}
-
-/**
- * Validate items.
- *
- * @param[in] items
- *   Pattern specification (list terminated by the END pattern item).
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_items_validate(struct rte_eth_dev *dev,
-				 const struct rte_flow_item items[],
-				 struct rte_flow_error *error,
-				 struct mlx5_flow_parse *parser)
-{
-	struct priv *priv = dev->data->dev_private;
-	const struct mlx5_flow_items *cur_item = mlx5_flow_items;
-	unsigned int i;
-	unsigned int last_voids = 0;
-	int ret = 0;
-
-	/* Initialise the offsets to start after verbs attribute. */
-	for (i = 0; i != hash_rxq_init_n; ++i)
-		parser->queue[i].offset = sizeof(struct ibv_flow_attr);
-	for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
-		const struct mlx5_flow_items *token = NULL;
-		unsigned int n;
-
-		if (items->type == RTE_FLOW_ITEM_TYPE_VOID) {
-			last_voids++;
-			continue;
-		}
-		for (i = 0;
-		     cur_item->items &&
-		     cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
-		     ++i) {
-			if (cur_item->items[i] == items->type) {
-				token = &mlx5_flow_items[items->type];
-				break;
-			}
-		}
-		if (!token) {
-			ret = -ENOTSUP;
-			goto exit_item_not_supported;
-		}
-		cur_item = token;
-		ret = mlx5_flow_item_validate(items,
-					      (const uint8_t *)cur_item->mask,
-					      cur_item->mask_sz);
-		if (ret)
-			goto exit_item_not_supported;
-		if (IS_TUNNEL(items->type)) {
-			if (parser->tunnel &&
-			    !((items - last_voids - 1)->type ==
-			      RTE_FLOW_ITEM_TYPE_GRE && items->type ==
-			      RTE_FLOW_ITEM_TYPE_MPLS)) {
-				rte_flow_error_set(error, ENOTSUP,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   items,
-						   "Cannot recognize multiple"
-						   " tunnel encapsulations.");
-				return -rte_errno;
-			}
-			if (items->type == RTE_FLOW_ITEM_TYPE_MPLS &&
-			    !priv->config.mpls_en) {
-				rte_flow_error_set(error, ENOTSUP,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   items,
-						   "MPLS not supported or"
-						   " disabled in firmware"
-						   " configuration.");
-				return -rte_errno;
-			}
-			if (!priv->config.tunnel_en &&
-			    parser->rss_conf.level > 1) {
-				rte_flow_error_set(error, ENOTSUP,
-					RTE_FLOW_ERROR_TYPE_ITEM,
-					items,
-					"RSS on tunnel is not supported");
-				return -rte_errno;
-			}
-			parser->inner = IBV_FLOW_SPEC_INNER;
-			parser->tunnel = flow_ptype[items->type];
-		}
-		if (parser->drop) {
-			parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
-		} else {
-			for (n = 0; n != hash_rxq_init_n; ++n)
-				parser->queue[n].offset += cur_item->dst_sz;
-		}
-		last_voids = 0;
-	}
-	if (parser->drop) {
-		parser->queue[HASH_RXQ_ETH].offset +=
-			sizeof(struct ibv_flow_spec_action_drop);
-	}
-	if (parser->mark) {
-		for (i = 0; i != hash_rxq_init_n; ++i)
-			parser->queue[i].offset +=
-				sizeof(struct ibv_flow_spec_action_tag);
-	}
-	if (parser->count) {
-		unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
-
-		for (i = 0; i != hash_rxq_init_n; ++i)
-			parser->queue[i].offset += size;
-	}
-	return 0;
-exit_item_not_supported:
-	return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
-				  items, "item not supported");
-}
-
-/**
- * Allocate memory space to store verbs flow attributes.
- *
- * @param[in] size
- *   Amount of byte to allocate.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   A verbs flow attribute on success, NULL otherwise and rte_errno is set.
- */
-static struct ibv_flow_attr *
-mlx5_flow_convert_allocate(unsigned int size, struct rte_flow_error *error)
-{
-	struct ibv_flow_attr *ibv_attr;
-
-	ibv_attr = rte_calloc(__func__, 1, size, 0);
-	if (!ibv_attr) {
-		rte_flow_error_set(error, ENOMEM,
-				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				   NULL,
-				   "cannot allocate verbs spec attributes");
-		return NULL;
-	}
-	return ibv_attr;
-}
-
-/**
- * Make inner packet matching with an higher priority from the non Inner
- * matching.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[in, out] parser
- *   Internal parser structure.
- * @param attr
- *   User flow attribute.
- */
-static void
-mlx5_flow_update_priority(struct rte_eth_dev *dev,
-			  struct mlx5_flow_parse *parser,
-			  const struct rte_flow_attr *attr)
-{
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-	uint16_t priority;
-
-	/*			8 priorities	>= 16 priorities
-	 * Control flow:	4-7		8-15
-	 * User normal flow:	1-3		4-7
-	 * User tunnel flow:	0-2		0-3
-	 */
-	priority = attr->priority * MLX5_VERBS_FLOW_PRIO_8;
-	if (priv->config.max_verbs_prio == MLX5_VERBS_FLOW_PRIO_8)
-		priority /= 2;
-	/*
-	 * Lower non-tunnel flow Verbs priority 1 if only support 8 Verbs
-	 * priorities, lower 4 otherwise.
-	 */
-	if (!parser->inner) {
-		if (priv->config.max_verbs_prio == MLX5_VERBS_FLOW_PRIO_8)
-			priority += 1;
-		else
-			priority += MLX5_VERBS_FLOW_PRIO_8 / 2;
-	}
-	if (parser->drop) {
-		parser->queue[HASH_RXQ_ETH].ibv_attr->priority = priority +
-				hash_rxq_init[HASH_RXQ_ETH].flow_priority;
-		return;
-	}
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		parser->queue[i].ibv_attr->priority = priority +
-				hash_rxq_init[i].flow_priority;
-	}
-}
-
-/**
- * Finalise verbs flow attributes.
- *
- * @param[in, out] parser
- *   Internal parser structure.
- */
-static void
-mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser)
-{
-	unsigned int i;
-	uint32_t inner = parser->inner;
-
-	/* Don't create extra flows for outer RSS. */
-	if (parser->tunnel && parser->rss_conf.level < 2)
-		return;
-	/*
-	 * Fill missing layers in verbs specifications, or compute the correct
-	 * offset to allocate the memory space for the attributes and
-	 * specifications.
-	 */
-	for (i = 0; i != hash_rxq_init_n - 1; ++i) {
-		union {
-			struct ibv_flow_spec_ipv4_ext ipv4;
-			struct ibv_flow_spec_ipv6 ipv6;
-			struct ibv_flow_spec_tcp_udp udp_tcp;
-			struct ibv_flow_spec_eth eth;
-		} specs;
-		void *dst;
-		uint16_t size;
-
-		if (i == parser->layer)
-			continue;
-		if (parser->layer == HASH_RXQ_ETH ||
-		    parser->layer == HASH_RXQ_TUNNEL) {
-			if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
-				size = sizeof(struct ibv_flow_spec_ipv4_ext);
-				specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
-					.type = inner | IBV_FLOW_SPEC_IPV4_EXT,
-					.size = size,
-				};
-			} else {
-				size = sizeof(struct ibv_flow_spec_ipv6);
-				specs.ipv6 = (struct ibv_flow_spec_ipv6){
-					.type = inner | IBV_FLOW_SPEC_IPV6,
-					.size = size,
-				};
-			}
-			if (parser->queue[i].ibv_attr) {
-				dst = (void *)((uintptr_t)
-					       parser->queue[i].ibv_attr +
-					       parser->queue[i].offset);
-				memcpy(dst, &specs, size);
-				++parser->queue[i].ibv_attr->num_of_specs;
-			}
-			parser->queue[i].offset += size;
-		}
-		if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
-		    (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
-			size = sizeof(struct ibv_flow_spec_tcp_udp);
-			specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
-				.type = inner | ((i == HASH_RXQ_UDPV4 ||
-					  i == HASH_RXQ_UDPV6) ?
-					 IBV_FLOW_SPEC_UDP :
-					 IBV_FLOW_SPEC_TCP),
-				.size = size,
-			};
-			if (parser->queue[i].ibv_attr) {
-				dst = (void *)((uintptr_t)
-					       parser->queue[i].ibv_attr +
-					       parser->queue[i].offset);
-				memcpy(dst, &specs, size);
-				++parser->queue[i].ibv_attr->num_of_specs;
-			}
-			parser->queue[i].offset += size;
-		}
-	}
-}
-
-/**
- * Update flows according to pattern and RSS hash fields.
- *
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_rss(struct mlx5_flow_parse *parser)
-{
-	unsigned int i;
-	enum hash_rxq_type start;
-	enum hash_rxq_type layer;
-	int outer = parser->tunnel && parser->rss_conf.level < 2;
-	uint64_t rss = parser->rss_conf.types;
-
-	layer = outer ? parser->out_layer : parser->layer;
-	if (layer == HASH_RXQ_TUNNEL)
-		layer = HASH_RXQ_ETH;
-	if (outer) {
-		/* Only one hash type for outer RSS. */
-		if (rss && layer == HASH_RXQ_ETH) {
-			start = HASH_RXQ_TCPV4;
-		} else if (rss && layer != HASH_RXQ_ETH &&
-			   !(rss & hash_rxq_init[layer].dpdk_rss_hf)) {
-			/* If RSS not match L4 pattern, try L3 RSS. */
-			if (layer < HASH_RXQ_IPV4)
-				layer = HASH_RXQ_IPV4;
-			else if (layer > HASH_RXQ_IPV4 && layer < HASH_RXQ_IPV6)
-				layer = HASH_RXQ_IPV6;
-			start = layer;
-		} else {
-			start = layer;
-		}
-		/* Scan first valid hash type. */
-		for (i = start; rss && i <= layer; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-			if (hash_rxq_init[i].dpdk_rss_hf & rss)
-				break;
-		}
-		if (rss && i <= layer)
-			parser->queue[layer].hash_fields =
-					hash_rxq_init[i].hash_fields;
-		/* Trim unused hash types. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (parser->queue[i].ibv_attr && i != layer) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			}
-		}
-	} else {
-		/* Expand for inner or normal RSS. */
-		if (rss && (layer == HASH_RXQ_ETH || layer == HASH_RXQ_IPV4))
-			start = HASH_RXQ_TCPV4;
-		else if (rss && layer == HASH_RXQ_IPV6)
-			start = HASH_RXQ_TCPV6;
-		else
-			start = layer;
-		/* For L4 pattern, try L3 RSS if no L4 RSS. */
-		/* Trim unused hash types. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-			if (i < start || i > layer) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-				continue;
-			}
-			if (!rss)
-				continue;
-			if (hash_rxq_init[i].dpdk_rss_hf & rss) {
-				parser->queue[i].hash_fields =
-						hash_rxq_init[i].hash_fields;
-			} else if (i != layer) {
-				/* Remove unused RSS expansion. */
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			} else if (layer < HASH_RXQ_IPV4 &&
-				   (hash_rxq_init[HASH_RXQ_IPV4].dpdk_rss_hf &
-				    rss)) {
-				/* Allow IPv4 RSS on L4 pattern. */
-				parser->queue[i].hash_fields =
-					hash_rxq_init[HASH_RXQ_IPV4]
-						.hash_fields;
-			} else if (i > HASH_RXQ_IPV4 && i < HASH_RXQ_IPV6 &&
-				   (hash_rxq_init[HASH_RXQ_IPV6].dpdk_rss_hf &
-				    rss)) {
-				/* Allow IPv4 RSS on L4 pattern. */
-				parser->queue[i].hash_fields =
-					hash_rxq_init[HASH_RXQ_IPV6]
-						.hash_fields;
-			}
-		}
-	}
-	return 0;
-}
-
-/**
- * Validate and convert a flow supported by the NIC.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[in] attr
- *   Flow rule attributes.
- * @param[in] pattern
- *   Pattern specification (list terminated by the END pattern item).
- * @param[in] actions
- *   Associated actions (list terminated by the END action).
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert(struct rte_eth_dev *dev,
-		  const struct rte_flow_attr *attr,
-		  const struct rte_flow_item items[],
-		  const struct rte_flow_action actions[],
-		  struct rte_flow_error *error,
-		  struct mlx5_flow_parse *parser)
-{
-	const struct mlx5_flow_items *cur_item = mlx5_flow_items;
-	unsigned int i;
-	int ret;
-
-	/* First step. Validate the attributes, items and actions. */
-	*parser = (struct mlx5_flow_parse){
-		.create = parser->create,
-		.layer = HASH_RXQ_ETH,
-		.mark_id = MLX5_FLOW_MARK_DEFAULT,
-	};
-	ret = mlx5_flow_convert_attributes(attr, error);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert_actions(dev, actions, error, parser);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert_items_validate(dev, items, error, parser);
-	if (ret)
-		return ret;
-	mlx5_flow_convert_finalise(parser);
-	/*
-	 * Second step.
-	 * Allocate the memory space to store verbs specifications.
-	 */
-	if (parser->drop) {
-		unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
-
-		parser->queue[HASH_RXQ_ETH].ibv_attr =
-			mlx5_flow_convert_allocate(offset, error);
-		if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
-			goto exit_enomem;
-		parser->queue[HASH_RXQ_ETH].offset =
-			sizeof(struct ibv_flow_attr);
-	} else {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			unsigned int offset;
-
-			offset = parser->queue[i].offset;
-			parser->queue[i].ibv_attr =
-				mlx5_flow_convert_allocate(offset, error);
-			if (!parser->queue[i].ibv_attr)
-				goto exit_enomem;
-			parser->queue[i].offset = sizeof(struct ibv_flow_attr);
-		}
-	}
-	/* Third step. Conversion parse, fill the specifications. */
-	parser->inner = 0;
-	parser->tunnel = 0;
-	parser->layer = HASH_RXQ_ETH;
-	for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
-		struct mlx5_flow_data data = {
-			.dev = dev,
-			.parser = parser,
-			.error = error,
-		};
-
-		if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
-			continue;
-		cur_item = &mlx5_flow_items[items->type];
-		ret = cur_item->convert(items,
-					(cur_item->default_mask ?
-					 cur_item->default_mask :
-					 cur_item->mask),
-					 &data);
-		if (ret)
-			goto exit_free;
-	}
-	if (!parser->drop) {
-		/* RSS check, remove unused hash types. */
-		ret = mlx5_flow_convert_rss(parser);
-		if (ret)
-			goto exit_free;
-		/* Complete missing specification. */
-		mlx5_flow_convert_finalise(parser);
-	}
-	mlx5_flow_update_priority(dev, parser, attr);
-	if (parser->mark)
-		mlx5_flow_create_flag_mark(parser, parser->mark_id);
-	if (parser->count && parser->create) {
-		mlx5_flow_create_count(dev, parser);
-		if (!parser->cs)
-			goto exit_count_error;
-	}
-exit_free:
-	/* Only verification is expected, all resources should be released. */
-	if (!parser->create) {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (parser->queue[i].ibv_attr) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			}
-		}
-	}
-	return ret;
-exit_enomem:
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser->queue[i].ibv_attr) {
-			rte_free(parser->queue[i].ibv_attr);
-			parser->queue[i].ibv_attr = NULL;
-		}
-	}
-	rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL, "cannot allocate verbs spec attributes");
-	return -rte_errno;
-exit_count_error:
-	rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL, "cannot create counter");
-	return -rte_errno;
-}
-
-/**
- * Copy the specification created into the flow.
- *
- * @param parser
- *   Internal parser structure.
- * @param src
- *   Create specification.
- * @param size
- *   Size in bytes of the specification to copy.
- */
-static void
-mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
-		      unsigned int size)
-{
-	unsigned int i;
-	void *dst;
-
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset);
-		memcpy(dst, src, size);
-		++parser->queue[i].ibv_attr->num_of_specs;
-		parser->queue[i].offset += size;
-	}
-}
-
-/**
- * Convert Ethernet item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_eth(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_eth *spec = item->spec;
-	const struct rte_flow_item_eth *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
-	struct ibv_flow_spec_eth eth = {
-		.type = parser->inner | IBV_FLOW_SPEC_ETH,
-		.size = eth_size,
-	};
-
-	parser->layer = HASH_RXQ_ETH;
-	if (spec) {
-		unsigned int i;
-
-		if (!mask)
-			mask = default_mask;
-		memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
-		memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
-		eth.val.ether_type = spec->type;
-		memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
-		memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
-		eth.mask.ether_type = mask->type;
-		/* Remove unwanted bits from values. */
-		for (i = 0; i < ETHER_ADDR_LEN; ++i) {
-			eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
-			eth.val.src_mac[i] &= eth.mask.src_mac[i];
-		}
-		eth.val.ether_type &= eth.mask.ether_type;
-	}
-	mlx5_flow_create_copy(parser, &eth, eth_size);
-	return 0;
-}
-
-/**
- * Convert VLAN item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_vlan(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_vlan *spec = item->spec;
-	const struct rte_flow_item_vlan *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	struct ibv_flow_spec_eth *eth;
-	const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
-	const char *msg = "VLAN cannot be empty";
-
-	if (spec) {
-		unsigned int i;
-		if (!mask)
-			mask = default_mask;
-
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-
-			eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				       parser->queue[i].offset - eth_size);
-			eth->val.vlan_tag = spec->tci;
-			eth->mask.vlan_tag = mask->tci;
-			eth->val.vlan_tag &= eth->mask.vlan_tag;
-			/*
-			 * From verbs perspective an empty VLAN is equivalent
-			 * to a packet without VLAN layer.
-			 */
-			if (!eth->mask.vlan_tag)
-				goto error;
-			/* Outer TPID cannot be matched. */
-			if (eth->mask.ether_type) {
-				msg = "VLAN TPID matching is not supported";
-				goto error;
-			}
-			eth->val.ether_type = spec->inner_type;
-			eth->mask.ether_type = mask->inner_type;
-			eth->val.ether_type &= eth->mask.ether_type;
-		}
-		return 0;
-	}
-error:
-	return rte_flow_error_set(data->error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
-				  item, msg);
-}
-
-/**
- * Convert IPv4 item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_ipv4(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-	struct priv *priv = data->dev->data->dev_private;
-	const struct rte_flow_item_ipv4 *spec = item->spec;
-	const struct rte_flow_item_ipv4 *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
-	struct ibv_flow_spec_ipv4_ext ipv4 = {
-		.type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
-		.size = ipv4_size,
-	};
-
-	if (parser->layer == HASH_RXQ_TUNNEL &&
-	    parser->tunnel == ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] &&
-	    !priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	parser->layer = HASH_RXQ_IPV4;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
-			.src_ip = spec->hdr.src_addr,
-			.dst_ip = spec->hdr.dst_addr,
-			.proto = spec->hdr.next_proto_id,
-			.tos = spec->hdr.type_of_service,
-		};
-		ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
-			.src_ip = mask->hdr.src_addr,
-			.dst_ip = mask->hdr.dst_addr,
-			.proto = mask->hdr.next_proto_id,
-			.tos = mask->hdr.type_of_service,
-		};
-		/* Remove unwanted bits from values. */
-		ipv4.val.src_ip &= ipv4.mask.src_ip;
-		ipv4.val.dst_ip &= ipv4.mask.dst_ip;
-		ipv4.val.proto &= ipv4.mask.proto;
-		ipv4.val.tos &= ipv4.mask.tos;
-	}
-	mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
-	return 0;
-}
-
-/**
- * Convert IPv6 item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_ipv6(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-	struct priv *priv = data->dev->data->dev_private;
-	const struct rte_flow_item_ipv6 *spec = item->spec;
-	const struct rte_flow_item_ipv6 *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
-	struct ibv_flow_spec_ipv6 ipv6 = {
-		.type = parser->inner | IBV_FLOW_SPEC_IPV6,
-		.size = ipv6_size,
-	};
-
-	if (parser->layer == HASH_RXQ_TUNNEL &&
-	    parser->tunnel == ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] &&
-	    !priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	parser->layer = HASH_RXQ_IPV6;
-	if (spec) {
-		unsigned int i;
-		uint32_t vtc_flow_val;
-		uint32_t vtc_flow_mask;
-
-		if (!mask)
-			mask = default_mask;
-		memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
-		       RTE_DIM(ipv6.val.src_ip));
-		memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
-		       RTE_DIM(ipv6.val.dst_ip));
-		memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
-		       RTE_DIM(ipv6.mask.src_ip));
-		memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
-		       RTE_DIM(ipv6.mask.dst_ip));
-		vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
-		vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
-		ipv6.val.flow_label =
-			rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
-					 IPV6_HDR_FL_SHIFT);
-		ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
-					 IPV6_HDR_TC_SHIFT;
-		ipv6.val.next_hdr = spec->hdr.proto;
-		ipv6.val.hop_limit = spec->hdr.hop_limits;
-		ipv6.mask.flow_label =
-			rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
-					 IPV6_HDR_FL_SHIFT);
-		ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
-					  IPV6_HDR_TC_SHIFT;
-		ipv6.mask.next_hdr = mask->hdr.proto;
-		ipv6.mask.hop_limit = mask->hdr.hop_limits;
-		/* Remove unwanted bits from values. */
-		for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
-			ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
-			ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
-		}
-		ipv6.val.flow_label &= ipv6.mask.flow_label;
-		ipv6.val.traffic_class &= ipv6.mask.traffic_class;
-		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
-		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
-	}
-	mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
-	return 0;
-}
-
-/**
- * Convert UDP item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_udp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_udp *spec = item->spec;
-	const struct rte_flow_item_udp *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
-	struct ibv_flow_spec_tcp_udp udp = {
-		.type = parser->inner | IBV_FLOW_SPEC_UDP,
-		.size = udp_size,
-	};
-
-	if (parser->layer == HASH_RXQ_IPV4)
-		parser->layer = HASH_RXQ_UDPV4;
-	else
-		parser->layer = HASH_RXQ_UDPV6;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		udp.val.dst_port = spec->hdr.dst_port;
-		udp.val.src_port = spec->hdr.src_port;
-		udp.mask.dst_port = mask->hdr.dst_port;
-		udp.mask.src_port = mask->hdr.src_port;
-		/* Remove unwanted bits from values. */
-		udp.val.src_port &= udp.mask.src_port;
-		udp.val.dst_port &= udp.mask.dst_port;
-	}
-	mlx5_flow_create_copy(parser, &udp, udp_size);
-	return 0;
-}
-
-/**
- * Convert TCP item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_tcp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_tcp *spec = item->spec;
-	const struct rte_flow_item_tcp *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
-	struct ibv_flow_spec_tcp_udp tcp = {
-		.type = parser->inner | IBV_FLOW_SPEC_TCP,
-		.size = tcp_size,
-	};
-
-	if (parser->layer == HASH_RXQ_IPV4)
-		parser->layer = HASH_RXQ_TCPV4;
-	else
-		parser->layer = HASH_RXQ_TCPV6;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		tcp.val.dst_port = spec->hdr.dst_port;
-		tcp.val.src_port = spec->hdr.src_port;
-		tcp.mask.dst_port = mask->hdr.dst_port;
-		tcp.mask.src_port = mask->hdr.src_port;
-		/* Remove unwanted bits from values. */
-		tcp.val.src_port &= tcp.mask.src_port;
-		tcp.val.dst_port &= tcp.mask.dst_port;
-	}
-	mlx5_flow_create_copy(parser, &tcp, tcp_size);
-	return 0;
-}
-
-/**
- * Convert VXLAN item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_vxlan(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_vxlan *spec = item->spec;
-	const struct rte_flow_item_vxlan *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel vxlan = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
-	union vni {
-		uint32_t vlan_id;
-		uint8_t vni[4];
-	} id;
-
-	id.vni[0] = 0;
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		memcpy(&id.vni[1], spec->vni, 3);
-		vxlan.val.tunnel_id = id.vlan_id;
-		memcpy(&id.vni[1], mask->vni, 3);
-		vxlan.mask.tunnel_id = id.vlan_id;
-		/* Remove unwanted bits from values. */
-		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
-	}
-	/*
-	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
-	 * layer is defined in the Verbs specification it is interpreted as
-	 * wildcard and all packets will match this rule, if it follows a full
-	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
-	 * before will also match this rule.
-	 * To avoid such situation, VNI 0 is currently refused.
-	 */
-	/* Only allow tunnel w/o tunnel id pattern after proper outer spec. */
-	if (parser->out_layer == HASH_RXQ_ETH && !vxlan.val.tunnel_id)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "VxLAN vni cannot be 0");
-	mlx5_flow_create_copy(parser, &vxlan, size);
-	return 0;
-}
-
-/**
- * Convert VXLAN-GPE item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_vxlan_gpe(const struct rte_flow_item *item,
-			   const void *default_mask,
-			   struct mlx5_flow_data *data)
-{
-	struct priv *priv = data->dev->data->dev_private;
-	const struct rte_flow_item_vxlan_gpe *spec = item->spec;
-	const struct rte_flow_item_vxlan_gpe *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel vxlan = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
-	union vni {
-		uint32_t vlan_id;
-		uint8_t vni[4];
-	} id;
-
-	if (!priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	id.vni[0] = 0;
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN_GPE)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		memcpy(&id.vni[1], spec->vni, 3);
-		vxlan.val.tunnel_id = id.vlan_id;
-		memcpy(&id.vni[1], mask->vni, 3);
-		vxlan.mask.tunnel_id = id.vlan_id;
-		if (spec->protocol)
-			return rte_flow_error_set(data->error, EINVAL,
-						  RTE_FLOW_ERROR_TYPE_ITEM,
-						  item,
-						  "VxLAN-GPE protocol not"
-						  " supported");
-		/* Remove unwanted bits from values. */
-		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
-	}
-	/*
-	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
-	 * layer is defined in the Verbs specification it is interpreted as
-	 * wildcard and all packets will match this rule, if it follows a full
-	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
-	 * before will also match this rule.
-	 * To avoid such situation, VNI 0 is currently refused.
-	 */
-	/* Only allow tunnel w/o tunnel id pattern after proper outer spec. */
-	if (parser->out_layer == HASH_RXQ_ETH && !vxlan.val.tunnel_id)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "VxLAN-GPE vni cannot be 0");
-	mlx5_flow_create_copy(parser, &vxlan, size);
-	return 0;
-}
-
-/**
- * Convert GRE item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_gre(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	struct mlx5_flow_parse *parser = data->parser;
-#ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	(void)default_mask;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel tunnel = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
-#else
-	const struct rte_flow_item_gre *spec = item->spec;
-	const struct rte_flow_item_gre *mask = item->mask;
-	unsigned int size = sizeof(struct ibv_flow_spec_gre);
-	struct ibv_flow_spec_gre tunnel = {
-		.type = parser->inner | IBV_FLOW_SPEC_GRE,
-		.size = size,
-	};
-#endif
-	struct ibv_flow_spec_ipv4_ext *ipv4;
-	struct ibv_flow_spec_ipv6 *ipv6;
-	unsigned int i;
-
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_GRE)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
-		tunnel.val.protocol = spec->protocol;
-		tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
-		tunnel.mask.protocol = mask->protocol;
-		/* Remove unwanted bits from values. */
-		tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
-		tunnel.val.protocol &= tunnel.mask.protocol;
-		tunnel.val.key &= tunnel.mask.key;
-	}
-#endif
-	/* Update encapsulation IP layer protocol. */
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		if (parser->out_layer == HASH_RXQ_IPV4) {
-			ipv4 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset -
-				sizeof(struct ibv_flow_spec_ipv4_ext));
-			if (ipv4->mask.proto && ipv4->val.proto != MLX5_GRE)
-				break;
-			ipv4->val.proto = MLX5_GRE;
-			ipv4->mask.proto = 0xff;
-		} else if (parser->out_layer == HASH_RXQ_IPV6) {
-			ipv6 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset -
-				sizeof(struct ibv_flow_spec_ipv6));
-			if (ipv6->mask.next_hdr &&
-			    ipv6->val.next_hdr != MLX5_GRE)
-				break;
-			ipv6->val.next_hdr = MLX5_GRE;
-			ipv6->mask.next_hdr = 0xff;
-		}
-	}
-	if (i != hash_rxq_init_n)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "IP protocol of GRE must be 47");
-	mlx5_flow_create_copy(parser, &tunnel, size);
-	return 0;
-}
-
-/**
- * Convert MPLS item to Verbs specification.
- * MPLS tunnel types currently supported are MPLS-in-GRE and MPLS-in-UDP.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_mpls(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-#ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	(void)default_mask;
-	return rte_flow_error_set(data->error, ENOTSUP,
-				  RTE_FLOW_ERROR_TYPE_ITEM,
-				  item,
-				  "MPLS is not supported by driver");
-#else
-	const struct rte_flow_item_mpls *spec = item->spec;
-	const struct rte_flow_item_mpls *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int size = sizeof(struct ibv_flow_spec_mpls);
-	struct ibv_flow_spec_mpls mpls = {
-		.type = IBV_FLOW_SPEC_MPLS,
-		.size = size,
-	};
-
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	if (parser->layer == HASH_RXQ_UDPV4 ||
-	    parser->layer == HASH_RXQ_UDPV6) {
-		parser->tunnel =
-			ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_UDP)];
-		parser->out_layer = parser->layer;
-	} else {
-		parser->tunnel =
-			ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_GRE)];
-		/* parser->out_layer stays as in GRE out_layer. */
-	}
-	parser->layer = HASH_RXQ_TUNNEL;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		/*
-		 * The verbs label field includes the entire MPLS header:
-		 * bits 0:19 - label value field.
-		 * bits 20:22 - traffic class field.
-		 * bits 23 - bottom of stack bit.
-		 * bits 24:31 - ttl field.
-		 */
-		mpls.val.label = *(const uint32_t *)spec;
-		mpls.mask.label = *(const uint32_t *)mask;
-		/* Remove unwanted bits from values. */
-		mpls.val.label &= mpls.mask.label;
-	}
-	mlx5_flow_create_copy(parser, &mpls, size);
-	return 0;
-#endif
-}
-
-/**
- * Convert mark/flag action to Verbs specification.
- *
- * @param parser
- *   Internal parser structure.
- * @param mark_id
- *   Mark identifier.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
-{
-	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
-	struct ibv_flow_spec_action_tag tag = {
-		.type = IBV_FLOW_SPEC_ACTION_TAG,
-		.size = size,
-		.tag_id = mlx5_flow_mark_set(mark_id),
-	};
-
-	assert(parser->mark);
-	mlx5_flow_create_copy(parser, &tag, size);
-	return 0;
-}
-
-/**
- * Convert count action to Verbs specification.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Pointer to MLX5 flow parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused,
-		       struct mlx5_flow_parse *parser __rte_unused)
-{
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	struct priv *priv = dev->data->dev_private;
-	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
-	struct ibv_counter_set_init_attr init_attr = {0};
-	struct ibv_flow_spec_counter_action counter = {
-		.type = IBV_FLOW_SPEC_ACTION_COUNT,
-		.size = size,
-		.counter_set_handle = 0,
-	};
-
-	init_attr.counter_set_id = 0;
-	parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr);
-	if (!parser->cs) {
-		rte_errno = EINVAL;
-		return -rte_errno;
-	}
-	counter.counter_set_handle = parser->cs->handle;
-	mlx5_flow_create_copy(parser, &counter, size);
-#endif
-	return 0;
-}
-
-/**
- * Complete flow rule creation with a drop queue.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev,
-				   struct mlx5_flow_parse *parser,
-				   struct rte_flow *flow,
-				   struct rte_flow_error *error)
-{
-	struct priv *priv = dev->data->dev_private;
-	struct ibv_flow_spec_action_drop *drop;
-	unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
-
-	assert(priv->pd);
-	assert(priv->ctx);
-	flow->drop = 1;
-	drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr +
-			parser->queue[HASH_RXQ_ETH].offset);
-	*drop = (struct ibv_flow_spec_action_drop){
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-			.size = size,
-	};
-	++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs;
-	parser->queue[HASH_RXQ_ETH].offset += size;
-	flow->frxq[HASH_RXQ_ETH].ibv_attr =
-		parser->queue[HASH_RXQ_ETH].ibv_attr;
-	if (parser->count)
-		flow->cs = parser->cs;
-	if (!dev->data->dev_started)
-		return 0;
-	parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
-	flow->frxq[HASH_RXQ_ETH].ibv_flow =
-		mlx5_glue->create_flow(priv->flow_drop_queue->qp,
-				       flow->frxq[HASH_RXQ_ETH].ibv_attr);
-	if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-		rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "flow rule creation failure");
-		goto error;
-	}
-	return 0;
-error:
-	assert(flow);
-	if (flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-		claim_zero(mlx5_glue->destroy_flow
-			   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-		flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
-	}
-	if (flow->frxq[HASH_RXQ_ETH].ibv_attr) {
-		rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
-		flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL;
-	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-		parser->cs = NULL;
-	}
-	return -rte_errno;
-}
-
-/**
- * Create hash Rx queues when RSS is enabled.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev,
-				  struct mlx5_flow_parse *parser,
-				  struct rte_flow *flow,
-				  struct rte_flow_error *error)
-{
-	unsigned int i;
-
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
-		parser->queue[i].ibv_attr = NULL;
-		flow->frxq[i].hash_fields = parser->queue[i].hash_fields;
-		if (!dev->data->dev_started)
-			continue;
-		flow->frxq[i].hrxq =
-			mlx5_hrxq_get(dev,
-				      parser->rss_conf.key,
-				      parser->rss_conf.key_len,
-				      flow->frxq[i].hash_fields,
-				      parser->rss_conf.queue,
-				      parser->rss_conf.queue_num,
-				      parser->tunnel,
-				      parser->rss_conf.level);
-		if (flow->frxq[i].hrxq)
-			continue;
-		flow->frxq[i].hrxq =
-			mlx5_hrxq_new(dev,
-				      parser->rss_conf.key,
-				      parser->rss_conf.key_len,
-				      flow->frxq[i].hash_fields,
-				      parser->rss_conf.queue,
-				      parser->rss_conf.queue_num,
-				      parser->tunnel,
-				      parser->rss_conf.level);
-		if (!flow->frxq[i].hrxq) {
-			return rte_flow_error_set(error, ENOMEM,
-						  RTE_FLOW_ERROR_TYPE_HANDLE,
-						  NULL,
-						  "cannot create hash rxq");
-		}
-	}
-	return 0;
-}
-
-/**
- * RXQ update after flow rule creation.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Pointer to the flow rule.
- */
-static void
-mlx5_flow_create_update_rxqs(struct rte_eth_dev *dev, struct rte_flow *flow)
-{
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-	unsigned int j;
-
-	if (!dev->data->dev_started)
-		return;
-	for (i = 0; i != flow->rss_conf.queue_num; ++i) {
-		struct mlx5_rxq_data *rxq_data = (*priv->rxqs)
-						 [(*flow->queues)[i]];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
-		uint8_t tunnel = PTYPE_IDX(flow->tunnel);
-
-		rxq_data->mark |= flow->mark;
-		if (!tunnel)
-			continue;
-		rxq_ctrl->tunnel_types[tunnel] += 1;
-		/* Clear tunnel type if more than one tunnel types set. */
-		for (j = 0; j != RTE_DIM(rxq_ctrl->tunnel_types); ++j) {
-			if (j == tunnel)
-				continue;
-			if (rxq_ctrl->tunnel_types[j] > 0) {
-				rxq_data->tunnel = 0;
-				break;
-			}
-		}
-		if (j == RTE_DIM(rxq_ctrl->tunnel_types))
-			rxq_data->tunnel = flow->tunnel;
-	}
-}
-
-/**
- * Dump flow hash RX queue detail.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Pointer to the rte_flow.
- * @param hrxq_idx
- *   Hash RX queue index.
- */
-static void
-mlx5_flow_dump(struct rte_eth_dev *dev __rte_unused,
-	       struct rte_flow *flow __rte_unused,
-	       unsigned int hrxq_idx __rte_unused)
-{
-#ifndef NDEBUG
-	uintptr_t spec_ptr;
-	uint16_t j;
-	char buf[256];
-	uint8_t off;
-	uint64_t extra_hash_fields = 0;
+/* Dev ops structure defined in mlx5.c */
+extern const struct eth_dev_ops mlx5_dev_ops;
+extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
-#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	if (flow->tunnel && flow->rss_conf.level > 1)
-		extra_hash_fields = (uint32_t)IBV_RX_HASH_INNER;
-#endif
-	spec_ptr = (uintptr_t)(flow->frxq[hrxq_idx].ibv_attr + 1);
-	for (j = 0, off = 0; j < flow->frxq[hrxq_idx].ibv_attr->num_of_specs;
-	     j++) {
-		struct ibv_flow_spec *spec = (void *)spec_ptr;
-		off += sprintf(buf + off, " %x(%hu)", spec->hdr.type,
-			       spec->hdr.size);
-		spec_ptr += spec->hdr.size;
-	}
-	DRV_LOG(DEBUG,
-		"port %u Verbs flow %p type %u: hrxq:%p qp:%p ind:%p,"
-		" hash:%" PRIx64 "/%u specs:%hhu(%hu), priority:%hu, type:%d,"
-		" flags:%x, comp_mask:%x specs:%s",
-		dev->data->port_id, (void *)flow, hrxq_idx,
-		(void *)flow->frxq[hrxq_idx].hrxq,
-		(void *)flow->frxq[hrxq_idx].hrxq->qp,
-		(void *)flow->frxq[hrxq_idx].hrxq->ind_table,
-		(flow->frxq[hrxq_idx].hash_fields | extra_hash_fields),
-		flow->rss_conf.queue_num,
-		flow->frxq[hrxq_idx].ibv_attr->num_of_specs,
-		flow->frxq[hrxq_idx].ibv_attr->size,
-		flow->frxq[hrxq_idx].ibv_attr->priority,
-		flow->frxq[hrxq_idx].ibv_attr->type,
-		flow->frxq[hrxq_idx].ibv_attr->flags,
-		flow->frxq[hrxq_idx].ibv_attr->comp_mask,
-		buf);
-#endif
-}
+struct rte_flow {
+	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
+};
 
-/**
- * Complete flow rule creation.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_action_queue(struct rte_eth_dev *dev,
-			      struct mlx5_flow_parse *parser,
-			      struct rte_flow *flow,
-			      struct rte_flow_error *error)
-{
-	struct priv *priv __rte_unused = dev->data->dev_private;
-	int ret;
-	unsigned int i;
-	unsigned int flows_n = 0;
+static const struct rte_flow_ops mlx5_flow_ops = {
+	.isolate = mlx5_flow_isolate,
+};
 
-	assert(priv->pd);
-	assert(priv->ctx);
-	assert(!parser->drop);
-	ret = mlx5_flow_create_action_queue_rss(dev, parser, flow, error);
-	if (ret)
-		goto error;
-	if (parser->count)
-		flow->cs = parser->cs;
-	if (!dev->data->dev_started)
-		return 0;
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!flow->frxq[i].hrxq)
-			continue;
-		flow->frxq[i].ibv_flow =
-			mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
-					       flow->frxq[i].ibv_attr);
-		mlx5_flow_dump(dev, flow, i);
-		if (!flow->frxq[i].ibv_flow) {
-			rte_flow_error_set(error, ENOMEM,
-					   RTE_FLOW_ERROR_TYPE_HANDLE,
-					   NULL, "flow rule creation failure");
-			goto error;
-		}
-		++flows_n;
-	}
-	if (!flows_n) {
-		rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "internal error in flow creation");
-		goto error;
-	}
-	mlx5_flow_create_update_rxqs(dev, flow);
-	return 0;
-error:
-	ret = rte_errno; /* Save rte_errno before cleanup. */
-	assert(flow);
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (flow->frxq[i].ibv_flow) {
-			struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
+/* Convert FDIR request to Generic flow. */
+struct mlx5_fdir {
+	struct rte_flow_attr attr;
+	struct rte_flow_action actions[2];
+	struct rte_flow_item items[4];
+	struct rte_flow_item_eth l2;
+	struct rte_flow_item_eth l2_mask;
+	union {
+		struct rte_flow_item_ipv4 ipv4;
+		struct rte_flow_item_ipv6 ipv6;
+	} l3;
+	union {
+		struct rte_flow_item_ipv4 ipv4;
+		struct rte_flow_item_ipv6 ipv6;
+	} l3_mask;
+	union {
+		struct rte_flow_item_udp udp;
+		struct rte_flow_item_tcp tcp;
+	} l4;
+	union {
+		struct rte_flow_item_udp udp;
+		struct rte_flow_item_tcp tcp;
+	} l4_mask;
+	struct rte_flow_action_queue queue;
+};
 
-			claim_zero(mlx5_glue->destroy_flow(ibv_flow));
-		}
-		if (flow->frxq[i].hrxq)
-			mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
-		if (flow->frxq[i].ibv_attr)
-			rte_free(flow->frxq[i].ibv_attr);
-	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-		parser->cs = NULL;
-	}
-	rte_errno = ret; /* Restore rte_errno. */
-	return -rte_errno;
-}
+/* Verbs specification header. */
+struct ibv_spec_header {
+	enum ibv_flow_spec_type type;
+	uint16_t size;
+};
 
 /**
  * Convert a flow.
@@ -2452,69 +95,17 @@ mlx5_flow_create_action_queue(struct rte_eth_dev *dev,
  *   A flow on success, NULL otherwise and rte_errno is set.
  */
 static struct rte_flow *
-mlx5_flow_list_create(struct rte_eth_dev *dev,
-		      struct mlx5_flows *list,
-		      const struct rte_flow_attr *attr,
-		      const struct rte_flow_item items[],
-		      const struct rte_flow_action actions[],
+mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
+		      struct mlx5_flows *list __rte_unused,
+		      const struct rte_flow_attr *attr __rte_unused,
+		      const struct rte_flow_item items[] __rte_unused,
+		      const struct rte_flow_action actions[] __rte_unused,
 		      struct rte_flow_error *error)
 {
-	struct mlx5_flow_parse parser = { .create = 1, };
-	struct rte_flow *flow = NULL;
-	unsigned int i;
-	int ret;
-
-	ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser);
-	if (ret)
-		goto exit;
-	flow = rte_calloc(__func__, 1,
-			  sizeof(*flow) +
-			  parser.rss_conf.queue_num * sizeof(uint16_t),
-			  0);
-	if (!flow) {
-		rte_flow_error_set(error, ENOMEM,
-				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				   NULL,
-				   "cannot allocate flow memory");
-		return NULL;
-	}
-	/* Copy configuration. */
-	flow->queues = (uint16_t (*)[])(flow + 1);
-	flow->tunnel = parser.tunnel;
-	flow->rss_conf = (struct rte_flow_action_rss){
-		.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
-		.level = parser.rss_conf.level,
-		.types = parser.rss_conf.types,
-		.key_len = parser.rss_conf.key_len,
-		.queue_num = parser.rss_conf.queue_num,
-		.key = memcpy(flow->rss_key, parser.rss_conf.key,
-			      sizeof(*parser.rss_conf.key) *
-			      parser.rss_conf.key_len),
-		.queue = memcpy(flow->queues, parser.rss_conf.queue,
-				sizeof(*parser.rss_conf.queue) *
-				parser.rss_conf.queue_num),
-	};
-	flow->mark = parser.mark;
-	/* finalise the flow. */
-	if (parser.drop)
-		ret = mlx5_flow_create_action_queue_drop(dev, &parser, flow,
-							 error);
-	else
-		ret = mlx5_flow_create_action_queue(dev, &parser, flow, error);
-	if (ret)
-		goto exit;
-	TAILQ_INSERT_TAIL(list, flow, next);
-	DRV_LOG(DEBUG, "port %u flow created %p", dev->data->port_id,
-		(void *)flow);
-	return flow;
-exit:
-	DRV_LOG(ERR, "port %u flow creation error: %s", dev->data->port_id,
-		error->message);
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser.queue[i].ibv_attr)
-			rte_free(parser.queue[i].ibv_attr);
-	}
-	rte_free(flow);
+	rte_flow_error_set(error, ENOTSUP,
+			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			   NULL,
+			   "action not supported");
 	return NULL;
 }
 
@@ -2525,15 +116,16 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
  * @see rte_flow_ops
  */
 int
-mlx5_flow_validate(struct rte_eth_dev *dev,
-		   const struct rte_flow_attr *attr,
-		   const struct rte_flow_item items[],
-		   const struct rte_flow_action actions[],
+mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
+		   const struct rte_flow_attr *attr __rte_unused,
+		   const struct rte_flow_item items[] __rte_unused,
+		   const struct rte_flow_action actions[] __rte_unused,
 		   struct rte_flow_error *error)
 {
-	struct mlx5_flow_parse parser = { .create = 0, };
-
-	return mlx5_flow_convert(dev, attr, items, actions, error, &parser);
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "action not supported");
 }
 
 /**
@@ -2543,16 +135,17 @@ mlx5_flow_validate(struct rte_eth_dev *dev,
  * @see rte_flow_ops
  */
 struct rte_flow *
-mlx5_flow_create(struct rte_eth_dev *dev,
-		 const struct rte_flow_attr *attr,
-		 const struct rte_flow_item items[],
-		 const struct rte_flow_action actions[],
+mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
+		 const struct rte_flow_attr *attr __rte_unused,
+		 const struct rte_flow_item items[] __rte_unused,
+		 const struct rte_flow_action actions[] __rte_unused,
 		 struct rte_flow_error *error)
 {
-	struct priv *priv = dev->data->dev_private;
-
-	return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions,
-				     error);
+	rte_flow_error_set(error, ENOTSUP,
+			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			   NULL,
+			   "action not supported");
+	return NULL;
 }
 
 /**
@@ -2566,99 +159,10 @@ mlx5_flow_create(struct rte_eth_dev *dev,
  *   Flow to destroy.
  */
 static void
-mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
-		       struct rte_flow *flow)
+mlx5_flow_list_destroy(struct rte_eth_dev *dev __rte_unused,
+		       struct mlx5_flows *list __rte_unused,
+		       struct rte_flow *flow __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-
-	if (flow->drop || !dev->data->dev_started)
-		goto free;
-	for (i = 0; flow->tunnel && i != flow->rss_conf.queue_num; ++i) {
-		/* Update queue tunnel type. */
-		struct mlx5_rxq_data *rxq_data = (*priv->rxqs)
-						 [(*flow->queues)[i]];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
-		uint8_t tunnel = PTYPE_IDX(flow->tunnel);
-
-		assert(rxq_ctrl->tunnel_types[tunnel] > 0);
-		rxq_ctrl->tunnel_types[tunnel] -= 1;
-		if (!rxq_ctrl->tunnel_types[tunnel]) {
-			/* Update tunnel type. */
-			uint8_t j;
-			uint8_t types = 0;
-			uint8_t last;
-
-			for (j = 0; j < RTE_DIM(rxq_ctrl->tunnel_types); j++)
-				if (rxq_ctrl->tunnel_types[j]) {
-					types += 1;
-					last = j;
-				}
-			/* Keep same if more than one tunnel types left. */
-			if (types == 1)
-				rxq_data->tunnel = ptype_ext[last];
-			else if (types == 0)
-				/* No tunnel type left. */
-				rxq_data->tunnel = 0;
-		}
-	}
-	for (i = 0; flow->mark && i != flow->rss_conf.queue_num; ++i) {
-		struct rte_flow *tmp;
-		int mark = 0;
-
-		/*
-		 * To remove the mark from the queue, the queue must not be
-		 * present in any other marked flow (RSS or not).
-		 */
-		TAILQ_FOREACH(tmp, list, next) {
-			unsigned int j;
-			uint16_t *tqs = NULL;
-			uint16_t tq_n = 0;
-
-			if (!tmp->mark)
-				continue;
-			for (j = 0; j != hash_rxq_init_n; ++j) {
-				if (!tmp->frxq[j].hrxq)
-					continue;
-				tqs = tmp->frxq[j].hrxq->ind_table->queues;
-				tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
-			}
-			if (!tq_n)
-				continue;
-			for (j = 0; (j != tq_n) && !mark; j++)
-				if (tqs[j] == (*flow->queues)[i])
-					mark = 1;
-		}
-		(*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
-	}
-free:
-	if (flow->drop) {
-		if (flow->frxq[HASH_RXQ_ETH].ibv_flow)
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-		rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
-	} else {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			struct mlx5_flow *frxq = &flow->frxq[i];
-
-			if (frxq->ibv_flow)
-				claim_zero(mlx5_glue->destroy_flow
-					   (frxq->ibv_flow));
-			if (frxq->hrxq)
-				mlx5_hrxq_release(dev, frxq->hrxq);
-			if (frxq->ibv_attr)
-				rte_free(frxq->ibv_attr);
-		}
-	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-	}
-	TAILQ_REMOVE(list, flow, next);
-	DRV_LOG(DEBUG, "port %u flow destroyed %p", dev->data->port_id,
-		(void *)flow);
-	rte_free(flow);
 }
 
 /**
@@ -2690,97 +194,9 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_flow_create_drop_queue(struct rte_eth_dev *dev)
+mlx5_flow_create_drop_queue(struct rte_eth_dev *dev __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_hrxq_drop *fdq = NULL;
-
-	assert(priv->pd);
-	assert(priv->ctx);
-	fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
-	if (!fdq) {
-		DRV_LOG(WARNING,
-			"port %u cannot allocate memory for drop queue",
-			dev->data->port_id);
-		rte_errno = ENOMEM;
-		return -rte_errno;
-	}
-	fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
-	if (!fdq->cq) {
-		DRV_LOG(WARNING, "port %u cannot allocate CQ for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->wq = mlx5_glue->create_wq
-		(priv->ctx,
-		 &(struct ibv_wq_init_attr){
-			.wq_type = IBV_WQT_RQ,
-			.max_wr = 1,
-			.max_sge = 1,
-			.pd = priv->pd,
-			.cq = fdq->cq,
-		 });
-	if (!fdq->wq) {
-		DRV_LOG(WARNING, "port %u cannot allocate WQ for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->ind_table = mlx5_glue->create_rwq_ind_table
-		(priv->ctx,
-		 &(struct ibv_rwq_ind_table_init_attr){
-			.log_ind_tbl_size = 0,
-			.ind_tbl = &fdq->wq,
-			.comp_mask = 0,
-		 });
-	if (!fdq->ind_table) {
-		DRV_LOG(WARNING,
-			"port %u cannot allocate indirection table for drop"
-			" queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->qp = mlx5_glue->create_qp_ex
-		(priv->ctx,
-		 &(struct ibv_qp_init_attr_ex){
-			.qp_type = IBV_QPT_RAW_PACKET,
-			.comp_mask =
-				IBV_QP_INIT_ATTR_PD |
-				IBV_QP_INIT_ATTR_IND_TABLE |
-				IBV_QP_INIT_ATTR_RX_HASH,
-			.rx_hash_conf = (struct ibv_rx_hash_conf){
-				.rx_hash_function =
-					IBV_RX_HASH_FUNC_TOEPLITZ,
-				.rx_hash_key_len = rss_hash_default_key_len,
-				.rx_hash_key = rss_hash_default_key,
-				.rx_hash_fields_mask = 0,
-				},
-			.rwq_ind_tbl = fdq->ind_table,
-			.pd = priv->pd
-		 });
-	if (!fdq->qp) {
-		DRV_LOG(WARNING, "port %u cannot allocate QP for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	priv->flow_drop_queue = fdq;
 	return 0;
-error:
-	if (fdq->qp)
-		claim_zero(mlx5_glue->destroy_qp(fdq->qp));
-	if (fdq->ind_table)
-		claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
-	if (fdq->wq)
-		claim_zero(mlx5_glue->destroy_wq(fdq->wq));
-	if (fdq->cq)
-		claim_zero(mlx5_glue->destroy_cq(fdq->cq));
-	if (fdq)
-		rte_free(fdq);
-	priv->flow_drop_queue = NULL;
-	return -rte_errno;
 }
 
 /**
@@ -2790,23 +206,8 @@ mlx5_flow_create_drop_queue(struct rte_eth_dev *dev)
  *   Pointer to Ethernet device.
  */
 void
-mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
+mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
-
-	if (!fdq)
-		return;
-	if (fdq->qp)
-		claim_zero(mlx5_glue->destroy_qp(fdq->qp));
-	if (fdq->ind_table)
-		claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
-	if (fdq->wq)
-		claim_zero(mlx5_glue->destroy_wq(fdq->wq));
-	if (fdq->cq)
-		claim_zero(mlx5_glue->destroy_cq(fdq->cq));
-	rte_free(fdq);
-	priv->flow_drop_queue = NULL;
 }
 
 /**
@@ -2818,70 +219,9 @@ mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
  *   Pointer to a TAILQ flow list.
  */
 void
-mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
+mlx5_flow_stop(struct rte_eth_dev *dev __rte_unused,
+	       struct mlx5_flows *list __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct rte_flow *flow;
-	unsigned int i;
-
-	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
-		struct mlx5_ind_table_ibv *ind_tbl = NULL;
-
-		if (flow->drop) {
-			if (!flow->frxq[HASH_RXQ_ETH].ibv_flow)
-				continue;
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-			flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
-			DRV_LOG(DEBUG, "port %u flow %p removed",
-				dev->data->port_id, (void *)flow);
-			/* Next flow. */
-			continue;
-		}
-		/* Verify the flow has not already been cleaned. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_flow)
-				continue;
-			/*
-			 * Indirection table may be necessary to remove the
-			 * flags in the Rx queues.
-			 * This helps to speed-up the process by avoiding
-			 * another loop.
-			 */
-			ind_tbl = flow->frxq[i].hrxq->ind_table;
-			break;
-		}
-		if (i == hash_rxq_init_n)
-			return;
-		if (flow->mark) {
-			assert(ind_tbl);
-			for (i = 0; i != ind_tbl->queues_n; ++i)
-				(*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
-		}
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_flow)
-				continue;
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[i].ibv_flow));
-			flow->frxq[i].ibv_flow = NULL;
-			mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
-			flow->frxq[i].hrxq = NULL;
-		}
-		DRV_LOG(DEBUG, "port %u flow %p removed", dev->data->port_id,
-			(void *)flow);
-	}
-	/* Cleanup Rx queue tunnel info. */
-	for (i = 0; i != priv->rxqs_n; ++i) {
-		struct mlx5_rxq_data *q = (*priv->rxqs)[i];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(q, struct mlx5_rxq_ctrl, rxq);
-
-		if (!q)
-			continue;
-		memset((void *)rxq_ctrl->tunnel_types, 0,
-		       sizeof(rxq_ctrl->tunnel_types));
-		q->tunnel = 0;
-	}
 }
 
 /**
@@ -2896,76 +236,9 @@ mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
+mlx5_flow_start(struct rte_eth_dev *dev __rte_unused,
+		struct mlx5_flows *list __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct rte_flow *flow;
-
-	TAILQ_FOREACH(flow, list, next) {
-		unsigned int i;
-
-		if (flow->drop) {
-			flow->frxq[HASH_RXQ_ETH].ibv_flow =
-				mlx5_glue->create_flow
-				(priv->flow_drop_queue->qp,
-				 flow->frxq[HASH_RXQ_ETH].ibv_attr);
-			if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p cannot be applied",
-					dev->data->port_id, (void *)flow);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-			DRV_LOG(DEBUG, "port %u flow %p applied",
-				dev->data->port_id, (void *)flow);
-			/* Next flow. */
-			continue;
-		}
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_attr)
-				continue;
-			flow->frxq[i].hrxq =
-				mlx5_hrxq_get(dev, flow->rss_conf.key,
-					      flow->rss_conf.key_len,
-					      flow->frxq[i].hash_fields,
-					      flow->rss_conf.queue,
-					      flow->rss_conf.queue_num,
-					      flow->tunnel,
-					      flow->rss_conf.level);
-			if (flow->frxq[i].hrxq)
-				goto flow_create;
-			flow->frxq[i].hrxq =
-				mlx5_hrxq_new(dev, flow->rss_conf.key,
-					      flow->rss_conf.key_len,
-					      flow->frxq[i].hash_fields,
-					      flow->rss_conf.queue,
-					      flow->rss_conf.queue_num,
-					      flow->tunnel,
-					      flow->rss_conf.level);
-			if (!flow->frxq[i].hrxq) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p cannot create hash"
-					" rxq",
-					dev->data->port_id, (void *)flow);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-flow_create:
-			mlx5_flow_dump(dev, flow, i);
-			flow->frxq[i].ibv_flow =
-				mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
-						       flow->frxq[i].ibv_attr);
-			if (!flow->frxq[i].ibv_flow) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p type %u cannot be"
-					" applied",
-					dev->data->port_id, (void *)flow, i);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-		}
-		mlx5_flow_create_update_rxqs(dev, flow);
-	}
 	return 0;
 }
 
@@ -3019,7 +292,6 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
-		.priority = MLX5_CTRL_FLOW_PRIORITY,
 	};
 	struct rte_flow_item items[] = {
 		{
@@ -3129,83 +401,6 @@ mlx5_flow_flush(struct rte_eth_dev *dev,
 	return 0;
 }
 
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-/**
- * Query flow counter.
- *
- * @param cs
- *   the counter set.
- * @param counter_value
- *   returned data from the counter.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_query_count(struct ibv_counter_set *cs,
-		      struct mlx5_flow_counter_stats *counter_stats,
-		      struct rte_flow_query_count *query_count,
-		      struct rte_flow_error *error)
-{
-	uint64_t counters[2];
-	struct ibv_query_counter_set_attr query_cs_attr = {
-		.cs = cs,
-		.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
-	};
-	struct ibv_counter_set_data query_out = {
-		.out = counters,
-		.outlen = 2 * sizeof(uint64_t),
-	};
-	int err = mlx5_glue->query_counter_set(&query_cs_attr, &query_out);
-
-	if (err)
-		return rte_flow_error_set(error, err,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "cannot read counter");
-	query_count->hits_set = 1;
-	query_count->bytes_set = 1;
-	query_count->hits = counters[0] - counter_stats->hits;
-	query_count->bytes = counters[1] - counter_stats->bytes;
-	if (query_count->reset) {
-		counter_stats->hits = counters[0];
-		counter_stats->bytes = counters[1];
-	}
-	return 0;
-}
-
-/**
- * Query a flows.
- *
- * @see rte_flow_query()
- * @see rte_flow_ops
- */
-int
-mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
-		struct rte_flow *flow,
-		const struct rte_flow_action *action __rte_unused,
-		void *data,
-		struct rte_flow_error *error)
-{
-	if (flow->cs) {
-		int ret;
-
-		ret = mlx5_flow_query_count(flow->cs,
-					    &flow->counter_stats,
-					    (struct rte_flow_query_count *)data,
-					    error);
-		if (ret)
-			return ret;
-	} else {
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "no counter found for flow");
-	}
-	return 0;
-}
-#endif
-
 /**
  * Isolated mode.
  *
@@ -3445,18 +640,11 @@ mlx5_fdir_filter_add(struct rte_eth_dev *dev,
 			.type = 0,
 		},
 	};
-	struct mlx5_flow_parse parser = {
-		.layer = HASH_RXQ_ETH,
-	};
 	struct rte_flow_error error;
 	struct rte_flow *flow;
 	int ret;
 
 	ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
-				attributes.actions, &error, &parser);
 	if (ret)
 		return ret;
 	flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr,
@@ -3482,96 +670,11 @@ mlx5_fdir_filter_add(struct rte_eth_dev *dev,
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx5_fdir_filter_delete(struct rte_eth_dev *dev,
-			const struct rte_eth_fdir_filter *fdir_filter)
+mlx5_fdir_filter_delete(struct rte_eth_dev *dev __rte_unused,
+			const struct rte_eth_fdir_filter *fdir_filter
+			__rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_fdir attributes = {
-		.attr.group = 0,
-	};
-	struct mlx5_flow_parse parser = {
-		.create = 1,
-		.layer = HASH_RXQ_ETH,
-	};
-	struct rte_flow_error error;
-	struct rte_flow *flow;
-	unsigned int i;
-	int ret;
-
-	ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
-				attributes.actions, &error, &parser);
-	if (ret)
-		goto exit;
-	/*
-	 * Special case for drop action which is only set in the
-	 * specifications when the flow is created.  In this situation the
-	 * drop specification is missing.
-	 */
-	if (parser.drop) {
-		struct ibv_flow_spec_action_drop *drop;
-
-		drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr +
-				parser.queue[HASH_RXQ_ETH].offset);
-		*drop = (struct ibv_flow_spec_action_drop){
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-			.size = sizeof(struct ibv_flow_spec_action_drop),
-		};
-		parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++;
-	}
-	TAILQ_FOREACH(flow, &priv->flows, next) {
-		struct ibv_flow_attr *attr;
-		struct ibv_spec_header *attr_h;
-		void *spec;
-		struct ibv_flow_attr *flow_attr;
-		struct ibv_spec_header *flow_h;
-		void *flow_spec;
-		unsigned int specs_n;
-		unsigned int queue_id = parser.drop ? HASH_RXQ_ETH :
-						      parser.layer;
-
-		attr = parser.queue[queue_id].ibv_attr;
-		flow_attr = flow->frxq[queue_id].ibv_attr;
-		/* Compare first the attributes. */
-		if (!flow_attr ||
-		    memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
-			continue;
-		if (attr->num_of_specs == 0)
-			continue;
-		spec = (void *)((uintptr_t)attr +
-				sizeof(struct ibv_flow_attr));
-		flow_spec = (void *)((uintptr_t)flow_attr +
-				     sizeof(struct ibv_flow_attr));
-		specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
-		for (i = 0; i != specs_n; ++i) {
-			attr_h = spec;
-			flow_h = flow_spec;
-			if (memcmp(spec, flow_spec,
-				   RTE_MIN(attr_h->size, flow_h->size)))
-				goto wrong_flow;
-			spec = (void *)((uintptr_t)spec + attr_h->size);
-			flow_spec = (void *)((uintptr_t)flow_spec +
-					     flow_h->size);
-		}
-		/* At this point, the flow match. */
-		break;
-wrong_flow:
-		/* The flow does not match. */
-		continue;
-	}
-	if (flow)
-		mlx5_flow_list_destroy(dev, &priv->flows, flow);
-exit:
-	if (ret)
-		ret = rte_errno; /* Save rte_errno before cleanup. */
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser.queue[i].ibv_attr)
-			rte_free(parser.queue[i].ibv_attr);
-	}
-	if (ret)
-		rte_errno = ret; /* Restore rte_errno. */
+	rte_errno = ENOTSUP;
 	return -rte_errno;
 }
 
@@ -3738,45 +841,7 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
  *   number of supported Verbs flow priority.
  */
 unsigned int
-mlx5_get_max_verbs_prio(struct rte_eth_dev *dev)
+mlx5_get_max_verbs_prio(struct rte_eth_dev *dev __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	unsigned int verb_priorities = MLX5_VERBS_FLOW_PRIO_8;
-	struct {
-		struct ibv_flow_attr attr;
-		struct ibv_flow_spec_eth eth;
-		struct ibv_flow_spec_action_drop drop;
-	} flow_attr = {
-		.attr = {
-			.num_of_specs = 2,
-		},
-		.eth = {
-			.type = IBV_FLOW_SPEC_ETH,
-			.size = sizeof(struct ibv_flow_spec_eth),
-		},
-		.drop = {
-			.size = sizeof(struct ibv_flow_spec_action_drop),
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-		},
-	};
-	struct ibv_flow *flow;
-
-	do {
-		flow_attr.attr.priority = verb_priorities - 1;
-		flow = mlx5_glue->create_flow(priv->flow_drop_queue->qp,
-					      &flow_attr.attr);
-		if (flow) {
-			claim_zero(mlx5_glue->destroy_flow(flow));
-			/* Try more priorities. */
-			verb_priorities *= 2;
-		} else {
-			/* Failed, restore last right number. */
-			verb_priorities /= 2;
-			break;
-		}
-	} while (1);
-	DRV_LOG(DEBUG, "port %u Verbs flow priorities: %d,"
-		" user flow priorities: %d",
-		dev->data->port_id, verb_priorities, MLX5_CTRL_FLOW_PRIORITY);
-	return verb_priorities;
+	return 8;
 }
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 01/20] net/mlx5: remove flow support

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:33PM +0200, Nelio Laranjeiro wrote:
> This start a series to re-work the flow engine in mlx5 to easily support
> flow conversion to Verbs or TC.  This is necessary to handle both regular
> flows and representors flows.
> 
> As the full file needs to be clean-up to re-write all items/actions
> processing, this patch starts to disable the regular code and only let the
> PMD to start in isolated mode.
> 
> After this patch flow API will not be usable.
> 
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
Acked-by: Yongseok Koh <yskoh@mellanox.com>

Thanks

[dpdk-dev] [PATCH v2 02/20] net/mlx5: handle drop queues are regular queues

[Nelio Laranjeiro]

Drop queues are essentially used in flows due to Verbs API, the
information if the fate of the flow is a drop or not is already present
in the flow.  Due to this, drop queues can be fully mapped on regular
queues.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5.c      |  24 ++--
 drivers/net/mlx5/mlx5.h      |  14 ++-
 drivers/net/mlx5/mlx5_flow.c |  94 +++++++-------
 drivers/net/mlx5/mlx5_rxq.c  | 232 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_rxtx.h |   8 ++
 5 files changed, 310 insertions(+), 62 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index d01c0f6cf..aba6c1f9f 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -260,7 +260,6 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 		priv->txqs_n = 0;
 		priv->txqs = NULL;
 	}
-	mlx5_flow_delete_drop_queue(dev);
 	mlx5_mprq_free_mp(dev);
 	mlx5_mr_release(dev);
 	if (priv->pd != NULL) {
@@ -1100,22 +1099,15 @@ mlx5_dev_spawn_one(struct rte_device *dpdk_dev,
 	mlx5_link_update(eth_dev, 0);
 	/* Store device configuration on private structure. */
 	priv->config = config;
-	/* Create drop queue. */
-	err = mlx5_flow_create_drop_queue(eth_dev);
-	if (err) {
-		DRV_LOG(ERR, "port %u drop queue allocation failed: %s",
-			eth_dev->data->port_id, strerror(rte_errno));
-		err = rte_errno;
-		goto error;
-	}
 	/* Supported Verbs flow priority number detection. */
-	if (verb_priorities == 0)
-		verb_priorities = mlx5_get_max_verbs_prio(eth_dev);
-	if (verb_priorities < MLX5_VERBS_FLOW_PRIO_8) {
-		DRV_LOG(ERR, "port %u wrong Verbs flow priorities: %u",
-			eth_dev->data->port_id, verb_priorities);
-		err = ENOTSUP;
-		goto error;
+	if (verb_priorities == 0) {
+		err = mlx5_verbs_max_prio(eth_dev);
+		if (err < 0) {
+			DRV_LOG(ERR, "port %u wrong Verbs flow priorities",
+				eth_dev->data->port_id);
+			goto error;
+		}
+		verb_priorities = err;
 	}
 	priv->config.max_verbs_prio = verb_priorities;
 	/*
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 6674a77d4..11aa0932f 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -131,9 +131,6 @@ enum mlx5_verbs_alloc_type {
 	MLX5_VERBS_ALLOC_TYPE_RX_QUEUE,
 };
 
-/* 8 Verbs priorities. */
-#define MLX5_VERBS_FLOW_PRIO_8 8
-
 /**
  * Verbs allocator needs a context to know in the callback which kind of
  * resources it is allocating.
@@ -145,6 +142,12 @@ struct mlx5_verbs_alloc_ctx {
 
 LIST_HEAD(mlx5_mr_list, mlx5_mr);
 
+/* Flow drop context necessary due to Verbs API. */
+struct mlx5_drop {
+	struct mlx5_hrxq *hrxq; /* Hash Rx queue queue. */
+	struct mlx5_rxq_ibv *rxq; /* Verbs Rx queue. */
+};
+
 struct priv {
 	LIST_ENTRY(priv) mem_event_cb; /* Called by memory event callback. */
 	struct rte_eth_dev_data *dev_data;  /* Pointer to device data. */
@@ -175,7 +178,7 @@ struct priv {
 	struct rte_intr_handle intr_handle; /* Interrupt handler. */
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
-	struct mlx5_hrxq_drop *flow_drop_queue; /* Flow drop queue. */
+	struct mlx5_drop drop; /* Flow drop queues. */
 	struct mlx5_flows flows; /* RTE Flow rules. */
 	struct mlx5_flows ctrl_flows; /* Control flow rules. */
 	struct {
@@ -306,7 +309,8 @@ int mlx5_traffic_restart(struct rte_eth_dev *dev);
 
 /* mlx5_flow.c */
 
-unsigned int mlx5_get_max_verbs_prio(struct rte_eth_dev *dev);
+int mlx5_verbs_max_prio(struct rte_eth_dev *dev);
+void mlx5_flow_print(struct rte_flow *flow);
 int mlx5_flow_validate(struct rte_eth_dev *dev,
 		       const struct rte_flow_attr *attr,
 		       const struct rte_flow_item items[],
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index a45cb06e1..8b5b695d4 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -75,6 +75,58 @@ struct ibv_spec_header {
 	uint16_t size;
 };
 
+ /**
+  * Get the maximum number of priority available.
+  *
+  * @param dev
+  *   Pointer to Ethernet device.
+  *
+  * @return
+  *   number of supported Verbs flow priority on success, a negative errno
+  *   value otherwise and rte_errno is set.
+  */
+int
+mlx5_verbs_max_prio(struct rte_eth_dev *dev)
+{
+	struct {
+		struct ibv_flow_attr attr;
+		struct ibv_flow_spec_eth eth;
+		struct ibv_flow_spec_action_drop drop;
+	} flow_attr = {
+		.attr = {
+			.num_of_specs = 2,
+		},
+		.eth = {
+			.type = IBV_FLOW_SPEC_ETH,
+			.size = sizeof(struct ibv_flow_spec_eth),
+		},
+		.drop = {
+			.size = sizeof(struct ibv_flow_spec_action_drop),
+			.type = IBV_FLOW_SPEC_ACTION_DROP,
+		},
+	};
+	struct ibv_flow *flow;
+	uint32_t verb_priorities;
+	struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
+
+	if (!drop) {
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	for (verb_priorities = 0; 1; verb_priorities++) {
+		flow_attr.attr.priority = verb_priorities;
+		flow = mlx5_glue->create_flow(drop->qp,
+					      &flow_attr.attr);
+		if (!flow)
+			break;
+		claim_zero(mlx5_glue->destroy_flow(flow));
+	}
+	mlx5_hrxq_drop_release(dev, drop);
+	DRV_LOG(INFO, "port %u flow maximum priority: %d",
+		dev->data->port_id, verb_priorities);
+	return verb_priorities;
+}
+
 /**
  * Convert a flow.
  *
@@ -184,32 +236,6 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
 	}
 }
 
-/**
- * Create drop queue.
- *
- * @param dev
- *   Pointer to Ethernet device.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-int
-mlx5_flow_create_drop_queue(struct rte_eth_dev *dev __rte_unused)
-{
-	return 0;
-}
-
-/**
- * Delete drop queue.
- *
- * @param dev
- *   Pointer to Ethernet device.
- */
-void
-mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev __rte_unused)
-{
-}
-
 /**
  * Remove all flows.
  *
@@ -292,6 +318,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
+		.priority = priv->config.max_verbs_prio - 1,
 	};
 	struct rte_flow_item items[] = {
 		{
@@ -830,18 +857,3 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
 	}
 	return 0;
 }
-
-/**
- * Detect number of Verbs flow priorities supported.
- *
- * @param dev
- *   Pointer to Ethernet device.
- *
- * @return
- *   number of supported Verbs flow priority.
- */
-unsigned int
-mlx5_get_max_verbs_prio(struct rte_eth_dev *dev __rte_unused)
-{
-	return 8;
-}
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 17db7c160..5f17dce50 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1949,3 +1949,235 @@ mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev)
 	}
 	return ret;
 }
+
+/**
+ * Create a drop Rx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_rxq_ibv *
+mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct ibv_cq *cq;
+	struct ibv_wq *wq = NULL;
+	struct mlx5_rxq_ibv *rxq;
+
+	if (priv->drop.rxq)
+		return priv->drop.rxq;
+	cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
+	if (!cq) {
+		DEBUG("port %u cannot allocate CQ for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	wq = mlx5_glue->create_wq(priv->ctx,
+		 &(struct ibv_wq_init_attr){
+			.wq_type = IBV_WQT_RQ,
+			.max_wr = 1,
+			.max_sge = 1,
+			.pd = priv->pd,
+			.cq = cq,
+		 });
+	if (!wq) {
+		DEBUG("port %u cannot allocate WQ for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	rxq = rte_calloc(__func__, 1, sizeof(*rxq), 0);
+	if (!rxq) {
+		DEBUG("port %u cannot allocate drop Rx queue memory",
+		      dev->data->port_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	rxq->cq = cq;
+	rxq->wq = wq;
+	priv->drop.rxq = rxq;
+	return rxq;
+error:
+	if (wq)
+		claim_zero(mlx5_glue->destroy_wq(wq));
+	if (cq)
+		claim_zero(mlx5_glue->destroy_cq(cq));
+	return NULL;
+}
+
+/**
+ * Release a drop Rx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param rxq
+ *   Pointer to the drop Verbs Rx queue.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+void
+mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev, struct mlx5_rxq_ibv *rxq)
+{
+	if (rxq->wq)
+		claim_zero(mlx5_glue->destroy_wq(rxq->wq));
+	if (rxq->cq)
+		claim_zero(mlx5_glue->destroy_cq(rxq->cq));
+	rte_free(rxq);
+	((struct priv *)dev->data->dev_private)->drop.rxq = NULL;
+}
+
+/**
+ * Create a drop indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_ind_table_ibv *
+mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	struct mlx5_rxq_ibv *rxq;
+	struct mlx5_ind_table_ibv tmpl;
+
+	rxq = mlx5_rxq_ibv_drop_new(dev);
+	if (!rxq)
+		return NULL;
+	tmpl.ind_table = mlx5_glue->create_rwq_ind_table
+		(priv->ctx,
+		 &(struct ibv_rwq_ind_table_init_attr){
+			.log_ind_tbl_size = 0,
+			.ind_tbl = &rxq->wq,
+			.comp_mask = 0,
+		 });
+	if (!tmpl.ind_table) {
+		DEBUG("port %u cannot allocate indirection table for drop"
+		      " queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	ind_tbl = rte_calloc(__func__, 1, sizeof(*ind_tbl), 0);
+	if (!ind_tbl) {
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	ind_tbl->ind_table = tmpl.ind_table;
+	return ind_tbl;
+error:
+	mlx5_rxq_ibv_drop_release(dev, rxq);
+	return NULL;
+}
+
+/**
+ * Release a drop indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param ind_tbl
+ *   Indirection table to release.
+ */
+void
+mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev,
+				struct mlx5_ind_table_ibv *ind_tbl)
+{
+	claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl->ind_table));
+	mlx5_rxq_ibv_drop_release
+		(dev, ((struct priv *)dev->data->dev_private)->drop.rxq);
+	rte_free(ind_tbl);
+}
+
+/**
+ * Create a drop Rx Hash queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_hrxq *
+mlx5_hrxq_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	struct ibv_qp *qp;
+	struct mlx5_hrxq *hrxq;
+
+	if (priv->drop.hrxq) {
+		rte_atomic32_inc(&priv->drop.hrxq->refcnt);
+		return priv->drop.hrxq;
+	}
+	ind_tbl = mlx5_ind_table_ibv_drop_new(dev);
+	if (!ind_tbl)
+		return NULL;
+	qp = mlx5_glue->create_qp_ex(priv->ctx,
+		 &(struct ibv_qp_init_attr_ex){
+			.qp_type = IBV_QPT_RAW_PACKET,
+			.comp_mask =
+				IBV_QP_INIT_ATTR_PD |
+				IBV_QP_INIT_ATTR_IND_TABLE |
+				IBV_QP_INIT_ATTR_RX_HASH,
+			.rx_hash_conf = (struct ibv_rx_hash_conf){
+				.rx_hash_function =
+					IBV_RX_HASH_FUNC_TOEPLITZ,
+				.rx_hash_key_len = rss_hash_default_key_len,
+				.rx_hash_key = rss_hash_default_key,
+				.rx_hash_fields_mask = 0,
+				},
+			.rwq_ind_tbl = ind_tbl->ind_table,
+			.pd = priv->pd
+		 });
+	if (!qp) {
+		DEBUG("port %u cannot allocate QP for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	hrxq = rte_calloc(__func__, 1, sizeof(*hrxq), 0);
+	if (!hrxq) {
+		DRV_LOG(WARNING,
+			"port %u cannot allocate memory for drop queue",
+			dev->data->port_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	hrxq->ind_table = ind_tbl;
+	hrxq->qp = qp;
+	priv->drop.hrxq = hrxq;
+	rte_atomic32_set(&hrxq->refcnt, 1);
+	return hrxq;
+error:
+	if (ind_tbl)
+		mlx5_ind_table_ibv_drop_release(dev, ind_tbl);
+	return NULL;
+}
+
+/**
+ * Release a drop hash Rx queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param hrxq
+ *   Pointer to Hash Rx queue to release.
+ */
+void
+mlx5_hrxq_drop_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
+		claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
+		mlx5_ind_table_ibv_drop_release(dev, hrxq->ind_table);
+		rte_free(hrxq);
+		priv->drop.hrxq = NULL;
+	}
+}
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index f53bb43c3..51f7f678b 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -245,6 +245,9 @@ struct mlx5_rxq_ibv *mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
 struct mlx5_rxq_ibv *mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
 int mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv);
 int mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv);
+struct mlx5_rxq_ibv *mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev);
+void mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev,
+			       struct mlx5_rxq_ibv *rxq);
 int mlx5_rxq_ibv_verify(struct rte_eth_dev *dev);
 struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx,
 				   uint16_t desc, unsigned int socket,
@@ -265,6 +268,9 @@ struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_get(struct rte_eth_dev *dev,
 int mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
 			       struct mlx5_ind_table_ibv *ind_tbl);
 int mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev);
+void mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev,
+				     struct mlx5_ind_table_ibv *ind_tbl);
 struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev,
 				const uint8_t *rss_key, uint32_t rss_key_len,
 				uint64_t hash_fields,
@@ -277,6 +283,8 @@ struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev,
 				uint32_t tunnel, uint32_t rss_level);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
 int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev);
+void mlx5_hrxq_drop_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq);
 uint64_t mlx5_get_rx_port_offloads(void);
 uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
 
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 02/20] net/mlx5: handle drop queues are regular queues

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:34PM +0200, Nelio Laranjeiro wrote:
> Drop queues are essentially used in flows due to Verbs API, the
> information if the fate of the flow is a drop or not is already present
> in the flow.  Due to this, drop queues can be fully mapped on regular
> queues.

The title doesn't look good. Maybe typo?
	net/mlx5: handle drop queues are regular queues
'are' -> 'as'?

> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
>  drivers/net/mlx5/mlx5.c      |  24 ++--
>  drivers/net/mlx5/mlx5.h      |  14 ++-
>  drivers/net/mlx5/mlx5_flow.c |  94 +++++++-------
>  drivers/net/mlx5/mlx5_rxq.c  | 232 +++++++++++++++++++++++++++++++++++
>  drivers/net/mlx5/mlx5_rxtx.h |   8 ++
>  5 files changed, 310 insertions(+), 62 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
> index d01c0f6cf..aba6c1f9f 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -260,7 +260,6 @@ mlx5_dev_close(struct rte_eth_dev *dev)
>  		priv->txqs_n = 0;
>  		priv->txqs = NULL;
>  	}
> -	mlx5_flow_delete_drop_queue(dev);
>  	mlx5_mprq_free_mp(dev);
>  	mlx5_mr_release(dev);
>  	if (priv->pd != NULL) {
> @@ -1100,22 +1099,15 @@ mlx5_dev_spawn_one(struct rte_device *dpdk_dev,
>  	mlx5_link_update(eth_dev, 0);
>  	/* Store device configuration on private structure. */
>  	priv->config = config;
> -	/* Create drop queue. */
> -	err = mlx5_flow_create_drop_queue(eth_dev);
> -	if (err) {
> -		DRV_LOG(ERR, "port %u drop queue allocation failed: %s",
> -			eth_dev->data->port_id, strerror(rte_errno));
> -		err = rte_errno;
> -		goto error;
> -	}
>  	/* Supported Verbs flow priority number detection. */
> -	if (verb_priorities == 0)
> -		verb_priorities = mlx5_get_max_verbs_prio(eth_dev);
> -	if (verb_priorities < MLX5_VERBS_FLOW_PRIO_8) {
> -		DRV_LOG(ERR, "port %u wrong Verbs flow priorities: %u",
> -			eth_dev->data->port_id, verb_priorities);
> -		err = ENOTSUP;
> -		goto error;
> +	if (verb_priorities == 0) {
> +		err = mlx5_verbs_max_prio(eth_dev);
> +		if (err < 0) {
> +			DRV_LOG(ERR, "port %u wrong Verbs flow priorities",
> +				eth_dev->data->port_id);
> +			goto error;
> +		}
> +		verb_priorities = err;
>  	}
>  	priv->config.max_verbs_prio = verb_priorities;
>  	/*
> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
> index 6674a77d4..11aa0932f 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -131,9 +131,6 @@ enum mlx5_verbs_alloc_type {
>  	MLX5_VERBS_ALLOC_TYPE_RX_QUEUE,
>  };
>  
> -/* 8 Verbs priorities. */
> -#define MLX5_VERBS_FLOW_PRIO_8 8
> -
>  /**
>   * Verbs allocator needs a context to know in the callback which kind of
>   * resources it is allocating.
> @@ -145,6 +142,12 @@ struct mlx5_verbs_alloc_ctx {
>  
>  LIST_HEAD(mlx5_mr_list, mlx5_mr);
>  
> +/* Flow drop context necessary due to Verbs API. */
> +struct mlx5_drop {
> +	struct mlx5_hrxq *hrxq; /* Hash Rx queue queue. */
> +	struct mlx5_rxq_ibv *rxq; /* Verbs Rx queue. */
> +};
> +
>  struct priv {
>  	LIST_ENTRY(priv) mem_event_cb; /* Called by memory event callback. */
>  	struct rte_eth_dev_data *dev_data;  /* Pointer to device data. */
> @@ -175,7 +178,7 @@ struct priv {
>  	struct rte_intr_handle intr_handle; /* Interrupt handler. */
>  	unsigned int (*reta_idx)[]; /* RETA index table. */
>  	unsigned int reta_idx_n; /* RETA index size. */
> -	struct mlx5_hrxq_drop *flow_drop_queue; /* Flow drop queue. */
> +	struct mlx5_drop drop; /* Flow drop queues. */

priv->drop sounds strange. Why did you change the name?
How about priv->flow_drop if you didn't like the full name?

>  	struct mlx5_flows flows; /* RTE Flow rules. */
>  	struct mlx5_flows ctrl_flows; /* Control flow rules. */
>  	struct {
> @@ -306,7 +309,8 @@ int mlx5_traffic_restart(struct rte_eth_dev *dev);
>  
>  /* mlx5_flow.c */
>  
> -unsigned int mlx5_get_max_verbs_prio(struct rte_eth_dev *dev);
> +int mlx5_verbs_max_prio(struct rte_eth_dev *dev);
> +void mlx5_flow_print(struct rte_flow *flow);
>  int mlx5_flow_validate(struct rte_eth_dev *dev,
>  		       const struct rte_flow_attr *attr,
>  		       const struct rte_flow_item items[],
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index a45cb06e1..8b5b695d4 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -75,6 +75,58 @@ struct ibv_spec_header {
>  	uint16_t size;
>  };
>  
> + /**
> +  * Get the maximum number of priority available.
> +  *
> +  * @param dev
> +  *   Pointer to Ethernet device.
> +  *
> +  * @return
> +  *   number of supported Verbs flow priority on success, a negative errno
> +  *   value otherwise and rte_errno is set.
> +  */
> +int
> +mlx5_verbs_max_prio(struct rte_eth_dev *dev)
> +{
> +	struct {
> +		struct ibv_flow_attr attr;
> +		struct ibv_flow_spec_eth eth;
> +		struct ibv_flow_spec_action_drop drop;
> +	} flow_attr = {
> +		.attr = {
> +			.num_of_specs = 2,
> +		},
> +		.eth = {
> +			.type = IBV_FLOW_SPEC_ETH,
> +			.size = sizeof(struct ibv_flow_spec_eth),
> +		},
> +		.drop = {
> +			.size = sizeof(struct ibv_flow_spec_action_drop),
> +			.type = IBV_FLOW_SPEC_ACTION_DROP,
> +		},
> +	};
> +	struct ibv_flow *flow;
> +	uint32_t verb_priorities;
> +	struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
> +
> +	if (!drop) {
> +		rte_errno = ENOTSUP;
> +		return -rte_errno;
> +	}
> +	for (verb_priorities = 0; 1; verb_priorities++) {
> +		flow_attr.attr.priority = verb_priorities;
> +		flow = mlx5_glue->create_flow(drop->qp,
> +					      &flow_attr.attr);
> +		if (!flow)
> +			break;
> +		claim_zero(mlx5_glue->destroy_flow(flow));
> +	}
> +	mlx5_hrxq_drop_release(dev, drop);
> +	DRV_LOG(INFO, "port %u flow maximum priority: %d",
> +		dev->data->port_id, verb_priorities);
> +	return verb_priorities;
> +}
> +
>  /**
>   * Convert a flow.
>   *
> @@ -184,32 +236,6 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
>  	}
>  }
>  
> -/**
> - * Create drop queue.
> - *
> - * @param dev
> - *   Pointer to Ethernet device.
> - *
> - * @return
> - *   0 on success, a negative errno value otherwise and rte_errno is set.
> - */
> -int
> -mlx5_flow_create_drop_queue(struct rte_eth_dev *dev __rte_unused)
> -{
> -	return 0;
> -}
> -
> -/**
> - * Delete drop queue.
> - *
> - * @param dev
> - *   Pointer to Ethernet device.
> - */
> -void
> -mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev __rte_unused)
> -{
> -}
> -
>  /**
>   * Remove all flows.
>   *
> @@ -292,6 +318,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
>  	struct priv *priv = dev->data->dev_private;
>  	const struct rte_flow_attr attr = {
>  		.ingress = 1,
> +		.priority = priv->config.max_verbs_prio - 1,
>  	};
>  	struct rte_flow_item items[] = {
>  		{
> @@ -830,18 +857,3 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
>  	}
>  	return 0;
>  }
> -
> -/**
> - * Detect number of Verbs flow priorities supported.
> - *
> - * @param dev
> - *   Pointer to Ethernet device.
> - *
> - * @return
> - *   number of supported Verbs flow priority.
> - */
> -unsigned int
> -mlx5_get_max_verbs_prio(struct rte_eth_dev *dev __rte_unused)
> -{
> -	return 8;
> -}
> diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
> index 17db7c160..5f17dce50 100644
> --- a/drivers/net/mlx5/mlx5_rxq.c
> +++ b/drivers/net/mlx5/mlx5_rxq.c
> @@ -1949,3 +1949,235 @@ mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev)
>  	}
>  	return ret;
>  }
> +
> +/**
> + * Create a drop Rx queue Verbs object.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + *
> + * @return
> + *   The Verbs object initialised, NULL otherwise and rte_errno is set.
> + */
> +struct mlx5_rxq_ibv *
> +mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev)
> +{
> +	struct priv *priv = dev->data->dev_private;
> +	struct ibv_cq *cq;
> +	struct ibv_wq *wq = NULL;
> +	struct mlx5_rxq_ibv *rxq;
> +
> +	if (priv->drop.rxq)
> +		return priv->drop.rxq;
> +	cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
> +	if (!cq) {
> +		DEBUG("port %u cannot allocate CQ for drop queue",
> +		      dev->data->port_id);
> +		rte_errno = errno;
> +		goto error;
> +	}
> +	wq = mlx5_glue->create_wq(priv->ctx,
> +		 &(struct ibv_wq_init_attr){
> +			.wq_type = IBV_WQT_RQ,
> +			.max_wr = 1,
> +			.max_sge = 1,
> +			.pd = priv->pd,
> +			.cq = cq,
> +		 });
> +	if (!wq) {
> +		DEBUG("port %u cannot allocate WQ for drop queue",
> +		      dev->data->port_id);
> +		rte_errno = errno;
> +		goto error;
> +	}
> +	rxq = rte_calloc(__func__, 1, sizeof(*rxq), 0);
> +	if (!rxq) {
> +		DEBUG("port %u cannot allocate drop Rx queue memory",
> +		      dev->data->port_id);
> +		rte_errno = ENOMEM;
> +		goto error;
> +	}
> +	rxq->cq = cq;
> +	rxq->wq = wq;
> +	priv->drop.rxq = rxq;
> +	return rxq;
> +error:
> +	if (wq)
> +		claim_zero(mlx5_glue->destroy_wq(wq));
> +	if (cq)
> +		claim_zero(mlx5_glue->destroy_cq(cq));
> +	return NULL;
> +}
> +
> +/**
> + * Release a drop Rx queue Verbs object.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + * @param rxq
> + *   Pointer to the drop Verbs Rx queue.
> + *
> + * @return
> + *   The Verbs object initialised, NULL otherwise and rte_errno is set.
> + */
> +void
> +mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev, struct mlx5_rxq_ibv *rxq)

If rxq for drop is saved in priv->drop.rxq, then why does it have to get rxq
pointer as an argument? Looks redundant.

> +{
> +	if (rxq->wq)
> +		claim_zero(mlx5_glue->destroy_wq(rxq->wq));
> +	if (rxq->cq)
> +		claim_zero(mlx5_glue->destroy_cq(rxq->cq));
> +	rte_free(rxq);
> +	((struct priv *)dev->data->dev_private)->drop.rxq = NULL;
> +}
> +
> +/**
> + * Create a drop indirection table.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + *
> + * @return
> + *   The Verbs object initialised, NULL otherwise and rte_errno is set.
> + */
> +struct mlx5_ind_table_ibv *
> +mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev)
> +{
> +	struct priv *priv = dev->data->dev_private;
> +	struct mlx5_ind_table_ibv *ind_tbl;
> +	struct mlx5_rxq_ibv *rxq;
> +	struct mlx5_ind_table_ibv tmpl;
> +
> +	rxq = mlx5_rxq_ibv_drop_new(dev);
> +	if (!rxq)
> +		return NULL;
> +	tmpl.ind_table = mlx5_glue->create_rwq_ind_table
> +		(priv->ctx,
> +		 &(struct ibv_rwq_ind_table_init_attr){
> +			.log_ind_tbl_size = 0,
> +			.ind_tbl = &rxq->wq,
> +			.comp_mask = 0,
> +		 });
> +	if (!tmpl.ind_table) {
> +		DEBUG("port %u cannot allocate indirection table for drop"
> +		      " queue",
> +		      dev->data->port_id);
> +		rte_errno = errno;
> +		goto error;
> +	}
> +	ind_tbl = rte_calloc(__func__, 1, sizeof(*ind_tbl), 0);
> +	if (!ind_tbl) {
> +		rte_errno = ENOMEM;
> +		goto error;
> +	}
> +	ind_tbl->ind_table = tmpl.ind_table;
> +	return ind_tbl;
> +error:
> +	mlx5_rxq_ibv_drop_release(dev, rxq);
> +	return NULL;
> +}
> +
> +/**
> + * Release a drop indirection table.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + * @param ind_tbl
> + *   Indirection table to release.
> + */
> +void
> +mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev,
> +				struct mlx5_ind_table_ibv *ind_tbl)

ind_tbl is a redundant argument. Can be referenced by
priv->drop.hrxq->ind_table.

> +{
> +	claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl->ind_table));
> +	mlx5_rxq_ibv_drop_release
> +		(dev, ((struct priv *)dev->data->dev_private)->drop.rxq);
> +	rte_free(ind_tbl);
> +}
> +
> +/**
> + * Create a drop Rx Hash queue.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + *
> + * @return
> + *   The Verbs object initialised, NULL otherwise and rte_errno is set.
> + */
> +struct mlx5_hrxq *
> +mlx5_hrxq_drop_new(struct rte_eth_dev *dev)
> +{
> +	struct priv *priv = dev->data->dev_private;
> +	struct mlx5_ind_table_ibv *ind_tbl;
> +	struct ibv_qp *qp;
> +	struct mlx5_hrxq *hrxq;
> +
> +	if (priv->drop.hrxq) {
> +		rte_atomic32_inc(&priv->drop.hrxq->refcnt);
> +		return priv->drop.hrxq;
> +	}
> +	ind_tbl = mlx5_ind_table_ibv_drop_new(dev);
> +	if (!ind_tbl)
> +		return NULL;
> +	qp = mlx5_glue->create_qp_ex(priv->ctx,
> +		 &(struct ibv_qp_init_attr_ex){
> +			.qp_type = IBV_QPT_RAW_PACKET,
> +			.comp_mask =
> +				IBV_QP_INIT_ATTR_PD |
> +				IBV_QP_INIT_ATTR_IND_TABLE |
> +				IBV_QP_INIT_ATTR_RX_HASH,
> +			.rx_hash_conf = (struct ibv_rx_hash_conf){
> +				.rx_hash_function =
> +					IBV_RX_HASH_FUNC_TOEPLITZ,
> +				.rx_hash_key_len = rss_hash_default_key_len,
> +				.rx_hash_key = rss_hash_default_key,
> +				.rx_hash_fields_mask = 0,
> +				},
> +			.rwq_ind_tbl = ind_tbl->ind_table,
> +			.pd = priv->pd
> +		 });
> +	if (!qp) {
> +		DEBUG("port %u cannot allocate QP for drop queue",
> +		      dev->data->port_id);
> +		rte_errno = errno;
> +		goto error;
> +	}
> +	hrxq = rte_calloc(__func__, 1, sizeof(*hrxq), 0);
> +	if (!hrxq) {
> +		DRV_LOG(WARNING,
> +			"port %u cannot allocate memory for drop queue",
> +			dev->data->port_id);
> +		rte_errno = ENOMEM;
> +		goto error;
> +	}
> +	hrxq->ind_table = ind_tbl;
> +	hrxq->qp = qp;
> +	priv->drop.hrxq = hrxq;
> +	rte_atomic32_set(&hrxq->refcnt, 1);
> +	return hrxq;
> +error:
> +	if (ind_tbl)
> +		mlx5_ind_table_ibv_drop_release(dev, ind_tbl);
> +	return NULL;
> +}
> +
> +/**
> + * Release a drop hash Rx queue.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + * @param hrxq
> + *   Pointer to Hash Rx queue to release.
> + */
> +void
> +mlx5_hrxq_drop_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)

hrxq is a redundant argument. Can be referenced by priv->drop.hrxq.

Thanks,
Yongseok

> +{
> +	struct priv *priv = dev->data->dev_private;
> +
> +	if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
> +		claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
> +		mlx5_ind_table_ibv_drop_release(dev, hrxq->ind_table);
> +		rte_free(hrxq);
> +		priv->drop.hrxq = NULL;
> +	}
> +}
> diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> index f53bb43c3..51f7f678b 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.h
> +++ b/drivers/net/mlx5/mlx5_rxtx.h
> @@ -245,6 +245,9 @@ struct mlx5_rxq_ibv *mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
>  struct mlx5_rxq_ibv *mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
>  int mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv);
>  int mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv);
> +struct mlx5_rxq_ibv *mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev);
> +void mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev,
> +			       struct mlx5_rxq_ibv *rxq);
>  int mlx5_rxq_ibv_verify(struct rte_eth_dev *dev);
>  struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx,
>  				   uint16_t desc, unsigned int socket,
> @@ -265,6 +268,9 @@ struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_get(struct rte_eth_dev *dev,
>  int mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
>  			       struct mlx5_ind_table_ibv *ind_tbl);
>  int mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev);
> +struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev);
> +void mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev,
> +				     struct mlx5_ind_table_ibv *ind_tbl);
>  struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev,
>  				const uint8_t *rss_key, uint32_t rss_key_len,
>  				uint64_t hash_fields,
> @@ -277,6 +283,8 @@ struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev,
>  				uint32_t tunnel, uint32_t rss_level);
>  int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
>  int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev);
> +struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev);
> +void mlx5_hrxq_drop_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq);
>  uint64_t mlx5_get_rx_port_offloads(void);
>  uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
>  
> -- 
> 2.18.0
> 

Re: [dpdk-dev] [PATCH v2 02/20] net/mlx5: handle drop queues are regular queues

[Nélio Laranjeiro]

On Mon, Jul 02, 2018 at 06:07:03PM -0700, Yongseok Koh wrote:
> On Wed, Jun 27, 2018 at 05:07:34PM +0200, Nelio Laranjeiro wrote:
> > Drop queues are essentially used in flows due to Verbs API, the
> > information if the fate of the flow is a drop or not is already present
> > in the flow.  Due to this, drop queues can be fully mapped on regular
> > queues.
> 
> The title doesn't look good. Maybe typo?
> 	net/mlx5: handle drop queues are regular queues
> 'are' -> 'as'?
[...]

Right,

> >  struct priv {
> >  	LIST_ENTRY(priv) mem_event_cb; /* Called by memory event callback. */
> >  	struct rte_eth_dev_data *dev_data;  /* Pointer to device data. */
> > @@ -175,7 +178,7 @@ struct priv {
> >  	struct rte_intr_handle intr_handle; /* Interrupt handler. */
> >  	unsigned int (*reta_idx)[]; /* RETA index table. */
> >  	unsigned int reta_idx_n; /* RETA index size. */
> > -	struct mlx5_hrxq_drop *flow_drop_queue; /* Flow drop queue. */
> > +	struct mlx5_drop drop; /* Flow drop queues. */
> 
> priv->drop sounds strange. Why did you change the name?
> How about priv->flow_drop if you didn't like the full name?
>[...]

flow_drop_queue is also confusing as it is a drop hash rx queue, it can
be used without a flow as a regular queue.
Renaming it to drop_hrxq.

> > +/**
> > + * Release a drop Rx queue Verbs object.
> > + *
> > + * @param dev
> > + *   Pointer to Ethernet device.
> > + * @param rxq
> > + *   Pointer to the drop Verbs Rx queue.
> > + *
> > + * @return
> > + *   The Verbs object initialised, NULL otherwise and rte_errno is set.
> > + */
> > +void
> > +mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev, struct mlx5_rxq_ibv *rxq)
> 
> If rxq for drop is saved in priv->drop.rxq, then why does it have to get rxq
> pointer as an argument? Looks redundant.
>[...]

Like for all hrxqs, indirection tables, rxqs,  which are stored in priv
inside a list or an array.  Priv is used as a storage place which is
only access through *_{new,get,release} functions.
This is also to keep a consistency between regular hrxqs, and drop hrxq also.

> > +void
> > +mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev,
> > +				struct mlx5_ind_table_ibv *ind_tbl)
> 
> ind_tbl is a redundant argument. Can be referenced by
> priv->drop.hrxq->ind_table.
>[...]

Ditto.

> > +/**
> > + * Release a drop hash Rx queue.
> > + *
> > + * @param dev
> > + *   Pointer to Ethernet device.
> > + * @param hrxq
> > + *   Pointer to Hash Rx queue to release.
> > + */
> > +void
> > +mlx5_hrxq_drop_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
>[...]
> 
> hrxq is a redundant argument. Can be referenced by priv->drop.hrxq.
>[...]
 
Ditto.

Thanks,

-- 
Nélio Laranjeiro
6WIND

Re: [dpdk-dev] [PATCH v2 02/20] net/mlx5: handle drop queues are regular queues

[Yongseok Koh]

On Tue, Jul 03, 2018 at 09:17:56AM +0200, Nélio Laranjeiro wrote:
> On Mon, Jul 02, 2018 at 06:07:03PM -0700, Yongseok Koh wrote:
> > On Wed, Jun 27, 2018 at 05:07:34PM +0200, Nelio Laranjeiro wrote:
[...]
> flow_drop_queue is also confusing as it is a drop hash rx queue, it can
> be used without a flow as a regular queue.
> Renaming it to drop_hrxq.

Not so much critical to me but the entry has a field having repeating name.
	priv->drop_hrxq.hrxq and priv->drop_hrxq.rxq
Sounds still confusing...

> > > +/**
> > > + * Release a drop Rx queue Verbs object.
> > > + *
> > > + * @param dev
> > > + *   Pointer to Ethernet device.
> > > + * @param rxq
> > > + *   Pointer to the drop Verbs Rx queue.
> > > + *
> > > + * @return
> > > + *   The Verbs object initialised, NULL otherwise and rte_errno is set.
> > > + */
> > > +void
> > > +mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev, struct mlx5_rxq_ibv *rxq)
> > 
> > If rxq for drop is saved in priv->drop.rxq, then why does it have to get rxq
> > pointer as an argument? Looks redundant.
> >[...]
> 
> Like for all hrxqs, indirection tables, rxqs,  which are stored in priv
> inside a list or an array.

However, the assumption is there's only one drop queue while the regular ones
have multiple instances.

> Priv is used as a storage place which is only access through
> *_{new,get,release} functions.

Yes, that's what I'm telling you. *_{new,get,release}() accesses priv, then why
the pointer (which is saved in priv) is needed as an argument?

> This is also to keep a consistency between regular hrxqs, and drop hrxq also.
Not sure why that consistency has to be kept.

int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx);
int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);

mlx5_rxq_release() takes index as the instances are stored in an array and
mlx5_hrxq_release() takes pointer as the instances are stored in a list.

Then, what if there's only one instance and no need to search?
Not taking such an argument sounds more consistent...

Thanks,
Yongseok

> 
> > > +void
> > > +mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev,
> > > +				struct mlx5_ind_table_ibv *ind_tbl)
> > 
> > ind_tbl is a redundant argument. Can be referenced by
> > priv->drop.hrxq->ind_table.
> >[...]
> 
> Ditto.
> 
> > > +/**
> > > + * Release a drop hash Rx queue.
> > > + *
> > > + * @param dev
> > > + *   Pointer to Ethernet device.
> > > + * @param hrxq
> > > + *   Pointer to Hash Rx queue to release.
> > > + */
> > > +void
> > > +mlx5_hrxq_drop_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
> >[...]
> > 
> > hrxq is a redundant argument. Can be referenced by priv->drop.hrxq.
> >[...]
>  
> Ditto.
> 
> Thanks,
> 
> -- 
> Nélio Laranjeiro
> 6WIND

Re: [dpdk-dev] [PATCH v2 02/20] net/mlx5: handle drop queues are regular queues

[Nélio Laranjeiro]

On Tue, Jul 03, 2018 at 10:05:05AM -0700, Yongseok Koh wrote:
> On Tue, Jul 03, 2018 at 09:17:56AM +0200, Nélio Laranjeiro wrote:
> > On Mon, Jul 02, 2018 at 06:07:03PM -0700, Yongseok Koh wrote:
> > > On Wed, Jun 27, 2018 at 05:07:34PM +0200, Nelio Laranjeiro wrote:
> [...]
> > flow_drop_queue is also confusing as it is a drop hash rx queue, it can
> > be used without a flow as a regular queue.
> > Renaming it to drop_hrxq.
> 
> Not so much critical to me but the entry has a field having repeating name.
> 	priv->drop_hrxq.hrxq and priv->drop_hrxq.rxq
> Sounds still confusing...
> 
> > > > +/**
> > > > + * Release a drop Rx queue Verbs object.
> > > > + *
> > > > + * @param dev
> > > > + *   Pointer to Ethernet device.
> > > > + * @param rxq
> > > > + *   Pointer to the drop Verbs Rx queue.
> > > > + *
> > > > + * @return
> > > > + *   The Verbs object initialised, NULL otherwise and rte_errno is set.
> > > > + */
> > > > +void
> > > > +mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev, struct mlx5_rxq_ibv *rxq)
> > > 
> > > If rxq for drop is saved in priv->drop.rxq, then why does it have to get rxq
> > > pointer as an argument? Looks redundant.
> > >[...]
> > 
> > Like for all hrxqs, indirection tables, rxqs,  which are stored in priv
> > inside a list or an array.
> 
> However, the assumption is there's only one drop queue while the regular ones
> have multiple instances.
> 
> > Priv is used as a storage place which is only access through
> > *_{new,get,release} functions.
> 
> Yes, that's what I'm telling you. *_{new,get,release}() accesses priv, then why
> the pointer (which is saved in priv) is needed as an argument?
> 
> > This is also to keep a consistency between regular hrxqs, and drop hrxq also.
> Not sure why that consistency has to be kept.
> 
> int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx);
> int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
> 
> mlx5_rxq_release() takes index as the instances are stored in an array and
> mlx5_hrxq_release() takes pointer as the instances are stored in a list.
> 
> Then, what if there's only one instance and no need to search?
> Not taking such an argument sounds more consistent...
>[...]

Even if I prefer to keep consistency about identical functionality (in
this case creating an hash Rx queue), I'll make the changes to please
you.

Regards,

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH v2 03/20] net/mlx5: replace verbs priorities by flow

[Nelio Laranjeiro]

Previous work introduce verbs priorities, whereas the PMD is making
translation between Flow priority into Verbs.  Rename this to make more
sense on what the PMD has to translate.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5.c      | 15 ++++-----------
 drivers/net/mlx5/mlx5.h      |  4 ++--
 drivers/net/mlx5/mlx5_flow.c | 22 +++++++++++-----------
 3 files changed, 17 insertions(+), 24 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index aba6c1f9f..c3c8dffae 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -703,7 +703,6 @@ mlx5_dev_spawn_one(struct rte_device *dpdk_dev,
 	unsigned int tunnel_en = 0;
 	unsigned int mpls_en = 0;
 	unsigned int swp = 0;
-	unsigned int verb_priorities = 0;
 	unsigned int mprq = 0;
 	unsigned int mprq_min_stride_size_n = 0;
 	unsigned int mprq_max_stride_size_n = 0;
@@ -1100,16 +1099,10 @@ mlx5_dev_spawn_one(struct rte_device *dpdk_dev,
 	/* Store device configuration on private structure. */
 	priv->config = config;
 	/* Supported Verbs flow priority number detection. */
-	if (verb_priorities == 0) {
-		err = mlx5_verbs_max_prio(eth_dev);
-		if (err < 0) {
-			DRV_LOG(ERR, "port %u wrong Verbs flow priorities",
-				eth_dev->data->port_id);
-			goto error;
-		}
-		verb_priorities = err;
-	}
-	priv->config.max_verbs_prio = verb_priorities;
+	err = mlx5_flow_priorities(eth_dev);
+	if (err < 0)
+		goto error;
+	priv->config.flow_prio = err;
 	/*
 	 * Once the device is added to the list of memory event
 	 * callback, its global MR cache table cannot be expanded
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 11aa0932f..ed8c1c9a2 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -114,7 +114,7 @@ struct mlx5_dev_config {
 		unsigned int min_rxqs_num;
 		/* Rx queue count threshold to enable MPRQ. */
 	} mprq; /* Configurations for Multi-Packet RQ. */
-	unsigned int max_verbs_prio; /* Number of Verb flow priorities. */
+	unsigned int flow_prio; /* Number of flow priorities. */
 	unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
 	unsigned int ind_table_max_size; /* Maximum indirection table size. */
 	int txq_inline; /* Maximum packet size for inlining. */
@@ -309,7 +309,7 @@ int mlx5_traffic_restart(struct rte_eth_dev *dev);
 
 /* mlx5_flow.c */
 
-int mlx5_verbs_max_prio(struct rte_eth_dev *dev);
+int mlx5_flow_priorities(struct rte_eth_dev *dev);
 void mlx5_flow_print(struct rte_flow *flow);
 int mlx5_flow_validate(struct rte_eth_dev *dev,
 		       const struct rte_flow_attr *attr,
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 8b5b695d4..5d3bc183d 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -82,11 +82,11 @@ struct ibv_spec_header {
   *   Pointer to Ethernet device.
   *
   * @return
-  *   number of supported Verbs flow priority on success, a negative errno
-  *   value otherwise and rte_errno is set.
+  *   number of supported flow priority on success, a negative errno value
+  *   otherwise and rte_errno is set.
   */
 int
-mlx5_verbs_max_prio(struct rte_eth_dev *dev)
+mlx5_flow_priorities(struct rte_eth_dev *dev)
 {
 	struct {
 		struct ibv_flow_attr attr;
@@ -106,25 +106,25 @@ mlx5_verbs_max_prio(struct rte_eth_dev *dev)
 		},
 	};
 	struct ibv_flow *flow;
-	uint32_t verb_priorities;
 	struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
+	uint16_t vprio[] = { 8, 16 };
+	int i;
 
 	if (!drop) {
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	for (verb_priorities = 0; 1; verb_priorities++) {
-		flow_attr.attr.priority = verb_priorities;
-		flow = mlx5_glue->create_flow(drop->qp,
-					      &flow_attr.attr);
+	for (i = 0; i != RTE_DIM(vprio); i++) {
+		flow_attr.attr.priority = vprio[i] - 1;
+		flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
 		if (!flow)
 			break;
 		claim_zero(mlx5_glue->destroy_flow(flow));
 	}
 	mlx5_hrxq_drop_release(dev, drop);
 	DRV_LOG(INFO, "port %u flow maximum priority: %d",
-		dev->data->port_id, verb_priorities);
-	return verb_priorities;
+		dev->data->port_id, vprio[i]);
+	return vprio[i];
 }
 
 /**
@@ -318,7 +318,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
-		.priority = priv->config.max_verbs_prio - 1,
+		.priority = priv->config.flow_prio - 1,
 	};
 	struct rte_flow_item items[] = {
 		{
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 03/20] net/mlx5: replace verbs priorities by flow

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:35PM +0200, Nelio Laranjeiro wrote:
> Previous work introduce verbs priorities, whereas the PMD is making
> translation between Flow priority into Verbs.  Rename this to make more
> sense on what the PMD has to translate.
> 
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
Acked-by: Yongseok Koh <yskoh@mellanox.com>

Thanks

[dpdk-dev] [PATCH v2 04/20] net/mlx5: support flow Ethernet item among with drop action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5.c      |   1 +
 drivers/net/mlx5/mlx5_flow.c | 671 ++++++++++++++++++++++++++++++++---
 2 files changed, 630 insertions(+), 42 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index c3c8dffae..665a3c31f 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -241,6 +241,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	/* In case mlx5_dev_stop() has not been called. */
 	mlx5_dev_interrupt_handler_uninstall(dev);
 	mlx5_traffic_disable(dev);
+	mlx5_flow_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
 	dev->rx_pkt_burst = removed_rx_burst;
 	dev->tx_pkt_burst = removed_tx_burst;
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 5d3bc183d..bc8c5de33 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -35,11 +35,50 @@
 extern const struct eth_dev_ops mlx5_dev_ops;
 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
+/* Pattern Layer bits. */
+#define MLX5_FLOW_LAYER_OUTER_L2 (1u << 0)
+#define MLX5_FLOW_LAYER_OUTER_L3_IPV4 (1u << 1)
+#define MLX5_FLOW_LAYER_OUTER_L3_IPV6 (1u << 2)
+#define MLX5_FLOW_LAYER_OUTER_L4_UDP (1u << 3)
+#define MLX5_FLOW_LAYER_OUTER_L4_TCP (1u << 4)
+#define MLX5_FLOW_LAYER_OUTER_VLAN (1u << 5)
+/* Masks. */
+#define MLX5_FLOW_LAYER_OUTER_L3 \
+	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
+#define MLX5_FLOW_LAYER_OUTER_L4 \
+	(MLX5_FLOW_LAYER_OUTER_L4_UDP | MLX5_FLOW_LAYER_OUTER_L4_TCP)
+
+/* Action fate on the packet. */
+#define MLX5_FLOW_FATE_DROP (1u << 0)
+
+/** Handles information leading to a drop fate. */
+struct mlx5_flow_verbs {
+	unsigned int size; /**< Size of the attribute. */
+	struct {
+		struct ibv_flow_attr *attr;
+		/**< Pointer to the Specification buffer. */
+		uint8_t *specs; /**< Pointer to the specifications. */
+	};
+	struct ibv_flow *flow; /**< Verbs flow pointer. */
+	struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */
+};
+
+/* Flow structure. */
 struct rte_flow {
 	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
+	struct rte_flow_attr attributes; /**< User flow attribute. */
+	uint32_t layers;
+	/**< Bit-fields of present layers see MLX5_FLOW_ITEMS_*. */
+	uint32_t fate;
+	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
+	struct mlx5_flow_verbs verbs; /* Verbs drop flow. */
 };
 
 static const struct rte_flow_ops mlx5_flow_ops = {
+	.validate = mlx5_flow_validate,
+	.create = mlx5_flow_create,
+	.destroy = mlx5_flow_destroy,
+	.flush = mlx5_flow_flush,
 	.isolate = mlx5_flow_isolate,
 };
 
@@ -128,12 +167,418 @@ mlx5_flow_priorities(struct rte_eth_dev *dev)
 }
 
 /**
- * Convert a flow.
+ * Flow debug purpose function only available when
+ * CONFIG_RTE_LIBRTE_MLX5_DEBUG=y
+ *
+ * @param flow
+ *   Pointer to the flow structure to display.
+ */
+void
+mlx5_flow_print(struct rte_flow *flow __rte_unused)
+{
+#ifndef NDEBUG
+	fprintf(stdout, "---------8<------------\n");
+	fprintf(stdout, "%s: flow information\n", MLX5_DRIVER_NAME);
+	fprintf(stdout, " attributes: group %u priority %u ingress %d egress %d"
+		" transfer %d\n", flow->attributes.group,
+		flow->attributes.priority,
+		flow->attributes.ingress,
+		flow->attributes.egress,
+		flow->attributes.transfer);
+	fprintf(stdout, " layers: %s/%s/%s\n",
+		flow->layers & MLX5_FLOW_LAYER_OUTER_L2 ? "l2" : "-",
+		flow->layers & MLX5_FLOW_LAYER_OUTER_L3 ? "l3" : "-",
+		flow->layers & MLX5_FLOW_LAYER_OUTER_L4 ? "l4" : "-");
+	if (flow->fate & MLX5_FLOW_FATE_DROP)
+		fprintf(stdout, " fate: drop queue\n");
+	if (flow->verbs.attr) {
+		struct ibv_spec_header *hdr =
+			(struct ibv_spec_header *)flow->verbs.specs;
+		const int n = flow->verbs.attr->num_of_specs;
+		int i;
+
+		fprintf(stdout, " Verbs attributes: specs_n %u\n",
+			flow->verbs.attr->num_of_specs);
+		for (i = 0; i != n; ++i) {
+			rte_hexdump(stdout, " ", hdr, hdr->size);
+			hdr = (struct ibv_spec_header *)
+				((uint8_t *)hdr + hdr->size);
+		}
+	}
+	fprintf(stdout, "--------->8------------\n");
+#endif
+}
+
+/**
+ * Validate Attributes provided by the user.
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to a TAILQ flow list.
+ * @param attr
+ *   Pointer to flow attributes
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_attributes(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
+		     struct rte_flow *flow, struct rte_flow_error *error)
+{
+	uint32_t priority_max =
+		((struct priv *)dev->data->dev_private)->config.flow_prio;
+
+	if (attr->group)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+					  NULL,
+					  "groups are not supported");
+	if (attr->priority >= priority_max)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+					  NULL,
+					  "priority value is not supported");
+	if (attr->egress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+					  NULL,
+					  "egress is not supported");
+	if (attr->transfer)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
+					  NULL,
+					  "transfer is not supported");
+	if (!attr->ingress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+					  NULL,
+					  "only ingress is supported");
+	flow->attributes = *attr;
+	return 0;
+}
+
+/**
+ * Check support for a given item.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param default_mask[in]
+ *   Bit-masks covering supported fields to compare with spec, last and mask in
+ *   \item.
+ * @param nic_mask[in]
+ *   Bit-masks covering supported fields by the NIC to compare with user mask.
+ * @param size
+ *   Bit-Mask size in bytes.
+ * @param error[out]
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_validate(const struct rte_flow_item *item,
+			const uint8_t *default_mask,
+			const uint8_t *nic_mask,
+			unsigned int size,
+			struct rte_flow_error *error)
+{
+	const uint8_t *mask = item->mask ? item->mask : default_mask;
+	unsigned int i;
+
+	assert(nic_mask);
+	for (i = 0; i < size; ++i)
+		if ((nic_mask[i] | mask[i]) != nic_mask[i])
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "mask enables non supported"
+						  " bits");
+	if (!item->spec && (item->mask || item->last))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "mask/last without a spec is not"
+					  " supported");
+	if (item->spec && item->last) {
+		uint8_t spec[size];
+		uint8_t last[size];
+		unsigned int i;
+		int ret;
+
+		for (i = 0; i < size; ++i) {
+			spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
+			last[i] = ((const uint8_t *)item->last)[i] & mask[i];
+		}
+		ret = memcmp(spec, last, size);
+		if (ret != 0)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "spec and last different"
+						  " is not supported");
+	}
+	return 0;
+}
+
+/**
+ * Add a verbs specification.
+ *
+ * @param flow
+ *   Pointer to flow structure.
+ * @param src
+ *   Create specification.
+ * @param size
+ *   Size in bytes of the specification to copy.
+ */
+static void
+mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
+{
+	if (flow->verbs.specs) {
+		void *dst;
+
+		dst = (void *)(flow->verbs.specs + flow->verbs.size);
+		memcpy(dst, src, size);
+		++flow->verbs.attr->num_of_specs;
+	}
+	flow->verbs.size += size;
+}
+
+/**
+ * Validate Ethernet layer and possibly create the Verbs specification.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param flow_size[in]
+ *   Size of the buffer to store the specification.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_eth *spec = item->spec;
+	const struct rte_flow_item_eth *mask = item->mask;
+	const struct rte_flow_item_eth nic_mask = {
+		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.type = RTE_BE16(0xffff),
+	};
+	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	struct ibv_flow_spec_eth eth = {
+		.type = IBV_FLOW_SPEC_ETH,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L2)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L2 layers already configured");
+	if (!mask)
+		mask = &rte_flow_item_eth_mask;
+	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+				      (const uint8_t *)&nic_mask,
+				      sizeof(struct rte_flow_item_eth),
+				      error);
+	if (ret)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		unsigned int i;
+
+		memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
+		memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
+		eth.val.ether_type = spec->type;
+		memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
+		memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
+		eth.mask.ether_type = mask->type;
+		/* Remove unwanted bits from values. */
+		for (i = 0; i < ETHER_ADDR_LEN; ++i) {
+			eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
+			eth.val.src_mac[i] &= eth.mask.src_mac[i];
+		}
+		eth.val.ether_type &= eth.mask.ether_type;
+	}
+	mlx5_flow_spec_verbs_add(flow, &eth, size);
+	return size;
+}
+
+/**
+ * Validate items provided by the user.
+ *
+ * @param items
+ *   Pointer to flow items array.
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param flow_size
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_items(const struct rte_flow_item items[],
+		struct rte_flow *flow, const size_t flow_size,
+		struct rte_flow_error *error)
+{
+	int remain = flow_size;
+	size_t size = 0;
+
+	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
+		int ret = 0;
+
+		switch (items->type) {
+		case RTE_FLOW_ITEM_TYPE_VOID:
+			break;
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			ret = mlx5_flow_item_eth(items, flow, remain, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  items,
+						  "item not supported");
+		}
+		if (ret < 0)
+			return ret;
+		if (remain > ret)
+			remain -= ret;
+		else
+			remain = 0;
+		size += ret;
+	}
+	if (!flow->layers) {
+		const struct rte_flow_item item = {
+			.type = RTE_FLOW_ITEM_TYPE_ETH,
+		};
+
+		return mlx5_flow_item_eth(&item, flow, flow_size, error);
+	}
+	return size;
+}
+
+/**
+ * Validate action drop provided by the user.
+ *
+ * @param actions
+ *   Pointer to flow actions array.
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param flow_size
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_action_drop(const struct rte_flow_action *actions,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
+{
+	unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
+	struct ibv_flow_spec_action_drop drop = {
+			.type = IBV_FLOW_SPEC_ACTION_DROP,
+			.size = size,
+	};
+
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "multiple fate actions are not"
+					  " supported");
+	if (size < flow_size)
+		mlx5_flow_spec_verbs_add(flow, &drop, size);
+	flow->fate |= MLX5_FLOW_FATE_DROP;
+	return size;
+}
+
+/**
+ * Validate actions provided by the user.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param actions
+ *   Pointer to flow actions array.
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param flow_size
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_actions(struct rte_eth_dev *dev __rte_unused,
+		  const struct rte_flow_action actions[],
+		  struct rte_flow *flow, const size_t flow_size,
+		  struct rte_flow_error *error)
+{
+	size_t size = 0;
+	int remain = flow_size;
+	int ret = 0;
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_DROP:
+			ret = mlx5_flow_action_drop(actions, flow, remain,
+						    error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+		if (ret < 0)
+			return ret;
+		if (remain > ret)
+			remain -= ret;
+		else
+			remain = 0;
+		size += ret;
+	}
+	if (!flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "no fate action found");
+	return size;
+}
+
+/**
+ * Validate the rule and return a flow structure filled accordingly.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param flow
+ *   Pointer an flow structure.
+ * @param flow_size
+ *   Size of the allocated space to store the flow information.
  * @param[in] attr
  *   Flow rule attributes.
  * @param[in] pattern
@@ -144,21 +589,39 @@ mlx5_flow_priorities(struct rte_eth_dev *dev)
  *   Perform verbose error reporting if not NULL.
  *
  * @return
- *   A flow on success, NULL otherwise and rte_errno is set.
+ *   The amount of bytes necessary to create the flow, a negative errno value
+ *   otherwise and rte_errno is set.
  */
-static struct rte_flow *
-mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
-		      struct mlx5_flows *list __rte_unused,
-		      const struct rte_flow_attr *attr __rte_unused,
-		      const struct rte_flow_item items[] __rte_unused,
-		      const struct rte_flow_action actions[] __rte_unused,
-		      struct rte_flow_error *error)
+static int
+mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
+		const size_t flow_size,
+		const struct rte_flow_attr *attr,
+		const struct rte_flow_item items[],
+		const struct rte_flow_action actions[],
+		struct rte_flow_error *error)
 {
-	rte_flow_error_set(error, ENOTSUP,
-			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL,
-			   "action not supported");
-	return NULL;
+	struct rte_flow local_flow = { .layers = 0, };
+	size_t size = sizeof(*flow) + sizeof(struct ibv_flow_attr);
+	int remain = (flow_size > size) ? flow_size - size : 0;
+	int ret;
+
+	if (!remain)
+		flow = &local_flow;
+	ret = mlx5_flow_attributes(dev, attr, flow, error);
+	if (ret < 0)
+		return ret;
+	ret = mlx5_flow_items(items, flow, remain, error);
+	if (ret < 0)
+		return ret;
+	size += ret;
+	remain = (flow_size > size) ? flow_size - size : 0;
+	ret = mlx5_flow_actions(dev, actions, flow, remain, error);
+	if (ret < 0)
+		return ret;
+	size += ret;
+	if (size <= flow_size)
+		flow->verbs.attr->priority = flow->attributes.priority;
+	return size;
 }
 
 /**
@@ -168,16 +631,142 @@ mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
  * @see rte_flow_ops
  */
 int
-mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
-		   const struct rte_flow_attr *attr __rte_unused,
-		   const struct rte_flow_item items[] __rte_unused,
-		   const struct rte_flow_action actions[] __rte_unused,
+mlx5_flow_validate(struct rte_eth_dev *dev,
+		   const struct rte_flow_attr *attr,
+		   const struct rte_flow_item items[],
+		   const struct rte_flow_action actions[],
 		   struct rte_flow_error *error)
 {
-	return rte_flow_error_set(error, ENOTSUP,
-				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				  NULL,
-				  "action not supported");
+	int ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+
+	if (ret < 0)
+		return ret;
+	return 0;
+}
+
+/**
+ * Remove the flow.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param flow
+ *   Pointer to flow structure.
+ */
+static void
+mlx5_flow_fate_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	if (flow->fate & MLX5_FLOW_FATE_DROP) {
+		if (flow->verbs.flow) {
+			claim_zero(mlx5_glue->destroy_flow(flow->verbs.flow));
+			flow->verbs.flow = NULL;
+		}
+	}
+	if (flow->verbs.hrxq) {
+		mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
+		flow->verbs.hrxq = NULL;
+	}
+}
+
+/**
+ * Apply the flow.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param flow
+ *   Pointer to flow structure.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_fate_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
+		     struct rte_flow_error *error)
+{
+	flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
+	if (!flow->verbs.hrxq)
+		return rte_flow_error_set
+			(error, errno,
+			 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			 NULL,
+			 "cannot allocate Drop queue");
+	flow->verbs.flow =
+		mlx5_glue->create_flow(flow->verbs.hrxq->qp, flow->verbs.attr);
+	if (!flow->verbs.flow) {
+		mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
+		flow->verbs.hrxq = NULL;
+		return rte_flow_error_set(error, errno,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "kernel module refuses to create"
+					  " flow");
+	}
+	return 0;
+}
+
+/**
+ * Create a flow.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param list
+ *   Pointer to a TAILQ flow list.
+ * @param[in] attr
+ *   Flow rule attributes.
+ * @param[in] items
+ *   Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ *   Associated actions (list terminated by the END action).
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   A flow on success, NULL otherwise and rte_errno is set.
+ */
+static struct rte_flow *
+mlx5_flow_list_create(struct rte_eth_dev *dev,
+		      struct mlx5_flows *list,
+		      const struct rte_flow_attr *attr,
+		      const struct rte_flow_item items[],
+		      const struct rte_flow_action actions[],
+		      struct rte_flow_error *error)
+{
+	struct rte_flow *flow;
+	size_t size;
+	int ret;
+
+	ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+	if (ret < 0)
+		return NULL;
+	size = ret;
+	flow = rte_zmalloc(__func__, size, 0);
+	if (!flow) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "cannot allocate memory");
+		return NULL;
+	}
+	flow->verbs.attr = (struct ibv_flow_attr *)(flow + 1);
+	flow->verbs.specs = (uint8_t *)(flow->verbs.attr + 1);
+	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
+	if (ret < 0)
+		goto error;
+	assert((size_t)ret == size);
+	if (dev->data->dev_started) {
+		ret = mlx5_flow_fate_apply(dev, flow, error);
+		if (ret < 0)
+			goto error;
+	}
+	TAILQ_INSERT_TAIL(list, flow, next);
+	return flow;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	mlx5_flow_fate_remove(dev, flow);
+	rte_free(flow);
+	rte_errno = ret; /* Restore rte_errno. */
+	return NULL;
 }
 
 /**
@@ -187,17 +776,15 @@ mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
  * @see rte_flow_ops
  */
 struct rte_flow *
-mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
-		 const struct rte_flow_attr *attr __rte_unused,
-		 const struct rte_flow_item items[] __rte_unused,
-		 const struct rte_flow_action actions[] __rte_unused,
+mlx5_flow_create(struct rte_eth_dev *dev,
+		 const struct rte_flow_attr *attr,
+		 const struct rte_flow_item items[],
+		 const struct rte_flow_action actions[],
 		 struct rte_flow_error *error)
 {
-	rte_flow_error_set(error, ENOTSUP,
-			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL,
-			   "action not supported");
-	return NULL;
+	return mlx5_flow_list_create
+		(dev, &((struct priv *)dev->data->dev_private)->flows,
+		 attr, items, actions, error);
 }
 
 /**
@@ -211,10 +798,12 @@ mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
  *   Flow to destroy.
  */
 static void
-mlx5_flow_list_destroy(struct rte_eth_dev *dev __rte_unused,
-		       struct mlx5_flows *list __rte_unused,
-		       struct rte_flow *flow __rte_unused)
+mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
+		       struct rte_flow *flow)
 {
+	mlx5_flow_fate_remove(dev, flow);
+	TAILQ_REMOVE(list, flow, next);
+	rte_free(flow);
 }
 
 /**
@@ -422,9 +1011,8 @@ int
 mlx5_flow_flush(struct rte_eth_dev *dev,
 		struct rte_flow_error *error __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->flows);
+	mlx5_flow_list_flush(dev,
+			     &((struct priv *)dev->data->dev_private)->flows);
 	return 0;
 }
 
@@ -737,9 +1325,8 @@ mlx5_fdir_filter_update(struct rte_eth_dev *dev,
 static void
 mlx5_fdir_filter_flush(struct rte_eth_dev *dev)
 {
-	struct priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->flows);
+	mlx5_flow_list_flush(dev,
+			     &((struct priv *)dev->data->dev_private)->flows);
 }
 
 /**
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 04/20] net/mlx5: support flow Ethernet item among with drop action

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:36PM +0200, Nelio Laranjeiro wrote:

net/mlx5: support flow Ethernet item among with drop action

Typo? "among" -> "along"?

> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
>  drivers/net/mlx5/mlx5.c      |   1 +
>  drivers/net/mlx5/mlx5_flow.c | 671 ++++++++++++++++++++++++++++++++---
>  2 files changed, 630 insertions(+), 42 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
> index c3c8dffae..665a3c31f 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -241,6 +241,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
>  	/* In case mlx5_dev_stop() has not been called. */
>  	mlx5_dev_interrupt_handler_uninstall(dev);
>  	mlx5_traffic_disable(dev);
> +	mlx5_flow_flush(dev, NULL);
>  	/* Prevent crashes when queues are still in use. */
>  	dev->rx_pkt_burst = removed_rx_burst;
>  	dev->tx_pkt_burst = removed_tx_burst;
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index 5d3bc183d..bc8c5de33 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -35,11 +35,50 @@
>  extern const struct eth_dev_ops mlx5_dev_ops;
>  extern const struct eth_dev_ops mlx5_dev_ops_isolate;
>  
> +/* Pattern Layer bits. */
> +#define MLX5_FLOW_LAYER_OUTER_L2 (1u << 0)
> +#define MLX5_FLOW_LAYER_OUTER_L3_IPV4 (1u << 1)
> +#define MLX5_FLOW_LAYER_OUTER_L3_IPV6 (1u << 2)
> +#define MLX5_FLOW_LAYER_OUTER_L4_UDP (1u << 3)
> +#define MLX5_FLOW_LAYER_OUTER_L4_TCP (1u << 4)
> +#define MLX5_FLOW_LAYER_OUTER_VLAN (1u << 5)
> +/* Masks. */
> +#define MLX5_FLOW_LAYER_OUTER_L3 \
> +	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
> +#define MLX5_FLOW_LAYER_OUTER_L4 \
> +	(MLX5_FLOW_LAYER_OUTER_L4_UDP | MLX5_FLOW_LAYER_OUTER_L4_TCP)
> +
> +/* Action fate on the packet. */
> +#define MLX5_FLOW_FATE_DROP (1u << 0)

I understand what you mean by 'fate' but 'action fate' sounds strange to me.
How about 'terminal action' instead? E.g. flow->terminal_actions or
flow->term_actions instead of flow->fate.

And, I'm not sure why you define a new (redundant) macro. You could use
RTE_FLOW_ACTION_TYPE_DROP instead. I don't see any reason for setting different
action flag in flow->fate. If you want to specify a group of terminal actions,
you could've done:

#define MLX5_FLOW_TERMINAL_ACTIONS \
	(RTE_FLOW_ACTION_TYPE_DROP | RTE_FLOW_ACTION_TYPE_RSS | \
	 RTE_FLOW_ACTION_TYPE_QUEUE)

> +/** Handles information leading to a drop fate. */
> +struct mlx5_flow_verbs {
> +	unsigned int size; /**< Size of the attribute. */
> +	struct {
> +		struct ibv_flow_attr *attr;
> +		/**< Pointer to the Specification buffer. */
> +		uint8_t *specs; /**< Pointer to the specifications. */
> +	};
> +	struct ibv_flow *flow; /**< Verbs flow pointer. */
> +	struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */
> +};
> +
> +/* Flow structure. */
>  struct rte_flow {
>  	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
> +	struct rte_flow_attr attributes; /**< User flow attribute. */
> +	uint32_t layers;
> +	/**< Bit-fields of present layers see MLX5_FLOW_ITEMS_*. */
> +	uint32_t fate;
> +	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
> +	struct mlx5_flow_verbs verbs; /* Verbs drop flow. */
>  };
>  
>  static const struct rte_flow_ops mlx5_flow_ops = {
> +	.validate = mlx5_flow_validate,
> +	.create = mlx5_flow_create,
> +	.destroy = mlx5_flow_destroy,
> +	.flush = mlx5_flow_flush,
>  	.isolate = mlx5_flow_isolate,
>  };
>  
> @@ -128,12 +167,418 @@ mlx5_flow_priorities(struct rte_eth_dev *dev)
>  }
>  
>  /**
> - * Convert a flow.
> + * Flow debug purpose function only available when
> + * CONFIG_RTE_LIBRTE_MLX5_DEBUG=y
> + *
> + * @param flow
> + *   Pointer to the flow structure to display.
> + */
> +void
> +mlx5_flow_print(struct rte_flow *flow __rte_unused)
> +{
> +#ifndef NDEBUG
> +	fprintf(stdout, "---------8<------------\n");
> +	fprintf(stdout, "%s: flow information\n", MLX5_DRIVER_NAME);
> +	fprintf(stdout, " attributes: group %u priority %u ingress %d egress %d"
> +		" transfer %d\n", flow->attributes.group,
> +		flow->attributes.priority,
> +		flow->attributes.ingress,
> +		flow->attributes.egress,
> +		flow->attributes.transfer);
> +	fprintf(stdout, " layers: %s/%s/%s\n",
> +		flow->layers & MLX5_FLOW_LAYER_OUTER_L2 ? "l2" : "-",
> +		flow->layers & MLX5_FLOW_LAYER_OUTER_L3 ? "l3" : "-",
> +		flow->layers & MLX5_FLOW_LAYER_OUTER_L4 ? "l4" : "-");
> +	if (flow->fate & MLX5_FLOW_FATE_DROP)
> +		fprintf(stdout, " fate: drop queue\n");
> +	if (flow->verbs.attr) {
> +		struct ibv_spec_header *hdr =
> +			(struct ibv_spec_header *)flow->verbs.specs;
> +		const int n = flow->verbs.attr->num_of_specs;
> +		int i;
> +
> +		fprintf(stdout, " Verbs attributes: specs_n %u\n",
> +			flow->verbs.attr->num_of_specs);
> +		for (i = 0; i != n; ++i) {
> +			rte_hexdump(stdout, " ", hdr, hdr->size);
> +			hdr = (struct ibv_spec_header *)
> +				((uint8_t *)hdr + hdr->size);
> +		}
> +	}
> +	fprintf(stdout, "--------->8------------\n");
> +#endif
> +}
> +
> +/**
> + * Validate Attributes provided by the user.

And it also stores attributes.

>   *
>   * @param dev
>   *   Pointer to Ethernet device.
> - * @param list
> - *   Pointer to a TAILQ flow list.
> + * @param attr
> + *   Pointer to flow attributes
> + * @param flow
> + *   Pointer to the rte_flow structure.
> + * @param error
> + *   Pointer to error structure.
> + *
> + * @return
> + *   0 on success, a negative errno value otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_attributes(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
> +		     struct rte_flow *flow, struct rte_flow_error *error)

Better to have mlx5_flow_attributes_validate() like mlx5_flow_item_validate()?

> +{
> +	uint32_t priority_max =
> +		((struct priv *)dev->data->dev_private)->config.flow_prio;
> +
> +	if (attr->group)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
> +					  NULL,
> +					  "groups are not supported");

"group is not supported"?

> +	if (attr->priority >= priority_max)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
> +					  NULL,
> +					  "priority value is not supported");

"priority is out of range" or something else?

> +	if (attr->egress)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
> +					  NULL,
> +					  "egress is not supported");
> +	if (attr->transfer)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
> +					  NULL,
> +					  "transfer is not supported");
> +	if (!attr->ingress)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
> +					  NULL,
> +					  "only ingress is supported");

Better to have "ingress should be specified" or something else?

> +	flow->attributes = *attr;
> +	return 0;
> +}
> +
> +/**
> + * Check support for a given item.
> + *
> + * @param item[in]
> + *   Item specification.
> + * @param default_mask[in]
> + *   Bit-masks covering supported fields to compare with spec, last and mask in
> + *   \item.

Remove the back-slash and the description isn't clear. From the code, it seems
to be a default mask to be used if item doesn't have mask.

> + * @param nic_mask[in]
> + *   Bit-masks covering supported fields by the NIC to compare with user mask.

s/Bit-masks/Bit-mask


> + * @param size
> + *   Bit-Mask size in bytes.

s/Bit-Mask/Bit-mask

> + * @param error[out]
> + *   Pointer to error structure.
> + *
> + * @return
> + *   0 on success, a negative errno value otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_item_validate(const struct rte_flow_item *item,
> +			const uint8_t *default_mask,
> +			const uint8_t *nic_mask,
> +			unsigned int size,
> +			struct rte_flow_error *error)
> +{
> +	const uint8_t *mask = item->mask ? item->mask : default_mask;

In the mlx5_flow_item_eth(), rte_flow_item_eth_mask is passed to default_mask if
item->mask is null, then why doing it again here?

> +	unsigned int i;
> +
> +	assert(nic_mask);
> +	for (i = 0; i < size; ++i)
> +		if ((nic_mask[i] | mask[i]) != nic_mask[i])
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  item,
> +						  "mask enables non supported"
> +						  " bits");

default_mask is determined by caller, which is rte_flow_item_eth_mask or
item->mask. And nic_mask is also given by PMD, which is a function local
structure in the caller func. Doesn't look like a clear definition. Intead of
having both default_mask and nic_mask here, you can take supported_mask, which
can be (rte_flow_item_{name}_mask | nic_mask) in the calling func. And check
validity of item->mask against supported_mask only if item->mask isn't null.

> +	if (!item->spec && (item->mask || item->last))
> +		return rte_flow_error_set(error, EINVAL,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "mask/last without a spec is not"
> +					  " supported");
> +	if (item->spec && item->last) {
> +		uint8_t spec[size];
> +		uint8_t last[size];
> +		unsigned int i;
> +		int ret;
> +
> +		for (i = 0; i < size; ++i) {
> +			spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
> +			last[i] = ((const uint8_t *)item->last)[i] & mask[i];
> +		}
> +		ret = memcmp(spec, last, size);
> +		if (ret != 0)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  item,
> +						  "spec and last different"
> +						  " is not supported");

Can you rephrase the error message?

> +	}
> +	return 0;
> +}
> +
> +/**
> + * Add a verbs specification.
> + *
> + * @param flow
> + *   Pointer to flow structure.
> + * @param src
> + *   Create specification.
> + * @param size
> + *   Size in bytes of the specification to copy.
> + */
> +static void
> +mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
> +{
> +	if (flow->verbs.specs) {
> +		void *dst;
> +
> +		dst = (void *)(flow->verbs.specs + flow->verbs.size);
> +		memcpy(dst, src, size);
> +		++flow->verbs.attr->num_of_specs;
> +	}
> +	flow->verbs.size += size;
> +}
> +
> +/**
> + * Validate Ethernet layer and possibly create the Verbs specification.
> + *
> + * @param item[in]
> + *   Item specification.
> + * @param flow[in, out]
> + *   Pointer to flow structure.
> + * @param flow_size[in]
> + *   Size of the buffer to store the specification.
> + * @param error
> + *   Pointer to error structure.
> + *
> + * @return
> + *   size in bytes necessary for the conversion, a negative errno value
> + *   otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
> +		   const size_t flow_size, struct rte_flow_error *error)
> +{
> +	const struct rte_flow_item_eth *spec = item->spec;
> +	const struct rte_flow_item_eth *mask = item->mask;
> +	const struct rte_flow_item_eth nic_mask = {
> +		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
> +		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
> +		.type = RTE_BE16(0xffff),
> +	};

If it is a device-specific one, shouldn't it be located in a header file?

> +	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
> +	struct ibv_flow_spec_eth eth = {
> +		.type = IBV_FLOW_SPEC_ETH,
> +		.size = size,
> +	};
> +	int ret;
> +
> +	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L2)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "L2 layers already configured");
> +	if (!mask)
> +		mask = &rte_flow_item_eth_mask;

What is the meaning of default_mask for mlx5_flow_item_validate()? Also refer
to my comments in mlx5_flow_item_validate().

> +	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> +				      (const uint8_t *)&nic_mask,
> +				      sizeof(struct rte_flow_item_eth),
> +				      error);
> +	if (ret)
> +		return ret;
> +	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2;
> +	if (size > flow_size)
> +		return size;
> +	if (spec) {
> +		unsigned int i;
> +
> +		memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
> +		memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
> +		eth.val.ether_type = spec->type;
> +		memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
> +		memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
> +		eth.mask.ether_type = mask->type;
> +		/* Remove unwanted bits from values. */
> +		for (i = 0; i < ETHER_ADDR_LEN; ++i) {
> +			eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
> +			eth.val.src_mac[i] &= eth.mask.src_mac[i];
> +		}
> +		eth.val.ether_type &= eth.mask.ether_type;
> +	}
> +	mlx5_flow_spec_verbs_add(flow, &eth, size);

Do you have a plan to change this redundant memcpy? I mean, writing directly to
the allocated memory for verbs spec instead of the local micro-scratchpad.

> +	return size;
> +}
> +
> +/**
> + * Validate items provided by the user.
> + *
> + * @param items
> + *   Pointer to flow items array.
> + * @param flow
> + *   Pointer to the rte_flow structure.
> + * @param flow_size
> + *   Size in bytes of the available space for to store the flow information.
> + * @param error
> + *   Pointer to error structure.
> + *
> + * @return
> + *   size in bytes necessary for the conversion, a negative errno value
> + *   otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_items(const struct rte_flow_item items[],
> +		struct rte_flow *flow, const size_t flow_size,
> +		struct rte_flow_error *error)
> +{
> +	int remain = flow_size;
> +	size_t size = 0;
> +
> +	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
> +		int ret = 0;
> +
> +		switch (items->type) {
> +		case RTE_FLOW_ITEM_TYPE_VOID:
> +			break;
> +		case RTE_FLOW_ITEM_TYPE_ETH:
> +			ret = mlx5_flow_item_eth(items, flow, remain, error);
> +			break;
> +		default:
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  items,
> +						  "item not supported");
> +		}
> +		if (ret < 0)
> +			return ret;
> +		if (remain > ret)
> +			remain -= ret;
> +		else
> +			remain = 0;
> +		size += ret;
> +	}
> +	if (!flow->layers) {
> +		const struct rte_flow_item item = {
> +			.type = RTE_FLOW_ITEM_TYPE_ETH,
> +		};
> +
> +		return mlx5_flow_item_eth(&item, flow, flow_size, error);
> +	}
> +	return size;
> +}
> +
> +/**
> + * Validate action drop provided by the user.
> + *
> + * @param actions
> + *   Pointer to flow actions array.
> + * @param flow
> + *   Pointer to the rte_flow structure.
> + * @param flow_size
> + *   Size in bytes of the available space for to store the flow information.
> + * @param error
> + *   Pointer to error structure.
> + *
> + * @return
> + *   size in bytes necessary for the conversion, a negative errno value
> + *   otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_action_drop(const struct rte_flow_action *actions,
> +		      struct rte_flow *flow, const size_t flow_size,
> +		      struct rte_flow_error *error)
> +{
> +	unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
> +	struct ibv_flow_spec_action_drop drop = {
> +			.type = IBV_FLOW_SPEC_ACTION_DROP,
> +			.size = size,
> +	};
> +
> +	if (flow->fate)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ACTION,
> +					  actions,
> +					  "multiple fate actions are not"
> +					  " supported");
> +	if (size < flow_size)
> +		mlx5_flow_spec_verbs_add(flow, &drop, size);
> +	flow->fate |= MLX5_FLOW_FATE_DROP;
> +	return size;
> +}
> +
> +/**
> + * Validate actions provided by the user.
> + *
> + * @param dev
> + *   Pointer to Ethernet device structure.
> + * @param actions
> + *   Pointer to flow actions array.
> + * @param flow
> + *   Pointer to the rte_flow structure.
> + * @param flow_size
> + *   Size in bytes of the available space for to store the flow information.
> + * @param error
> + *   Pointer to error structure.
> + *
> + * @return
> + *   size in bytes necessary for the conversion, a negative errno value
> + *   otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_actions(struct rte_eth_dev *dev __rte_unused,
> +		  const struct rte_flow_action actions[],
> +		  struct rte_flow *flow, const size_t flow_size,
> +		  struct rte_flow_error *error)
> +{
> +	size_t size = 0;
> +	int remain = flow_size;
> +	int ret = 0;
> +
> +	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
> +		switch (actions->type) {
> +		case RTE_FLOW_ACTION_TYPE_VOID:
> +			break;
> +		case RTE_FLOW_ACTION_TYPE_DROP:
> +			ret = mlx5_flow_action_drop(actions, flow, remain,
> +						    error);
> +			break;
> +		default:
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ACTION,
> +						  actions,
> +						  "action not supported");
> +		}
> +		if (ret < 0)
> +			return ret;
> +		if (remain > ret)
> +			remain -= ret;
> +		else
> +			remain = 0;
> +		size += ret;
> +	}
> +	if (!flow->fate)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +					  NULL,
> +					  "no fate action found");
> +	return size;
> +}
> +
> +/**
> + * Validate the rule and return a flow structure filled accordingly.

Please add more detailed description. This is the most important function in
mlx5_flow.c, it deserves more lines of explanation. Make sure to explain about
the multiple functionality it has - validate, get required size and execute
translation only if it gets allocated memory. Plus, how to validate a flow, e.g.
setting flow_size=0 and flow can be null in that case.

> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + * @param flow
> + *   Pointer an flow structure.

s/an/to
shouldn't it be [out]?
And please mention flow can be null for testing call.

> + * @param flow_size
> + *   Size of the allocated space to store the flow information.

Please mention what happens if the size isn't enough or announce it can be
intentionally zero in order to validate the flow and to know the required size
for translation.

>   * @param[in] attr
>   *   Flow rule attributes.
>   * @param[in] pattern
> @@ -144,21 +589,39 @@ mlx5_flow_priorities(struct rte_eth_dev *dev)
>   *   Perform verbose error reporting if not NULL.
>   *
>   * @return
> - *   A flow on success, NULL otherwise and rte_errno is set.
> + *   The amount of bytes necessary to create the flow, a negative errno value
> + *   otherwise and rte_errno is set.

'amount of bytes' -> 'size of memory'
And not clear what is 'otherwise' because there's no definition of success.

>   */
> -static struct rte_flow *
> -mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
> -		      struct mlx5_flows *list __rte_unused,
> -		      const struct rte_flow_attr *attr __rte_unused,
> -		      const struct rte_flow_item items[] __rte_unused,
> -		      const struct rte_flow_action actions[] __rte_unused,
> -		      struct rte_flow_error *error)
> +static int
> +mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
> +		const size_t flow_size,
> +		const struct rte_flow_attr *attr,
> +		const struct rte_flow_item items[],
> +		const struct rte_flow_action actions[],
> +		struct rte_flow_error *error)
>  {
> -	rte_flow_error_set(error, ENOTSUP,
> -			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> -			   NULL,
> -			   "action not supported");
> -	return NULL;
> +	struct rte_flow local_flow = { .layers = 0, };
> +	size_t size = sizeof(*flow) + sizeof(struct ibv_flow_attr);
> +	int remain = (flow_size > size) ? flow_size - size : 0;
> +	int ret;
> +
> +	if (!remain)
> +		flow = &local_flow;
> +	ret = mlx5_flow_attributes(dev, attr, flow, error);
> +	if (ret < 0)
> +		return ret;
> +	ret = mlx5_flow_items(items, flow, remain, error);
> +	if (ret < 0)
> +		return ret;
> +	size += ret;
> +	remain = (flow_size > size) ? flow_size - size : 0;
> +	ret = mlx5_flow_actions(dev, actions, flow, remain, error);
> +	if (ret < 0)
> +		return ret;
> +	size += ret;
> +	if (size <= flow_size)
> +		flow->verbs.attr->priority = flow->attributes.priority;
> +	return size;
>  }
>  
>  /**
> @@ -168,16 +631,142 @@ mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
>   * @see rte_flow_ops
>   */
>  int
> -mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
> -		   const struct rte_flow_attr *attr __rte_unused,
> -		   const struct rte_flow_item items[] __rte_unused,
> -		   const struct rte_flow_action actions[] __rte_unused,
> +mlx5_flow_validate(struct rte_eth_dev *dev,
> +		   const struct rte_flow_attr *attr,
> +		   const struct rte_flow_item items[],
> +		   const struct rte_flow_action actions[],
>  		   struct rte_flow_error *error)
>  {
> -	return rte_flow_error_set(error, ENOTSUP,
> -				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> -				  NULL,
> -				  "action not supported");
> +	int ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
> +
> +	if (ret < 0)
> +		return ret;
> +	return 0;
> +}
> +
> +/**
> + * Remove the flow.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + * @param flow
> + *   Pointer to flow structure.
> + */
> +static void
> +mlx5_flow_fate_remove(struct rte_eth_dev *dev, struct rte_flow *flow)

Sounds weird. Is it removing a fate of a flow?
Description is just 'removing a flow'.

> +{
> +	if (flow->fate & MLX5_FLOW_FATE_DROP) {
> +		if (flow->verbs.flow) {
> +			claim_zero(mlx5_glue->destroy_flow(flow->verbs.flow));
> +			flow->verbs.flow = NULL;
> +		}
> +	}
> +	if (flow->verbs.hrxq) {
> +		mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
> +		flow->verbs.hrxq = NULL;
> +	}
> +}
> +
> +/**
> + * Apply the flow.
> + *
> + * @param dev
> + *   Pointer to Ethernet device structure.
> + * @param flow
> + *   Pointer to flow structure.
> + * @param error
> + *   Pointer to error structure.
> + *
> + * @return
> + *   0 on success, a negative errno value otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_fate_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
> +		     struct rte_flow_error *error)

Same here. Either change the name or add more description.

> +{
> +	flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
> +	if (!flow->verbs.hrxq)
> +		return rte_flow_error_set
> +			(error, errno,
> +			 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +			 NULL,
> +			 "cannot allocate Drop queue");
> +	flow->verbs.flow =
> +		mlx5_glue->create_flow(flow->verbs.hrxq->qp, flow->verbs.attr);
> +	if (!flow->verbs.flow) {
> +		mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
> +		flow->verbs.hrxq = NULL;
> +		return rte_flow_error_set(error, errno,
> +					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +					  NULL,
> +					  "kernel module refuses to create"
> +					  " flow");
> +	}
> +	return 0;
> +}
> +
> +/**
> + * Create a flow.

And "insert to the provided list".

> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + * @param list
> + *   Pointer to a TAILQ flow list.
> + * @param[in] attr
> + *   Flow rule attributes.
> + * @param[in] items
> + *   Pattern specification (list terminated by the END pattern item).
> + * @param[in] actions
> + *   Associated actions (list terminated by the END action).
> + * @param[out] error
> + *   Perform verbose error reporting if not NULL.
> + *
> + * @return
> + *   A flow on success, NULL otherwise and rte_errno is set.
> + */
> +static struct rte_flow *
> +mlx5_flow_list_create(struct rte_eth_dev *dev,
> +		      struct mlx5_flows *list,
> +		      const struct rte_flow_attr *attr,
> +		      const struct rte_flow_item items[],
> +		      const struct rte_flow_action actions[],
> +		      struct rte_flow_error *error)
> +{
> +	struct rte_flow *flow;
> +	size_t size;
> +	int ret;
> +
> +	ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
> +	if (ret < 0)
> +		return NULL;
> +	size = ret;
> +	flow = rte_zmalloc(__func__, size, 0);
> +	if (!flow) {
> +		rte_flow_error_set(error, ENOMEM,
> +				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +				   NULL,
> +				   "cannot allocate memory");
> +		return NULL;
> +	}
> +	flow->verbs.attr = (struct ibv_flow_attr *)(flow + 1);
> +	flow->verbs.specs = (uint8_t *)(flow->verbs.attr + 1);
> +	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
> +	if (ret < 0)
> +		goto error;
> +	assert((size_t)ret == size);
> +	if (dev->data->dev_started) {
> +		ret = mlx5_flow_fate_apply(dev, flow, error);
> +		if (ret < 0)
> +			goto error;
> +	}
> +	TAILQ_INSERT_TAIL(list, flow, next);
> +	return flow;
> +error:
> +	ret = rte_errno; /* Save rte_errno before cleanup. */
> +	mlx5_flow_fate_remove(dev, flow);
> +	rte_free(flow);
> +	rte_errno = ret; /* Restore rte_errno. */
> +	return NULL;
>  }
>  
>  /**
> @@ -187,17 +776,15 @@ mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
>   * @see rte_flow_ops
>   */
>  struct rte_flow *
> -mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
> -		 const struct rte_flow_attr *attr __rte_unused,
> -		 const struct rte_flow_item items[] __rte_unused,
> -		 const struct rte_flow_action actions[] __rte_unused,
> +mlx5_flow_create(struct rte_eth_dev *dev,
> +		 const struct rte_flow_attr *attr,
> +		 const struct rte_flow_item items[],
> +		 const struct rte_flow_action actions[],
>  		 struct rte_flow_error *error)
>  {
> -	rte_flow_error_set(error, ENOTSUP,
> -			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> -			   NULL,
> -			   "action not supported");
> -	return NULL;
> +	return mlx5_flow_list_create
> +		(dev, &((struct priv *)dev->data->dev_private)->flows,
> +		 attr, items, actions, error);
>  }
>  
>  /**
> @@ -211,10 +798,12 @@ mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
>   *   Flow to destroy.
>   */
>  static void
> -mlx5_flow_list_destroy(struct rte_eth_dev *dev __rte_unused,
> -		       struct mlx5_flows *list __rte_unused,
> -		       struct rte_flow *flow __rte_unused)
> +mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
> +		       struct rte_flow *flow)
>  {
> +	mlx5_flow_fate_remove(dev, flow);
> +	TAILQ_REMOVE(list, flow, next);
> +	rte_free(flow);
>  }
>  
>  /**
> @@ -422,9 +1011,8 @@ int
>  mlx5_flow_flush(struct rte_eth_dev *dev,
>  		struct rte_flow_error *error __rte_unused)
>  {
> -	struct priv *priv = dev->data->dev_private;
> -
> -	mlx5_flow_list_flush(dev, &priv->flows);
> +	mlx5_flow_list_flush(dev,
> +			     &((struct priv *)dev->data->dev_private)->flows);

Can you keep priv and make this line simpler?

>  	return 0;
>  }
>  
> @@ -737,9 +1325,8 @@ mlx5_fdir_filter_update(struct rte_eth_dev *dev,
>  static void
>  mlx5_fdir_filter_flush(struct rte_eth_dev *dev)
>  {
> -	struct priv *priv = dev->data->dev_private;
> -
> -	mlx5_flow_list_flush(dev, &priv->flows);
> +	mlx5_flow_list_flush(dev,
> +			     &((struct priv *)dev->data->dev_private)->flows);

Same here.

>  }
>  
>  /**
> -- 
> 2.18.0
> 

Re: [dpdk-dev] [PATCH v2 04/20] net/mlx5: support flow Ethernet item among with drop action

[Nélio Laranjeiro]

On Tue, Jul 03, 2018 at 03:27:18PM -0700, Yongseok Koh wrote:
> On Wed, Jun 27, 2018 at 05:07:36PM +0200, Nelio Laranjeiro wrote:
> 
> net/mlx5: support flow Ethernet item among with drop action
> 
> Typo? "among" -> "along"?
> 
> > Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> > ---
> >  drivers/net/mlx5/mlx5.c      |   1 +
> >  drivers/net/mlx5/mlx5_flow.c | 671 ++++++++++++++++++++++++++++++++---
> >  2 files changed, 630 insertions(+), 42 deletions(-)
> > 
> > diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
> > index c3c8dffae..665a3c31f 100644
> > --- a/drivers/net/mlx5/mlx5.c
> > +++ b/drivers/net/mlx5/mlx5.c
> > @@ -241,6 +241,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
> >  	/* In case mlx5_dev_stop() has not been called. */
> >  	mlx5_dev_interrupt_handler_uninstall(dev);
> >  	mlx5_traffic_disable(dev);
> > +	mlx5_flow_flush(dev, NULL);
> >  	/* Prevent crashes when queues are still in use. */
> >  	dev->rx_pkt_burst = removed_rx_burst;
> >  	dev->tx_pkt_burst = removed_tx_burst;
> > diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> > index 5d3bc183d..bc8c5de33 100644
> > --- a/drivers/net/mlx5/mlx5_flow.c
> > +++ b/drivers/net/mlx5/mlx5_flow.c
> > @@ -35,11 +35,50 @@
> >  extern const struct eth_dev_ops mlx5_dev_ops;
> >  extern const struct eth_dev_ops mlx5_dev_ops_isolate;
> >  
> > +/* Pattern Layer bits. */
> > +#define MLX5_FLOW_LAYER_OUTER_L2 (1u << 0)
> > +#define MLX5_FLOW_LAYER_OUTER_L3_IPV4 (1u << 1)
> > +#define MLX5_FLOW_LAYER_OUTER_L3_IPV6 (1u << 2)
> > +#define MLX5_FLOW_LAYER_OUTER_L4_UDP (1u << 3)
> > +#define MLX5_FLOW_LAYER_OUTER_L4_TCP (1u << 4)
> > +#define MLX5_FLOW_LAYER_OUTER_VLAN (1u << 5)
> > +/* Masks. */
> > +#define MLX5_FLOW_LAYER_OUTER_L3 \
> > +	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
> > +#define MLX5_FLOW_LAYER_OUTER_L4 \
> > +	(MLX5_FLOW_LAYER_OUTER_L4_UDP | MLX5_FLOW_LAYER_OUTER_L4_TCP)
> > +
> > +/* Action fate on the packet. */
> > +#define MLX5_FLOW_FATE_DROP (1u << 0)
> 
> I understand what you mean by 'fate' but 'action fate' sounds strange to me.
> How about 'terminal action' instead? E.g. flow->terminal_actions or
> flow->term_actions instead of flow->fate.

It uses the same terminology as in the flow API [1] to make it consistent
and easier to find.
In addition "terminal" in the API is also used for actions terminating
the rule .e.g. JUMP.

> And, I'm not sure why you define a new (redundant) macro. You could use
> RTE_FLOW_ACTION_TYPE_DROP instead. I don't see any reason for setting different
> action flag in flow->fate. If you want to specify a group of terminal actions,
> you could've done:
> 
> #define MLX5_FLOW_TERMINAL_ACTIONS \
> 	(RTE_FLOW_ACTION_TYPE_DROP | RTE_FLOW_ACTION_TYPE_RSS | \
> 	 RTE_FLOW_ACTION_TYPE_QUEUE)

You are missing a point, RTE_FLOW_ACTION_TYPE_* are values not bits (see
enum rte_flow_action_type), doing this won't have the expected behavior.

>[...] 
> > +/**
> > + * Validate Attributes provided by the user.
> 
> And it also stores attributes.
> 
> >   *
> >   * @param dev
> >   *   Pointer to Ethernet device.
> > - * @param list
> > - *   Pointer to a TAILQ flow list.
> > + * @param attr
> > + *   Pointer to flow attributes
> > + * @param flow
> > + *   Pointer to the rte_flow structure.
> > + * @param error
> > + *   Pointer to error structure.
> > + *
> > + * @return
> > + *   0 on success, a negative errno value otherwise and rte_errno is set.
> > + */
> > +static int
> > +mlx5_flow_attributes(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
> > +		     struct rte_flow *flow, struct rte_flow_error *error)
> 
> Better to have mlx5_flow_attributes_validate() like mlx5_flow_item_validate()?

If it stores, it cannot be named as validate.

> > +{
> > +	uint32_t priority_max =
> > +		((struct priv *)dev->data->dev_private)->config.flow_prio;
> > +
> > +	if (attr->group)
> > +		return rte_flow_error_set(error, ENOTSUP,
> > +					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
> > +					  NULL,
> > +					  "groups are not supported");
> 
> "group is not supported"?
> 
> > +	if (attr->priority >= priority_max)
> > +		return rte_flow_error_set(error, ENOTSUP,
> > +					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
> > +					  NULL,
> > +					  "priority value is not supported");
> 
> "priority is out of range" or something else?
> 
> > +	if (attr->egress)
> > +		return rte_flow_error_set(error, ENOTSUP,
> > +					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
> > +					  NULL,
> > +					  "egress is not supported");
> > +	if (attr->transfer)
> > +		return rte_flow_error_set(error, ENOTSUP,
> > +					  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
> > +					  NULL,
> > +					  "transfer is not supported");
> > +	if (!attr->ingress)
> > +		return rte_flow_error_set(error, ENOTSUP,
> > +					  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
> > +					  NULL,
> > +					  "only ingress is supported");
> 
> Better to have "ingress should be specified" or something else?

Done,

> > +	flow->attributes = *attr;
> > +	return 0;
> > +}
> > +
> > +/**
> > + * Check support for a given item.
> > + *
> > + * @param item[in]
> > + *   Item specification.
> > + * @param default_mask[in]
> > + *   Bit-masks covering supported fields to compare with spec, last and mask in
> > + *   \item.
> 
> Remove the back-slash and the description isn't clear. From the code, it seems
> to be a default mask to be used if item doesn't have mask.
> 
> > + * @param nic_mask[in]
> > + *   Bit-masks covering supported fields by the NIC to compare with user mask.
> 
> s/Bit-masks/Bit-mask

All over the literature it is bit-masks.

> > + * @param size
> > + *   Bit-Mask size in bytes.
> 
> s/Bit-Mask/Bit-mask
>
> > + * @param error[out]
> > + *   Pointer to error structure.
> > + *
> > + * @return
> > + *   0 on success, a negative errno value otherwise and rte_errno is set.
> > + */
> > +static int
> > +mlx5_flow_item_validate(const struct rte_flow_item *item,
> > +			const uint8_t *default_mask,
> > +			const uint8_t *nic_mask,
> > +			unsigned int size,
> > +			struct rte_flow_error *error)
> > +{
> > +	const uint8_t *mask = item->mask ? item->mask : default_mask;
> 
> In the mlx5_flow_item_eth(), rte_flow_item_eth_mask is passed to default_mask if
> item->mask is null, then why doing it again here?

You are right, it is not necessary here again.

> 
> > +	unsigned int i;
> > +
> > +	assert(nic_mask);
> > +	for (i = 0; i < size; ++i)
> > +		if ((nic_mask[i] | mask[i]) != nic_mask[i])
> > +			return rte_flow_error_set(error, ENOTSUP,
> > +						  RTE_FLOW_ERROR_TYPE_ITEM,
> > +						  item,
> > +						  "mask enables non supported"
> > +						  " bits");
> 
> default_mask is determined by caller, which is rte_flow_item_eth_mask or
> item->mask. And nic_mask is also given by PMD, which is a function local
> structure in the caller func. Doesn't look like a clear definition. Intead of
> having both default_mask and nic_mask here, you can take supported_mask, which
> can be (rte_flow_item_{name}_mask | nic_mask) in the calling func. And check
> validity of item->mask against supported_mask only if item->mask isn't null.

Moving the default_mask to the callee duplicates a lot of code all over
the item functions and there are several, in addition we need to
confront the user/default mask to the supported NIC mask to be able to
inform him when he tries to match bits unsupported by the NIC.  Doing
this also increase the amount of code.
This means that this for loop should be copy/paste inside each callee
function.

> > +	if (!item->spec && (item->mask || item->last))
> > +		return rte_flow_error_set(error, EINVAL,
> > +					  RTE_FLOW_ERROR_TYPE_ITEM,
> > +					  item,
> > +					  "mask/last without a spec is not"
> > +					  " supported");
> > +	if (item->spec && item->last) {
> > +		uint8_t spec[size];
> > +		uint8_t last[size];
> > +		unsigned int i;
> > +		int ret;
> > +
> > +		for (i = 0; i < size; ++i) {
> > +			spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
> > +			last[i] = ((const uint8_t *)item->last)[i] & mask[i];
> > +		}
> > +		ret = memcmp(spec, last, size);
> > +		if (ret != 0)
> > +			return rte_flow_error_set(error, ENOTSUP,
> > +						  RTE_FLOW_ERROR_TYPE_ITEM,
> > +						  item,
> > +						  "spec and last different"
> > +						  " is not supported");
> 
> Can you rephrase the error message?

Done,

>[...] 
> > +static int
> > +mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
> > +		   const size_t flow_size, struct rte_flow_error *error)
> > +{
> > +	const struct rte_flow_item_eth *spec = item->spec;
> > +	const struct rte_flow_item_eth *mask = item->mask;
> > +	const struct rte_flow_item_eth nic_mask = {
> > +		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
> > +		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
> > +		.type = RTE_BE16(0xffff),
> > +	};
> 
> If it is a device-specific one, shouldn't it be located in a header file?

I would agree if it was used in several places.

> > +	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
> > +	struct ibv_flow_spec_eth eth = {
> > +		.type = IBV_FLOW_SPEC_ETH,
> > +		.size = size,
> > +	};
> > +	int ret;
> > +
> > +	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L2)
> > +		return rte_flow_error_set(error, ENOTSUP,
> > +					  RTE_FLOW_ERROR_TYPE_ITEM,
> > +					  item,
> > +					  "L2 layers already configured");
> > +	if (!mask)
> > +		mask = &rte_flow_item_eth_mask;
> 
> What is the meaning of default_mask for mlx5_flow_item_validate()? Also refer
> to my comments in mlx5_flow_item_validate().

An item should always have a mask defined, if the user does not set a
mask, we need to pick one and in such situation it is the flow default
mask.  But flow default mask may enable matching bits unsupported by the
NIC.

> > +	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> > +				      (const uint8_t *)&nic_mask,
> > +				      sizeof(struct rte_flow_item_eth),
> > +				      error);
> > +	if (ret)
> > +		return ret;
> > +	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2;
> > +	if (size > flow_size)
> > +		return size;
> > +	if (spec) {
> > +		unsigned int i;
> > +
> > +		memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
> > +		memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
> > +		eth.val.ether_type = spec->type;
> > +		memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
> > +		memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
> > +		eth.mask.ether_type = mask->type;
> > +		/* Remove unwanted bits from values. */
> > +		for (i = 0; i < ETHER_ADDR_LEN; ++i) {
> > +			eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
> > +			eth.val.src_mac[i] &= eth.mask.src_mac[i];
> > +		}
> > +		eth.val.ether_type &= eth.mask.ether_type;
> > +	}
> > +	mlx5_flow_spec_verbs_add(flow, &eth, size);
> 
> Do you have a plan to change this redundant memcpy? I mean, writing directly to
> the allocated memory for verbs spec instead of the local micro-scratchpad.

Unfortunately I don't have time for this for this release, but it is not
the only improvement to make, I'll share with you privately.

>[...]
> > +
> > +/**
> > + * Validate the rule and return a flow structure filled accordingly.
> 
> Please add more detailed description. This is the most important function in
> mlx5_flow.c, it deserves more lines of explanation. Make sure to explain about
> the multiple functionality it has - validate, get required size and execute
> translation only if it gets allocated memory. Plus, how to validate a flow, e.g.
> setting flow_size=0 and flow can be null in that case.

Agreed,
 
> > + *
> > + * @param dev
> > + *   Pointer to Ethernet device.
> > + * @param flow
> > + *   Pointer an flow structure.
> 
> s/an/to
> shouldn't it be [out]?

[in, out] in fact.

> And please mention flow can be null for testing call.

It can be anything until the flow_size is accurate.  The testing is not
based on the flow nor the size, but in the fact the conversion may use
more size than the one provided.

> > + * @param flow_size
> > + *   Size of the allocated space to store the flow information.
> 
> Please mention what happens if the size isn't enough or announce it can be
> intentionally zero in order to validate the flow and to know the required size
> for translation.
> 
> >   * @param[in] attr
> >   *   Flow rule attributes.
> >   * @param[in] pattern
> > @@ -144,21 +589,39 @@ mlx5_flow_priorities(struct rte_eth_dev *dev)
> >   *   Perform verbose error reporting if not NULL.
> >   *
> >   * @return
> > - *   A flow on success, NULL otherwise and rte_errno is set.
> > + *   The amount of bytes necessary to create the flow, a negative errno value
> > + *   otherwise and rte_errno is set.
> 
> 'amount of bytes' -> 'size of memory'
> And not clear what is 'otherwise' because there's no definition of success.

Reworded.

>[...]
> > +/**
> > + * Remove the flow.
> > + *
> > + * @param dev
> > + *   Pointer to Ethernet device.
> > + * @param flow
> > + *   Pointer to flow structure.
> > + */
> > +static void
> > +mlx5_flow_fate_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
> 
> Sounds weird. Is it removing a fate of a flow?
> Description is just 'removing a flow'.
>[...]
> > +/**
> > + * Apply the flow.
> > + *
> > + * @param dev
> > + *   Pointer to Ethernet device structure.
> > + * @param flow
> > + *   Pointer to flow structure.
> > + * @param error
> > + *   Pointer to error structure.
> > + *
> > + * @return
> > + *   0 on success, a negative errno value otherwise and rte_errno is set.
> > + */
> > +static int
> > +mlx5_flow_fate_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
> > +		     struct rte_flow_error *error)
> 
> Same here. Either change the name or add more description.

Agreed, "fate" removed in both functions.

>[...]
> > +/**
> > + * Create a flow.
> 
> And "insert to the provided list".

Added.

> > + *
> > + * @param dev
> > + *   Pointer to Ethernet device.
> > + * @param list
> > + *   Pointer to a TAILQ flow list.
> > + * @param[in] attr
> > + *   Flow rule attributes.
> > + * @param[in] items
> > + *   Pattern specification (list terminated by the END pattern item).
> > + * @param[in] actions
> > + *   Associated actions (list terminated by the END action).
> > + * @param[out] error
> > + *   Perform verbose error reporting if not NULL.
> > + *
> > + * @return
> > + *   A flow on success, NULL otherwise and rte_errno is set.
> > + */
> > +static struct rte_flow *
> > +mlx5_flow_list_create(struct rte_eth_dev *dev,
> > +		      struct mlx5_flows *list,
> > +		      const struct rte_flow_attr *attr,
> > +		      const struct rte_flow_item items[],
> > +		      const struct rte_flow_action actions[],
> > +		      struct rte_flow_error *error)
> > +{
>[...]
> >  /**
> > @@ -422,9 +1011,8 @@ int
> >  mlx5_flow_flush(struct rte_eth_dev *dev,
> >  		struct rte_flow_error *error __rte_unused)
> >  {
> > -	struct priv *priv = dev->data->dev_private;
> > -
> > -	mlx5_flow_list_flush(dev, &priv->flows);
> > +	mlx5_flow_list_flush(dev,
> > +			     &((struct priv *)dev->data->dev_private)->flows);
> 
> Can you keep priv and make this line simpler?
> 
> >  	return 0;
> >  }
> >  
> > @@ -737,9 +1325,8 @@ mlx5_fdir_filter_update(struct rte_eth_dev *dev,
> >  static void
> >  mlx5_fdir_filter_flush(struct rte_eth_dev *dev)
> >  {
> > -	struct priv *priv = dev->data->dev_private;
> > -
> > -	mlx5_flow_list_flush(dev, &priv->flows);
> > +	mlx5_flow_list_flush(dev,
> > +			     &((struct priv *)dev->data->dev_private)->flows);
> 
> Same here.

It was a request from Adrien. (done anyway).

Thanks,

[1] https://doc.dpdk.org/guides/prog_guide/rte_flow.html?highlight=rte_flow#actions

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH v2 05/20] net/mlx5: add flow queue action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 100 +++++++++++++++++++++++++++++++----
 1 file changed, 89 insertions(+), 11 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index bc8c5de33..7ac1ba24c 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -50,6 +50,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
 /* Action fate on the packet. */
 #define MLX5_FLOW_FATE_DROP (1u << 0)
+#define MLX5_FLOW_FATE_QUEUE (1u << 1)
 
 /** Handles information leading to a drop fate. */
 struct mlx5_flow_verbs {
@@ -71,7 +72,8 @@ struct rte_flow {
 	/**< Bit-fields of present layers see MLX5_FLOW_ITEMS_*. */
 	uint32_t fate;
 	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
-	struct mlx5_flow_verbs verbs; /* Verbs drop flow. */
+	struct mlx5_flow_verbs verbs; /* Verbs flow. */
+	uint16_t queue; /**< Destination queue to redirect traffic to. */
 };
 
 static const struct rte_flow_ops mlx5_flow_ops = {
@@ -191,6 +193,8 @@ mlx5_flow_print(struct rte_flow *flow __rte_unused)
 		flow->layers & MLX5_FLOW_LAYER_OUTER_L4 ? "l4" : "-");
 	if (flow->fate & MLX5_FLOW_FATE_DROP)
 		fprintf(stdout, " fate: drop queue\n");
+	else if (flow->fate & MLX5_FLOW_FATE_QUEUE)
+		fprintf(stdout, " fate: target queue %u\n", flow->queue);
 	if (flow->verbs.attr) {
 		struct ibv_spec_header *hdr =
 			(struct ibv_spec_header *)flow->verbs.specs;
@@ -512,6 +516,53 @@ mlx5_flow_action_drop(const struct rte_flow_action *actions,
 	return size;
 }
 
+/**
+ * Validate action queue provided by the user.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param actions
+ *   Pointer to flow actions array.
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param flow_size
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_action_queue(struct rte_eth_dev *dev,
+		       const struct rte_flow_action *actions,
+		       struct rte_flow *flow,
+		       struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_action_queue *queue = actions->conf;
+
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "multiple fate actions are not"
+					  " supported");
+	if (queue->index >= priv->rxqs_n)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &queue->index,
+					  "queue index out of range");
+	if (!(*priv->rxqs)[queue->index])
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &queue->index,
+					  "queue is not configured");
+	flow->queue = queue->index;
+	flow->fate |= MLX5_FLOW_FATE_QUEUE;
+	return 0;
+}
+
 /**
  * Validate actions provided by the user.
  *
@@ -531,7 +582,7 @@ mlx5_flow_action_drop(const struct rte_flow_action *actions,
  *   otherwise and rte_errno is set.
  */
 static int
-mlx5_flow_actions(struct rte_eth_dev *dev __rte_unused,
+mlx5_flow_actions(struct rte_eth_dev *dev,
 		  const struct rte_flow_action actions[],
 		  struct rte_flow *flow, const size_t flow_size,
 		  struct rte_flow_error *error)
@@ -548,6 +599,9 @@ mlx5_flow_actions(struct rte_eth_dev *dev __rte_unused,
 			ret = mlx5_flow_action_drop(actions, flow, remain,
 						    error);
 			break;
+		case RTE_FLOW_ACTION_TYPE_QUEUE:
+			ret = mlx5_flow_action_queue(dev, actions, flow, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -662,7 +716,10 @@ mlx5_flow_fate_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
 		}
 	}
 	if (flow->verbs.hrxq) {
-		mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
+		if (flow->fate & MLX5_FLOW_FATE_DROP)
+			mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
+		else if (flow->fate & MLX5_FLOW_FATE_QUEUE)
+			mlx5_hrxq_release(dev, flow->verbs.hrxq);
 		flow->verbs.hrxq = NULL;
 	}
 }
@@ -684,17 +741,38 @@ static int
 mlx5_flow_fate_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 		     struct rte_flow_error *error)
 {
-	flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
-	if (!flow->verbs.hrxq)
-		return rte_flow_error_set
-			(error, errno,
-			 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			 NULL,
-			 "cannot allocate Drop queue");
+	if (flow->fate & MLX5_FLOW_FATE_DROP) {
+		flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
+		if (!flow->verbs.hrxq)
+			return rte_flow_error_set
+				(error, errno,
+				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				 NULL,
+				 "cannot allocate Drop queue");
+	} else if (flow->fate & MLX5_FLOW_FATE_QUEUE) {
+		struct mlx5_hrxq *hrxq;
+
+		hrxq = mlx5_hrxq_get(dev, rss_hash_default_key,
+				     rss_hash_default_key_len, 0,
+				     &flow->queue, 1, 0, 0);
+		if (!hrxq)
+			hrxq = mlx5_hrxq_new(dev, rss_hash_default_key,
+					     rss_hash_default_key_len, 0,
+					     &flow->queue, 1, 0, 0);
+		if (!hrxq)
+			return rte_flow_error_set(error, rte_errno,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"cannot create flow");
+		flow->verbs.hrxq = hrxq;
+	}
 	flow->verbs.flow =
 		mlx5_glue->create_flow(flow->verbs.hrxq->qp, flow->verbs.attr);
 	if (!flow->verbs.flow) {
-		mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
+		if (flow->fate & MLX5_FLOW_FATE_DROP)
+			mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
+		else
+			mlx5_hrxq_release(dev, flow->verbs.hrxq);
 		flow->verbs.hrxq = NULL;
 		return rte_flow_error_set(error, errno,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 05/20] net/mlx5: add flow queue action

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:37PM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
Acked-by: Yongseok Koh <yskoh@mellanox.com>

Thanks

[dpdk-dev] [PATCH v2 06/20] net/mlx5: add flow stop/start

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 7ac1ba24c..6593eafa0 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -912,9 +912,12 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
  *   Pointer to a TAILQ flow list.
  */
 void
-mlx5_flow_stop(struct rte_eth_dev *dev __rte_unused,
-	       struct mlx5_flows *list __rte_unused)
+mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
 {
+	struct rte_flow *flow;
+
+	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
+		mlx5_flow_fate_remove(dev, flow);
 }
 
 /**
@@ -929,10 +932,23 @@ mlx5_flow_stop(struct rte_eth_dev *dev __rte_unused,
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_flow_start(struct rte_eth_dev *dev __rte_unused,
-		struct mlx5_flows *list __rte_unused)
+mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
 {
+	struct rte_flow *flow;
+	struct rte_flow_error error;
+	int ret = 0;
+
+	TAILQ_FOREACH(flow, list, next) {
+		ret = mlx5_flow_fate_apply(dev, flow, &error);
+		if (ret < 0)
+			goto error;
+	}
 	return 0;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	mlx5_flow_stop(dev, list);
+	rte_errno = ret; /* Restore rte_errno. */
+	return -rte_errno;
 }
 
 /**
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 06/20] net/mlx5: add flow stop/start

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:38PM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
Acked-by: Yongseok Koh <yskoh@mellanox.com>

Thanks

[dpdk-dev] [PATCH v2 07/20] net/mlx5: add flow VLAN item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 123 +++++++++++++++++++++++++++++++++++
 1 file changed, 123 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 6593eafa0..6a576ddd9 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -420,6 +420,126 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Update the VLAN tag in the Ethernet spec.
+ *
+ * @param attr[in, out]
+ *   Pointer to Verbs attributes structure.
+ * @param eth[in]
+ *   Verbs structure containing the VLAN information to copy.
+ */
+static void
+mlx5_flow_item_vlan_update(struct ibv_flow_attr *attr,
+			   struct ibv_flow_spec_eth *eth)
+{
+	unsigned int i;
+	enum ibv_flow_spec_type search = IBV_FLOW_SPEC_ETH;
+	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
+		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
+
+	for (i = 0; i != attr->num_of_specs; ++i) {
+		if (hdr->type == search) {
+			struct ibv_flow_spec_eth *e =
+				(struct ibv_flow_spec_eth *)hdr;
+
+			e->val.vlan_tag = eth->val.vlan_tag;
+			e->mask.vlan_tag = eth->mask.vlan_tag;
+			break;
+		}
+		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
+	}
+}
+
+/**
+ * Validate VLAN layer and possibly create/modify the Verbs specification.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param flow_size[in]
+ *   Size of the buffer to store the specification.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_vlan *spec = item->spec;
+	const struct rte_flow_item_vlan *mask = item->mask;
+	const struct rte_flow_item_vlan nic_mask = {
+		.tci = RTE_BE16(0x0fff),
+	};
+	unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	struct ibv_flow_spec_eth eth = {
+		.type = IBV_FLOW_SPEC_ETH,
+		.size = size,
+	};
+	int ret;
+	const uint32_t lm = MLX5_FLOW_LAYER_OUTER_L3 |
+			MLX5_FLOW_LAYER_OUTER_L4;
+	const uint32_t vlanm = MLX5_FLOW_LAYER_OUTER_VLAN;
+	const uint32_t l2m = MLX5_FLOW_LAYER_OUTER_L2;
+
+	if (flow->layers & vlanm)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L2 layers already configured");
+	else if ((flow->layers & lm) != 0)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L2 layer cannot follow L3/L4 layer");
+	if (!mask)
+		mask = &rte_flow_item_vlan_mask;
+	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+				      (const uint8_t *)&nic_mask,
+				      sizeof(struct rte_flow_item_vlan), error);
+	if (ret)
+		return ret;
+	if (spec) {
+		eth.val.vlan_tag = spec->tci;
+		eth.mask.vlan_tag = mask->tci;
+		eth.val.vlan_tag &= eth.mask.vlan_tag;
+		eth.val.ether_type = spec->inner_type;
+		eth.mask.ether_type = mask->inner_type;
+		eth.val.ether_type &= eth.mask.ether_type;
+	}
+	/*
+	 * From verbs perspective an empty VLAN is equivalent
+	 * to a packet without VLAN layer.
+	 */
+	if (!eth.mask.vlan_tag)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+					  item->spec,
+					  "VLAN cannot be empty");
+	/* Outer TPID cannot be matched. */
+	if (eth.mask.ether_type)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+					  item->spec,
+					  "VLAN TPID matching is not"
+					  " supported");
+	if (!(flow->layers & l2m)) {
+		if (size <= flow_size)
+			mlx5_flow_spec_verbs_add(flow, &eth, size);
+	} else {
+		if (flow->verbs.attr)
+			mlx5_flow_item_vlan_update(flow->verbs.attr, &eth);
+		size = 0; /**< Only an update is done in eth specification. */
+	}
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2 |
+		MLX5_FLOW_LAYER_OUTER_VLAN;
+	return size;
+}
+
 /**
  * Validate items provided by the user.
  *
@@ -453,6 +573,9 @@ mlx5_flow_items(const struct rte_flow_item items[],
 		case RTE_FLOW_ITEM_TYPE_ETH:
 			ret = mlx5_flow_item_eth(items, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			ret = mlx5_flow_item_vlan(items, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 07/20] net/mlx5: add flow VLAN item

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:39PM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
>  drivers/net/mlx5/mlx5_flow.c | 123 +++++++++++++++++++++++++++++++++++
>  1 file changed, 123 insertions(+)
> 
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index 6593eafa0..6a576ddd9 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -420,6 +420,126 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
>  	return size;
>  }
>  
> +/**
> + * Update the VLAN tag in the Ethernet spec.
> + *
> + * @param attr[in, out]
> + *   Pointer to Verbs attributes structure.
> + * @param eth[in]
> + *   Verbs structure containing the VLAN information to copy.
> + */
> +static void
> +mlx5_flow_item_vlan_update(struct ibv_flow_attr *attr,
> +			   struct ibv_flow_spec_eth *eth)
> +{
> +	unsigned int i;
> +	enum ibv_flow_spec_type search = IBV_FLOW_SPEC_ETH;
> +	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
> +		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
> +
> +	for (i = 0; i != attr->num_of_specs; ++i) {
> +		if (hdr->type == search) {
> +			struct ibv_flow_spec_eth *e =
> +				(struct ibv_flow_spec_eth *)hdr;
> +
> +			e->val.vlan_tag = eth->val.vlan_tag;
> +			e->mask.vlan_tag = eth->mask.vlan_tag;

You'll have to overwrite ether_type as well, otherwise the e->ether_type would
have TPID.

> +			break;
> +		}
> +		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
> +	}
> +}
> +
> +/**
> + * Validate VLAN layer and possibly create/modify the Verbs specification.
> + *
> + * @param item[in]
> + *   Item specification.
> + * @param flow[in, out]
> + *   Pointer to flow structure.
> + * @param flow_size[in]
> + *   Size of the buffer to store the specification.
> + * @param error
> + *   Pointer to error structure.
> + *
> + * @return
> + *   size in bytes necessary for the conversion, a negative errno value
> + *   otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
> +		    const size_t flow_size, struct rte_flow_error *error)
> +{
> +	const struct rte_flow_item_vlan *spec = item->spec;
> +	const struct rte_flow_item_vlan *mask = item->mask;
> +	const struct rte_flow_item_vlan nic_mask = {
> +		.tci = RTE_BE16(0x0fff),
> +	};
> +	unsigned int size = sizeof(struct ibv_flow_spec_eth);
> +	struct ibv_flow_spec_eth eth = {
> +		.type = IBV_FLOW_SPEC_ETH,
> +		.size = size,
> +	};
> +	int ret;
> +	const uint32_t lm = MLX5_FLOW_LAYER_OUTER_L3 |
> +			MLX5_FLOW_LAYER_OUTER_L4;

What does 'lm' stand for? Better to use l34m?

> +	const uint32_t vlanm = MLX5_FLOW_LAYER_OUTER_VLAN;
> +	const uint32_t l2m = MLX5_FLOW_LAYER_OUTER_L2;
> +
> +	if (flow->layers & vlanm)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "L2 layers already configured");

L2? Not VLAN?

> +	else if ((flow->layers & lm) != 0)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "L2 layer cannot follow L3/L4 layer");
> +	if (!mask)
> +		mask = &rte_flow_item_vlan_mask;
> +	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> +				      (const uint8_t *)&nic_mask,
> +				      sizeof(struct rte_flow_item_vlan), error);
> +	if (ret)
> +		return ret;
> +	if (spec) {
> +		eth.val.vlan_tag = spec->tci;
> +		eth.mask.vlan_tag = mask->tci;
> +		eth.val.vlan_tag &= eth.mask.vlan_tag;
> +		eth.val.ether_type = spec->inner_type;
> +		eth.mask.ether_type = mask->inner_type;
> +		eth.val.ether_type &= eth.mask.ether_type;
> +	}
> +	/*
> +	 * From verbs perspective an empty VLAN is equivalent
> +	 * to a packet without VLAN layer.
> +	 */
> +	if (!eth.mask.vlan_tag)
> +		return rte_flow_error_set(error, EINVAL,
> +					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
> +					  item->spec,
> +					  "VLAN cannot be empty");
> +	/* Outer TPID cannot be matched. */
> +	if (eth.mask.ether_type)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
> +					  item->spec,
> +					  "VLAN TPID matching is not"
> +					  " supported");

Not sure 100% but I don't think ether_type means TPID but the inner packet type
coming after the VLAN ID. E.g.

	/dmac/smac/0x8100/TCI/0x0800/ipv4...
                                 ^
                                 |

> +	if (!(flow->layers & l2m)) {
> +		if (size <= flow_size)
> +			mlx5_flow_spec_verbs_add(flow, &eth, size);
> +	} else {
> +		if (flow->verbs.attr)
> +			mlx5_flow_item_vlan_update(flow->verbs.attr, &eth);
> +		size = 0; /**< Only an update is done in eth specification. */

Any specific reason to use doxygen style comment only here?

> +	}
> +	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2 |
> +		MLX5_FLOW_LAYER_OUTER_VLAN;
> +	return size;
> +}
> +
>  /**
>   * Validate items provided by the user.
>   *
> @@ -453,6 +573,9 @@ mlx5_flow_items(const struct rte_flow_item items[],
>  		case RTE_FLOW_ITEM_TYPE_ETH:
>  			ret = mlx5_flow_item_eth(items, flow, remain, error);
>  			break;
> +		case RTE_FLOW_ITEM_TYPE_VLAN:
> +			ret = mlx5_flow_item_vlan(items, flow, remain, error);
> +			break;
>  		default:
>  			return rte_flow_error_set(error, ENOTSUP,
>  						  RTE_FLOW_ERROR_TYPE_ITEM,
> -- 
> 2.18.0
> 

Re: [dpdk-dev] [PATCH v2 07/20] net/mlx5: add flow VLAN item

[Nélio Laranjeiro]

On Tue, Jul 03, 2018 at 04:56:21PM -0700, Yongseok Koh wrote:
> On Wed, Jun 27, 2018 at 05:07:39PM +0200, Nelio Laranjeiro wrote:
> > Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> > ---
> >  drivers/net/mlx5/mlx5_flow.c | 123 +++++++++++++++++++++++++++++++++++
> >  1 file changed, 123 insertions(+)
> > 
> > diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> > index 6593eafa0..6a576ddd9 100644
> > --- a/drivers/net/mlx5/mlx5_flow.c
> > +++ b/drivers/net/mlx5/mlx5_flow.c
> > @@ -420,6 +420,126 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
> >  	return size;
> >  }
> >  
> > +/**
> > + * Update the VLAN tag in the Ethernet spec.
> > + *
> > + * @param attr[in, out]
> > + *   Pointer to Verbs attributes structure.
> > + * @param eth[in]
> > + *   Verbs structure containing the VLAN information to copy.
> > + */
> > +static void
> > +mlx5_flow_item_vlan_update(struct ibv_flow_attr *attr,
> > +			   struct ibv_flow_spec_eth *eth)
> > +{
> > +	unsigned int i;
> > +	enum ibv_flow_spec_type search = IBV_FLOW_SPEC_ETH;
> > +	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
> > +		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
> > +
> > +	for (i = 0; i != attr->num_of_specs; ++i) {
> > +		if (hdr->type == search) {
> > +			struct ibv_flow_spec_eth *e =
> > +				(struct ibv_flow_spec_eth *)hdr;
> > +
> > +			e->val.vlan_tag = eth->val.vlan_tag;
> > +			e->mask.vlan_tag = eth->mask.vlan_tag;
> 
> You'll have to overwrite ether_type as well, otherwise the e->ether_type would
> have TPID.

Right,

> > +			break;
> > +		}
> > +		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
> > +	}
> > +}
> > +
> > +/**
> > + * Validate VLAN layer and possibly create/modify the Verbs specification.
> > + *
> > + * @param item[in]
> > + *   Item specification.
> > + * @param flow[in, out]
> > + *   Pointer to flow structure.
> > + * @param flow_size[in]
> > + *   Size of the buffer to store the specification.
> > + * @param error
> > + *   Pointer to error structure.
> > + *
> > + * @return
> > + *   size in bytes necessary for the conversion, a negative errno value
> > + *   otherwise and rte_errno is set.
> > + */
> > +static int
> > +mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
> > +		    const size_t flow_size, struct rte_flow_error *error)
> > +{
> > +	const struct rte_flow_item_vlan *spec = item->spec;
> > +	const struct rte_flow_item_vlan *mask = item->mask;
> > +	const struct rte_flow_item_vlan nic_mask = {
> > +		.tci = RTE_BE16(0x0fff),
> > +	};
> > +	unsigned int size = sizeof(struct ibv_flow_spec_eth);
> > +	struct ibv_flow_spec_eth eth = {
> > +		.type = IBV_FLOW_SPEC_ETH,
> > +		.size = size,
> > +	};
> > +	int ret;
> > +	const uint32_t lm = MLX5_FLOW_LAYER_OUTER_L3 |
> > +			MLX5_FLOW_LAYER_OUTER_L4;
> 
> What does 'lm' stand for? Better to use l34m?

l34m seems better,

> > +	const uint32_t vlanm = MLX5_FLOW_LAYER_OUTER_VLAN;
> > +	const uint32_t l2m = MLX5_FLOW_LAYER_OUTER_L2;
> > +
> > +	if (flow->layers & vlanm)
> > +		return rte_flow_error_set(error, ENOTSUP,
> > +					  RTE_FLOW_ERROR_TYPE_ITEM,
> > +					  item,
> > +					  "L2 layers already configured");
> 
> L2? Not VLAN?

Indeed it is VLAN.

> > +	else if ((flow->layers & lm) != 0)
> > +		return rte_flow_error_set(error, ENOTSUP,
> > +					  RTE_FLOW_ERROR_TYPE_ITEM,
> > +					  item,
> > +					  "L2 layer cannot follow L3/L4 layer");
> > +	if (!mask)
> > +		mask = &rte_flow_item_vlan_mask;
> > +	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> > +				      (const uint8_t *)&nic_mask,
> > +				      sizeof(struct rte_flow_item_vlan), error);
> > +	if (ret)
> > +		return ret;
> > +	if (spec) {
> > +		eth.val.vlan_tag = spec->tci;
> > +		eth.mask.vlan_tag = mask->tci;
> > +		eth.val.vlan_tag &= eth.mask.vlan_tag;
> > +		eth.val.ether_type = spec->inner_type;
> > +		eth.mask.ether_type = mask->inner_type;
> > +		eth.val.ether_type &= eth.mask.ether_type;
> > +	}
> > +	/*
> > +	 * From verbs perspective an empty VLAN is equivalent
> > +	 * to a packet without VLAN layer.
> > +	 */
> > +	if (!eth.mask.vlan_tag)
> > +		return rte_flow_error_set(error, EINVAL,
> > +					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
> > +					  item->spec,
> > +					  "VLAN cannot be empty");
> > +	/* Outer TPID cannot be matched. */
> > +	if (eth.mask.ether_type)
> > +		return rte_flow_error_set(error, ENOTSUP,
> > +					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
> > +					  item->spec,
> > +					  "VLAN TPID matching is not"
> > +					  " supported");
> 
> Not sure 100% but I don't think ether_type means TPID but the inner packet type
> coming after the VLAN ID. E.g.
> 
> 	/dmac/smac/0x8100/TCI/0x0800/ipv4...
>                                  ^
>                                  |

I remember to have faced such issue several months ago, from what I
understood it is configurable but there is no way in Verbs to make it.
Currently (after testing) it matches the above logic, I will remove this
check.

> > +	if (!(flow->layers & l2m)) {
> > +		if (size <= flow_size)
> > +			mlx5_flow_spec_verbs_add(flow, &eth, size);
> > +	} else {
> > +		if (flow->verbs.attr)
> > +			mlx5_flow_item_vlan_update(flow->verbs.attr, &eth);
> > +		size = 0; /**< Only an update is done in eth specification. */
> 
> Any specific reason to use doxygen style comment only here?

No reason, I'll remove it.

Thanks,

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH v2 08/20] net/mlx5: add flow IPv4 item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 83 ++++++++++++++++++++++++++++++++++++
 1 file changed, 83 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 6a576ddd9..8e7a0bb5a 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -540,6 +540,86 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Validate IPv4 layer and possibly create the Verbs specification.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param flow_size[in]
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_ipv4 *spec = item->spec;
+	const struct rte_flow_item_ipv4 *mask = item->mask;
+	const struct rte_flow_item_ipv4 nic_mask = {
+		.hdr = {
+			.src_addr = RTE_BE32(0xffffffff),
+			.dst_addr = RTE_BE32(0xffffffff),
+			.type_of_service = 0xff,
+			.next_proto_id = 0xff,
+		},
+	};
+	unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
+	struct ibv_flow_spec_ipv4_ext ipv4 = {
+		.type = IBV_FLOW_SPEC_IPV4_EXT,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "multiple L3 layers not supported");
+	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 cannot follow an L4 layer.");
+	if (!mask)
+		mask = &rte_flow_item_ipv4_mask;
+	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+				      (const uint8_t *)&nic_mask,
+				      sizeof(struct rte_flow_item_ipv4), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
+			.src_ip = spec->hdr.src_addr,
+			.dst_ip = spec->hdr.dst_addr,
+			.proto = spec->hdr.next_proto_id,
+			.tos = spec->hdr.type_of_service,
+		};
+		ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
+			.src_ip = mask->hdr.src_addr,
+			.dst_ip = mask->hdr.dst_addr,
+			.proto = mask->hdr.next_proto_id,
+			.tos = mask->hdr.type_of_service,
+		};
+		/* Remove unwanted bits from values. */
+		ipv4.val.src_ip &= ipv4.mask.src_ip;
+		ipv4.val.dst_ip &= ipv4.mask.dst_ip;
+		ipv4.val.proto &= ipv4.mask.proto;
+		ipv4.val.tos &= ipv4.mask.tos;
+	}
+	mlx5_flow_spec_verbs_add(flow, &ipv4, size);
+	return size;
+}
+
 /**
  * Validate items provided by the user.
  *
@@ -576,6 +656,9 @@ mlx5_flow_items(const struct rte_flow_item items[],
 		case RTE_FLOW_ITEM_TYPE_VLAN:
 			ret = mlx5_flow_item_vlan(items, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			ret = mlx5_flow_item_ipv4(items, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 08/20] net/mlx5: add flow IPv4 item

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:40PM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
>  drivers/net/mlx5/mlx5_flow.c | 83 ++++++++++++++++++++++++++++++++++++
>  1 file changed, 83 insertions(+)
> 
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index 6a576ddd9..8e7a0bb5a 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -540,6 +540,86 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
>  	return size;
>  }
>  
> +/**
> + * Validate IPv4 layer and possibly create the Verbs specification.

Instead of 'possibly', please describe what it does in details like I asked for
the previous patch:
	net/mlx5: support flow Ethernet item among with drop action

Thanks,
Yongseok

> + *
> + * @param item[in]
> + *   Item specification.
> + * @param flow[in, out]
> + *   Pointer to flow structure.
> + * @param flow_size[in]
> + *   Size in bytes of the available space for to store the flow information.
> + * @param error
> + *   Pointer to error structure.
> + *
> + * @return
> + *   size in bytes necessary for the conversion, a negative errno value
> + *   otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
> +		    const size_t flow_size, struct rte_flow_error *error)
> +{
> +	const struct rte_flow_item_ipv4 *spec = item->spec;
> +	const struct rte_flow_item_ipv4 *mask = item->mask;
> +	const struct rte_flow_item_ipv4 nic_mask = {
> +		.hdr = {
> +			.src_addr = RTE_BE32(0xffffffff),
> +			.dst_addr = RTE_BE32(0xffffffff),
> +			.type_of_service = 0xff,
> +			.next_proto_id = 0xff,
> +		},
> +	};
> +	unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
> +	struct ibv_flow_spec_ipv4_ext ipv4 = {
> +		.type = IBV_FLOW_SPEC_IPV4_EXT,
> +		.size = size,
> +	};
> +	int ret;
> +
> +	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "multiple L3 layers not supported");
> +	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "L3 cannot follow an L4 layer.");
> +	if (!mask)
> +		mask = &rte_flow_item_ipv4_mask;
> +	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> +				      (const uint8_t *)&nic_mask,
> +				      sizeof(struct rte_flow_item_ipv4), error);
> +	if (ret < 0)
> +		return ret;
> +	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
> +	if (size > flow_size)
> +		return size;
> +	if (spec) {
> +		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
> +			.src_ip = spec->hdr.src_addr,
> +			.dst_ip = spec->hdr.dst_addr,
> +			.proto = spec->hdr.next_proto_id,
> +			.tos = spec->hdr.type_of_service,
> +		};
> +		ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
> +			.src_ip = mask->hdr.src_addr,
> +			.dst_ip = mask->hdr.dst_addr,
> +			.proto = mask->hdr.next_proto_id,
> +			.tos = mask->hdr.type_of_service,
> +		};
> +		/* Remove unwanted bits from values. */
> +		ipv4.val.src_ip &= ipv4.mask.src_ip;
> +		ipv4.val.dst_ip &= ipv4.mask.dst_ip;
> +		ipv4.val.proto &= ipv4.mask.proto;
> +		ipv4.val.tos &= ipv4.mask.tos;
> +	}
> +	mlx5_flow_spec_verbs_add(flow, &ipv4, size);
> +	return size;
> +}
> +
>  /**
>   * Validate items provided by the user.
>   *
> @@ -576,6 +656,9 @@ mlx5_flow_items(const struct rte_flow_item items[],
>  		case RTE_FLOW_ITEM_TYPE_VLAN:
>  			ret = mlx5_flow_item_vlan(items, flow, remain, error);
>  			break;
> +		case RTE_FLOW_ITEM_TYPE_IPV4:
> +			ret = mlx5_flow_item_ipv4(items, flow, remain, error);
> +			break;
>  		default:
>  			return rte_flow_error_set(error, ENOTSUP,
>  						  RTE_FLOW_ERROR_TYPE_ITEM,
> -- 
> 2.18.0
> 

[dpdk-dev] [PATCH v2 09/20] net/mlx5: add flow IPv6 item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 108 +++++++++++++++++++++++++++++++++++
 1 file changed, 108 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 8e7a0bb5a..4e018400a 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -620,6 +620,111 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Validate IPv6 layer and possibly create the Verbs specification.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param flow_size[in]
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_ipv6 *spec = item->spec;
+	const struct rte_flow_item_ipv6 *mask = item->mask;
+	const struct rte_flow_item_ipv6 nic_mask = {
+		.hdr = {
+			.src_addr =
+				"\xff\xff\xff\xff\xff\xff\xff\xff"
+				"\xff\xff\xff\xff\xff\xff\xff\xff",
+			.dst_addr =
+				"\xff\xff\xff\xff\xff\xff\xff\xff"
+				"\xff\xff\xff\xff\xff\xff\xff\xff",
+			.vtc_flow = RTE_BE32(0xffffffff),
+			.proto = 0xff,
+			.hop_limits = 0xff,
+		},
+	};
+	unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
+	struct ibv_flow_spec_ipv6 ipv6 = {
+		.type = IBV_FLOW_SPEC_IPV6,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "multiple L3 layers not supported");
+	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 cannot follow an L4 layer.");
+	if (!mask)
+		mask = &rte_flow_item_ipv6_mask;
+	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+				      (const uint8_t *)&nic_mask,
+				      sizeof(struct rte_flow_item_ipv6), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		unsigned int i;
+		uint32_t vtc_flow_val;
+		uint32_t vtc_flow_mask;
+
+		memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
+		       RTE_DIM(ipv6.val.src_ip));
+		memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
+		       RTE_DIM(ipv6.val.dst_ip));
+		memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
+		       RTE_DIM(ipv6.mask.src_ip));
+		memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
+		       RTE_DIM(ipv6.mask.dst_ip));
+		vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
+		vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
+		ipv6.val.flow_label =
+			rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
+					 IPV6_HDR_FL_SHIFT);
+		ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
+					 IPV6_HDR_TC_SHIFT;
+		ipv6.val.next_hdr = spec->hdr.proto;
+		ipv6.val.hop_limit = spec->hdr.hop_limits;
+		ipv6.mask.flow_label =
+			rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
+					 IPV6_HDR_FL_SHIFT);
+		ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
+					  IPV6_HDR_TC_SHIFT;
+		ipv6.mask.next_hdr = mask->hdr.proto;
+		ipv6.mask.hop_limit = mask->hdr.hop_limits;
+		/* Remove unwanted bits from values. */
+		for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
+			ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
+			ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
+		}
+		ipv6.val.flow_label &= ipv6.mask.flow_label;
+		ipv6.val.traffic_class &= ipv6.mask.traffic_class;
+		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
+		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
+	}
+	mlx5_flow_spec_verbs_add(flow, &ipv6, size);
+	return size;
+}
+
 /**
  * Validate items provided by the user.
  *
@@ -659,6 +764,9 @@ mlx5_flow_items(const struct rte_flow_item items[],
 		case RTE_FLOW_ITEM_TYPE_IPV4:
 			ret = mlx5_flow_item_ipv4(items, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			ret = mlx5_flow_item_ipv6(items, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 09/20] net/mlx5: add flow IPv6 item

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:41PM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
Acked-by: Yongseok Koh <yskoh@mellanox.com>

Thanks

[dpdk-dev] [PATCH v2 10/20] net/mlx5: add flow UDP item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 65 ++++++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 4e018400a..06b4b7f41 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -725,6 +725,68 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Validate UDP layer and possibly create the Verbs specification.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param flow_size[in]
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_udp *spec = item->spec;
+	const struct rte_flow_item_udp *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
+	struct ibv_flow_spec_tcp_udp udp = {
+		.type = IBV_FLOW_SPEC_UDP,
+		.size = size,
+	};
+	int ret;
+
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 is mandatory to filter on L4");
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L4 layer is already present");
+	if (!mask)
+		mask = &rte_flow_item_udp_mask;
+	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+				      (const uint8_t *)&rte_flow_item_udp_mask,
+				      sizeof(struct rte_flow_item_udp), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		udp.val.dst_port = spec->hdr.dst_port;
+		udp.val.src_port = spec->hdr.src_port;
+		udp.mask.dst_port = mask->hdr.dst_port;
+		udp.mask.src_port = mask->hdr.src_port;
+		/* Remove unwanted bits from values. */
+		udp.val.src_port &= udp.mask.src_port;
+		udp.val.dst_port &= udp.mask.dst_port;
+	}
+	mlx5_flow_spec_verbs_add(flow, &udp, size);
+	return size;
+}
+
 /**
  * Validate items provided by the user.
  *
@@ -767,6 +829,9 @@ mlx5_flow_items(const struct rte_flow_item items[],
 		case RTE_FLOW_ITEM_TYPE_IPV6:
 			ret = mlx5_flow_item_ipv6(items, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			ret = mlx5_flow_item_udp(items, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 10/20] net/mlx5: add flow UDP item

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:42PM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
Acked-by: Yongseok Koh <yskoh@mellanox.com>

Thanks

[dpdk-dev] [PATCH v2 11/20] net/mlx5: add flow TCP item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 65 ++++++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 06b4b7f41..57f072c03 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -787,6 +787,68 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Validate TCP layer and possibly create the Verbs specification.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param flow_size[in]
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_tcp *spec = item->spec;
+	const struct rte_flow_item_tcp *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
+	struct ibv_flow_spec_tcp_udp tcp = {
+		.type = IBV_FLOW_SPEC_TCP,
+		.size = size,
+	};
+	int ret;
+
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 is mandatory to filter on L4");
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L4 layer is already present");
+	if (!mask)
+		mask = &rte_flow_item_tcp_mask;
+	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+				      (const uint8_t *)&rte_flow_item_tcp_mask,
+				      sizeof(struct rte_flow_item_tcp), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		tcp.val.dst_port = spec->hdr.dst_port;
+		tcp.val.src_port = spec->hdr.src_port;
+		tcp.mask.dst_port = mask->hdr.dst_port;
+		tcp.mask.src_port = mask->hdr.src_port;
+		/* Remove unwanted bits from values. */
+		tcp.val.src_port &= tcp.mask.src_port;
+		tcp.val.dst_port &= tcp.mask.dst_port;
+	}
+	mlx5_flow_spec_verbs_add(flow, &tcp, size);
+	return size;
+}
+
 /**
  * Validate items provided by the user.
  *
@@ -832,6 +894,9 @@ mlx5_flow_items(const struct rte_flow_item items[],
 		case RTE_FLOW_ITEM_TYPE_UDP:
 			ret = mlx5_flow_item_udp(items, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			ret = mlx5_flow_item_tcp(items, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 11/20] net/mlx5: add flow TCP item

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:43PM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
Acked-by: Yongseok Koh <yskoh@mellanox.com>

Thanks

[dpdk-dev] [PATCH v2 12/20] net/mlx5: add mark/flag flow action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 209 +++++++++++++++++++++++++++++++++++
 1 file changed, 209 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 57f072c03..a39157533 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -52,6 +52,10 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_FLOW_FATE_DROP (1u << 0)
 #define MLX5_FLOW_FATE_QUEUE (1u << 1)
 
+/* Modify a packet. */
+#define MLX5_FLOW_MOD_FLAG (1u << 0)
+#define MLX5_FLOW_MOD_MARK (1u << 1)
+
 /** Handles information leading to a drop fate. */
 struct mlx5_flow_verbs {
 	unsigned int size; /**< Size of the attribute. */
@@ -70,6 +74,8 @@ struct rte_flow {
 	struct rte_flow_attr attributes; /**< User flow attribute. */
 	uint32_t layers;
 	/**< Bit-fields of present layers see MLX5_FLOW_ITEMS_*. */
+	uint32_t modifier;
+	/**< Bit-fields of present modifier see MLX5_FLOW_MOD_*. */
 	uint32_t fate;
 	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
 	struct mlx5_flow_verbs verbs; /* Verbs flow. */
@@ -954,6 +960,12 @@ mlx5_flow_action_drop(const struct rte_flow_action *actions,
 					  actions,
 					  "multiple fate actions are not"
 					  " supported");
+	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "drop is not compatible with"
+					  " flag/mark action");
 	if (size < flow_size)
 		mlx5_flow_spec_verbs_add(flow, &drop, size);
 	flow->fate |= MLX5_FLOW_FATE_DROP;
@@ -1007,6 +1019,144 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/**
+ * Validate action flag provided by the user.
+ *
+ * @param actions
+ *   Pointer to flow actions array.
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param flow_size
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_action_flag(const struct rte_flow_action *actions,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
+{
+	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
+	struct ibv_flow_spec_action_tag tag = {
+		.type = IBV_FLOW_SPEC_ACTION_TAG,
+		.size = size,
+		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
+	};
+
+	if (flow->modifier & MLX5_FLOW_MOD_FLAG)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "flag action already present");
+	if (flow->fate & MLX5_FLOW_FATE_DROP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "flag is not compatible with drop"
+					  " action");
+	if (flow->modifier & MLX5_FLOW_MOD_MARK)
+		return 0;
+	flow->modifier |= MLX5_FLOW_MOD_FLAG;
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &tag, size);
+	return size;
+}
+
+/**
+ * Update verbs specification to modify the flag to mark.
+ *
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param mark_id
+ *   Mark identifier to replace the flag.
+ */
+static void
+mlx5_flow_verbs_mark_update(struct rte_flow *flow, uint32_t mark_id)
+{
+	struct ibv_spec_header *hdr;
+	int i;
+
+	/* Update Verbs specification. */
+	hdr = (struct ibv_spec_header *)flow->verbs.specs;
+	for (i = 0; i != flow->verbs.attr->num_of_specs; ++i) {
+		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
+			struct ibv_flow_spec_action_tag *t =
+				(struct ibv_flow_spec_action_tag *)hdr;
+
+			t->tag_id = mlx5_flow_mark_set(mark_id);
+		}
+		hdr = (struct ibv_spec_header *)((uintptr_t)hdr + hdr->size);
+	}
+}
+
+/**
+ * Validate action mark provided by the user.
+ *
+ * @param actions
+ *   Pointer to flow actions array.
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param flow_size[in]
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_action_mark(const struct rte_flow_action *actions,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
+{
+	const struct rte_flow_action_mark *mark = actions->conf;
+	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
+	struct ibv_flow_spec_action_tag tag = {
+		.type = IBV_FLOW_SPEC_ACTION_TAG,
+		.size = size,
+	};
+
+	if (!mark)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "configuration cannot be null");
+	if (mark->id >= MLX5_FLOW_MARK_MAX)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &mark->id,
+					  "mark must be between 0 and"
+					  " 16777199");
+	if (flow->modifier & MLX5_FLOW_MOD_MARK)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "mark action already present");
+	if (flow->fate & MLX5_FLOW_FATE_DROP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "mark is not compatible with drop"
+					  " action");
+	if (flow->modifier & MLX5_FLOW_MOD_FLAG) {
+		mlx5_flow_verbs_mark_update(flow, mark->id);
+		size = 0; /**< Only an update is done in the specification. */
+	} else {
+		tag.tag_id = mlx5_flow_mark_set(mark->id);
+		if (size <= flow_size) {
+			tag.tag_id = mlx5_flow_mark_set(mark->id);
+			mlx5_flow_spec_verbs_add(flow, &tag, size);
+		}
+	}
+	flow->modifier |= MLX5_FLOW_MOD_MARK;
+	return size;
+}
+
 /**
  * Validate actions provided by the user.
  *
@@ -1039,6 +1189,14 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 		switch (actions->type) {
 		case RTE_FLOW_ACTION_TYPE_VOID:
 			break;
+		case RTE_FLOW_ACTION_TYPE_FLAG:
+			ret = mlx5_flow_action_flag(actions, flow, remain,
+						    error);
+			break;
+		case RTE_FLOW_ACTION_TYPE_MARK:
+			ret = mlx5_flow_action_mark(actions, flow, remain,
+						    error);
+			break;
 		case RTE_FLOW_ACTION_TYPE_DROP:
 			ret = mlx5_flow_action_drop(actions, flow, remain,
 						    error);
@@ -1122,6 +1280,23 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Mark the Rx queues mark flag if the flow has a mark or flag modifier.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param flow
+ *   Pointer to flow structure.
+ */
+static void
+mlx5_flow_rxq_mark(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	(*priv->rxqs)[flow->queue]->mark |=
+		flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK);
+}
+
 /**
  * Validate a flow supported by the NIC.
  *
@@ -1281,6 +1456,7 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
 		if (ret < 0)
 			goto error;
 	}
+	mlx5_flow_rxq_mark(dev, flow);
 	TAILQ_INSERT_TAIL(list, flow, next);
 	return flow;
 error:
@@ -1323,8 +1499,31 @@ static void
 mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
 		       struct rte_flow *flow)
 {
+	struct priv *priv = dev->data->dev_private;
+	struct rte_flow *rflow;
+	const uint32_t mask = MLX5_FLOW_MOD_FLAG & MLX5_FLOW_MOD_MARK;
+	int mark = 0;
+
 	mlx5_flow_fate_remove(dev, flow);
 	TAILQ_REMOVE(list, flow, next);
+	if (!(flow->modifier & mask)) {
+		rte_free(flow);
+		return;
+	}
+	/*
+	 * When a flow is removed and this flow has a flag/mark modifier, all
+	 * flows needs to be parse to verify if the Rx queue use by the flow
+	 * still need to track the flag/mark request.
+	 */
+	TAILQ_FOREACH(rflow, &priv->flows, next) {
+		if (!(rflow->modifier & mask))
+			continue;
+		if (flow->queue == rflow->queue) {
+			mark = 1;
+			break;
+		}
+	}
+	(*priv->rxqs)[flow->queue]->mark = !!mark;
 	rte_free(flow);
 }
 
@@ -1358,10 +1557,19 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
 void
 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
 {
+	struct priv *priv = dev->data->dev_private;
 	struct rte_flow *flow;
+	unsigned int i;
+	unsigned int idx;
 
 	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
 		mlx5_flow_fate_remove(dev, flow);
+	for (idx = 0, i = 0; idx != priv->rxqs_n; ++i) {
+		if (!(*priv->rxqs)[idx])
+			continue;
+		(*priv->rxqs)[idx]->mark = 0;
+		++idx;
+	}
 }
 
 /**
@@ -1386,6 +1594,7 @@ mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
 		ret = mlx5_flow_fate_apply(dev, flow, &error);
 		if (ret < 0)
 			goto error;
+		mlx5_flow_rxq_mark(dev, flow);
 	}
 	return 0;
 error:
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 12/20] net/mlx5: add mark/flag flow action

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:44PM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
>  drivers/net/mlx5/mlx5_flow.c | 209 +++++++++++++++++++++++++++++++++++
>  1 file changed, 209 insertions(+)
> 
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index 57f072c03..a39157533 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -52,6 +52,10 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
>  #define MLX5_FLOW_FATE_DROP (1u << 0)
>  #define MLX5_FLOW_FATE_QUEUE (1u << 1)
>  
> +/* Modify a packet. */
> +#define MLX5_FLOW_MOD_FLAG (1u << 0)
> +#define MLX5_FLOW_MOD_MARK (1u << 1)
> +
>  /** Handles information leading to a drop fate. */
>  struct mlx5_flow_verbs {
>  	unsigned int size; /**< Size of the attribute. */
> @@ -70,6 +74,8 @@ struct rte_flow {
>  	struct rte_flow_attr attributes; /**< User flow attribute. */
>  	uint32_t layers;
>  	/**< Bit-fields of present layers see MLX5_FLOW_ITEMS_*. */
> +	uint32_t modifier;
> +	/**< Bit-fields of present modifier see MLX5_FLOW_MOD_*. */

Why do you think flag and mark modify a packet? I don't think modifier is an
appropriate name.

>  	uint32_t fate;
>  	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
>  	struct mlx5_flow_verbs verbs; /* Verbs flow. */
> @@ -954,6 +960,12 @@ mlx5_flow_action_drop(const struct rte_flow_action *actions,
>  					  actions,
>  					  "multiple fate actions are not"
>  					  " supported");
> +	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK))
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ACTION,
> +					  actions,
> +					  "drop is not compatible with"
> +					  " flag/mark action");
>  	if (size < flow_size)
>  		mlx5_flow_spec_verbs_add(flow, &drop, size);
>  	flow->fate |= MLX5_FLOW_FATE_DROP;
> @@ -1007,6 +1019,144 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
>  	return 0;
>  }
>  
> +/**
> + * Validate action flag provided by the user.
> + *
> + * @param actions
> + *   Pointer to flow actions array.
> + * @param flow
> + *   Pointer to the rte_flow structure.
> + * @param flow_size
> + *   Size in bytes of the available space for to store the flow information.
> + * @param error
> + *   Pointer to error structure.
> + *
> + * @return
> + *   size in bytes necessary for the conversion, a negative errno value
> + *   otherwise and rte_errno is set.

Like I asked for the previous patches, please be more verbose for function
description and explanation of args and return value.

> + */
> +static int
> +mlx5_flow_action_flag(const struct rte_flow_action *actions,
> +		      struct rte_flow *flow, const size_t flow_size,
> +		      struct rte_flow_error *error)
> +{
> +	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
> +	struct ibv_flow_spec_action_tag tag = {
> +		.type = IBV_FLOW_SPEC_ACTION_TAG,
> +		.size = size,
> +		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
> +	};
> +
> +	if (flow->modifier & MLX5_FLOW_MOD_FLAG)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ACTION,
> +					  actions,
> +					  "flag action already present");
> +	if (flow->fate & MLX5_FLOW_FATE_DROP)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ACTION,
> +					  actions,
> +					  "flag is not compatible with drop"
> +					  " action");
> +	if (flow->modifier & MLX5_FLOW_MOD_MARK)
> +		return 0;
> +	flow->modifier |= MLX5_FLOW_MOD_FLAG;
> +	if (size <= flow_size)
> +		mlx5_flow_spec_verbs_add(flow, &tag, size);
> +	return size;
> +}
> +
> +/**
> + * Update verbs specification to modify the flag to mark.
> + *
> + * @param flow
> + *   Pointer to the rte_flow structure.
> + * @param mark_id
> + *   Mark identifier to replace the flag.
> + */
> +static void
> +mlx5_flow_verbs_mark_update(struct rte_flow *flow, uint32_t mark_id)
> +{
> +	struct ibv_spec_header *hdr;
> +	int i;
> +
> +	/* Update Verbs specification. */
> +	hdr = (struct ibv_spec_header *)flow->verbs.specs;
> +	for (i = 0; i != flow->verbs.attr->num_of_specs; ++i) {

flow->verbs.attr/specs can be null in case of validation call. But you don't
need to fix it because it is anyway changed and fixed when you add RSS action.

> +		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
> +			struct ibv_flow_spec_action_tag *t =
> +				(struct ibv_flow_spec_action_tag *)hdr;
> +
> +			t->tag_id = mlx5_flow_mark_set(mark_id);
> +		}
> +		hdr = (struct ibv_spec_header *)((uintptr_t)hdr + hdr->size);
> +	}
> +}
> +
> +/**
> + * Validate action mark provided by the user.
> + *
> + * @param actions
> + *   Pointer to flow actions array.
> + * @param flow
> + *   Pointer to the rte_flow structure.
> + * @param flow_size[in]
> + *   Size in bytes of the available space for to store the flow information.
> + * @param error
> + *   Pointer to error structure.
> + *
> + * @return
> + *   size in bytes necessary for the conversion, a negative errno value
> + *   otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_action_mark(const struct rte_flow_action *actions,
> +		      struct rte_flow *flow, const size_t flow_size,
> +		      struct rte_flow_error *error)
> +{
> +	const struct rte_flow_action_mark *mark = actions->conf;
> +	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
> +	struct ibv_flow_spec_action_tag tag = {
> +		.type = IBV_FLOW_SPEC_ACTION_TAG,
> +		.size = size,
> +	};
> +
> +	if (!mark)
> +		return rte_flow_error_set(error, EINVAL,
> +					  RTE_FLOW_ERROR_TYPE_ACTION,
> +					  actions,
> +					  "configuration cannot be null");
> +	if (mark->id >= MLX5_FLOW_MARK_MAX)
> +		return rte_flow_error_set(error, EINVAL,
> +					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
> +					  &mark->id,
> +					  "mark must be between 0 and"
> +					  " 16777199");

Use %d and (MLX5_FLOW_MARK_MAX - 1), instead of fixed string.

> +	if (flow->modifier & MLX5_FLOW_MOD_MARK)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ACTION,
> +					  actions,
> +					  "mark action already present");
> +	if (flow->fate & MLX5_FLOW_FATE_DROP)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ACTION,
> +					  actions,
> +					  "mark is not compatible with drop"
> +					  " action");
> +	if (flow->modifier & MLX5_FLOW_MOD_FLAG) {
> +		mlx5_flow_verbs_mark_update(flow, mark->id);
> +		size = 0; /**< Only an update is done in the specification. */
> +	} else {
> +		tag.tag_id = mlx5_flow_mark_set(mark->id);
> +		if (size <= flow_size) {
> +			tag.tag_id = mlx5_flow_mark_set(mark->id);
> +			mlx5_flow_spec_verbs_add(flow, &tag, size);
> +		}
> +	}
> +	flow->modifier |= MLX5_FLOW_MOD_MARK;
> +	return size;
> +}
> +
>  /**
>   * Validate actions provided by the user.
>   *
> @@ -1039,6 +1189,14 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
>  		switch (actions->type) {
>  		case RTE_FLOW_ACTION_TYPE_VOID:
>  			break;
> +		case RTE_FLOW_ACTION_TYPE_FLAG:
> +			ret = mlx5_flow_action_flag(actions, flow, remain,
> +						    error);
> +			break;
> +		case RTE_FLOW_ACTION_TYPE_MARK:
> +			ret = mlx5_flow_action_mark(actions, flow, remain,
> +						    error);
> +			break;
>  		case RTE_FLOW_ACTION_TYPE_DROP:
>  			ret = mlx5_flow_action_drop(actions, flow, remain,
>  						    error);
> @@ -1122,6 +1280,23 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
>  	return size;
>  }
>  
> +/**
> + * Mark the Rx queues mark flag if the flow has a mark or flag modifier.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + * @param flow
> + *   Pointer to flow structure.
> + */
> +static void
> +mlx5_flow_rxq_mark(struct rte_eth_dev *dev, struct rte_flow *flow)
> +{
> +	struct priv *priv = dev->data->dev_private;
> +
> +	(*priv->rxqs)[flow->queue]->mark |=
> +		flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK);

This has to be !!(...) as rxq->mark has only 1 bit. But, it is also fixed by
coming RSS patches. Not sure what's benefit of splitting patches in this way.

> +}
> +
>  /**
>   * Validate a flow supported by the NIC.
>   *
> @@ -1281,6 +1456,7 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
>  		if (ret < 0)
>  			goto error;
>  	}
> +	mlx5_flow_rxq_mark(dev, flow);
>  	TAILQ_INSERT_TAIL(list, flow, next);
>  	return flow;
>  error:
> @@ -1323,8 +1499,31 @@ static void
>  mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
>  		       struct rte_flow *flow)
>  {
> +	struct priv *priv = dev->data->dev_private;
> +	struct rte_flow *rflow;
> +	const uint32_t mask = MLX5_FLOW_MOD_FLAG & MLX5_FLOW_MOD_MARK;
> +	int mark = 0;
> +
>  	mlx5_flow_fate_remove(dev, flow);
>  	TAILQ_REMOVE(list, flow, next);
> +	if (!(flow->modifier & mask)) {
> +		rte_free(flow);
> +		return;
> +	}
> +	/*
> +	 * When a flow is removed and this flow has a flag/mark modifier, all
> +	 * flows needs to be parse to verify if the Rx queue use by the flow
> +	 * still need to track the flag/mark request.
> +	 */

When a flow is created, mlx5_flow_rxq_mark() is called. Is there a specific
reason for not writing a separate function in order to drop rxq->mark bit?

> +	TAILQ_FOREACH(rflow, &priv->flows, next) {
> +		if (!(rflow->modifier & mask))
> +			continue;
> +		if (flow->queue == rflow->queue) {
> +			mark = 1;
> +			break;
> +		}
> +	}
> +	(*priv->rxqs)[flow->queue]->mark = !!mark;

mark can be either 0 or 1, then !!mark == mark anyway.

>  	rte_free(flow);
>  }
>  
> @@ -1358,10 +1557,19 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
>  void
>  mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
>  {
> +	struct priv *priv = dev->data->dev_private;
>  	struct rte_flow *flow;
> +	unsigned int i;
> +	unsigned int idx;
>  
>  	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
>  		mlx5_flow_fate_remove(dev, flow);
> +	for (idx = 0, i = 0; idx != priv->rxqs_n; ++i) {
> +		if (!(*priv->rxqs)[idx])
> +			continue;
> +		(*priv->rxqs)[idx]->mark = 0;
> +		++idx;
> +	}

Same question here but looks like this part is being moved to
mlx5_flow_rxqs_clear() in the future.

>  }
>  
>  /**
> @@ -1386,6 +1594,7 @@ mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
>  		ret = mlx5_flow_fate_apply(dev, flow, &error);
>  		if (ret < 0)
>  			goto error;
> +		mlx5_flow_rxq_mark(dev, flow);
>  	}
>  	return 0;
>  error:
> -- 
> 2.18.0
> 

Re: [dpdk-dev] [PATCH v2 12/20] net/mlx5: add mark/flag flow action

[Nélio Laranjeiro]

On Wed, Jul 04, 2018 at 01:34:19AM -0700, Yongseok Koh wrote:
> On Wed, Jun 27, 2018 at 05:07:44PM +0200, Nelio Laranjeiro wrote:
> > Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> > ---
> >  drivers/net/mlx5/mlx5_flow.c | 209 +++++++++++++++++++++++++++++++++++
> >  1 file changed, 209 insertions(+)
> > 
> > diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> > index 57f072c03..a39157533 100644
> > --- a/drivers/net/mlx5/mlx5_flow.c
> > +++ b/drivers/net/mlx5/mlx5_flow.c
> > @@ -52,6 +52,10 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
> >  #define MLX5_FLOW_FATE_DROP (1u << 0)
> >  #define MLX5_FLOW_FATE_QUEUE (1u << 1)
> >  
> > +/* Modify a packet. */
> > +#define MLX5_FLOW_MOD_FLAG (1u << 0)
> > +#define MLX5_FLOW_MOD_MARK (1u << 1)
> > +
> >  /** Handles information leading to a drop fate. */
> >  struct mlx5_flow_verbs {
> >  	unsigned int size; /**< Size of the attribute. */
> > @@ -70,6 +74,8 @@ struct rte_flow {
> >  	struct rte_flow_attr attributes; /**< User flow attribute. */
> >  	uint32_t layers;
> >  	/**< Bit-fields of present layers see MLX5_FLOW_ITEMS_*. */
> > +	uint32_t modifier;
> > +	/**< Bit-fields of present modifier see MLX5_FLOW_MOD_*. */
> 
> Why do you think flag and mark modify a packet? I don't think modifier is an
> appropriate name.

API terminology: "Actions that modify matching traffic contents or its
properties. This includes adding/removing encapsulation, encryption,
compression and marks."

> >  	uint32_t fate;
> >  	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
> >  	struct mlx5_flow_verbs verbs; /* Verbs flow. */
> > @@ -954,6 +960,12 @@ mlx5_flow_action_drop(const struct rte_flow_action *actions,
> >  					  actions,
> >  					  "multiple fate actions are not"
> >  					  " supported");
> > +	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK))
> > +		return rte_flow_error_set(error, ENOTSUP,
> > +					  RTE_FLOW_ERROR_TYPE_ACTION,
> > +					  actions,
> > +					  "drop is not compatible with"
> > +					  " flag/mark action");
> >  	if (size < flow_size)
> >  		mlx5_flow_spec_verbs_add(flow, &drop, size);
> >  	flow->fate |= MLX5_FLOW_FATE_DROP;
> > @@ -1007,6 +1019,144 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
> >  	return 0;
> >  }
> >  
> > +/**
> > + * Validate action flag provided by the user.
> > + *
> > + * @param actions
> > + *   Pointer to flow actions array.
> > + * @param flow
> > + *   Pointer to the rte_flow structure.
> > + * @param flow_size
> > + *   Size in bytes of the available space for to store the flow information.
> > + * @param error
> > + *   Pointer to error structure.
> > + *
> > + * @return
> > + *   size in bytes necessary for the conversion, a negative errno value
> > + *   otherwise and rte_errno is set.
> 
> Like I asked for the previous patches, please be more verbose for function
> description and explanation of args and return value.

I've update the documentation of all patches it would be strange to see
some with correct comments and some without :)

> > + */
> > +static int
> > +mlx5_flow_action_flag(const struct rte_flow_action *actions,
> > +		      struct rte_flow *flow, const size_t flow_size,
> > +		      struct rte_flow_error *error)
> > +{
> > +	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
> > +	struct ibv_flow_spec_action_tag tag = {
> > +		.type = IBV_FLOW_SPEC_ACTION_TAG,
> > +		.size = size,
> > +		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
> > +	};
> > +
> > +	if (flow->modifier & MLX5_FLOW_MOD_FLAG)
> > +		return rte_flow_error_set(error, ENOTSUP,
> > +					  RTE_FLOW_ERROR_TYPE_ACTION,
> > +					  actions,
> > +					  "flag action already present");
> > +	if (flow->fate & MLX5_FLOW_FATE_DROP)
> > +		return rte_flow_error_set(error, ENOTSUP,
> > +					  RTE_FLOW_ERROR_TYPE_ACTION,
> > +					  actions,
> > +					  "flag is not compatible with drop"
> > +					  " action");
> > +	if (flow->modifier & MLX5_FLOW_MOD_MARK)
> > +		return 0;
> > +	flow->modifier |= MLX5_FLOW_MOD_FLAG;
> > +	if (size <= flow_size)
> > +		mlx5_flow_spec_verbs_add(flow, &tag, size);
> > +	return size;
> > +}
> > +
> > +/**
> > + * Update verbs specification to modify the flag to mark.
> > + *
> > + * @param flow
> > + *   Pointer to the rte_flow structure.
> > + * @param mark_id
> > + *   Mark identifier to replace the flag.
> > + */
> > +static void
> > +mlx5_flow_verbs_mark_update(struct rte_flow *flow, uint32_t mark_id)
> > +{
> > +	struct ibv_spec_header *hdr;
> > +	int i;
> > +
> > +	/* Update Verbs specification. */
> > +	hdr = (struct ibv_spec_header *)flow->verbs.specs;
> > +	for (i = 0; i != flow->verbs.attr->num_of_specs; ++i) {
> 
> flow->verbs.attr/specs can be null in case of validation call. But you don't
> need to fix it because it is anyway changed and fixed when you add RSS action.

You are right, but it still need to be fixed, if for some reason a
bisect is used this may break the bug research.

> > +		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
> > +			struct ibv_flow_spec_action_tag *t =
> > +				(struct ibv_flow_spec_action_tag *)hdr;
> > +
> > +			t->tag_id = mlx5_flow_mark_set(mark_id);
> > +		}
> > +		hdr = (struct ibv_spec_header *)((uintptr_t)hdr + hdr->size);
> > +	}
> > +}
> > +
> > +/**
> > + * Validate action mark provided by the user.
> > + *
> > + * @param actions
> > + *   Pointer to flow actions array.
> > + * @param flow
> > + *   Pointer to the rte_flow structure.
> > + * @param flow_size[in]
> > + *   Size in bytes of the available space for to store the flow information.
> > + * @param error
> > + *   Pointer to error structure.
> > + *
> > + * @return
> > + *   size in bytes necessary for the conversion, a negative errno value
> > + *   otherwise and rte_errno is set.
> > + */
> > +static int
> > +mlx5_flow_action_mark(const struct rte_flow_action *actions,
> > +		      struct rte_flow *flow, const size_t flow_size,
> > +		      struct rte_flow_error *error)
> > +{
> > +	const struct rte_flow_action_mark *mark = actions->conf;
> > +	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
> > +	struct ibv_flow_spec_action_tag tag = {
> > +		.type = IBV_FLOW_SPEC_ACTION_TAG,
> > +		.size = size,
> > +	};
> > +
> > +	if (!mark)
> > +		return rte_flow_error_set(error, EINVAL,
> > +					  RTE_FLOW_ERROR_TYPE_ACTION,
> > +					  actions,
> > +					  "configuration cannot be null");
> > +	if (mark->id >= MLX5_FLOW_MARK_MAX)
> > +		return rte_flow_error_set(error, EINVAL,
> > +					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
> > +					  &mark->id,
> > +					  "mark must be between 0 and"
> > +					  " 16777199");
> 
> Use %d and (MLX5_FLOW_MARK_MAX - 1), instead of fixed string.

It needs an snprintf, rte_flow_error_set() does not accept formatting
strings.

>[...]
> > +/**
> > + * Mark the Rx queues mark flag if the flow has a mark or flag modifier.
> > + *
> > + * @param dev
> > + *   Pointer to Ethernet device.
> > + * @param flow
> > + *   Pointer to flow structure.
> > + */
> > +static void
> > +mlx5_flow_rxq_mark(struct rte_eth_dev *dev, struct rte_flow *flow)
> > +{
> > +	struct priv *priv = dev->data->dev_private;
> > +
> > +	(*priv->rxqs)[flow->queue]->mark |=
> > +		flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK);
> 
> This has to be !!(...) as rxq->mark has only 1 bit. But, it is also fixed by
> coming RSS patches. Not sure what's benefit of splitting patches in this way.

Same answer as above, even if fixed after, it still need a fix here.

> > +}
> > +
> >  /**
> >   * Validate a flow supported by the NIC.
> >   *
> > @@ -1281,6 +1456,7 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
> >  		if (ret < 0)
> >  			goto error;
> >  	}
> > +	mlx5_flow_rxq_mark(dev, flow);
> >  	TAILQ_INSERT_TAIL(list, flow, next);
> >  	return flow;
> >  error:
> > @@ -1323,8 +1499,31 @@ static void
> >  mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
> >  		       struct rte_flow *flow)
> >  {
> > +	struct priv *priv = dev->data->dev_private;
> > +	struct rte_flow *rflow;
> > +	const uint32_t mask = MLX5_FLOW_MOD_FLAG & MLX5_FLOW_MOD_MARK;
> > +	int mark = 0;
> > +
> >  	mlx5_flow_fate_remove(dev, flow);
> >  	TAILQ_REMOVE(list, flow, next);
> > +	if (!(flow->modifier & mask)) {
> > +		rte_free(flow);
> > +		return;
> > +	}
> > +	/*
> > +	 * When a flow is removed and this flow has a flag/mark modifier, all
> > +	 * flows needs to be parse to verify if the Rx queue use by the flow
> > +	 * still need to track the flag/mark request.
> > +	 */
> 
> When a flow is created, mlx5_flow_rxq_mark() is called. Is there a specific
> reason for not writing a separate function in order to drop rxq->mark bit?
>
> > +	TAILQ_FOREACH(rflow, &priv->flows, next) {
> > +		if (!(rflow->modifier & mask))
> > +			continue;
> > +		if (flow->queue == rflow->queue) {
> > +			mark = 1;
> > +			break;
> > +		}
> > +	}
> > +	(*priv->rxqs)[flow->queue]->mark = !!mark;
> 
> mark can be either 0 or 1, then !!mark == mark anyway.
> 
> >  	rte_free(flow);
> >  }
> >  
> > @@ -1358,10 +1557,19 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
> >  void
> >  mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
> >  {
> > +	struct priv *priv = dev->data->dev_private;
> >  	struct rte_flow *flow;
> > +	unsigned int i;
> > +	unsigned int idx;
> >  
> >  	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
> >  		mlx5_flow_fate_remove(dev, flow);
> > +	for (idx = 0, i = 0; idx != priv->rxqs_n; ++i) {
> > +		if (!(*priv->rxqs)[idx])
> > +			continue;
> > +		(*priv->rxqs)[idx]->mark = 0;
> > +		++idx;
> > +	}
> 
> Same question here but looks like this part is being moved to
> mlx5_flow_rxqs_clear() in the future.

Addressing both question here, for the flow_stop() and flow_destroy()
the process is different, for the stop, the flow remains with the mark
bit set but all queues must me cleared, there is no comparison to make.
As you can see, it don't even get a flow, it directly unset the mask bit
in the Rx queues.
For the destroy the issue is different, several flows may be using the
same Rx queues, if one of them will remains and has a mark, then the
associated queues must keep their mark bit set.
As the process is different, it would end in two distinct functions and
each one used by a single function.

For the mlx5_flow_rxq_mark(), the situation is different, the same
process is make when a flow is created and the flow are started.

> >  }
> >  
> >  /**
> > @@ -1386,6 +1594,7 @@ mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
> >  		ret = mlx5_flow_fate_apply(dev, flow, &error);
> >  		if (ret < 0)
> >  			goto error;
> > +		mlx5_flow_rxq_mark(dev, flow);
> >  	}
> >  	return 0;
> >  error:
> > -- 
> > 2.18.0

Thanks,

-- 
Nélio Laranjeiro
6WIND

Re: [dpdk-dev] [PATCH v2 12/20] net/mlx5: add mark/flag flow action

[Yongseok Koh]

On Thu, Jul 05, 2018 at 10:47:35AM +0200, Nélio Laranjeiro wrote:
> On Wed, Jul 04, 2018 at 01:34:19AM -0700, Yongseok Koh wrote:
> > On Wed, Jun 27, 2018 at 05:07:44PM +0200, Nelio Laranjeiro wrote:
> > > Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> > > ---
> > >  drivers/net/mlx5/mlx5_flow.c | 209 +++++++++++++++++++++++++++++++++++
> > >  1 file changed, 209 insertions(+)
> > > 
> > > diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> > > index 57f072c03..a39157533 100644
> > > --- a/drivers/net/mlx5/mlx5_flow.c
> > > +++ b/drivers/net/mlx5/mlx5_flow.c
> > > @@ -52,6 +52,10 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
> > >  #define MLX5_FLOW_FATE_DROP (1u << 0)
> > >  #define MLX5_FLOW_FATE_QUEUE (1u << 1)
> > >  
> > > +/* Modify a packet. */
> > > +#define MLX5_FLOW_MOD_FLAG (1u << 0)
> > > +#define MLX5_FLOW_MOD_MARK (1u << 1)
> > > +
> > >  /** Handles information leading to a drop fate. */
> > >  struct mlx5_flow_verbs {
> > >  	unsigned int size; /**< Size of the attribute. */
> > > @@ -70,6 +74,8 @@ struct rte_flow {
> > >  	struct rte_flow_attr attributes; /**< User flow attribute. */
> > >  	uint32_t layers;
> > >  	/**< Bit-fields of present layers see MLX5_FLOW_ITEMS_*. */
> > > +	uint32_t modifier;
> > > +	/**< Bit-fields of present modifier see MLX5_FLOW_MOD_*. */
> > 
> > Why do you think flag and mark modify a packet? I don't think modifier is an
> > appropriate name.
> 
> API terminology: "Actions that modify matching traffic contents or its
> properties. This includes adding/removing encapsulation, encryption,
> compression and marks."
> 
> > >  	uint32_t fate;
> > >  	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
> > >  	struct mlx5_flow_verbs verbs; /* Verbs flow. */
> > > @@ -954,6 +960,12 @@ mlx5_flow_action_drop(const struct rte_flow_action *actions,
> > >  					  actions,
> > >  					  "multiple fate actions are not"
> > >  					  " supported");
> > > +	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK))
> > > +		return rte_flow_error_set(error, ENOTSUP,
> > > +					  RTE_FLOW_ERROR_TYPE_ACTION,
> > > +					  actions,
> > > +					  "drop is not compatible with"
> > > +					  " flag/mark action");
> > >  	if (size < flow_size)
> > >  		mlx5_flow_spec_verbs_add(flow, &drop, size);
> > >  	flow->fate |= MLX5_FLOW_FATE_DROP;
> > > @@ -1007,6 +1019,144 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
> > >  	return 0;
> > >  }
> > >  
> > > +/**
> > > + * Validate action flag provided by the user.
> > > + *
> > > + * @param actions
> > > + *   Pointer to flow actions array.
> > > + * @param flow
> > > + *   Pointer to the rte_flow structure.
> > > + * @param flow_size
> > > + *   Size in bytes of the available space for to store the flow information.
> > > + * @param error
> > > + *   Pointer to error structure.
> > > + *
> > > + * @return
> > > + *   size in bytes necessary for the conversion, a negative errno value
> > > + *   otherwise and rte_errno is set.
> > 
> > Like I asked for the previous patches, please be more verbose for function
> > description and explanation of args and return value.
> 
> I've update the documentation of all patches it would be strange to see
> some with correct comments and some without :)
> 
> > > + */
> > > +static int
> > > +mlx5_flow_action_flag(const struct rte_flow_action *actions,
> > > +		      struct rte_flow *flow, const size_t flow_size,
> > > +		      struct rte_flow_error *error)
> > > +{
> > > +	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
> > > +	struct ibv_flow_spec_action_tag tag = {
> > > +		.type = IBV_FLOW_SPEC_ACTION_TAG,
> > > +		.size = size,
> > > +		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
> > > +	};
> > > +
> > > +	if (flow->modifier & MLX5_FLOW_MOD_FLAG)
> > > +		return rte_flow_error_set(error, ENOTSUP,
> > > +					  RTE_FLOW_ERROR_TYPE_ACTION,
> > > +					  actions,
> > > +					  "flag action already present");
> > > +	if (flow->fate & MLX5_FLOW_FATE_DROP)
> > > +		return rte_flow_error_set(error, ENOTSUP,
> > > +					  RTE_FLOW_ERROR_TYPE_ACTION,
> > > +					  actions,
> > > +					  "flag is not compatible with drop"
> > > +					  " action");
> > > +	if (flow->modifier & MLX5_FLOW_MOD_MARK)
> > > +		return 0;
> > > +	flow->modifier |= MLX5_FLOW_MOD_FLAG;
> > > +	if (size <= flow_size)
> > > +		mlx5_flow_spec_verbs_add(flow, &tag, size);
> > > +	return size;
> > > +}
> > > +
> > > +/**
> > > + * Update verbs specification to modify the flag to mark.
> > > + *
> > > + * @param flow
> > > + *   Pointer to the rte_flow structure.
> > > + * @param mark_id
> > > + *   Mark identifier to replace the flag.
> > > + */
> > > +static void
> > > +mlx5_flow_verbs_mark_update(struct rte_flow *flow, uint32_t mark_id)
> > > +{
> > > +	struct ibv_spec_header *hdr;
> > > +	int i;
> > > +
> > > +	/* Update Verbs specification. */
> > > +	hdr = (struct ibv_spec_header *)flow->verbs.specs;
> > > +	for (i = 0; i != flow->verbs.attr->num_of_specs; ++i) {
> > 
> > flow->verbs.attr/specs can be null in case of validation call. But you don't
> > need to fix it because it is anyway changed and fixed when you add RSS action.
> 
> You are right, but it still need to be fixed, if for some reason a
> bisect is used this may break the bug research.
> 
> > > +		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
> > > +			struct ibv_flow_spec_action_tag *t =
> > > +				(struct ibv_flow_spec_action_tag *)hdr;
> > > +
> > > +			t->tag_id = mlx5_flow_mark_set(mark_id);
> > > +		}
> > > +		hdr = (struct ibv_spec_header *)((uintptr_t)hdr + hdr->size);
> > > +	}
> > > +}
> > > +
> > > +/**
> > > + * Validate action mark provided by the user.
> > > + *
> > > + * @param actions
> > > + *   Pointer to flow actions array.
> > > + * @param flow
> > > + *   Pointer to the rte_flow structure.
> > > + * @param flow_size[in]
> > > + *   Size in bytes of the available space for to store the flow information.
> > > + * @param error
> > > + *   Pointer to error structure.
> > > + *
> > > + * @return
> > > + *   size in bytes necessary for the conversion, a negative errno value
> > > + *   otherwise and rte_errno is set.
> > > + */
> > > +static int
> > > +mlx5_flow_action_mark(const struct rte_flow_action *actions,
> > > +		      struct rte_flow *flow, const size_t flow_size,
> > > +		      struct rte_flow_error *error)
> > > +{
> > > +	const struct rte_flow_action_mark *mark = actions->conf;
> > > +	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
> > > +	struct ibv_flow_spec_action_tag tag = {
> > > +		.type = IBV_FLOW_SPEC_ACTION_TAG,
> > > +		.size = size,
> > > +	};
> > > +
> > > +	if (!mark)
> > > +		return rte_flow_error_set(error, EINVAL,
> > > +					  RTE_FLOW_ERROR_TYPE_ACTION,
> > > +					  actions,
> > > +					  "configuration cannot be null");
> > > +	if (mark->id >= MLX5_FLOW_MARK_MAX)
> > > +		return rte_flow_error_set(error, EINVAL,
> > > +					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
> > > +					  &mark->id,
> > > +					  "mark must be between 0 and"
> > > +					  " 16777199");
> > 
> > Use %d and (MLX5_FLOW_MARK_MAX - 1), instead of fixed string.
> 
> It needs an snprintf, rte_flow_error_set() does not accept formatting
> strings.

I think the following would work but never mind. I'm okay with leaving it as is.
No need to make a change here.

#define STRINGIFY(x) #x
#define TOSTRING(x) STRINGIFY(x)
					"mark must be between 0 and "
					TOSTRING(MLX5_FLOW_MARK_MAX - 1));

> >[...]
> > > +/**
> > > + * Mark the Rx queues mark flag if the flow has a mark or flag modifier.
> > > + *
> > > + * @param dev
> > > + *   Pointer to Ethernet device.
> > > + * @param flow
> > > + *   Pointer to flow structure.
> > > + */
> > > +static void
> > > +mlx5_flow_rxq_mark(struct rte_eth_dev *dev, struct rte_flow *flow)
> > > +{
> > > +	struct priv *priv = dev->data->dev_private;
> > > +
> > > +	(*priv->rxqs)[flow->queue]->mark |=
> > > +		flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK);
> > 
> > This has to be !!(...) as rxq->mark has only 1 bit. But, it is also fixed by
> > coming RSS patches. Not sure what's benefit of splitting patches in this way.
> 
> Same answer as above, even if fixed after, it still need a fix here.
> 
> > > +}
> > > +
> > >  /**
> > >   * Validate a flow supported by the NIC.
> > >   *
> > > @@ -1281,6 +1456,7 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
> > >  		if (ret < 0)
> > >  			goto error;
> > >  	}
> > > +	mlx5_flow_rxq_mark(dev, flow);
> > >  	TAILQ_INSERT_TAIL(list, flow, next);
> > >  	return flow;
> > >  error:
> > > @@ -1323,8 +1499,31 @@ static void
> > >  mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
> > >  		       struct rte_flow *flow)
> > >  {
> > > +	struct priv *priv = dev->data->dev_private;
> > > +	struct rte_flow *rflow;
> > > +	const uint32_t mask = MLX5_FLOW_MOD_FLAG & MLX5_FLOW_MOD_MARK;
> > > +	int mark = 0;
> > > +
> > >  	mlx5_flow_fate_remove(dev, flow);
> > >  	TAILQ_REMOVE(list, flow, next);
> > > +	if (!(flow->modifier & mask)) {
> > > +		rte_free(flow);
> > > +		return;
> > > +	}
> > > +	/*
> > > +	 * When a flow is removed and this flow has a flag/mark modifier, all
> > > +	 * flows needs to be parse to verify if the Rx queue use by the flow
> > > +	 * still need to track the flag/mark request.
> > > +	 */
> > 
> > When a flow is created, mlx5_flow_rxq_mark() is called. Is there a specific
> > reason for not writing a separate function in order to drop rxq->mark bit?
> >
> > > +	TAILQ_FOREACH(rflow, &priv->flows, next) {
> > > +		if (!(rflow->modifier & mask))
> > > +			continue;
> > > +		if (flow->queue == rflow->queue) {
> > > +			mark = 1;
> > > +			break;
> > > +		}
> > > +	}
> > > +	(*priv->rxqs)[flow->queue]->mark = !!mark;
> > 
> > mark can be either 0 or 1, then !!mark == mark anyway.
> > 
> > >  	rte_free(flow);
> > >  }
> > >  
> > > @@ -1358,10 +1557,19 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
> > >  void
> > >  mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
> > >  {
> > > +	struct priv *priv = dev->data->dev_private;
> > >  	struct rte_flow *flow;
> > > +	unsigned int i;
> > > +	unsigned int idx;
> > >  
> > >  	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
> > >  		mlx5_flow_fate_remove(dev, flow);
> > > +	for (idx = 0, i = 0; idx != priv->rxqs_n; ++i) {
> > > +		if (!(*priv->rxqs)[idx])
> > > +			continue;
> > > +		(*priv->rxqs)[idx]->mark = 0;
> > > +		++idx;
> > > +	}
> > 
> > Same question here but looks like this part is being moved to
> > mlx5_flow_rxqs_clear() in the future.
> 
> Addressing both question here, for the flow_stop() and flow_destroy()
> the process is different, for the stop, the flow remains with the mark
> bit set but all queues must me cleared, there is no comparison to make.
> As you can see, it don't even get a flow, it directly unset the mask bit
> in the Rx queues.
> For the destroy the issue is different, several flows may be using the
> same Rx queues, if one of them will remains and has a mark, then the
> associated queues must keep their mark bit set.
> As the process is different, it would end in two distinct functions and
> each one used by a single function.
> 
> For the mlx5_flow_rxq_mark(), the situation is different, the same
> process is make when a flow is created and the flow are started.

I knew the differences but I just wanted to ask if having a separate function
can be a viable option, e.g.,

mlx5_flow_rxq_mark_set()
mlx5_flow_rxq_mark_clear()
mlx5_flow_rxq_mark_trim()


Thanks,
Yongseok

Re: [dpdk-dev] [PATCH v2 12/20] net/mlx5: add mark/flag flow action

[Nélio Laranjeiro]

On Thu, Jul 05, 2018 at 12:56:09PM -0700, Yongseok Koh wrote:
>[...]
> > > > +	if (mark->id >= MLX5_FLOW_MARK_MAX)
> > > > +		return rte_flow_error_set(error, EINVAL,
> > > > +					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
> > > > +					  &mark->id,
> > > > +					  "mark must be between 0 and"
> > > > +					  " 16777199");
> > > 
> > > Use %d and (MLX5_FLOW_MARK_MAX - 1), instead of fixed string.
> > 
> > It needs an snprintf, rte_flow_error_set() does not accept formatting
> > strings.
> 
> I think the following would work but never mind. I'm okay with leaving it as is.
> No need to make a change here.
> 
> #define STRINGIFY(x) #x
> #define TOSTRING(x) STRINGIFY(x)
> 					"mark must be between 0 and "
> 					TOSTRING(MLX5_FLOW_MARK_MAX - 1));
> 

It was to avoid adding a macro, but indeed there is the same kind in
mlx4, I'll port them for mlx5.

> > >[...]
> > Addressing both question here, for the flow_stop() and flow_destroy()
> > the process is different, for the stop, the flow remains with the mark
> > bit set but all queues must me cleared, there is no comparison to make.
> > As you can see, it don't even get a flow, it directly unset the mask bit
> > in the Rx queues.
> > For the destroy the issue is different, several flows may be using the
> > same Rx queues, if one of them will remains and has a mark, then the
> > associated queues must keep their mark bit set.
> > As the process is different, it would end in two distinct functions and
> > each one used by a single function.
> > 
> > For the mlx5_flow_rxq_mark(), the situation is different, the same
> > process is make when a flow is created and the flow are started.
> 
> I knew the differences but I just wanted to ask if having a separate function
> can be a viable option, e.g.,
> 
> mlx5_flow_rxq_mark_set()
> mlx5_flow_rxq_mark_clear()
> mlx5_flow_rxq_mark_trim()

Certainly, the point is those functions have a short life as few patches
letter they will be removed.
I suppose you prefer to have them and I don't think it will take too
much time to add such function, it will make part of the next revision
;).

Thanks,

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH v2 13/20] net/mlx5: add RSS flow action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 1211 +++++++++++++++++++++++++---------
 1 file changed, 899 insertions(+), 312 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index a39157533..08e0a6556 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -51,13 +51,148 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* Action fate on the packet. */
 #define MLX5_FLOW_FATE_DROP (1u << 0)
 #define MLX5_FLOW_FATE_QUEUE (1u << 1)
+#define MLX5_FLOW_FATE_RSS (1u << 2)
 
 /* Modify a packet. */
 #define MLX5_FLOW_MOD_FLAG (1u << 0)
 #define MLX5_FLOW_MOD_MARK (1u << 1)
 
+/* Priority reserved for default flows. */
+#define MLX5_FLOW_PRIO_RSVD ((uint32_t)-1)
+
+enum mlx5_expansion {
+	MLX5_EXPANSION_ROOT,
+	MLX5_EXPANSION_ROOT2,
+	MLX5_EXPANSION_OUTER_ETH,
+	MLX5_EXPANSION_OUTER_IPV4,
+	MLX5_EXPANSION_OUTER_IPV4_UDP,
+	MLX5_EXPANSION_OUTER_IPV4_TCP,
+	MLX5_EXPANSION_OUTER_IPV6,
+	MLX5_EXPANSION_OUTER_IPV6_UDP,
+	MLX5_EXPANSION_OUTER_IPV6_TCP,
+	MLX5_EXPANSION_VXLAN,
+	MLX5_EXPANSION_VXLAN_GPE,
+	MLX5_EXPANSION_GRE,
+	MLX5_EXPANSION_MPLS,
+	MLX5_EXPANSION_ETH,
+	MLX5_EXPANSION_IPV4,
+	MLX5_EXPANSION_IPV4_UDP,
+	MLX5_EXPANSION_IPV4_TCP,
+	MLX5_EXPANSION_IPV6,
+	MLX5_EXPANSION_IPV6_UDP,
+	MLX5_EXPANSION_IPV6_TCP,
+};
+
+/** Supported expansion of items. */
+static const struct rte_flow_expand_node mlx5_support_expansion[] = {
+	[MLX5_EXPANSION_ROOT] = {
+		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_ETH,
+					      MLX5_EXPANSION_IPV4,
+					      MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_END,
+	},
+	[MLX5_EXPANSION_ROOT2] = {
+		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_OUTER_ETH,
+					      MLX5_EXPANSION_OUTER_IPV4,
+					      MLX5_EXPANSION_OUTER_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_END,
+	},
+	[MLX5_EXPANSION_OUTER_ETH] = {
+		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_OUTER_IPV4,
+					      MLX5_EXPANSION_OUTER_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+		.rss_types = 0,
+	},
+	[MLX5_EXPANSION_OUTER_IPV4] = {
+		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_OUTER_IPV4_UDP,
+					      MLX5_EXPANSION_OUTER_IPV4_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV4,
+		.rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+			ETH_RSS_NONFRAG_IPV4_OTHER,
+	},
+	[MLX5_EXPANSION_OUTER_IPV4_UDP] = {
+		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_VXLAN),
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
+	},
+	[MLX5_EXPANSION_OUTER_IPV4_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
+	},
+	[MLX5_EXPANSION_OUTER_IPV6] = {
+		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_OUTER_IPV6_UDP,
+					      MLX5_EXPANSION_OUTER_IPV6_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV6,
+		.rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
+			ETH_RSS_NONFRAG_IPV6_OTHER,
+	},
+	[MLX5_EXPANSION_OUTER_IPV6_UDP] = {
+		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_VXLAN),
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
+	},
+	[MLX5_EXPANSION_OUTER_IPV6_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
+	},
+	[MLX5_EXPANSION_VXLAN] = {
+		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_ETH),
+		.type = RTE_FLOW_ITEM_TYPE_VXLAN,
+	},
+	[MLX5_EXPANSION_VXLAN_GPE] = {
+		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_ETH,
+					      MLX5_EXPANSION_IPV4,
+					      MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
+	},
+	[MLX5_EXPANSION_GRE] = {
+		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_IPV4),
+		.type = RTE_FLOW_ITEM_TYPE_GRE,
+	},
+	[MLX5_EXPANSION_ETH] = {
+		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_IPV4,
+					      MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+	},
+	[MLX5_EXPANSION_IPV4] = {
+		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_IPV4_UDP,
+					      MLX5_EXPANSION_IPV4_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV4,
+		.rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+			ETH_RSS_NONFRAG_IPV4_OTHER,
+	},
+	[MLX5_EXPANSION_IPV4_UDP] = {
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
+	},
+	[MLX5_EXPANSION_IPV4_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
+	},
+	[MLX5_EXPANSION_IPV6] = {
+		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_IPV6_UDP,
+					      MLX5_EXPANSION_IPV6_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV6,
+		.rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
+			ETH_RSS_NONFRAG_IPV6_OTHER,
+	},
+	[MLX5_EXPANSION_IPV6_UDP] = {
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
+	},
+	[MLX5_EXPANSION_IPV6_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
+	},
+};
+
 /** Handles information leading to a drop fate. */
 struct mlx5_flow_verbs {
+	LIST_ENTRY(mlx5_flow_verbs) next;
+	uint32_t layers;
+	/**< Bit-fields of expanded layers see MLX5_FLOW_ITEMS_*. */
+	uint32_t modifier;
+	/**< Bit-fields of expanded modifier see MLX5_FLOW_MOD_*. */
 	unsigned int size; /**< Size of the attribute. */
 	struct {
 		struct ibv_flow_attr *attr;
@@ -66,20 +201,26 @@ struct mlx5_flow_verbs {
 	};
 	struct ibv_flow *flow; /**< Verbs flow pointer. */
 	struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */
+	uint64_t hash_fields; /**< Verbs hash Rx queue hash fields. */
 };
 
 /* Flow structure. */
 struct rte_flow {
 	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
 	struct rte_flow_attr attributes; /**< User flow attribute. */
+	uint32_t expand:1; /**< Flow is expanded due to RSS configuration. */
 	uint32_t layers;
 	/**< Bit-fields of present layers see MLX5_FLOW_ITEMS_*. */
 	uint32_t modifier;
 	/**< Bit-fields of present modifier see MLX5_FLOW_MOD_*. */
 	uint32_t fate;
 	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
-	struct mlx5_flow_verbs verbs; /* Verbs flow. */
-	uint16_t queue; /**< Destination queue to redirect traffic to. */
+	LIST_HEAD(verbs, mlx5_flow_verbs) verbs; /**< Verbs flows list. */
+	struct mlx5_flow_verbs *cur_verbs;
+	/**< Current Verbs flow structure being filled. */
+	struct rte_flow_action_rss rss;/**< RSS context. */
+	uint8_t key[40]; /**< RSS hash key. */
+	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
 };
 
 static const struct rte_flow_ops mlx5_flow_ops = {
@@ -122,16 +263,27 @@ struct ibv_spec_header {
 	uint16_t size;
 };
 
- /**
-  * Get the maximum number of priority available.
-  *
-  * @param dev
-  *   Pointer to Ethernet device.
-  *
-  * @return
-  *   number of supported flow priority on success, a negative errno value
-  *   otherwise and rte_errno is set.
-  */
+/* Map of Verbs to Flow priority with 8 Verbs priorities. */
+static const uint32_t priority_map_3[][3] = {
+	{ 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
+};
+
+/* Map of Verbs to Flow priority with 16 Verbs priorities. */
+static const uint32_t priority_map_5[][3] = {
+	{ 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
+	{ 9, 10, 11 }, { 12, 13, 14 },
+};
+
+/**
+ * Get the maximum number of priority available.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   number of supported flow priority on success, a negative errno
+ *   value otherwise and rte_errno is set.
+ */
 int
 mlx5_flow_priorities(struct rte_eth_dev *dev)
 {
@@ -156,6 +308,7 @@ mlx5_flow_priorities(struct rte_eth_dev *dev)
 	struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
 	uint16_t vprio[] = { 8, 16 };
 	int i;
+	int priority = 0;
 
 	if (!drop) {
 		rte_errno = ENOTSUP;
@@ -167,11 +320,54 @@ mlx5_flow_priorities(struct rte_eth_dev *dev)
 		if (!flow)
 			break;
 		claim_zero(mlx5_glue->destroy_flow(flow));
+		priority = vprio[i];
+	}
+	switch (priority) {
+	case 8:
+		priority = 3;
+		break;
+	case 16:
+		priority = 5;
+		break;
+	default:
+		rte_errno = ENOTSUP;
+		DRV_LOG(ERR,
+			"port %u verbs maximum priority: %d expected 8/16",
+			dev->data->port_id, vprio[i]);
+		return -rte_errno;
 	}
 	mlx5_hrxq_drop_release(dev, drop);
 	DRV_LOG(INFO, "port %u flow maximum priority: %d",
-		dev->data->port_id, vprio[i]);
-	return vprio[i];
+		dev->data->port_id, priority);
+	return priority;
+}
+
+/**
+ * Adjust flow priority.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param flow
+ *   Pointer to an rte flow.
+ *
+ * @return
+ *   The priority adjusted.
+ */
+static int
+mlx5_flow_priority(struct rte_eth_dev *dev, uint32_t priority,
+		   uint32_t subpriority)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	switch (priv->config.flow_prio) {
+	case 3:
+		priority = priority_map_3[priority][subpriority];
+		break;
+	case 5:
+		priority = priority_map_5[priority][subpriority];
+		break;
+	}
+	return priority;
 }
 
 /**
@@ -185,6 +381,8 @@ void
 mlx5_flow_print(struct rte_flow *flow __rte_unused)
 {
 #ifndef NDEBUG
+	struct mlx5_flow_verbs *verbs = LIST_FIRST(&flow->verbs);
+
 	fprintf(stdout, "---------8<------------\n");
 	fprintf(stdout, "%s: flow information\n", MLX5_DRIVER_NAME);
 	fprintf(stdout, " attributes: group %u priority %u ingress %d egress %d"
@@ -193,26 +391,36 @@ mlx5_flow_print(struct rte_flow *flow __rte_unused)
 		flow->attributes.ingress,
 		flow->attributes.egress,
 		flow->attributes.transfer);
-	fprintf(stdout, " layers: %s/%s/%s\n",
-		flow->layers & MLX5_FLOW_LAYER_OUTER_L2 ? "l2" : "-",
-		flow->layers & MLX5_FLOW_LAYER_OUTER_L3 ? "l3" : "-",
-		flow->layers & MLX5_FLOW_LAYER_OUTER_L4 ? "l4" : "-");
-	if (flow->fate & MLX5_FLOW_FATE_DROP)
+	if (flow->fate & MLX5_FLOW_FATE_DROP) {
 		fprintf(stdout, " fate: drop queue\n");
-	else if (flow->fate & MLX5_FLOW_FATE_QUEUE)
-		fprintf(stdout, " fate: target queue %u\n", flow->queue);
-	if (flow->verbs.attr) {
-		struct ibv_spec_header *hdr =
-			(struct ibv_spec_header *)flow->verbs.specs;
-		const int n = flow->verbs.attr->num_of_specs;
-		int i;
-
-		fprintf(stdout, " Verbs attributes: specs_n %u\n",
-			flow->verbs.attr->num_of_specs);
-		for (i = 0; i != n; ++i) {
-			rte_hexdump(stdout, " ", hdr, hdr->size);
-			hdr = (struct ibv_spec_header *)
-				((uint8_t *)hdr + hdr->size);
+	} else {
+		uint16_t i;
+
+		fprintf(stdout, " fate: target queues");
+		for (i = 0; i != flow->rss.queue_num; ++i)
+			fprintf(stdout, " %u", (*flow->queue)[i]);
+		fprintf(stdout, "\n");
+	}
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		uint32_t layers = flow->layers | verbs->layers;
+
+		fprintf(stdout, " layers: %s/%s/%s\n",
+			layers & MLX5_FLOW_LAYER_OUTER_L2 ? "l2" : "-",
+			layers & MLX5_FLOW_LAYER_OUTER_L3 ? "l3" : "-",
+			layers & MLX5_FLOW_LAYER_OUTER_L4 ? "l4" : "-");
+		if (verbs->attr) {
+			struct ibv_spec_header *hdr =
+				(struct ibv_spec_header *)verbs->specs;
+			const int n = verbs->attr->num_of_specs;
+			int i;
+
+			fprintf(stdout, " Verbs attributes: specs_n %u\n",
+				verbs->attr->num_of_specs);
+			for (i = 0; i != n; ++i) {
+				rte_hexdump(stdout, " ", hdr, hdr->size);
+				hdr = (struct ibv_spec_header *)
+					((uint8_t *)hdr + hdr->size);
+			}
 		}
 	}
 	fprintf(stdout, "--------->8------------\n");
@@ -239,18 +447,20 @@ mlx5_flow_attributes(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 		     struct rte_flow *flow, struct rte_flow_error *error)
 {
 	uint32_t priority_max =
-		((struct priv *)dev->data->dev_private)->config.flow_prio;
+		((struct priv *)dev->data->dev_private)->config.flow_prio - 1;
 
 	if (attr->group)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 					  NULL,
 					  "groups are not supported");
-	if (attr->priority >= priority_max)
+	if (attr->priority != MLX5_FLOW_PRIO_RSVD &&
+	    attr->priority >= priority_max)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 					  NULL,
-					  "priority value is not supported");
+					  "requested priority value is not"
+					  " supported");
 	if (attr->egress)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
@@ -267,6 +477,8 @@ mlx5_flow_attributes(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 					  NULL,
 					  "only ingress is supported");
 	flow->attributes = *attr;
+	if (attr->priority == MLX5_FLOW_PRIO_RSVD)
+		flow->attributes.priority = priority_max;
 	return 0;
 }
 
@@ -346,14 +558,51 @@ mlx5_flow_item_validate(const struct rte_flow_item *item,
 static void
 mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
 {
-	if (flow->verbs.specs) {
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+
+	if (verbs->specs) {
 		void *dst;
 
-		dst = (void *)(flow->verbs.specs + flow->verbs.size);
+		dst = (void *)(verbs->specs + verbs->size);
 		memcpy(dst, src, size);
-		++flow->verbs.attr->num_of_specs;
+		++verbs->attr->num_of_specs;
 	}
-	flow->verbs.size += size;
+	verbs->size += size;
+}
+
+/**
+ * Update layer bit-field.
+ *
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param layers
+ *   Bit-fields of layers to add see MLX5_FLOW_ITEMS_*.
+ */
+static void
+mlx5_flow_layers_update(struct rte_flow *flow, uint32_t layers)
+{
+	if (flow->expand) {
+		if (flow->cur_verbs)
+			flow->cur_verbs->layers |= layers;
+	} else {
+		flow->layers |= layers;
+	}
+}
+
+/**
+ * Get layers bit-field.
+ *
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ */
+static uint32_t
+mlx5_flow_layers(struct rte_flow *flow)
+{
+	uint32_t layers = flow->layers;
+
+	if (flow->expand && flow->cur_verbs)
+		layers |= flow->cur_verbs->layers;
+	return layers;
 }
 
 /**
@@ -388,22 +637,26 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 		.type = IBV_FLOW_SPEC_ETH,
 		.size = size,
 	};
+	const uint32_t layers = mlx5_flow_layers(flow);
 	int ret;
 
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L2)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L2 layers already configured");
-	if (!mask)
-		mask = &rte_flow_item_eth_mask;
-	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
-				      (const uint8_t *)&nic_mask,
-				      sizeof(struct rte_flow_item_eth),
-				      error);
-	if (ret)
-		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2;
+	if (!flow->expand) {
+		if (layers & MLX5_FLOW_LAYER_OUTER_L2)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "L2 layers already"
+						  " configured");
+		if (!mask)
+			mask = &rte_flow_item_eth_mask;
+		ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+					      (const uint8_t *)&nic_mask,
+					      sizeof(struct rte_flow_item_eth),
+					      error);
+		if (ret)
+			return ret;
+	}
+	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L2);
 	if (size > flow_size)
 		return size;
 	if (spec) {
@@ -482,6 +735,7 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 		.tci = RTE_BE16(0x0fff),
 	};
 	unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 	struct ibv_flow_spec_eth eth = {
 		.type = IBV_FLOW_SPEC_ETH,
 		.size = size,
@@ -491,24 +745,30 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 			MLX5_FLOW_LAYER_OUTER_L4;
 	const uint32_t vlanm = MLX5_FLOW_LAYER_OUTER_VLAN;
 	const uint32_t l2m = MLX5_FLOW_LAYER_OUTER_L2;
+	const uint32_t layers = mlx5_flow_layers(flow);
 
-	if (flow->layers & vlanm)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L2 layers already configured");
-	else if ((flow->layers & lm) != 0)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L2 layer cannot follow L3/L4 layer");
-	if (!mask)
-		mask = &rte_flow_item_vlan_mask;
-	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
-				      (const uint8_t *)&nic_mask,
-				      sizeof(struct rte_flow_item_vlan), error);
-	if (ret)
-		return ret;
+	if (!flow->expand) {
+		if (layers & vlanm)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "L2 layers already"
+						  " configured");
+		else if ((layers & lm) != 0)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "L2 layer cannot follow"
+						  " L3/L4 layer");
+		if (!mask)
+			mask = &rte_flow_item_vlan_mask;
+		ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+					      (const uint8_t *)&nic_mask,
+					      sizeof(struct rte_flow_item_vlan),
+					      error);
+		if (ret)
+			return ret;
+	}
 	if (spec) {
 		eth.val.vlan_tag = spec->tci;
 		eth.mask.vlan_tag = mask->tci;
@@ -517,32 +777,34 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 		eth.mask.ether_type = mask->inner_type;
 		eth.val.ether_type &= eth.mask.ether_type;
 	}
-	/*
-	 * From verbs perspective an empty VLAN is equivalent
-	 * to a packet without VLAN layer.
-	 */
-	if (!eth.mask.vlan_tag)
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
-					  item->spec,
-					  "VLAN cannot be empty");
-	/* Outer TPID cannot be matched. */
-	if (eth.mask.ether_type)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
-					  item->spec,
-					  "VLAN TPID matching is not"
-					  " supported");
-	if (!(flow->layers & l2m)) {
+	if (!flow->expand) {
+		/*
+		 * From verbs perspective an empty VLAN is equivalent
+		 * to a packet without VLAN layer.
+		 */
+		if (!eth.mask.vlan_tag)
+			return rte_flow_error_set(error, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+						  item->spec,
+						  "VLAN cannot be empty");
+		/* Outer TPID cannot be matched. */
+		if (eth.mask.ether_type)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+						  item->spec,
+						  "VLAN TPID matching is not"
+						  " supported");
+	}
+	if (!(layers & l2m)) {
 		if (size <= flow_size)
 			mlx5_flow_spec_verbs_add(flow, &eth, size);
 	} else {
-		if (flow->verbs.attr)
-			mlx5_flow_item_vlan_update(flow->verbs.attr, &eth);
+		if (verbs->attr)
+			mlx5_flow_item_vlan_update(verbs->attr, &eth);
 		size = 0; /**< Only an update is done in eth specification. */
 	}
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2 |
-		MLX5_FLOW_LAYER_OUTER_VLAN;
+	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L2 |
+				MLX5_FLOW_LAYER_OUTER_VLAN);
 	return size;
 }
 
@@ -582,25 +844,31 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 		.size = size,
 	};
 	int ret;
+	const uint32_t layers = mlx5_flow_layers(flow);
 
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "multiple L3 layers not supported");
-	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 cannot follow an L4 layer.");
-	if (!mask)
-		mask = &rte_flow_item_ipv4_mask;
-	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
-				      (const uint8_t *)&nic_mask,
-				      sizeof(struct rte_flow_item_ipv4), error);
-	if (ret < 0)
-		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+	if (!flow->expand) {
+		if (layers & MLX5_FLOW_LAYER_OUTER_L3)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "multiple L3 layers not"
+						  " supported");
+		else if (layers & MLX5_FLOW_LAYER_OUTER_L4)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "L3 cannot follow an L4"
+						  " layer");
+		if (!mask)
+			mask = &rte_flow_item_ipv4_mask;
+		ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+					      (const uint8_t *)&nic_mask,
+					      sizeof(struct rte_flow_item_ipv4),
+					      error);
+		if (ret < 0)
+			return ret;
+	}
+	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L3_IPV4);
 	if (size > flow_size)
 		return size;
 	if (spec) {
@@ -667,25 +935,31 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 		.size = size,
 	};
 	int ret;
+	const uint32_t layers = mlx5_flow_layers(flow);
 
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "multiple L3 layers not supported");
-	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 cannot follow an L4 layer.");
-	if (!mask)
-		mask = &rte_flow_item_ipv6_mask;
-	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
-				      (const uint8_t *)&nic_mask,
-				      sizeof(struct rte_flow_item_ipv6), error);
-	if (ret < 0)
-		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+	if (!flow->expand) {
+		if (layers & MLX5_FLOW_LAYER_OUTER_L3)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "multiple L3 layers not"
+						  " supported");
+		else if (layers & MLX5_FLOW_LAYER_OUTER_L4)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "L3 cannot follow an L4"
+						  " layer");
+		if (!mask)
+			mask = &rte_flow_item_ipv6_mask;
+		ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
+					      (const uint8_t *)&nic_mask,
+					      sizeof(struct rte_flow_item_ipv6),
+					      error);
+		if (ret < 0)
+			return ret;
+	}
+	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L3_IPV6);
 	if (size > flow_size)
 		return size;
 	if (spec) {
@@ -759,25 +1033,31 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 		.size = size,
 	};
 	int ret;
+	const uint32_t layers = mlx5_flow_layers(flow);
 
-	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 is mandatory to filter on L4");
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L4 layer is already present");
-	if (!mask)
-		mask = &rte_flow_item_udp_mask;
-	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
-				      (const uint8_t *)&rte_flow_item_udp_mask,
-				      sizeof(struct rte_flow_item_udp), error);
-	if (ret < 0)
-		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
+	if (!flow->expand) {
+		if (!(layers & MLX5_FLOW_LAYER_OUTER_L3))
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "L3 is mandatory to filter"
+						  " on L4");
+		if (layers & MLX5_FLOW_LAYER_OUTER_L4)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "L4 layer is already"
+						  " present");
+		if (!mask)
+			mask = &rte_flow_item_udp_mask;
+		ret = mlx5_flow_item_validate
+			(item, (const uint8_t *)mask,
+			 (const uint8_t *)&rte_flow_item_udp_mask,
+			 sizeof(struct rte_flow_item_udp), error);
+		if (ret < 0)
+			return ret;
+	}
+	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L4_UDP);
 	if (size > flow_size)
 		return size;
 	if (spec) {
@@ -821,25 +1101,31 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 		.size = size,
 	};
 	int ret;
+	const uint32_t layers = mlx5_flow_layers(flow);
 
-	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 is mandatory to filter on L4");
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L4 layer is already present");
-	if (!mask)
-		mask = &rte_flow_item_tcp_mask;
-	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
-				      (const uint8_t *)&rte_flow_item_tcp_mask,
-				      sizeof(struct rte_flow_item_tcp), error);
-	if (ret < 0)
-		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
+	if (!flow->expand) {
+		if (!(layers & MLX5_FLOW_LAYER_OUTER_L3))
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "L3 is mandatory to filter"
+						  " on L4");
+		if (layers & MLX5_FLOW_LAYER_OUTER_L4)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "L4 layer is already"
+						  " present");
+		if (!mask)
+			mask = &rte_flow_item_tcp_mask;
+		ret = mlx5_flow_item_validate
+			(item, (const uint8_t *)mask,
+			 (const uint8_t *)&rte_flow_item_tcp_mask,
+			 sizeof(struct rte_flow_item_tcp), error);
+		if (ret < 0)
+			return ret;
+	}
+	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L4_TCP);
 	if (size > flow_size)
 		return size;
 	if (spec) {
@@ -954,18 +1240,20 @@ mlx5_flow_action_drop(const struct rte_flow_action *actions,
 			.size = size,
 	};
 
-	if (flow->fate)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  actions,
-					  "multiple fate actions are not"
-					  " supported");
-	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK))
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  actions,
-					  "drop is not compatible with"
-					  " flag/mark action");
+	if (!flow->expand) {
+		if (flow->fate)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "multiple fate actions are"
+						  " not supported");
+		if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK))
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "drop is not compatible with"
+						  " flag/mark action");
+	}
 	if (size < flow_size)
 		mlx5_flow_spec_verbs_add(flow, &drop, size);
 	flow->fate |= MLX5_FLOW_FATE_DROP;
@@ -998,6 +1286,8 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_action_queue *queue = actions->conf;
 
+	if (flow->expand)
+		return 0;
 	if (flow->fate)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -1014,11 +1304,162 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 					  &queue->index,
 					  "queue is not configured");
-	flow->queue = queue->index;
+	if (flow->queue)
+		(*flow->queue)[0] = queue->index;
+	flow->rss.queue_num = 1;
 	flow->fate |= MLX5_FLOW_FATE_QUEUE;
 	return 0;
 }
 
+/**
+ * Store the Verbs hash fields and priority according to the layer and types.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param flow
+ *   Pointer to flow structure.
+ * @param types
+ *   RSS types for this flow (see ETH_RSS_*).
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_action_rss_verbs_attr(struct rte_eth_dev *dev, struct rte_flow *flow,
+				uint32_t types)
+{
+	const uint32_t layers = mlx5_flow_layers(flow);
+	uint64_t hash_fields;
+	uint32_t priority;
+
+	if ((types & ETH_RSS_NONFRAG_IPV4_TCP) &&
+	    (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)) {
+		hash_fields = IBV_RX_HASH_SRC_IPV4 |
+			IBV_RX_HASH_DST_IPV4 |
+			IBV_RX_HASH_SRC_PORT_TCP |
+			IBV_RX_HASH_DST_PORT_TCP;
+		priority = 0;
+	} else if ((types & ETH_RSS_NONFRAG_IPV4_UDP) &&
+		 (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)) {
+		hash_fields = IBV_RX_HASH_SRC_IPV4 |
+			IBV_RX_HASH_DST_IPV4 |
+			IBV_RX_HASH_SRC_PORT_UDP |
+			IBV_RX_HASH_DST_PORT_UDP;
+		priority = 0;
+	} else if ((types & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4)) &&
+		 (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)) {
+		hash_fields = IBV_RX_HASH_SRC_IPV4 |
+			IBV_RX_HASH_DST_IPV4;
+		priority = 1;
+	} else if ((types & ETH_RSS_NONFRAG_IPV6_TCP) &&
+		 (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)) {
+		hash_fields = IBV_RX_HASH_SRC_IPV6 |
+			IBV_RX_HASH_DST_IPV6 |
+			IBV_RX_HASH_SRC_PORT_TCP |
+			IBV_RX_HASH_DST_PORT_TCP;
+		priority = 0;
+	} else if ((types & ETH_RSS_NONFRAG_IPV6_UDP) &&
+		 (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)) {
+		hash_fields = IBV_RX_HASH_SRC_IPV6 |
+			IBV_RX_HASH_DST_IPV6 |
+			IBV_RX_HASH_SRC_PORT_UDP |
+			IBV_RX_HASH_DST_PORT_UDP;
+		priority = 0;
+	} else if ((types & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6)) &&
+		 (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)) {
+		hash_fields = IBV_RX_HASH_SRC_IPV6 |
+			IBV_RX_HASH_DST_IPV6;
+		priority = 1;
+	} else {
+		hash_fields = 0;
+		priority = 2;
+	}
+	flow->cur_verbs->hash_fields = hash_fields;
+	flow->cur_verbs->attr->priority =
+		mlx5_flow_priority(dev, flow->attributes.priority, priority);
+	return 0;
+}
+
+/**
+ * Validate action queue provided by the user.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param actions
+ *   Pointer to flow actions array.
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param error
+ *   Pointer to error structure.
+ */
+static int
+mlx5_flow_action_rss(struct rte_eth_dev *dev,
+		     const struct rte_flow_action *actions,
+		     struct rte_flow *flow,
+		     struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_action_rss *rss = actions->conf;
+	unsigned int i;
+
+	if (flow->expand)
+		return 0;
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "multiple fate actions are not"
+					  " supported");
+	if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
+	    rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->func,
+					  "RSS hash function not supported");
+	if (rss->level > 1)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->level,
+					  "tunnel RSS is not supported");
+	if (rss->key_len < rss_hash_default_key_len)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->key_len,
+					  "RSS hash key too small");
+	if (rss->key_len > rss_hash_default_key_len)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->key_len,
+					  "RSS hash key too large");
+	if (rss->queue_num > priv->config.ind_table_max_size)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->queue_num,
+					  "number of queues too large");
+	if (rss->types & MLX5_RSS_HF_MASK)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->types,
+					  "some RSS protocols are not"
+					  " supported");
+	for (i = 0; i != rss->queue_num; ++i) {
+		if (!(*priv->rxqs)[rss->queue[i]])
+			return rte_flow_error_set
+				(error, EINVAL,
+				 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+				 &rss->queue[i],
+				 "queue is not configured");
+	}
+	if (flow->queue)
+		memcpy((*flow->queue), rss->queue,
+		       rss->queue_num * sizeof(uint16_t));
+	flow->rss.queue_num = rss->queue_num;
+	memcpy(flow->key, rss->key, rss_hash_default_key_len);
+	flow->rss.types = rss->types;
+	flow->fate |= MLX5_FLOW_FATE_RSS;
+	return 0;
+}
+
 /**
  * Validate action flag provided by the user.
  *
@@ -1046,43 +1487,59 @@ mlx5_flow_action_flag(const struct rte_flow_action *actions,
 		.size = size,
 		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
 	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 
-	if (flow->modifier & MLX5_FLOW_MOD_FLAG)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  actions,
-					  "flag action already present");
-	if (flow->fate & MLX5_FLOW_FATE_DROP)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  actions,
-					  "flag is not compatible with drop"
-					  " action");
-	if (flow->modifier & MLX5_FLOW_MOD_MARK)
-		return 0;
+	if (!flow->expand) {
+		if (flow->modifier & MLX5_FLOW_MOD_FLAG)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "flag action already present");
+		if (flow->fate & MLX5_FLOW_FATE_DROP)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "flag is not compatible with"
+						  " drop action");
+	}
+	/*
+	 * The two only possible cases, a mark has already been added in the
+	 * specification, in such case, the flag is already present in
+	 * addition of the mark.
+	 * Second case, has it is not possible to have two flags, it just
+	 * needs to add it.
+	 */
+	if (verbs) {
+		verbs->modifier |= MLX5_FLOW_MOD_FLAG;
+		if (verbs->modifier & MLX5_FLOW_MOD_MARK)
+			size = 0;
+		else if (size <= flow_size)
+			mlx5_flow_spec_verbs_add(flow, &tag, size);
+	} else {
+		if (flow->modifier & MLX5_FLOW_MOD_MARK)
+			size = 0;
+	}
 	flow->modifier |= MLX5_FLOW_MOD_FLAG;
-	if (size <= flow_size)
-		mlx5_flow_spec_verbs_add(flow, &tag, size);
 	return size;
 }
 
 /**
  * Update verbs specification to modify the flag to mark.
  *
- * @param flow
- *   Pointer to the rte_flow structure.
+ * @param verbs
+ *   Pointer to the mlx5_flow_verbs structure.
  * @param mark_id
  *   Mark identifier to replace the flag.
  */
 static void
-mlx5_flow_verbs_mark_update(struct rte_flow *flow, uint32_t mark_id)
+mlx5_flow_verbs_mark_update(struct mlx5_flow_verbs *verbs, uint32_t mark_id)
 {
 	struct ibv_spec_header *hdr;
 	int i;
 
 	/* Update Verbs specification. */
-	hdr = (struct ibv_spec_header *)flow->verbs.specs;
-	for (i = 0; i != flow->verbs.attr->num_of_specs; ++i) {
+	hdr = (struct ibv_spec_header *)verbs->specs;
+	for (i = 0; i != verbs->attr->num_of_specs; ++i) {
 		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
 			struct ibv_flow_spec_action_tag *t =
 				(struct ibv_flow_spec_action_tag *)hdr;
@@ -1120,38 +1577,52 @@ mlx5_flow_action_mark(const struct rte_flow_action *actions,
 		.type = IBV_FLOW_SPEC_ACTION_TAG,
 		.size = size,
 	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 
-	if (!mark)
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  actions,
-					  "configuration cannot be null");
-	if (mark->id >= MLX5_FLOW_MARK_MAX)
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
-					  &mark->id,
-					  "mark must be between 0 and"
-					  " 16777199");
-	if (flow->modifier & MLX5_FLOW_MOD_MARK)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  actions,
-					  "mark action already present");
-	if (flow->fate & MLX5_FLOW_FATE_DROP)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION,
-					  actions,
-					  "mark is not compatible with drop"
-					  " action");
-	if (flow->modifier & MLX5_FLOW_MOD_FLAG) {
-		mlx5_flow_verbs_mark_update(flow, mark->id);
-		size = 0; /**< Only an update is done in the specification. */
-	} else {
-		tag.tag_id = mlx5_flow_mark_set(mark->id);
-		if (size <= flow_size) {
+	if (!flow->expand) {
+		if (!mark)
+			return rte_flow_error_set(error, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "configuration cannot be"
+						  " null");
+		if (mark->id >= MLX5_FLOW_MARK_MAX)
+			return rte_flow_error_set
+				(error, EINVAL,
+				 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+				 &mark->id,
+				 "mark must be between 0 and 16777199");
+		if (flow->modifier & MLX5_FLOW_MOD_MARK)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "mark action already"
+						  " present");
+		if (flow->fate & MLX5_FLOW_FATE_DROP)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "mark is not compatible with"
+						  " drop action");
+	}
+	/*
+	 * The two only possible cases, a flag has already been added in the
+	 * specification, in such case, it needs to be update to add the id.
+	 * Second case, has it is not possible to have two mark, it just
+	 * needs to add it.
+	 */
+	if (verbs) {
+		verbs->modifier |= MLX5_FLOW_MOD_MARK;
+		if (verbs->modifier & MLX5_FLOW_MOD_FLAG) {
+			mlx5_flow_verbs_mark_update(verbs, mark->id);
+			size = 0;
+		} else if (size <= flow_size) {
 			tag.tag_id = mlx5_flow_mark_set(mark->id);
 			mlx5_flow_spec_verbs_add(flow, &tag, size);
 		}
+	} else {
+		if (flow->modifier & MLX5_FLOW_MOD_FLAG)
+			size = 0;
 	}
 	flow->modifier |= MLX5_FLOW_MOD_MARK;
 	return size;
@@ -1185,6 +1656,15 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 	int remain = flow_size;
 	int ret = 0;
 
+	/*
+	 * FLAG/MARK are the only actions having a specification in Verbs and
+	 * not making part of the packet fate.  Due to this specificity and to
+	 * avoid extra variable, their bit in the flow->modifier bit-field are
+	 * disabled here to compute the exact necessary memory those action
+	 * needs.
+	 */
+	flow->modifier &= ~(MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK);
+	/* Process the actions. */
 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 		switch (actions->type) {
 		case RTE_FLOW_ACTION_TYPE_VOID:
@@ -1204,6 +1684,9 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 		case RTE_FLOW_ACTION_TYPE_QUEUE:
 			ret = mlx5_flow_action_queue(dev, actions, flow, error);
 			break;
+		case RTE_FLOW_ACTION_TYPE_RSS:
+			ret = mlx5_flow_action_rss(dev, actions, flow, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -1257,27 +1740,92 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 		struct rte_flow_error *error)
 {
 	struct rte_flow local_flow = { .layers = 0, };
-	size_t size = sizeof(*flow) + sizeof(struct ibv_flow_attr);
+	size_t size = sizeof(*flow);
 	int remain = (flow_size > size) ? flow_size - size : 0;
+	struct rte_flow_expand_rss *buf;
 	int ret;
+	uint32_t i;
 
 	if (!remain)
 		flow = &local_flow;
 	ret = mlx5_flow_attributes(dev, attr, flow, error);
 	if (ret < 0)
 		return ret;
-	ret = mlx5_flow_items(items, flow, remain, error);
-	if (ret < 0)
-		return ret;
-	size += ret;
-	remain = (flow_size > size) ? flow_size - size : 0;
-	ret = mlx5_flow_actions(dev, actions, flow, remain, error);
+	ret = mlx5_flow_actions(dev, actions, &local_flow, 0, error);
 	if (ret < 0)
 		return ret;
-	size += ret;
+	ret = rte_flow_expand_rss(NULL, 0, items, local_flow.rss.types,
+				  mlx5_support_expansion,
+				  local_flow.rss.level < 2 ?
+				  MLX5_EXPANSION_ROOT : MLX5_EXPANSION_ROOT2);
+	assert(ret > 0);
+	buf = rte_calloc(__func__, 1, ret, 0);
+	if (!buf) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "not enough memory to expand the RSS flow");
+		goto error;
+	}
+	ret = rte_flow_expand_rss(buf, ret, items, local_flow.rss.types,
+				  mlx5_support_expansion,
+				  local_flow.rss.level < 2 ?
+				  MLX5_EXPANSION_ROOT : MLX5_EXPANSION_ROOT2);
+	assert(ret > 0);
+	size += RTE_ALIGN_CEIL(local_flow.rss.queue_num * sizeof(uint16_t),
+			       sizeof(void *));
 	if (size <= flow_size)
-		flow->verbs.attr->priority = flow->attributes.priority;
+		flow->queue = (void *)(flow + 1);
+	LIST_INIT(&flow->verbs);
+	flow->layers = 0;
+	flow->modifier = 0;
+	flow->fate = 0;
+	for (i = 0; i != buf->entries; ++i) {
+		size_t off = size;
+
+		size += sizeof(struct ibv_flow_attr) +
+			sizeof(struct mlx5_flow_verbs);
+		remain = (flow_size > size) ? flow_size - size : 0;
+		if (remain) {
+			flow->cur_verbs = (void *)((uintptr_t)flow + off);
+			flow->cur_verbs->attr = (void *)(flow->cur_verbs + 1);
+			flow->cur_verbs->specs =
+				(void *)(flow->cur_verbs->attr + 1);
+		}
+		ret = mlx5_flow_items
+			((const struct rte_flow_item *)buf->patterns[i],
+			 flow, remain, error);
+		if (ret < 0)
+			goto error;
+		size += ret;
+		if (remain > ret)
+			remain -= ret;
+		else
+			remain = 0;
+		ret = mlx5_flow_actions(dev, actions, flow, remain, error);
+		if (ret < 0)
+			goto error;
+		size += ret;
+		if (remain > ret)
+			remain -= ret;
+		else
+			remain = 0;
+		if (size <= flow_size) {
+			flow->cur_verbs->attr->priority =
+				flow->attributes.priority;
+			ret = mlx5_flow_action_rss_verbs_attr(dev, flow,
+							      flow->rss.types);
+			if (ret < 0)
+				goto error;
+			LIST_INSERT_HEAD(&flow->verbs, flow->cur_verbs, next);
+		}
+		flow->expand = !!(buf->entries > 1);
+	}
+	rte_free(buf);
 	return size;
+error:
+	rte_free(buf);
+	return ret;
 }
 
 /**
@@ -1292,9 +1840,13 @@ static void
 mlx5_flow_rxq_mark(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
 	struct priv *priv = dev->data->dev_private;
+	const uint32_t mask = MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK;
+	uint32_t i;
 
-	(*priv->rxqs)[flow->queue]->mark |=
-		flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK);
+	if (!(flow->modifier & mask))
+		return;
+	for (i = 0; i != flow->rss.queue_num; ++i)
+		(*priv->rxqs)[(*flow->queue)[i]]->mark = 1;
 }
 
 /**
@@ -1328,18 +1880,20 @@ mlx5_flow_validate(struct rte_eth_dev *dev,
 static void
 mlx5_flow_fate_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
-	if (flow->fate & MLX5_FLOW_FATE_DROP) {
-		if (flow->verbs.flow) {
-			claim_zero(mlx5_glue->destroy_flow(flow->verbs.flow));
-			flow->verbs.flow = NULL;
+	struct mlx5_flow_verbs *verbs;
+
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (verbs->flow) {
+			claim_zero(mlx5_glue->destroy_flow(verbs->flow));
+			verbs->flow = NULL;
+		}
+		if (verbs->hrxq) {
+			if (flow->fate & MLX5_FLOW_FATE_DROP)
+				mlx5_hrxq_drop_release(dev, verbs->hrxq);
+			else
+				mlx5_hrxq_release(dev, verbs->hrxq);
+			verbs->hrxq = NULL;
 		}
-	}
-	if (flow->verbs.hrxq) {
-		if (flow->fate & MLX5_FLOW_FATE_DROP)
-			mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
-		else if (flow->fate & MLX5_FLOW_FATE_QUEUE)
-			mlx5_hrxq_release(dev, flow->verbs.hrxq);
-		flow->verbs.hrxq = NULL;
 	}
 }
 
@@ -1360,46 +1914,68 @@ static int
 mlx5_flow_fate_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 		     struct rte_flow_error *error)
 {
-	if (flow->fate & MLX5_FLOW_FATE_DROP) {
-		flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
-		if (!flow->verbs.hrxq)
-			return rte_flow_error_set
-				(error, errno,
-				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				 NULL,
-				 "cannot allocate Drop queue");
-	} else if (flow->fate & MLX5_FLOW_FATE_QUEUE) {
-		struct mlx5_hrxq *hrxq;
-
-		hrxq = mlx5_hrxq_get(dev, rss_hash_default_key,
-				     rss_hash_default_key_len, 0,
-				     &flow->queue, 1, 0, 0);
-		if (!hrxq)
-			hrxq = mlx5_hrxq_new(dev, rss_hash_default_key,
-					     rss_hash_default_key_len, 0,
-					     &flow->queue, 1, 0, 0);
-		if (!hrxq)
-			return rte_flow_error_set(error, rte_errno,
-					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					NULL,
-					"cannot create flow");
-		flow->verbs.hrxq = hrxq;
-	}
-	flow->verbs.flow =
-		mlx5_glue->create_flow(flow->verbs.hrxq->qp, flow->verbs.attr);
-	if (!flow->verbs.flow) {
-		if (flow->fate & MLX5_FLOW_FATE_DROP)
-			mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
-		else
-			mlx5_hrxq_release(dev, flow->verbs.hrxq);
-		flow->verbs.hrxq = NULL;
-		return rte_flow_error_set(error, errno,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "kernel module refuses to create"
-					  " flow");
+	struct mlx5_flow_verbs *verbs;
+	int err;
+
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (flow->fate & MLX5_FLOW_FATE_DROP) {
+			verbs->hrxq = mlx5_hrxq_drop_new(dev);
+			if (!verbs->hrxq) {
+				rte_flow_error_set
+					(error, errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					 NULL,
+					 "cannot get drop hash queue");
+				goto error;
+			}
+		} else {
+			struct mlx5_hrxq *hrxq;
+
+			hrxq = mlx5_hrxq_get(dev, flow->key,
+					     rss_hash_default_key_len,
+					     verbs->hash_fields,
+					     (*flow->queue),
+					     flow->rss.queue_num, 0, 0);
+			if (!hrxq)
+				hrxq = mlx5_hrxq_new(dev, flow->key,
+						     rss_hash_default_key_len,
+						     verbs->hash_fields,
+						     (*flow->queue),
+						     flow->rss.queue_num, 0, 0);
+			if (!hrxq) {
+				rte_flow_error_set
+					(error, rte_errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					 NULL,
+					 "cannot get hash queue");
+				goto error;
+			}
+			verbs->hrxq = hrxq;
+		}
+		verbs->flow =
+			mlx5_glue->create_flow(verbs->hrxq->qp, verbs->attr);
+		if (!verbs->flow) {
+			rte_flow_error_set(error, errno,
+					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					   NULL,
+					   "hardware refuses to create flow");
+			goto error;
+		}
 	}
 	return 0;
+error:
+	err = rte_errno; /* Save rte_errno before cleanup. */
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (verbs->hrxq) {
+			if (flow->fate & MLX5_FLOW_FATE_DROP)
+				mlx5_hrxq_drop_release(dev, verbs->hrxq);
+			else
+				mlx5_hrxq_release(dev, verbs->hrxq);
+			verbs->hrxq = NULL;
+		}
+	}
+	rte_errno = err; /* Restore rte_errno. */
+	return -rte_errno;
 }
 
 /**
@@ -1429,42 +2005,43 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
 		      const struct rte_flow_action actions[],
 		      struct rte_flow_error *error)
 {
-	struct rte_flow *flow;
-	size_t size;
+	struct rte_flow *flow = NULL;
+	size_t size = 0;
 	int ret;
 
-	ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
 	if (ret < 0)
 		return NULL;
 	size = ret;
-	flow = rte_zmalloc(__func__, size, 0);
+	flow = rte_calloc(__func__, 1, size, 0);
 	if (!flow) {
 		rte_flow_error_set(error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL,
-				   "cannot allocate memory");
+				   "not enough memory to create flow");
 		return NULL;
 	}
-	flow->verbs.attr = (struct ibv_flow_attr *)(flow + 1);
-	flow->verbs.specs = (uint8_t *)(flow->verbs.attr + 1);
 	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
-	if (ret < 0)
-		goto error;
+	if (ret < 0) {
+		rte_free(flow);
+		return NULL;
+	}
 	assert((size_t)ret == size);
 	if (dev->data->dev_started) {
 		ret = mlx5_flow_fate_apply(dev, flow, error);
-		if (ret < 0)
-			goto error;
+		if (ret < 0) {
+			ret = rte_errno; /* Save rte_errno before cleanup. */
+			if (flow) {
+				mlx5_flow_fate_remove(dev, flow);
+				rte_free(flow);
+			}
+			rte_errno = ret; /* Restore rte_errno. */
+			return NULL;
+		}
 	}
 	mlx5_flow_rxq_mark(dev, flow);
 	TAILQ_INSERT_TAIL(list, flow, next);
 	return flow;
-error:
-	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_fate_remove(dev, flow);
-	rte_free(flow);
-	rte_errno = ret; /* Restore rte_errno. */
-	return NULL;
 }
 
 /**
@@ -1502,7 +2079,7 @@ mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
 	struct priv *priv = dev->data->dev_private;
 	struct rte_flow *rflow;
 	const uint32_t mask = MLX5_FLOW_MOD_FLAG & MLX5_FLOW_MOD_MARK;
-	int mark = 0;
+	unsigned int i;
 
 	mlx5_flow_fate_remove(dev, flow);
 	TAILQ_REMOVE(list, flow, next);
@@ -1512,18 +2089,28 @@ mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
 	}
 	/*
 	 * When a flow is removed and this flow has a flag/mark modifier, all
-	 * flows needs to be parse to verify if the Rx queue use by the flow
+	 * flows needs to be parse to verify if the Rx queues use by the flow
 	 * still need to track the flag/mark request.
 	 */
-	TAILQ_FOREACH(rflow, &priv->flows, next) {
-		if (!(rflow->modifier & mask))
-			continue;
-		if (flow->queue == rflow->queue) {
-			mark = 1;
-			break;
+	for (i = 0; i != flow->rss.queue_num; ++i) {
+		int mark = 0;
+
+		TAILQ_FOREACH(rflow, &priv->flows, next) {
+			unsigned int j;
+
+			if (!(rflow->modifier & mask))
+				continue;
+			for (j = 0; j != rflow->rss.queue_num; ++j) {
+				if ((*flow->queue)[i] == (*rflow->queue)[j]) {
+					mark = 1;
+					break;
+				}
+			}
+			if (mark)
+				break;
 		}
+		(*priv->rxqs)[i]->mark = !!mark;
 	}
-	(*priv->rxqs)[flow->queue]->mark = !!mark;
 	rte_free(flow);
 }
 
@@ -1654,7 +2241,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
-		.priority = priv->config.flow_prio - 1,
+		.priority = MLX5_FLOW_PRIO_RSVD,
 	};
 	struct rte_flow_item items[] = {
 		{
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 13/20] net/mlx5: add RSS flow action

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:45PM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
>  drivers/net/mlx5/mlx5_flow.c | 1211 +++++++++++++++++++++++++---------
>  1 file changed, 899 insertions(+), 312 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index a39157533..08e0a6556 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -51,13 +51,148 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
>  /* Action fate on the packet. */
>  #define MLX5_FLOW_FATE_DROP (1u << 0)
>  #define MLX5_FLOW_FATE_QUEUE (1u << 1)
> +#define MLX5_FLOW_FATE_RSS (1u << 2)
>  
>  /* Modify a packet. */
>  #define MLX5_FLOW_MOD_FLAG (1u << 0)
>  #define MLX5_FLOW_MOD_MARK (1u << 1)
>  
> +/* Priority reserved for default flows. */
> +#define MLX5_FLOW_PRIO_RSVD ((uint32_t)-1)
> +
> +enum mlx5_expansion {
> +	MLX5_EXPANSION_ROOT,
> +	MLX5_EXPANSION_ROOT2,

How about MLX5_EXPANSION_OUTER_ROOT?

> +	MLX5_EXPANSION_OUTER_ETH,
> +	MLX5_EXPANSION_OUTER_IPV4,
> +	MLX5_EXPANSION_OUTER_IPV4_UDP,
> +	MLX5_EXPANSION_OUTER_IPV4_TCP,
> +	MLX5_EXPANSION_OUTER_IPV6,
> +	MLX5_EXPANSION_OUTER_IPV6_UDP,
> +	MLX5_EXPANSION_OUTER_IPV6_TCP,
> +	MLX5_EXPANSION_VXLAN,
> +	MLX5_EXPANSION_VXLAN_GPE,
> +	MLX5_EXPANSION_GRE,
> +	MLX5_EXPANSION_MPLS,
> +	MLX5_EXPANSION_ETH,
> +	MLX5_EXPANSION_IPV4,
> +	MLX5_EXPANSION_IPV4_UDP,
> +	MLX5_EXPANSION_IPV4_TCP,
> +	MLX5_EXPANSION_IPV6,
> +	MLX5_EXPANSION_IPV6_UDP,
> +	MLX5_EXPANSION_IPV6_TCP,
> +};
> +
> +/** Supported expansion of items. */
> +static const struct rte_flow_expand_node mlx5_support_expansion[] = {
> +	[MLX5_EXPANSION_ROOT] = {
> +		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_ETH,
> +					      MLX5_EXPANSION_IPV4,
> +					      MLX5_EXPANSION_IPV6),
> +		.type = RTE_FLOW_ITEM_TYPE_END,
> +	},
> +	[MLX5_EXPANSION_ROOT2] = {
> +		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_OUTER_ETH,
> +					      MLX5_EXPANSION_OUTER_IPV4,
> +					      MLX5_EXPANSION_OUTER_IPV6),
> +		.type = RTE_FLOW_ITEM_TYPE_END,
> +	},
> +	[MLX5_EXPANSION_OUTER_ETH] = {
> +		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_OUTER_IPV4,
> +					      MLX5_EXPANSION_OUTER_IPV6),
> +		.type = RTE_FLOW_ITEM_TYPE_ETH,
> +		.rss_types = 0,
> +	},
> +	[MLX5_EXPANSION_OUTER_IPV4] = {
> +		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_OUTER_IPV4_UDP,
> +					      MLX5_EXPANSION_OUTER_IPV4_TCP),
> +		.type = RTE_FLOW_ITEM_TYPE_IPV4,
> +		.rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
> +			ETH_RSS_NONFRAG_IPV4_OTHER,
> +	},
> +	[MLX5_EXPANSION_OUTER_IPV4_UDP] = {
> +		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_VXLAN),
> +		.type = RTE_FLOW_ITEM_TYPE_UDP,
> +		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
> +	},
> +	[MLX5_EXPANSION_OUTER_IPV4_TCP] = {
> +		.type = RTE_FLOW_ITEM_TYPE_TCP,
> +		.rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
> +	},
> +	[MLX5_EXPANSION_OUTER_IPV6] = {
> +		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_OUTER_IPV6_UDP,
> +					      MLX5_EXPANSION_OUTER_IPV6_TCP),
> +		.type = RTE_FLOW_ITEM_TYPE_IPV6,
> +		.rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
> +			ETH_RSS_NONFRAG_IPV6_OTHER,
> +	},
> +	[MLX5_EXPANSION_OUTER_IPV6_UDP] = {
> +		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_VXLAN),
> +		.type = RTE_FLOW_ITEM_TYPE_UDP,
> +		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
> +	},
> +	[MLX5_EXPANSION_OUTER_IPV6_TCP] = {
> +		.type = RTE_FLOW_ITEM_TYPE_TCP,
> +		.rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
> +	},
> +	[MLX5_EXPANSION_VXLAN] = {
> +		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_ETH),
> +		.type = RTE_FLOW_ITEM_TYPE_VXLAN,
> +	},
> +	[MLX5_EXPANSION_VXLAN_GPE] = {
> +		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_ETH,
> +					      MLX5_EXPANSION_IPV4,
> +					      MLX5_EXPANSION_IPV6),
> +		.type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
> +	},
> +	[MLX5_EXPANSION_GRE] = {
> +		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_IPV4),
> +		.type = RTE_FLOW_ITEM_TYPE_GRE,
> +	},
> +	[MLX5_EXPANSION_ETH] = {
> +		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_IPV4,
> +					      MLX5_EXPANSION_IPV6),
> +		.type = RTE_FLOW_ITEM_TYPE_ETH,
> +	},
> +	[MLX5_EXPANSION_IPV4] = {
> +		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_IPV4_UDP,
> +					      MLX5_EXPANSION_IPV4_TCP),
> +		.type = RTE_FLOW_ITEM_TYPE_IPV4,
> +		.rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
> +			ETH_RSS_NONFRAG_IPV4_OTHER,
> +	},
> +	[MLX5_EXPANSION_IPV4_UDP] = {
> +		.type = RTE_FLOW_ITEM_TYPE_UDP,
> +		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
> +	},
> +	[MLX5_EXPANSION_IPV4_TCP] = {
> +		.type = RTE_FLOW_ITEM_TYPE_TCP,
> +		.rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
> +	},
> +	[MLX5_EXPANSION_IPV6] = {
> +		.next = RTE_FLOW_EXPAND_ITEMS(MLX5_EXPANSION_IPV6_UDP,
> +					      MLX5_EXPANSION_IPV6_TCP),
> +		.type = RTE_FLOW_ITEM_TYPE_IPV6,
> +		.rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
> +			ETH_RSS_NONFRAG_IPV6_OTHER,
> +	},
> +	[MLX5_EXPANSION_IPV6_UDP] = {
> +		.type = RTE_FLOW_ITEM_TYPE_UDP,
> +		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
> +	},
> +	[MLX5_EXPANSION_IPV6_TCP] = {
> +		.type = RTE_FLOW_ITEM_TYPE_TCP,
> +		.rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
> +	},
> +};
> +
>  /** Handles information leading to a drop fate. */
>  struct mlx5_flow_verbs {
> +	LIST_ENTRY(mlx5_flow_verbs) next;
> +	uint32_t layers;
> +	/**< Bit-fields of expanded layers see MLX5_FLOW_ITEMS_*. */
> +	uint32_t modifier;
> +	/**< Bit-fields of expanded modifier see MLX5_FLOW_MOD_*. */
>  	unsigned int size; /**< Size of the attribute. */
>  	struct {
>  		struct ibv_flow_attr *attr;
> @@ -66,20 +201,26 @@ struct mlx5_flow_verbs {
>  	};
>  	struct ibv_flow *flow; /**< Verbs flow pointer. */
>  	struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */
> +	uint64_t hash_fields; /**< Verbs hash Rx queue hash fields. */
>  };
>  
>  /* Flow structure. */
>  struct rte_flow {
>  	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
>  	struct rte_flow_attr attributes; /**< User flow attribute. */
> +	uint32_t expand:1; /**< Flow is expanded due to RSS configuration. */

Suggest 'expanded'.

>  	uint32_t layers;
>  	/**< Bit-fields of present layers see MLX5_FLOW_ITEMS_*. */
>  	uint32_t modifier;
>  	/**< Bit-fields of present modifier see MLX5_FLOW_MOD_*. */
>  	uint32_t fate;
>  	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
> -	struct mlx5_flow_verbs verbs; /* Verbs flow. */
> -	uint16_t queue; /**< Destination queue to redirect traffic to. */
> +	LIST_HEAD(verbs, mlx5_flow_verbs) verbs; /**< Verbs flows list. */
> +	struct mlx5_flow_verbs *cur_verbs;
> +	/**< Current Verbs flow structure being filled. */
> +	struct rte_flow_action_rss rss;/**< RSS context. */
> +	uint8_t key[40]; /**< RSS hash key. */

Let's define a macro for '40'.

> +	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
>  };
>  
>  static const struct rte_flow_ops mlx5_flow_ops = {
> @@ -122,16 +263,27 @@ struct ibv_spec_header {
>  	uint16_t size;
>  };
>  
> - /**
> -  * Get the maximum number of priority available.
> -  *
> -  * @param dev
> -  *   Pointer to Ethernet device.
> -  *
> -  * @return
> -  *   number of supported flow priority on success, a negative errno value
> -  *   otherwise and rte_errno is set.
> -  */
> +/* Map of Verbs to Flow priority with 8 Verbs priorities. */
> +static const uint32_t priority_map_3[][3] = {
> +	{ 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
> +};
> +
> +/* Map of Verbs to Flow priority with 16 Verbs priorities. */
> +static const uint32_t priority_map_5[][3] = {
> +	{ 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
> +	{ 9, 10, 11 }, { 12, 13, 14 },
> +};

How about 

enum mlx5_sub_priority {
	MLX5_SUB_PRIORITY_0 = 0,
	MLX5_SUB_PRIORITY_1,
	MLX5_SUB_PRIORITY_2,
	MLX5_SUB_PRIORITY_MAX,
};

static const uint32_t priority_map_3[][MLX5_SUB_PRIORITY_MAX] = {

> +
> +/**
> + * Get the maximum number of priority available.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + *
> + * @return
> + *   number of supported flow priority on success, a negative errno
> + *   value otherwise and rte_errno is set.
> + */
>  int
>  mlx5_flow_priorities(struct rte_eth_dev *dev)

mlx5_flow_priorities() vs mlx5_flow_priority(), similar name but different
functionality. Better to rename it, e.g. mlx5_flow_get_max_priority() and
mlx5_flow_adjust_priority()

>  {
> @@ -156,6 +308,7 @@ mlx5_flow_priorities(struct rte_eth_dev *dev)
>  	struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
>  	uint16_t vprio[] = { 8, 16 };
>  	int i;
> +	int priority = 0;
>  
>  	if (!drop) {
>  		rte_errno = ENOTSUP;
> @@ -167,11 +320,54 @@ mlx5_flow_priorities(struct rte_eth_dev *dev)
>  		if (!flow)
>  			break;
>  		claim_zero(mlx5_glue->destroy_flow(flow));
> +		priority = vprio[i];
> +	}
> +	switch (priority) {
> +	case 8:
> +		priority = 3;

How about,
	priority = RTE_DIM(priority_map_3);

> +		break;
> +	case 16:
> +		priority = 5;

	priority = RTE_DIM(priority_map_5);

> +		break;
> +	default:
> +		rte_errno = ENOTSUP;
> +		DRV_LOG(ERR,
> +			"port %u verbs maximum priority: %d expected 8/16",
> +			dev->data->port_id, vprio[i]);
> +		return -rte_errno;
>  	}
>  	mlx5_hrxq_drop_release(dev, drop);
>  	DRV_LOG(INFO, "port %u flow maximum priority: %d",
> -		dev->data->port_id, vprio[i]);
> -	return vprio[i];
> +		dev->data->port_id, priority);
> +	return priority;
> +}
> +
> +/** > + * Adjust flow priority.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + * @param flow
> + *   Pointer to an rte flow.
> + *
> + * @return
> + *   The priority adjusted.
> + */
> +static int
> +mlx5_flow_priority(struct rte_eth_dev *dev, uint32_t priority,
> +		   uint32_t subpriority)
> +{
> +	struct priv *priv = dev->data->dev_private;
> +
> +	switch (priv->config.flow_prio) {
> +	case 3:

	case RTE_DIM(priority_map_3):

> +		priority = priority_map_3[priority][subpriority];
> +		break;
> +	case 5:

	case RTE_DIM(priority_map_5):

> +		priority = priority_map_5[priority][subpriority];
> +		break;
> +	}
> +	return priority;
>  }
>  
>  /**
> @@ -185,6 +381,8 @@ void
>  mlx5_flow_print(struct rte_flow *flow __rte_unused)
>  {
>  #ifndef NDEBUG
> +	struct mlx5_flow_verbs *verbs = LIST_FIRST(&flow->verbs);
> +
>  	fprintf(stdout, "---------8<------------\n");
>  	fprintf(stdout, "%s: flow information\n", MLX5_DRIVER_NAME);
>  	fprintf(stdout, " attributes: group %u priority %u ingress %d egress %d"
> @@ -193,26 +391,36 @@ mlx5_flow_print(struct rte_flow *flow __rte_unused)
>  		flow->attributes.ingress,
>  		flow->attributes.egress,
>  		flow->attributes.transfer);
> -	fprintf(stdout, " layers: %s/%s/%s\n",
> -		flow->layers & MLX5_FLOW_LAYER_OUTER_L2 ? "l2" : "-",
> -		flow->layers & MLX5_FLOW_LAYER_OUTER_L3 ? "l3" : "-",
> -		flow->layers & MLX5_FLOW_LAYER_OUTER_L4 ? "l4" : "-");
> -	if (flow->fate & MLX5_FLOW_FATE_DROP)
> +	if (flow->fate & MLX5_FLOW_FATE_DROP) {
>  		fprintf(stdout, " fate: drop queue\n");
> -	else if (flow->fate & MLX5_FLOW_FATE_QUEUE)
> -		fprintf(stdout, " fate: target queue %u\n", flow->queue);
> -	if (flow->verbs.attr) {
> -		struct ibv_spec_header *hdr =
> -			(struct ibv_spec_header *)flow->verbs.specs;
> -		const int n = flow->verbs.attr->num_of_specs;
> -		int i;
> -
> -		fprintf(stdout, " Verbs attributes: specs_n %u\n",
> -			flow->verbs.attr->num_of_specs);
> -		for (i = 0; i != n; ++i) {
> -			rte_hexdump(stdout, " ", hdr, hdr->size);
> -			hdr = (struct ibv_spec_header *)
> -				((uint8_t *)hdr + hdr->size);
> +	} else {
> +		uint16_t i;
> +
> +		fprintf(stdout, " fate: target queues");
> +		for (i = 0; i != flow->rss.queue_num; ++i)
> +			fprintf(stdout, " %u", (*flow->queue)[i]);
> +		fprintf(stdout, "\n");
> +	}
> +	LIST_FOREACH(verbs, &flow->verbs, next) {
> +		uint32_t layers = flow->layers | verbs->layers;
> +
> +		fprintf(stdout, " layers: %s/%s/%s\n",
> +			layers & MLX5_FLOW_LAYER_OUTER_L2 ? "l2" : "-",
> +			layers & MLX5_FLOW_LAYER_OUTER_L3 ? "l3" : "-",
> +			layers & MLX5_FLOW_LAYER_OUTER_L4 ? "l4" : "-");
> +		if (verbs->attr) {
> +			struct ibv_spec_header *hdr =
> +				(struct ibv_spec_header *)verbs->specs;
> +			const int n = verbs->attr->num_of_specs;
> +			int i;
> +
> +			fprintf(stdout, " Verbs attributes: specs_n %u\n",
> +				verbs->attr->num_of_specs);
> +			for (i = 0; i != n; ++i) {
> +				rte_hexdump(stdout, " ", hdr, hdr->size);
> +				hdr = (struct ibv_spec_header *)
> +					((uint8_t *)hdr + hdr->size);
> +			}
>  		}
>  	}
>  	fprintf(stdout, "--------->8------------\n");
> @@ -239,18 +447,20 @@ mlx5_flow_attributes(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
>  		     struct rte_flow *flow, struct rte_flow_error *error)
>  {
>  	uint32_t priority_max =
> -		((struct priv *)dev->data->dev_private)->config.flow_prio;
> +		((struct priv *)dev->data->dev_private)->config.flow_prio - 1;
>  
>  	if (attr->group)
>  		return rte_flow_error_set(error, ENOTSUP,
>  					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
>  					  NULL,
>  					  "groups are not supported");
> -	if (attr->priority >= priority_max)
> +	if (attr->priority != MLX5_FLOW_PRIO_RSVD &&
> +	    attr->priority >= priority_max)
>  		return rte_flow_error_set(error, ENOTSUP,
>  					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
>  					  NULL,
> -					  "priority value is not supported");
> +					  "requested priority value is not"
> +					  " supported");
>  	if (attr->egress)
>  		return rte_flow_error_set(error, ENOTSUP,
>  					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
> @@ -267,6 +477,8 @@ mlx5_flow_attributes(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
>  					  NULL,
>  					  "only ingress is supported");
>  	flow->attributes = *attr;
> +	if (attr->priority == MLX5_FLOW_PRIO_RSVD)
> +		flow->attributes.priority = priority_max;
>  	return 0;
>  }
>  
> @@ -346,14 +558,51 @@ mlx5_flow_item_validate(const struct rte_flow_item *item,
>  static void
>  mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
>  {
> -	if (flow->verbs.specs) {
> +	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
> +
> +	if (verbs->specs) {
>  		void *dst;
>  
> -		dst = (void *)(flow->verbs.specs + flow->verbs.size);
> +		dst = (void *)(verbs->specs + verbs->size);
>  		memcpy(dst, src, size);
> -		++flow->verbs.attr->num_of_specs;
> +		++verbs->attr->num_of_specs;
>  	}
> -	flow->verbs.size += size;
> +	verbs->size += size;
> +}
> +
> +/**
> + * Update layer bit-field.
> + *
> + * @param flow[in, out]
> + *   Pointer to flow structure.
> + * @param layers
> + *   Bit-fields of layers to add see MLX5_FLOW_ITEMS_*.

Where is MLX5_FLOW_ITEMS_*? Isn't it MLX5_FLOW_LAYER_*?
There are several occurrences.

> + */
> +static void
> +mlx5_flow_layers_update(struct rte_flow *flow, uint32_t layers)
> +{
> +	if (flow->expand) {
> +		if (flow->cur_verbs)
> +			flow->cur_verbs->layers |= layers;

If flow->cur_verbs is null, does that mean it is a testing call? Then, is it
unnecessary to update layers for the testing call? Confusing..

> +	} else {
> +		flow->layers |= layers;
> +	}
> +}
> +
> +/**
> + * Get layers bit-field.
> + *
> + * @param flow[in, out]
> + *   Pointer to flow structure.
> + */
> +static uint32_t
> +mlx5_flow_layers(struct rte_flow *flow)
> +{
> +	uint32_t layers = flow->layers;
> +
> +	if (flow->expand && flow->cur_verbs)

If flow is expanded and it is a testing call, then flow->layers is used?

> +		layers |= flow->cur_verbs->layers;
> +	return layers;

This part is so unclear to me, hard to understand. There are two 'layers'
fields, one in rte_flow and the other in mlx5_flow_verbs. It seems
rte_flow->layers is used only when the flow isn't expanded. If a flow is
expanded, flow->expand is set after processing the first entry in the expanded
list. In mlx5_flow_merge(),

	for (i = 0; i != buf->entries; ++i) {

		...

		flow->expand = !!(buf->entries > 1);
	}

Why is flow->expand set at the end of the loop? Is this in order to avoid
validation for the expanded flows? mlx5_flow_item_xxx() executes validation only
if flow->expand is zero, why?

And why does mlx5_flow_layers() have to return (flow->layers |
flow->cur_verbs->layers) if expanded?

If there are 3 entries in the rte_flow_expand_rss,
	eth
	eth / ipv4 / udp
	eth / ipv6 / udp

Then, the 2nd and 3rd don't have MLX5_FLOW_LAYER_OUTER_L2 in layers field?
Please explain in details and add comments appropriately.

>  }
>  
>  /**
> @@ -388,22 +637,26 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
>  		.type = IBV_FLOW_SPEC_ETH,
>  		.size = size,
>  	};
> +	const uint32_t layers = mlx5_flow_layers(flow);
>  	int ret;
>  
> -	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L2)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ITEM,
> -					  item,
> -					  "L2 layers already configured");
> -	if (!mask)
> -		mask = &rte_flow_item_eth_mask;
> -	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> -				      (const uint8_t *)&nic_mask,
> -				      sizeof(struct rte_flow_item_eth),
> -				      error);
> -	if (ret)
> -		return ret;
> -	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2;
> +	if (!flow->expand) {
> +		if (layers & MLX5_FLOW_LAYER_OUTER_L2)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  item,
> +						  "L2 layers already"
> +						  " configured");
> +		if (!mask)
> +			mask = &rte_flow_item_eth_mask;
> +		ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> +					      (const uint8_t *)&nic_mask,
> +					      sizeof(struct rte_flow_item_eth),
> +					      error);
> +		if (ret)
> +			return ret;
> +	}
> +	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L2);
>  	if (size > flow_size)
>  		return size;
>  	if (spec) {
> @@ -482,6 +735,7 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
>  		.tci = RTE_BE16(0x0fff),
>  	};
>  	unsigned int size = sizeof(struct ibv_flow_spec_eth);
> +	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
>  	struct ibv_flow_spec_eth eth = {
>  		.type = IBV_FLOW_SPEC_ETH,
>  		.size = size,
> @@ -491,24 +745,30 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
>  			MLX5_FLOW_LAYER_OUTER_L4;
>  	const uint32_t vlanm = MLX5_FLOW_LAYER_OUTER_VLAN;
>  	const uint32_t l2m = MLX5_FLOW_LAYER_OUTER_L2;
> +	const uint32_t layers = mlx5_flow_layers(flow);
>  
> -	if (flow->layers & vlanm)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ITEM,
> -					  item,
> -					  "L2 layers already configured");
> -	else if ((flow->layers & lm) != 0)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ITEM,
> -					  item,
> -					  "L2 layer cannot follow L3/L4 layer");
> -	if (!mask)
> -		mask = &rte_flow_item_vlan_mask;
> -	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> -				      (const uint8_t *)&nic_mask,
> -				      sizeof(struct rte_flow_item_vlan), error);
> -	if (ret)
> -		return ret;
> +	if (!flow->expand) {
> +		if (layers & vlanm)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  item,
> +						  "L2 layers already"
> +						  " configured");
> +		else if ((layers & lm) != 0)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  item,
> +						  "L2 layer cannot follow"
> +						  " L3/L4 layer");
> +		if (!mask)
> +			mask = &rte_flow_item_vlan_mask;
> +		ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> +					      (const uint8_t *)&nic_mask,
> +					      sizeof(struct rte_flow_item_vlan),
> +					      error);
> +		if (ret)
> +			return ret;
> +	}
>  	if (spec) {
>  		eth.val.vlan_tag = spec->tci;
>  		eth.mask.vlan_tag = mask->tci;
> @@ -517,32 +777,34 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
>  		eth.mask.ether_type = mask->inner_type;
>  		eth.val.ether_type &= eth.mask.ether_type;
>  	}
> -	/*
> -	 * From verbs perspective an empty VLAN is equivalent
> -	 * to a packet without VLAN layer.
> -	 */
> -	if (!eth.mask.vlan_tag)
> -		return rte_flow_error_set(error, EINVAL,
> -					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
> -					  item->spec,
> -					  "VLAN cannot be empty");
> -	/* Outer TPID cannot be matched. */
> -	if (eth.mask.ether_type)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
> -					  item->spec,
> -					  "VLAN TPID matching is not"
> -					  " supported");
> -	if (!(flow->layers & l2m)) {
> +	if (!flow->expand) {
> +		/*
> +		 * From verbs perspective an empty VLAN is equivalent
> +		 * to a packet without VLAN layer.
> +		 */
> +		if (!eth.mask.vlan_tag)
> +			return rte_flow_error_set(error, EINVAL,
> +						  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
> +						  item->spec,
> +						  "VLAN cannot be empty");
> +		/* Outer TPID cannot be matched. */
> +		if (eth.mask.ether_type)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
> +						  item->spec,
> +						  "VLAN TPID matching is not"
> +						  " supported");
> +	}
> +	if (!(layers & l2m)) {
>  		if (size <= flow_size)
>  			mlx5_flow_spec_verbs_add(flow, &eth, size);
>  	} else {
> -		if (flow->verbs.attr)
> -			mlx5_flow_item_vlan_update(flow->verbs.attr, &eth);
> +		if (verbs->attr)
> +			mlx5_flow_item_vlan_update(verbs->attr, &eth);
>  		size = 0; /**< Only an update is done in eth specification. */
>  	}
> -	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2 |
> -		MLX5_FLOW_LAYER_OUTER_VLAN;
> +	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L2 |
> +				MLX5_FLOW_LAYER_OUTER_VLAN);
>  	return size;
>  }
>  
> @@ -582,25 +844,31 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
>  		.size = size,
>  	};
>  	int ret;
> +	const uint32_t layers = mlx5_flow_layers(flow);
>  
> -	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ITEM,
> -					  item,
> -					  "multiple L3 layers not supported");
> -	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ITEM,
> -					  item,
> -					  "L3 cannot follow an L4 layer.");
> -	if (!mask)
> -		mask = &rte_flow_item_ipv4_mask;
> -	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> -				      (const uint8_t *)&nic_mask,
> -				      sizeof(struct rte_flow_item_ipv4), error);
> -	if (ret < 0)
> -		return ret;
> -	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
> +	if (!flow->expand) {
> +		if (layers & MLX5_FLOW_LAYER_OUTER_L3)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  item,
> +						  "multiple L3 layers not"
> +						  " supported");
> +		else if (layers & MLX5_FLOW_LAYER_OUTER_L4)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  item,
> +						  "L3 cannot follow an L4"
> +						  " layer");
> +		if (!mask)
> +			mask = &rte_flow_item_ipv4_mask;
> +		ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> +					      (const uint8_t *)&nic_mask,
> +					      sizeof(struct rte_flow_item_ipv4),
> +					      error);
> +		if (ret < 0)
> +			return ret;
> +	}
> +	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L3_IPV4);
>  	if (size > flow_size)
>  		return size;
>  	if (spec) {
> @@ -667,25 +935,31 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
>  		.size = size,
>  	};
>  	int ret;
> +	const uint32_t layers = mlx5_flow_layers(flow);
>  
> -	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ITEM,
> -					  item,
> -					  "multiple L3 layers not supported");
> -	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ITEM,
> -					  item,
> -					  "L3 cannot follow an L4 layer.");
> -	if (!mask)
> -		mask = &rte_flow_item_ipv6_mask;
> -	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> -				      (const uint8_t *)&nic_mask,
> -				      sizeof(struct rte_flow_item_ipv6), error);
> -	if (ret < 0)
> -		return ret;
> -	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
> +	if (!flow->expand) {
> +		if (layers & MLX5_FLOW_LAYER_OUTER_L3)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  item,
> +						  "multiple L3 layers not"
> +						  " supported");
> +		else if (layers & MLX5_FLOW_LAYER_OUTER_L4)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  item,
> +						  "L3 cannot follow an L4"
> +						  " layer");
> +		if (!mask)
> +			mask = &rte_flow_item_ipv6_mask;
> +		ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> +					      (const uint8_t *)&nic_mask,
> +					      sizeof(struct rte_flow_item_ipv6),
> +					      error);
> +		if (ret < 0)
> +			return ret;
> +	}
> +	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L3_IPV6);
>  	if (size > flow_size)
>  		return size;
>  	if (spec) {
> @@ -759,25 +1033,31 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
>  		.size = size,
>  	};
>  	int ret;
> +	const uint32_t layers = mlx5_flow_layers(flow);
>  
> -	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ITEM,
> -					  item,
> -					  "L3 is mandatory to filter on L4");
> -	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ITEM,
> -					  item,
> -					  "L4 layer is already present");
> -	if (!mask)
> -		mask = &rte_flow_item_udp_mask;
> -	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> -				      (const uint8_t *)&rte_flow_item_udp_mask,
> -				      sizeof(struct rte_flow_item_udp), error);
> -	if (ret < 0)
> -		return ret;
> -	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
> +	if (!flow->expand) {
> +		if (!(layers & MLX5_FLOW_LAYER_OUTER_L3))
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  item,
> +						  "L3 is mandatory to filter"
> +						  " on L4");
> +		if (layers & MLX5_FLOW_LAYER_OUTER_L4)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  item,
> +						  "L4 layer is already"
> +						  " present");
> +		if (!mask)
> +			mask = &rte_flow_item_udp_mask;
> +		ret = mlx5_flow_item_validate
> +			(item, (const uint8_t *)mask,
> +			 (const uint8_t *)&rte_flow_item_udp_mask,
> +			 sizeof(struct rte_flow_item_udp), error);
> +		if (ret < 0)
> +			return ret;
> +	}
> +	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L4_UDP);
>  	if (size > flow_size)
>  		return size;
>  	if (spec) {
> @@ -821,25 +1101,31 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
>  		.size = size,
>  	};
>  	int ret;
> +	const uint32_t layers = mlx5_flow_layers(flow);
>  
> -	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ITEM,
> -					  item,
> -					  "L3 is mandatory to filter on L4");
> -	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ITEM,
> -					  item,
> -					  "L4 layer is already present");
> -	if (!mask)
> -		mask = &rte_flow_item_tcp_mask;
> -	ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> -				      (const uint8_t *)&rte_flow_item_tcp_mask,
> -				      sizeof(struct rte_flow_item_tcp), error);
> -	if (ret < 0)
> -		return ret;
> -	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
> +	if (!flow->expand) {
> +		if (!(layers & MLX5_FLOW_LAYER_OUTER_L3))
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  item,
> +						  "L3 is mandatory to filter"
> +						  " on L4");
> +		if (layers & MLX5_FLOW_LAYER_OUTER_L4)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  item,
> +						  "L4 layer is already"
> +						  " present");
> +		if (!mask)
> +			mask = &rte_flow_item_tcp_mask;
> +		ret = mlx5_flow_item_validate
> +			(item, (const uint8_t *)mask,
> +			 (const uint8_t *)&rte_flow_item_tcp_mask,
> +			 sizeof(struct rte_flow_item_tcp), error);
> +		if (ret < 0)
> +			return ret;
> +	}
> +	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L4_TCP);
>  	if (size > flow_size)
>  		return size;
>  	if (spec) {
> @@ -954,18 +1240,20 @@ mlx5_flow_action_drop(const struct rte_flow_action *actions,
>  			.size = size,
>  	};
>  
> -	if (flow->fate)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ACTION,
> -					  actions,
> -					  "multiple fate actions are not"
> -					  " supported");
> -	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK))
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ACTION,
> -					  actions,
> -					  "drop is not compatible with"
> -					  " flag/mark action");
> +	if (!flow->expand) {
> +		if (flow->fate)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ACTION,
> +						  actions,
> +						  "multiple fate actions are"
> +						  " not supported");
> +		if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK))
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ACTION,
> +						  actions,
> +						  "drop is not compatible with"
> +						  " flag/mark action");
> +	}
>  	if (size < flow_size)
>  		mlx5_flow_spec_verbs_add(flow, &drop, size);
>  	flow->fate |= MLX5_FLOW_FATE_DROP;
> @@ -998,6 +1286,8 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
>  	struct priv *priv = dev->data->dev_private;
>  	const struct rte_flow_action_queue *queue = actions->conf;
>  
> +	if (flow->expand)
> +		return 0;
>  	if (flow->fate)
>  		return rte_flow_error_set(error, ENOTSUP,
>  					  RTE_FLOW_ERROR_TYPE_ACTION,
> @@ -1014,11 +1304,162 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
>  					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
>  					  &queue->index,
>  					  "queue is not configured");
> -	flow->queue = queue->index;
> +	if (flow->queue)
> +		(*flow->queue)[0] = queue->index;
> +	flow->rss.queue_num = 1;
>  	flow->fate |= MLX5_FLOW_FATE_QUEUE;
>  	return 0;
>  }
>  
> +/**
> + * Store the Verbs hash fields and priority according to the layer and types.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + * @param flow
> + *   Pointer to flow structure.
> + * @param types
> + *   RSS types for this flow (see ETH_RSS_*).
> + *
> + * @return
> + *   0 on success, a negative errno value otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_action_rss_verbs_attr(struct rte_eth_dev *dev, struct rte_flow *flow,
> +				uint32_t types)
> +{
> +	const uint32_t layers = mlx5_flow_layers(flow);
> +	uint64_t hash_fields;
> +	uint32_t priority;
> +
> +	if ((types & ETH_RSS_NONFRAG_IPV4_TCP) &&
> +	    (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)) {
> +		hash_fields = IBV_RX_HASH_SRC_IPV4 |
> +			IBV_RX_HASH_DST_IPV4 |
> +			IBV_RX_HASH_SRC_PORT_TCP |
> +			IBV_RX_HASH_DST_PORT_TCP;
> +		priority = 0;
> +	} else if ((types & ETH_RSS_NONFRAG_IPV4_UDP) &&
> +		 (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)) {
> +		hash_fields = IBV_RX_HASH_SRC_IPV4 |
> +			IBV_RX_HASH_DST_IPV4 |
> +			IBV_RX_HASH_SRC_PORT_UDP |
> +			IBV_RX_HASH_DST_PORT_UDP;
> +		priority = 0;
> +	} else if ((types & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4)) &&
> +		 (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)) {
> +		hash_fields = IBV_RX_HASH_SRC_IPV4 |
> +			IBV_RX_HASH_DST_IPV4;
> +		priority = 1;
> +	} else if ((types & ETH_RSS_NONFRAG_IPV6_TCP) &&
> +		 (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)) {
> +		hash_fields = IBV_RX_HASH_SRC_IPV6 |
> +			IBV_RX_HASH_DST_IPV6 |
> +			IBV_RX_HASH_SRC_PORT_TCP |
> +			IBV_RX_HASH_DST_PORT_TCP;
> +		priority = 0;
> +	} else if ((types & ETH_RSS_NONFRAG_IPV6_UDP) &&
> +		 (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)) {
> +		hash_fields = IBV_RX_HASH_SRC_IPV6 |
> +			IBV_RX_HASH_DST_IPV6 |
> +			IBV_RX_HASH_SRC_PORT_UDP |
> +			IBV_RX_HASH_DST_PORT_UDP;
> +		priority = 0;
> +	} else if ((types & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6)) &&
> +		 (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)) {
> +		hash_fields = IBV_RX_HASH_SRC_IPV6 |
> +			IBV_RX_HASH_DST_IPV6;
> +		priority = 1;
> +	} else {
> +		hash_fields = 0;
> +		priority = 2;

How about 
		delta = MLX5_SUB_PRIORITY_2;

> +	}
> +	flow->cur_verbs->hash_fields = hash_fields;
> +	flow->cur_verbs->attr->priority =
> +		mlx5_flow_priority(dev, flow->attributes.priority, priority);
> +	return 0;
> +}
> +
> +/**
> + * Validate action queue provided by the user.
> + *
> + * @param dev
> + *   Pointer to Ethernet device structure.
> + * @param actions
> + *   Pointer to flow actions array.
> + * @param flow
> + *   Pointer to the rte_flow structure.
> + * @param error
> + *   Pointer to error structure.

Missing return value.

> + */
> +static int
> +mlx5_flow_action_rss(struct rte_eth_dev *dev,
> +		     const struct rte_flow_action *actions,
> +		     struct rte_flow *flow,
> +		     struct rte_flow_error *error)
> +{
> +	struct priv *priv = dev->data->dev_private;
> +	const struct rte_flow_action_rss *rss = actions->conf;
> +	unsigned int i;
> +
> +	if (flow->expand)
> +		return 0;
> +	if (flow->fate)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ACTION,
> +					  actions,
> +					  "multiple fate actions are not"
> +					  " supported");
> +	if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
> +	    rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
> +					  &rss->func,
> +					  "RSS hash function not supported");
> +	if (rss->level > 1)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
> +					  &rss->level,
> +					  "tunnel RSS is not supported");
> +	if (rss->key_len < rss_hash_default_key_len)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
> +					  &rss->key_len,
> +					  "RSS hash key too small");
> +	if (rss->key_len > rss_hash_default_key_len)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
> +					  &rss->key_len,
> +					  "RSS hash key too large");
> +	if (rss->queue_num > priv->config.ind_table_max_size)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
> +					  &rss->queue_num,
> +					  "number of queues too large");
> +	if (rss->types & MLX5_RSS_HF_MASK)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
> +					  &rss->types,
> +					  "some RSS protocols are not"
> +					  " supported");
> +	for (i = 0; i != rss->queue_num; ++i) {
> +		if (!(*priv->rxqs)[rss->queue[i]])
> +			return rte_flow_error_set
> +				(error, EINVAL,
> +				 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
> +				 &rss->queue[i],
> +				 "queue is not configured");
> +	}
> +	if (flow->queue)
> +		memcpy((*flow->queue), rss->queue,
> +		       rss->queue_num * sizeof(uint16_t));
> +	flow->rss.queue_num = rss->queue_num;
> +	memcpy(flow->key, rss->key, rss_hash_default_key_len);
> +	flow->rss.types = rss->types;
> +	flow->fate |= MLX5_FLOW_FATE_RSS;
> +	return 0;
> +}
> +
>  /**
>   * Validate action flag provided by the user.
>   *
> @@ -1046,43 +1487,59 @@ mlx5_flow_action_flag(const struct rte_flow_action *actions,
>  		.size = size,
>  		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
>  	};
> +	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
>  
> -	if (flow->modifier & MLX5_FLOW_MOD_FLAG)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ACTION,
> -					  actions,
> -					  "flag action already present");
> -	if (flow->fate & MLX5_FLOW_FATE_DROP)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ACTION,
> -					  actions,
> -					  "flag is not compatible with drop"
> -					  " action");
> -	if (flow->modifier & MLX5_FLOW_MOD_MARK)
> -		return 0;
> +	if (!flow->expand) {
> +		if (flow->modifier & MLX5_FLOW_MOD_FLAG)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ACTION,
> +						  actions,
> +						  "flag action already present");
> +		if (flow->fate & MLX5_FLOW_FATE_DROP)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ACTION,
> +						  actions,
> +						  "flag is not compatible with"
> +						  " drop action");
> +	}
> +	/*
> +	 * The two only possible cases, a mark has already been added in the
> +	 * specification, in such case, the flag is already present in
> +	 * addition of the mark.
> +	 * Second case, has it is not possible to have two flags, it just
> +	 * needs to add it.
> +	 */

Can you rephrase the 'second case'? Maybe 'has' -> 'as'?

> +	if (verbs) {
> +		verbs->modifier |= MLX5_FLOW_MOD_FLAG;
> +		if (verbs->modifier & MLX5_FLOW_MOD_MARK)
> +			size = 0;
> +		else if (size <= flow_size)
> +			mlx5_flow_spec_verbs_add(flow, &tag, size);
> +	} else {
> +		if (flow->modifier & MLX5_FLOW_MOD_MARK)
> +			size = 0;
> +	}
>  	flow->modifier |= MLX5_FLOW_MOD_FLAG;
> -	if (size <= flow_size)
> -		mlx5_flow_spec_verbs_add(flow, &tag, size);
>  	return size;
>  }
>  
>  /**
>   * Update verbs specification to modify the flag to mark.
>   *
> - * @param flow
> - *   Pointer to the rte_flow structure.
> + * @param verbs
> + *   Pointer to the mlx5_flow_verbs structure.
>   * @param mark_id
>   *   Mark identifier to replace the flag.
>   */
>  static void
> -mlx5_flow_verbs_mark_update(struct rte_flow *flow, uint32_t mark_id)
> +mlx5_flow_verbs_mark_update(struct mlx5_flow_verbs *verbs, uint32_t mark_id)
>  {
>  	struct ibv_spec_header *hdr;
>  	int i;
>  
>  	/* Update Verbs specification. */
> -	hdr = (struct ibv_spec_header *)flow->verbs.specs;
> -	for (i = 0; i != flow->verbs.attr->num_of_specs; ++i) {
> +	hdr = (struct ibv_spec_header *)verbs->specs;
> +	for (i = 0; i != verbs->attr->num_of_specs; ++i) {
>  		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
>  			struct ibv_flow_spec_action_tag *t =
>  				(struct ibv_flow_spec_action_tag *)hdr;
> @@ -1120,38 +1577,52 @@ mlx5_flow_action_mark(const struct rte_flow_action *actions,
>  		.type = IBV_FLOW_SPEC_ACTION_TAG,
>  		.size = size,
>  	};
> +	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
>  
> -	if (!mark)
> -		return rte_flow_error_set(error, EINVAL,
> -					  RTE_FLOW_ERROR_TYPE_ACTION,
> -					  actions,
> -					  "configuration cannot be null");
> -	if (mark->id >= MLX5_FLOW_MARK_MAX)
> -		return rte_flow_error_set(error, EINVAL,
> -					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
> -					  &mark->id,
> -					  "mark must be between 0 and"
> -					  " 16777199");
> -	if (flow->modifier & MLX5_FLOW_MOD_MARK)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ACTION,
> -					  actions,
> -					  "mark action already present");
> -	if (flow->fate & MLX5_FLOW_FATE_DROP)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ACTION,
> -					  actions,
> -					  "mark is not compatible with drop"
> -					  " action");
> -	if (flow->modifier & MLX5_FLOW_MOD_FLAG) {
> -		mlx5_flow_verbs_mark_update(flow, mark->id);
> -		size = 0; /**< Only an update is done in the specification. */
> -	} else {
> -		tag.tag_id = mlx5_flow_mark_set(mark->id);
> -		if (size <= flow_size) {
> +	if (!flow->expand) {
> +		if (!mark)
> +			return rte_flow_error_set(error, EINVAL,
> +						  RTE_FLOW_ERROR_TYPE_ACTION,
> +						  actions,
> +						  "configuration cannot be"
> +						  " null");
> +		if (mark->id >= MLX5_FLOW_MARK_MAX)
> +			return rte_flow_error_set
> +				(error, EINVAL,
> +				 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
> +				 &mark->id,
> +				 "mark must be between 0 and 16777199");
> +		if (flow->modifier & MLX5_FLOW_MOD_MARK)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ACTION,
> +						  actions,
> +						  "mark action already"
> +						  " present");
> +		if (flow->fate & MLX5_FLOW_FATE_DROP)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ACTION,
> +						  actions,
> +						  "mark is not compatible with"
> +						  " drop action");
> +	}
> +	/*
> +	 * The two only possible cases, a flag has already been added in the
> +	 * specification, in such case, it needs to be update to add the id.
> +	 * Second case, has it is not possible to have two mark, it just
> +	 * needs to add it.
> +	 */

Can you rephrase the 'second case'? Maybe 'has' -> 'as'?

> +	if (verbs) {
> +		verbs->modifier |= MLX5_FLOW_MOD_MARK;
> +		if (verbs->modifier & MLX5_FLOW_MOD_FLAG) {
> +			mlx5_flow_verbs_mark_update(verbs, mark->id);
> +			size = 0;
> +		} else if (size <= flow_size) {

If verbs isn't null (not testing call), isn't it guaranteed there's enough
space? Is it still needed to check the size?

>  			tag.tag_id = mlx5_flow_mark_set(mark->id);
>  			mlx5_flow_spec_verbs_add(flow, &tag, size);
>  		}
> +	} else {
> +		if (flow->modifier & MLX5_FLOW_MOD_FLAG)
> +			size = 0;
>  	}
>  	flow->modifier |= MLX5_FLOW_MOD_MARK;
>  	return size;
> @@ -1185,6 +1656,15 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
>  	int remain = flow_size;
>  	int ret = 0;
>  
> +	/*
> +	 * FLAG/MARK are the only actions having a specification in Verbs and
> +	 * not making part of the packet fate.  Due to this specificity and to
> +	 * avoid extra variable, their bit in the flow->modifier bit-field are
> +	 * disabled here to compute the exact necessary memory those action
> +	 * needs.
> +	 */
> +	flow->modifier &= ~(MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK);

Can't understand this well. Is this for the case where the flow is expanded? If
so, why don't you reset flow->modifier in the for loop of mlx5_flow_merge()?

> +	/* Process the actions. */
>  	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
>  		switch (actions->type) {
>  		case RTE_FLOW_ACTION_TYPE_VOID:
> @@ -1204,6 +1684,9 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
>  		case RTE_FLOW_ACTION_TYPE_QUEUE:
>  			ret = mlx5_flow_action_queue(dev, actions, flow, error);
>  			break;
> +		case RTE_FLOW_ACTION_TYPE_RSS:
> +			ret = mlx5_flow_action_rss(dev, actions, flow, error);
> +			break;
>  		default:
>  			return rte_flow_error_set(error, ENOTSUP,
>  						  RTE_FLOW_ERROR_TYPE_ACTION,
> @@ -1257,27 +1740,92 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
>  		struct rte_flow_error *error)
>  {
>  	struct rte_flow local_flow = { .layers = 0, };
> -	size_t size = sizeof(*flow) + sizeof(struct ibv_flow_attr);
> +	size_t size = sizeof(*flow);
>  	int remain = (flow_size > size) ? flow_size - size : 0;
> +	struct rte_flow_expand_rss *buf;
>  	int ret;
> +	uint32_t i;
>  
>  	if (!remain)
>  		flow = &local_flow;
>  	ret = mlx5_flow_attributes(dev, attr, flow, error);
>  	if (ret < 0)
>  		return ret;
> -	ret = mlx5_flow_items(items, flow, remain, error);
> -	if (ret < 0)
> -		return ret;
> -	size += ret;
> -	remain = (flow_size > size) ? flow_size - size : 0;
> -	ret = mlx5_flow_actions(dev, actions, flow, remain, error);
> +	ret = mlx5_flow_actions(dev, actions, &local_flow, 0, error);
>  	if (ret < 0)
>  		return ret;
> -	size += ret;
> +	ret = rte_flow_expand_rss(NULL, 0, items, local_flow.rss.types,
> +				  mlx5_support_expansion,
> +				  local_flow.rss.level < 2 ?
> +				  MLX5_EXPANSION_ROOT : MLX5_EXPANSION_ROOT2);
> +	assert(ret > 0);
> +	buf = rte_calloc(__func__, 1, ret, 0);
> +	if (!buf) {
> +		rte_flow_error_set(error, ENOMEM,
> +				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +				   NULL,
> +				   "not enough memory to expand the RSS flow");
> +		goto error;
> +	}

I'm pretty sure you've already fixed this bug. Validation can't return ENOMEM.

> +	ret = rte_flow_expand_rss(buf, ret, items, local_flow.rss.types,
> +				  mlx5_support_expansion,
> +				  local_flow.rss.level < 2 ?
> +				  MLX5_EXPANSION_ROOT : MLX5_EXPANSION_ROOT2);
> +	assert(ret > 0);
> +	size += RTE_ALIGN_CEIL(local_flow.rss.queue_num * sizeof(uint16_t),
> +			       sizeof(void *));
>  	if (size <= flow_size)
> -		flow->verbs.attr->priority = flow->attributes.priority;
> +		flow->queue = (void *)(flow + 1);
> +	LIST_INIT(&flow->verbs);
> +	flow->layers = 0;
> +	flow->modifier = 0;
> +	flow->fate = 0;
> +	for (i = 0; i != buf->entries; ++i) {
> +		size_t off = size;
> +
> +		size += sizeof(struct ibv_flow_attr) +
> +			sizeof(struct mlx5_flow_verbs);
> +		remain = (flow_size > size) ? flow_size - size : 0;
> +		if (remain) {
> +			flow->cur_verbs = (void *)((uintptr_t)flow + off);
> +			flow->cur_verbs->attr = (void *)(flow->cur_verbs + 1);
> +			flow->cur_verbs->specs =
> +				(void *)(flow->cur_verbs->attr + 1);
> +		}
> +		ret = mlx5_flow_items
> +			((const struct rte_flow_item *)buf->patterns[i],
> +			 flow, remain, error);
> +		if (ret < 0)
> +			goto error;
> +		size += ret;
> +		if (remain > ret)
> +			remain -= ret;
> +		else
> +			remain = 0;
> +		ret = mlx5_flow_actions(dev, actions, flow, remain, error);
> +		if (ret < 0)
> +			goto error;
> +		size += ret;
> +		if (remain > ret)
> +			remain -= ret;
> +		else
> +			remain = 0;
> +		if (size <= flow_size) {
> +			flow->cur_verbs->attr->priority =
> +				flow->attributes.priority;
> +			ret = mlx5_flow_action_rss_verbs_attr(dev, flow,
> +							      flow->rss.types);
> +			if (ret < 0)
> +				goto error;
> +			LIST_INSERT_HEAD(&flow->verbs, flow->cur_verbs, next);
> +		}
> +		flow->expand = !!(buf->entries > 1);
> +	}
> +	rte_free(buf);
>  	return size;
> +error:
> +	rte_free(buf);
> +	return ret;
>  }
>  
>  /**
> @@ -1292,9 +1840,13 @@ static void
>  mlx5_flow_rxq_mark(struct rte_eth_dev *dev, struct rte_flow *flow)
>  {
>  	struct priv *priv = dev->data->dev_private;
> +	const uint32_t mask = MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK;
> +	uint32_t i;
>  
> -	(*priv->rxqs)[flow->queue]->mark |=
> -		flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK);
> +	if (!(flow->modifier & mask))
> +		return;
> +	for (i = 0; i != flow->rss.queue_num; ++i)
> +		(*priv->rxqs)[(*flow->queue)[i]]->mark = 1;
>  }
>  
>  /**
> @@ -1328,18 +1880,20 @@ mlx5_flow_validate(struct rte_eth_dev *dev,
>  static void
>  mlx5_flow_fate_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
>  {
> -	if (flow->fate & MLX5_FLOW_FATE_DROP) {
> -		if (flow->verbs.flow) {
> -			claim_zero(mlx5_glue->destroy_flow(flow->verbs.flow));
> -			flow->verbs.flow = NULL;
> +	struct mlx5_flow_verbs *verbs;
> +
> +	LIST_FOREACH(verbs, &flow->verbs, next) {
> +		if (verbs->flow) {
> +			claim_zero(mlx5_glue->destroy_flow(verbs->flow));
> +			verbs->flow = NULL;
> +		}
> +		if (verbs->hrxq) {
> +			if (flow->fate & MLX5_FLOW_FATE_DROP)
> +				mlx5_hrxq_drop_release(dev, verbs->hrxq);
> +			else
> +				mlx5_hrxq_release(dev, verbs->hrxq);
> +			verbs->hrxq = NULL;
>  		}
> -	}
> -	if (flow->verbs.hrxq) {
> -		if (flow->fate & MLX5_FLOW_FATE_DROP)
> -			mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
> -		else if (flow->fate & MLX5_FLOW_FATE_QUEUE)
> -			mlx5_hrxq_release(dev, flow->verbs.hrxq);
> -		flow->verbs.hrxq = NULL;
>  	}
>  }
>  
> @@ -1360,46 +1914,68 @@ static int
>  mlx5_flow_fate_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
>  		     struct rte_flow_error *error)
>  {
> -	if (flow->fate & MLX5_FLOW_FATE_DROP) {
> -		flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
> -		if (!flow->verbs.hrxq)
> -			return rte_flow_error_set
> -				(error, errno,
> -				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> -				 NULL,
> -				 "cannot allocate Drop queue");
> -	} else if (flow->fate & MLX5_FLOW_FATE_QUEUE) {
> -		struct mlx5_hrxq *hrxq;
> -
> -		hrxq = mlx5_hrxq_get(dev, rss_hash_default_key,
> -				     rss_hash_default_key_len, 0,
> -				     &flow->queue, 1, 0, 0);
> -		if (!hrxq)
> -			hrxq = mlx5_hrxq_new(dev, rss_hash_default_key,
> -					     rss_hash_default_key_len, 0,
> -					     &flow->queue, 1, 0, 0);
> -		if (!hrxq)
> -			return rte_flow_error_set(error, rte_errno,
> -					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> -					NULL,
> -					"cannot create flow");
> -		flow->verbs.hrxq = hrxq;
> -	}
> -	flow->verbs.flow =
> -		mlx5_glue->create_flow(flow->verbs.hrxq->qp, flow->verbs.attr);
> -	if (!flow->verbs.flow) {
> -		if (flow->fate & MLX5_FLOW_FATE_DROP)
> -			mlx5_hrxq_drop_release(dev, flow->verbs.hrxq);
> -		else
> -			mlx5_hrxq_release(dev, flow->verbs.hrxq);
> -		flow->verbs.hrxq = NULL;
> -		return rte_flow_error_set(error, errno,
> -					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> -					  NULL,
> -					  "kernel module refuses to create"
> -					  " flow");
> +	struct mlx5_flow_verbs *verbs;
> +	int err;
> +
> +	LIST_FOREACH(verbs, &flow->verbs, next) {
> +		if (flow->fate & MLX5_FLOW_FATE_DROP) {
> +			verbs->hrxq = mlx5_hrxq_drop_new(dev);
> +			if (!verbs->hrxq) {
> +				rte_flow_error_set
> +					(error, errno,
> +					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +					 NULL,
> +					 "cannot get drop hash queue");
> +				goto error;
> +			}
> +		} else {
> +			struct mlx5_hrxq *hrxq;
> +
> +			hrxq = mlx5_hrxq_get(dev, flow->key,
> +					     rss_hash_default_key_len,
> +					     verbs->hash_fields,
> +					     (*flow->queue),
> +					     flow->rss.queue_num, 0, 0);
> +			if (!hrxq)
> +				hrxq = mlx5_hrxq_new(dev, flow->key,
> +						     rss_hash_default_key_len,
> +						     verbs->hash_fields,
> +						     (*flow->queue),
> +						     flow->rss.queue_num, 0, 0);
> +			if (!hrxq) {
> +				rte_flow_error_set
> +					(error, rte_errno,
> +					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +					 NULL,
> +					 "cannot get hash queue");
> +				goto error;
> +			}
> +			verbs->hrxq = hrxq;
> +		}
> +		verbs->flow =
> +			mlx5_glue->create_flow(verbs->hrxq->qp, verbs->attr);
> +		if (!verbs->flow) {
> +			rte_flow_error_set(error, errno,
> +					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +					   NULL,
> +					   "hardware refuses to create flow");
> +			goto error;
> +		}
>  	}
>  	return 0;
> +error:
> +	err = rte_errno; /* Save rte_errno before cleanup. */
> +	LIST_FOREACH(verbs, &flow->verbs, next) {
> +		if (verbs->hrxq) {
> +			if (flow->fate & MLX5_FLOW_FATE_DROP)
> +				mlx5_hrxq_drop_release(dev, verbs->hrxq);
> +			else
> +				mlx5_hrxq_release(dev, verbs->hrxq);
> +			verbs->hrxq = NULL;
> +		}
> +	}
> +	rte_errno = err; /* Restore rte_errno. */
> +	return -rte_errno;
>  }
>  
>  /**
> @@ -1429,42 +2005,43 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
>  		      const struct rte_flow_action actions[],
>  		      struct rte_flow_error *error)
>  {
> -	struct rte_flow *flow;
> -	size_t size;
> +	struct rte_flow *flow = NULL;
> +	size_t size = 0;
>  	int ret;
>  
> -	ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
> +	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
>  	if (ret < 0)
>  		return NULL;
>  	size = ret;
> -	flow = rte_zmalloc(__func__, size, 0);
> +	flow = rte_calloc(__func__, 1, size, 0);
>  	if (!flow) {
>  		rte_flow_error_set(error, ENOMEM,
>  				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
>  				   NULL,
> -				   "cannot allocate memory");
> +				   "not enough memory to create flow");
>  		return NULL;
>  	}
> -	flow->verbs.attr = (struct ibv_flow_attr *)(flow + 1);
> -	flow->verbs.specs = (uint8_t *)(flow->verbs.attr + 1);
>  	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
> -	if (ret < 0)
> -		goto error;
> +	if (ret < 0) {
> +		rte_free(flow);
> +		return NULL;
> +	}
>  	assert((size_t)ret == size);
>  	if (dev->data->dev_started) {
>  		ret = mlx5_flow_fate_apply(dev, flow, error);
> -		if (ret < 0)
> -			goto error;
> +		if (ret < 0) {
> +			ret = rte_errno; /* Save rte_errno before cleanup. */
> +			if (flow) {
> +				mlx5_flow_fate_remove(dev, flow);
> +				rte_free(flow);
> +			}
> +			rte_errno = ret; /* Restore rte_errno. */
> +			return NULL;
> +		}
>  	}
>  	mlx5_flow_rxq_mark(dev, flow);
>  	TAILQ_INSERT_TAIL(list, flow, next);
>  	return flow;
> -error:
> -	ret = rte_errno; /* Save rte_errno before cleanup. */
> -	mlx5_flow_fate_remove(dev, flow);
> -	rte_free(flow);
> -	rte_errno = ret; /* Restore rte_errno. */
> -	return NULL;
>  }
>  
>  /**
> @@ -1502,7 +2079,7 @@ mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
>  	struct priv *priv = dev->data->dev_private;
>  	struct rte_flow *rflow;
>  	const uint32_t mask = MLX5_FLOW_MOD_FLAG & MLX5_FLOW_MOD_MARK;
> -	int mark = 0;
> +	unsigned int i;
>  
>  	mlx5_flow_fate_remove(dev, flow);
>  	TAILQ_REMOVE(list, flow, next);
> @@ -1512,18 +2089,28 @@ mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
>  	}
>  	/*
>  	 * When a flow is removed and this flow has a flag/mark modifier, all
> -	 * flows needs to be parse to verify if the Rx queue use by the flow
> +	 * flows needs to be parse to verify if the Rx queues use by the flow
>  	 * still need to track the flag/mark request.
>  	 */
> -	TAILQ_FOREACH(rflow, &priv->flows, next) {
> -		if (!(rflow->modifier & mask))
> -			continue;
> -		if (flow->queue == rflow->queue) {
> -			mark = 1;
> -			break;
> +	for (i = 0; i != flow->rss.queue_num; ++i) {
> +		int mark = 0;
> +
> +		TAILQ_FOREACH(rflow, &priv->flows, next) {
> +			unsigned int j;
> +
> +			if (!(rflow->modifier & mask))
> +				continue;
> +			for (j = 0; j != rflow->rss.queue_num; ++j) {
> +				if ((*flow->queue)[i] == (*rflow->queue)[j]) {
> +					mark = 1;
> +					break;
> +				}
> +			}
> +			if (mark)
> +				break;
>  		}
> +		(*priv->rxqs)[i]->mark = !!mark;
>  	}
> -	(*priv->rxqs)[flow->queue]->mark = !!mark;
>  	rte_free(flow);
>  }
>  
> @@ -1654,7 +2241,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
>  	struct priv *priv = dev->data->dev_private;
>  	const struct rte_flow_attr attr = {
>  		.ingress = 1,
> -		.priority = priv->config.flow_prio - 1,
> +		.priority = MLX5_FLOW_PRIO_RSVD,
>  	};
>  	struct rte_flow_item items[] = {
>  		{
> -- 
> 2.18.0
> 

Re: [dpdk-dev] [PATCH v2 13/20] net/mlx5: add RSS flow action

[Nélio Laranjeiro]

Hi Yongseok,

I am only addressing your questions concerns here, almost all other
points I also agree with them.

On Thu, Jul 05, 2018 at 07:16:35PM -0700, Yongseok Koh wrote:
> On Wed, Jun 27, 2018 at 05:07:45PM +0200, Nelio Laranjeiro wrote:
> > Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> > ---
>[...]
> 
> > + */
> > +static void
> > +mlx5_flow_layers_update(struct rte_flow *flow, uint32_t layers)
> > +{
> > +	if (flow->expand) {
> > +		if (flow->cur_verbs)
> > +			flow->cur_verbs->layers |= layers;
> 
> If flow->cur_verbs is null, does that mean it is a testing call? Then, is it
> unnecessary to update layers for the testing call? Confusing..

No it may also happen if the buffer was too small, in any case the code
continues its validation.

> > +	} else {
> > +		flow->layers |= layers;
> > +	}
> > +}
> > +
> > +/**
> > + * Get layers bit-field.
> > + *
> > + * @param flow[in, out]
> > + *   Pointer to flow structure.
> > + */
> > +static uint32_t
> > +mlx5_flow_layers(struct rte_flow *flow)
> > +{
> > +	uint32_t layers = flow->layers;
> > +
> > +	if (flow->expand && flow->cur_verbs)
> 
> If flow is expanded and it is a testing call, then flow->layers is used?
>
> > +		layers |= flow->cur_verbs->layers;
> > +	return layers;
> 
> This part is so unclear to me, hard to understand. There are two 'layers'
> fields, one in rte_flow and the other in mlx5_flow_verbs. It seems
> rte_flow->layers is used only when the flow isn't expanded. If a flow is
> expanded, flow->expand is set after processing the first entry in the expanded
> list. In mlx5_flow_merge(),
> 
> 	for (i = 0; i != buf->entries; ++i) {
> 
> 		...
> 
> 		flow->expand = !!(buf->entries > 1);
> 	}
> 
> Why is flow->expand set at the end of the loop? Is this in order to avoid
> validation for the expanded flows?
> mlx5_flow_item_xxx() executes validation only if flow->expand is zero,
> why?

Expanded flows are PMD internal cooking to match the user request, they
are fully added by the PMD and thus must be valid.  There is not need to
validate their position nor redundancy and for the spec, last, mask they
are provided as wildcards which means those pointer are null.

> And why does mlx5_flow_layers() have to return (flow->layers |
> flow->cur_verbs->layers) if expanded?

You are right, and indeed it is a bug.  With the fix, only the expanded
flow will use the verbs->layers, whereas the first conversion will use
the flow->layers.
 
> If there are 3 entries in the rte_flow_expand_rss,
> 	eth
> 	eth / ipv4 / udp
> 	eth / ipv6 / udp
> 
> Then, the 2nd and 3rd don't have MLX5_FLOW_LAYER_OUTER_L2 in layers field?
> Please explain in details and add comments appropriately.

The 2nd and 3rd should have the same layers bits as the 1st rule in
addition of their own bits set .e.g 2nd rule will have L2 + L3_IPv4 +
L4_UDP the 3rd L3 + L3_IPv6 + L4_UDP.


I will try to sanitize it, the issue being for a single rte_flow we can
have several verbs flows with the expansion, for each of them we need to
update the priority and this based on the more specific layer of each,
it needs to be stored in some place.  I think the biggest issue you are
raising here is the lack of correct variable naming and documentation,
I'll try to improve it at most.

>[...]
> > +	if (verbs) {
> > +		verbs->modifier |= MLX5_FLOW_MOD_MARK;
> > +		if (verbs->modifier & MLX5_FLOW_MOD_FLAG) {
> > +			mlx5_flow_verbs_mark_update(verbs, mark->id);
> > +			size = 0;
> > +		} else if (size <= flow_size) {
> 
> If verbs isn't null (not testing call), isn't it guaranteed there's enough
> space? Is it still needed to check the size?

Unfortunately not, verbs variable may be valid and pointing into a zone
in the buffer but it does not mean there is enough space to store the
mark specification in the buffer.

> >  			tag.tag_id = mlx5_flow_mark_set(mark->id);
> >  			mlx5_flow_spec_verbs_add(flow, &tag, size);
> >  		}
> > +	} else {
> > +		if (flow->modifier & MLX5_FLOW_MOD_FLAG)
> > +			size = 0;
> >  	}
> >  	flow->modifier |= MLX5_FLOW_MOD_MARK;
> >  	return size;
> > @@ -1185,6 +1656,15 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
> >  	int remain = flow_size;
> >  	int ret = 0;
> >  
> > +	/*
> > +	 * FLAG/MARK are the only actions having a specification in Verbs and
> > +	 * not making part of the packet fate.  Due to this specificity and to
> > +	 * avoid extra variable, their bit in the flow->modifier bit-field are
> > +	 * disabled here to compute the exact necessary memory those action
> > +	 * needs.
> > +	 */
> > +	flow->modifier &= ~(MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK);
> 
> Can't understand this well. Is this for the case where the flow is expanded? If
> so, why don't you reset flow->modifier in the for loop of mlx5_flow_merge()?

Yes it is, I'll move it.

>[...]
> > +	assert(ret > 0);
> > +	buf = rte_calloc(__func__, 1, ret, 0);
> > +	if (!buf) {
> > +		rte_flow_error_set(error, ENOMEM,
> > +				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> > +				   NULL,
> > +				   "not enough memory to expand the RSS flow");
> > +		goto error;
> > +	}
> 
> I'm pretty sure you've already fixed this bug. Validation can't return ENOMEM.

You know me well ;)

Thanks,

-- 
Nélio Laranjeiro
6WIND

Re: [dpdk-dev] [PATCH v2 13/20] net/mlx5: add RSS flow action

[Yongseok Koh]

> On Jul 6, 2018, at 8:59 AM, Nélio Laranjeiro <nelio.laranjeiro@6wind.com> wrote:
> 
> Hi Yongseok,
> 
> I am only addressing your questions concerns here, almost all other
> points I also agree with them.
> 
> On Thu, Jul 05, 2018 at 07:16:35PM -0700, Yongseok Koh wrote:
>> On Wed, Jun 27, 2018 at 05:07:45PM +0200, Nelio Laranjeiro wrote:
>>> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
>>> ---
>> [...]
>> 
>>> + */
>>> +static void
>>> +mlx5_flow_layers_update(struct rte_flow *flow, uint32_t layers)
>>> +{
>>> +	if (flow->expand) {
>>> +		if (flow->cur_verbs)
>>> +			flow->cur_verbs->layers |= layers;
>> 
>> If flow->cur_verbs is null, does that mean it is a testing call? Then, is it
>> unnecessary to update layers for the testing call? Confusing..
> 
> No it may also happen if the buffer was too small, in any case the code
> continues its validation.

Okay, understand. Thanks.
But another question was, if it is a testing call (flow->cur_verbs is null) with
flow->expand being set, then no 'layers' isn't updated in this code. Is it okay?

Thanks,
Yongseok

>>> +	} else {
>>> +		flow->layers |= layers;
>>> +	}
>>> +}
>>> +

Re: [dpdk-dev] [PATCH v2 13/20] net/mlx5: add RSS flow action

[Nélio Laranjeiro]

On Fri, Jul 06, 2018 at 05:35:22PM +0000, Yongseok Koh wrote:
> 
> > On Jul 6, 2018, at 8:59 AM, Nélio Laranjeiro <nelio.laranjeiro@6wind.com> wrote:
> > 
> > Hi Yongseok,
> > 
> > I am only addressing your questions concerns here, almost all other
> > points I also agree with them.
> > 
> > On Thu, Jul 05, 2018 at 07:16:35PM -0700, Yongseok Koh wrote:
> >> On Wed, Jun 27, 2018 at 05:07:45PM +0200, Nelio Laranjeiro wrote:
> >>> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> >>> ---
> >> [...]
> >> 
> >>> + */
> >>> +static void
> >>> +mlx5_flow_layers_update(struct rte_flow *flow, uint32_t layers)
> >>> +{
> >>> +	if (flow->expand) {
> >>> +		if (flow->cur_verbs)
> >>> +			flow->cur_verbs->layers |= layers;
> >> 
> >> If flow->cur_verbs is null, does that mean it is a testing call? Then, is it
> >> unnecessary to update layers for the testing call? Confusing..
> > 
> > No it may also happen if the buffer was too small, in any case the code
> > continues its validation.
> 
> Okay, understand. Thanks.
> But another question was, if it is a testing call (flow->cur_verbs is null) with
> flow->expand being set, then no 'layers' isn't updated in this code. Is it okay?

yes it was ok, after I've fixed the issue in the layers themselves,
again no layer position was done when the expanded was enabled.

Thanks,

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH v2 14/20] net/mlx5: remove useless arguments in hrxq API

[Nelio Laranjeiro]

RSS level is necessary to had a bit in the hash_fields which is already
provided in this API, for the tunnel, it is necessary to request such
queue to compute the checksum on the inner most, this last one should
always be activated.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c |  4 ++--
 drivers/net/mlx5/mlx5_rxq.c  | 39 +++++++++---------------------------
 drivers/net/mlx5/mlx5_rxtx.h |  8 ++------
 3 files changed, 13 insertions(+), 38 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 08e0a6556..7dda88641 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -1935,13 +1935,13 @@ mlx5_flow_fate_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 					     rss_hash_default_key_len,
 					     verbs->hash_fields,
 					     (*flow->queue),
-					     flow->rss.queue_num, 0, 0);
+					     flow->rss.queue_num);
 			if (!hrxq)
 				hrxq = mlx5_hrxq_new(dev, flow->key,
 						     rss_hash_default_key_len,
 						     verbs->hash_fields,
 						     (*flow->queue),
-						     flow->rss.queue_num, 0, 0);
+						     flow->rss.queue_num);
 			if (!hrxq) {
 				rte_flow_error_set
 					(error, rte_errno,
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 5f17dce50..8f65e4299 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1730,10 +1730,6 @@ mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev)
  *   first queue index will be taken for the indirection table.
  * @param queues_n
  *   Number of queues.
- * @param tunnel
- *   Tunnel type, implies tunnel offloading like inner checksum if available.
- * @param rss_level
- *   RSS hash on tunnel level.
  *
  * @return
  *   The Verbs object initialised, NULL otherwise and rte_errno is set.
@@ -1742,17 +1738,13 @@ struct mlx5_hrxq *
 mlx5_hrxq_new(struct rte_eth_dev *dev,
 	      const uint8_t *rss_key, uint32_t rss_key_len,
 	      uint64_t hash_fields,
-	      const uint16_t *queues, uint32_t queues_n,
-	      uint32_t tunnel, uint32_t rss_level)
+	      const uint16_t *queues, uint32_t queues_n)
 {
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
 	struct mlx5_ind_table_ibv *ind_tbl;
 	struct ibv_qp *qp;
 	int err;
-#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	struct mlx5dv_qp_init_attr qp_init_attr = {0};
-#endif
 
 	queues_n = hash_fields ? queues_n : 1;
 	ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n);
@@ -1767,11 +1759,6 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 		rss_key = rss_hash_default_key;
 	}
 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	if (tunnel) {
-		qp_init_attr.comp_mask =
-				MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
-		qp_init_attr.create_flags = MLX5DV_QP_CREATE_TUNNEL_OFFLOADS;
-	}
 	qp = mlx5_glue->dv_create_qp
 		(priv->ctx,
 		 &(struct ibv_qp_init_attr_ex){
@@ -1787,14 +1774,17 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 				.rx_hash_key = rss_key ?
 					       (void *)(uintptr_t)rss_key :
 					       rss_hash_default_key,
-				.rx_hash_fields_mask = hash_fields |
-					(tunnel && rss_level > 1 ?
-					(uint32_t)IBV_RX_HASH_INNER : 0),
+				.rx_hash_fields_mask = hash_fields,
 			},
 			.rwq_ind_tbl = ind_tbl->ind_table,
 			.pd = priv->pd,
 		 },
-		 &qp_init_attr);
+		 &(struct mlx5dv_qp_init_attr){
+			.comp_mask = (hash_fields & IBV_RX_HASH_INNER) ?
+				 MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS :
+				 0,
+			.create_flags = MLX5DV_QP_CREATE_TUNNEL_OFFLOADS,
+		 });
 #else
 	qp = mlx5_glue->create_qp_ex
 		(priv->ctx,
@@ -1828,8 +1818,6 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 	hrxq->qp = qp;
 	hrxq->rss_key_len = rss_key_len;
 	hrxq->hash_fields = hash_fields;
-	hrxq->tunnel = tunnel;
-	hrxq->rss_level = rss_level;
 	memcpy(hrxq->rss_key, rss_key, rss_key_len);
 	rte_atomic32_inc(&hrxq->refcnt);
 	LIST_INSERT_HEAD(&priv->hrxqs, hrxq, next);
@@ -1855,10 +1843,6 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
  *   first queue index will be taken for the indirection table.
  * @param queues_n
  *   Number of queues.
- * @param tunnel
- *   Tunnel type, implies tunnel offloading like inner checksum if available.
- * @param rss_level
- *   RSS hash on tunnel level
  *
  * @return
  *   An hash Rx queue on success.
@@ -1867,8 +1851,7 @@ struct mlx5_hrxq *
 mlx5_hrxq_get(struct rte_eth_dev *dev,
 	      const uint8_t *rss_key, uint32_t rss_key_len,
 	      uint64_t hash_fields,
-	      const uint16_t *queues, uint32_t queues_n,
-	      uint32_t tunnel, uint32_t rss_level)
+	      const uint16_t *queues, uint32_t queues_n)
 {
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
@@ -1883,10 +1866,6 @@ mlx5_hrxq_get(struct rte_eth_dev *dev,
 			continue;
 		if (hrxq->hash_fields != hash_fields)
 			continue;
-		if (hrxq->tunnel != tunnel)
-			continue;
-		if (hrxq->rss_level != rss_level)
-			continue;
 		ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n);
 		if (!ind_tbl)
 			continue;
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 51f7f678b..bb67c32a6 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -157,8 +157,6 @@ struct mlx5_hrxq {
 	struct mlx5_ind_table_ibv *ind_table; /* Indirection table. */
 	struct ibv_qp *qp; /* Verbs queue pair. */
 	uint64_t hash_fields; /* Verbs Hash fields. */
-	uint32_t tunnel; /* Tunnel type. */
-	uint32_t rss_level; /* RSS on tunnel level. */
 	uint32_t rss_key_len; /* Hash key length in bytes. */
 	uint8_t rss_key[]; /* Hash key. */
 };
@@ -274,13 +272,11 @@ void mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev,
 struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev,
 				const uint8_t *rss_key, uint32_t rss_key_len,
 				uint64_t hash_fields,
-				const uint16_t *queues, uint32_t queues_n,
-				uint32_t tunnel, uint32_t rss_level);
+				const uint16_t *queues, uint32_t queues_n);
 struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev,
 				const uint8_t *rss_key, uint32_t rss_key_len,
 				uint64_t hash_fields,
-				const uint16_t *queues, uint32_t queues_n,
-				uint32_t tunnel, uint32_t rss_level);
+				const uint16_t *queues, uint32_t queues_n);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
 int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev);
 struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev);
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 14/20] net/mlx5: remove useless arguments in hrxq API

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:46PM +0200, Nelio Laranjeiro wrote:
> RSS level is necessary to had a bit in the hash_fields which is already
> provided in this API, for the tunnel, it is necessary to request such
> queue to compute the checksum on the inner most, this last one should
> always be activated.
> 
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
Acked-by: Yongseok Koh <yskoh@mellanox.com>

Thanks

[dpdk-dev] [PATCH v2 15/20] net/mlx5: support inner RSS computation

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 131 +++++++++++++++++++++++++----------
 drivers/net/mlx5/mlx5_rxtx.h |   1 -
 2 files changed, 96 insertions(+), 36 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 7dda88641..eedf0c461 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -219,6 +219,8 @@ struct rte_flow {
 	struct mlx5_flow_verbs *cur_verbs;
 	/**< Current Verbs flow structure being filled. */
 	struct rte_flow_action_rss rss;/**< RSS context. */
+	uint32_t ptype;
+	/**< Store tunnel packet type data to store in Rx queue. */
 	uint8_t key[40]; /**< RSS hash key. */
 	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
 };
@@ -1320,13 +1322,15 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
  *   Pointer to flow structure.
  * @param types
  *   RSS types for this flow (see ETH_RSS_*).
+ * @param level
+ *   RSS level.
  *
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 static int
 mlx5_flow_action_rss_verbs_attr(struct rte_eth_dev *dev, struct rte_flow *flow,
-				uint32_t types)
+				uint32_t types, uint32_t level)
 {
 	const uint32_t layers = mlx5_flow_layers(flow);
 	uint64_t hash_fields;
@@ -1374,6 +1378,8 @@ mlx5_flow_action_rss_verbs_attr(struct rte_eth_dev *dev, struct rte_flow *flow,
 		hash_fields = 0;
 		priority = 2;
 	}
+	if (hash_fields && level == 2)
+		hash_fields |= IBV_RX_HASH_INNER;
 	flow->cur_verbs->hash_fields = hash_fields;
 	flow->cur_verbs->attr->priority =
 		mlx5_flow_priority(dev, flow->attributes.priority, priority);
@@ -1416,7 +1422,7 @@ mlx5_flow_action_rss(struct rte_eth_dev *dev,
 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 					  &rss->func,
 					  "RSS hash function not supported");
-	if (rss->level > 1)
+	if (rss->level > 2)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 					  &rss->level,
@@ -1456,6 +1462,7 @@ mlx5_flow_action_rss(struct rte_eth_dev *dev,
 	flow->rss.queue_num = rss->queue_num;
 	memcpy(flow->key, rss->key, rss_hash_default_key_len);
 	flow->rss.types = rss->types;
+	flow->rss.level = rss->level;
 	flow->fate |= MLX5_FLOW_FATE_RSS;
 	return 0;
 }
@@ -1814,7 +1821,8 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 			flow->cur_verbs->attr->priority =
 				flow->attributes.priority;
 			ret = mlx5_flow_action_rss_verbs_attr(dev, flow,
-							      flow->rss.types);
+							      flow->rss.types,
+							      flow->rss.level);
 			if (ret < 0)
 				goto error;
 			LIST_INSERT_HEAD(&flow->verbs, flow->cur_verbs, next);
@@ -1828,27 +1836,6 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 	return ret;
 }
 
-/**
- * Mark the Rx queues mark flag if the flow has a mark or flag modifier.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Pointer to flow structure.
- */
-static void
-mlx5_flow_rxq_mark(struct rte_eth_dev *dev, struct rte_flow *flow)
-{
-	struct priv *priv = dev->data->dev_private;
-	const uint32_t mask = MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK;
-	uint32_t i;
-
-	if (!(flow->modifier & mask))
-		return;
-	for (i = 0; i != flow->rss.queue_num; ++i)
-		(*priv->rxqs)[(*flow->queue)[i]]->mark = 1;
-}
-
 /**
  * Validate a flow supported by the NIC.
  *
@@ -1978,6 +1965,88 @@ mlx5_flow_fate_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 	return -rte_errno;
 }
 
+/**
+ * Set the Tunnel packet type and the Mark in the Rx queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param idx
+ *   Rx queue index.
+ */
+static void
+mlx5_flow_rxq(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct rte_flow *flow;
+	const uint32_t mark_m = MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK;
+	uint32_t ptype = 0;
+	uint32_t mark = 0;
+
+	TAILQ_FOREACH(flow, &priv->flows, next) {
+		unsigned int i;
+
+		for (i = 0; i != flow->rss.queue_num; ++i) {
+			if ((*flow->queue)[i] == idx) {
+				mark |= !!(flow->modifier & mark_m);
+				if (ptype == 0)
+					ptype = flow->ptype;
+				else if (ptype != flow->ptype)
+					ptype = (uint32_t)-1;
+				break;
+			}
+		}
+	}
+	if (ptype == (uint32_t)-1)
+		ptype = 0;
+	(*priv->rxqs)[idx]->tunnel = ptype;
+	(*priv->rxqs)[idx]->mark = mark;
+}
+
+/**
+ * Set the tunnel packet type and Mark in the Rx queues, if several packet
+ * types are possible, the information in the Rx queue is just cleared.
+ * Mark is not impacted by this case.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+static void
+mlx5_flow_rxqs(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int idx;
+	unsigned int n;
+
+	for (idx = 0, n = 0; n != priv->rxqs_n; ++idx) {
+		if (!(*priv->rxqs)[idx])
+			continue;
+		mlx5_flow_rxq(dev, idx);
+		n++;
+	}
+}
+
+/**
+ * Clear tunnel ptypes and Mark in Rx queues.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+static void
+mlx5_flow_rxqs_clear(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int idx;
+	unsigned int n;
+
+	for (idx = 0, n = 0; n != priv->rxqs_n; ++idx) {
+		if (!(*priv->rxqs)[idx])
+			continue;
+		(*priv->rxqs)[idx]->tunnel = 0;
+		(*priv->rxqs)[idx]->mark = 0;
+		n++;
+	}
+}
+
 /**
  * Create a flow.
  *
@@ -2039,8 +2108,8 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
 			return NULL;
 		}
 	}
-	mlx5_flow_rxq_mark(dev, flow);
 	TAILQ_INSERT_TAIL(list, flow, next);
+	mlx5_flow_rxqs(dev);
 	return flow;
 }
 
@@ -2144,19 +2213,11 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
 void
 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
 {
-	struct priv *priv = dev->data->dev_private;
 	struct rte_flow *flow;
-	unsigned int i;
-	unsigned int idx;
 
 	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
 		mlx5_flow_fate_remove(dev, flow);
-	for (idx = 0, i = 0; idx != priv->rxqs_n; ++i) {
-		if (!(*priv->rxqs)[idx])
-			continue;
-		(*priv->rxqs)[idx]->mark = 0;
-		++idx;
-	}
+	mlx5_flow_rxqs_clear(dev);
 }
 
 /**
@@ -2181,8 +2242,8 @@ mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
 		ret = mlx5_flow_fate_apply(dev, flow, &error);
 		if (ret < 0)
 			goto error;
-		mlx5_flow_rxq_mark(dev, flow);
 	}
+	mlx5_flow_rxqs(dev);
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index bb67c32a6..57504ceb2 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -136,7 +136,6 @@ struct mlx5_rxq_ctrl {
 	struct priv *priv; /* Back pointer to private data. */
 	struct mlx5_rxq_data rxq; /* Data path structure. */
 	unsigned int socket; /* CPU socket ID for allocations. */
-	uint32_t tunnel_types[16]; /* Tunnel type counter. */
 	unsigned int irq:1; /* Whether IRQ is enabled. */
 	uint16_t idx; /* Queue index. */
 };
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 15/20] net/mlx5: support inner RSS computation

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:47PM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
>  drivers/net/mlx5/mlx5_flow.c | 131 +++++++++++++++++++++++++----------
>  drivers/net/mlx5/mlx5_rxtx.h |   1 -
>  2 files changed, 96 insertions(+), 36 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index 7dda88641..eedf0c461 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -219,6 +219,8 @@ struct rte_flow {
>  	struct mlx5_flow_verbs *cur_verbs;
>  	/**< Current Verbs flow structure being filled. */
>  	struct rte_flow_action_rss rss;/**< RSS context. */
> +	uint32_t ptype;
> +	/**< Store tunnel packet type data to store in Rx queue. */
>  	uint8_t key[40]; /**< RSS hash key. */
>  	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
>  };
> @@ -1320,13 +1322,15 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
>   *   Pointer to flow structure.
>   * @param types
>   *   RSS types for this flow (see ETH_RSS_*).
> + * @param level
> + *   RSS level.
>   *
>   * @return
>   *   0 on success, a negative errno value otherwise and rte_errno is set.
>   */
>  static int
>  mlx5_flow_action_rss_verbs_attr(struct rte_eth_dev *dev, struct rte_flow *flow,
> -				uint32_t types)
> +				uint32_t types, uint32_t level)
>  {
>  	const uint32_t layers = mlx5_flow_layers(flow);
>  	uint64_t hash_fields;
> @@ -1374,6 +1378,8 @@ mlx5_flow_action_rss_verbs_attr(struct rte_eth_dev *dev, struct rte_flow *flow,
>  		hash_fields = 0;
>  		priority = 2;
>  	}
> +	if (hash_fields && level == 2)
> +		hash_fields |= IBV_RX_HASH_INNER;
>  	flow->cur_verbs->hash_fields = hash_fields;
>  	flow->cur_verbs->attr->priority =
>  		mlx5_flow_priority(dev, flow->attributes.priority, priority);
> @@ -1416,7 +1422,7 @@ mlx5_flow_action_rss(struct rte_eth_dev *dev,
>  					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
>  					  &rss->func,
>  					  "RSS hash function not supported");
> -	if (rss->level > 1)
> +	if (rss->level > 2)
>  		return rte_flow_error_set(error, ENOTSUP,
>  					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
>  					  &rss->level,
> @@ -1456,6 +1462,7 @@ mlx5_flow_action_rss(struct rte_eth_dev *dev,
>  	flow->rss.queue_num = rss->queue_num;
>  	memcpy(flow->key, rss->key, rss_hash_default_key_len);
>  	flow->rss.types = rss->types;
> +	flow->rss.level = rss->level;
>  	flow->fate |= MLX5_FLOW_FATE_RSS;
>  	return 0;
>  }
> @@ -1814,7 +1821,8 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
>  			flow->cur_verbs->attr->priority =
>  				flow->attributes.priority;
>  			ret = mlx5_flow_action_rss_verbs_attr(dev, flow,
> -							      flow->rss.types);
> +							      flow->rss.types,
> +							      flow->rss.level);
>  			if (ret < 0)
>  				goto error;
>  			LIST_INSERT_HEAD(&flow->verbs, flow->cur_verbs, next);
> @@ -1828,27 +1836,6 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
>  	return ret;
>  }
>  
> -/**
> - * Mark the Rx queues mark flag if the flow has a mark or flag modifier.
> - *
> - * @param dev
> - *   Pointer to Ethernet device.
> - * @param flow
> - *   Pointer to flow structure.
> - */
> -static void
> -mlx5_flow_rxq_mark(struct rte_eth_dev *dev, struct rte_flow *flow)
> -{
> -	struct priv *priv = dev->data->dev_private;
> -	const uint32_t mask = MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK;
> -	uint32_t i;
> -
> -	if (!(flow->modifier & mask))
> -		return;
> -	for (i = 0; i != flow->rss.queue_num; ++i)
> -		(*priv->rxqs)[(*flow->queue)[i]]->mark = 1;
> -}
> -
>  /**
>   * Validate a flow supported by the NIC.
>   *
> @@ -1978,6 +1965,88 @@ mlx5_flow_fate_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
>  	return -rte_errno;
>  }
>  
> +/**
> + * Set the Tunnel packet type and the Mark in the Rx queue.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + * @param idx
> + *   Rx queue index.
> + */
> +static void
> +mlx5_flow_rxq(struct rte_eth_dev *dev, uint16_t idx)
> +{
> +	struct priv *priv = dev->data->dev_private;
> +	struct rte_flow *flow;
> +	const uint32_t mark_m = MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK;
> +	uint32_t ptype = 0;
> +	uint32_t mark = 0;
> +
> +	TAILQ_FOREACH(flow, &priv->flows, next) {
> +		unsigned int i;
> +
> +		for (i = 0; i != flow->rss.queue_num; ++i) {
> +			if ((*flow->queue)[i] == idx) {
> +				mark |= !!(flow->modifier & mark_m);
> +				if (ptype == 0)
> +					ptype = flow->ptype;
> +				else if (ptype != flow->ptype)

I know someone else named it but flow->ptype sounds ambiguous.
flow->tunnel_ptype?

> +					ptype = (uint32_t)-1;
> +				break;
> +			}
> +		}
> +	}
> +	if (ptype == (uint32_t)-1)
> +		ptype = 0;
> +	(*priv->rxqs)[idx]->tunnel = ptype;
> +	(*priv->rxqs)[idx]->mark = mark;
> +}
> +
> +/**
> + * Set the tunnel packet type and Mark in the Rx queues, if several packet
> + * types are possible, the information in the Rx queue is just cleared.
> + * Mark is not impacted by this case.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + */
> +static void
> +mlx5_flow_rxqs(struct rte_eth_dev *dev)

Can you rename mlx5_flow_rxqs() and mlx5_flow_rxq()? Not clear about what each
func does from its name. Maybe, mlx5_flow_set_rxq_flag()?

> +{
> +	struct priv *priv = dev->data->dev_private;
> +	unsigned int idx;
> +	unsigned int n;
> +
> +	for (idx = 0, n = 0; n != priv->rxqs_n; ++idx) {
> +		if (!(*priv->rxqs)[idx])
> +			continue;
> +		mlx5_flow_rxq(dev, idx);
> +		n++;
> +	}
> +}
> +
> +/**
> + * Clear tunnel ptypes and Mark in Rx queues.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + */
> +static void
> +mlx5_flow_rxqs_clear(struct rte_eth_dev *dev)

This one is stranger. Sounds like it is clearing Rx queues.

> +{
> +	struct priv *priv = dev->data->dev_private;
> +	unsigned int idx;
> +	unsigned int n;
> +
> +	for (idx = 0, n = 0; n != priv->rxqs_n; ++idx) {
> +		if (!(*priv->rxqs)[idx])
> +			continue;
> +		(*priv->rxqs)[idx]->tunnel = 0;
> +		(*priv->rxqs)[idx]->mark = 0;
> +		n++;
> +	}
> +}
> +
>  /**
>   * Create a flow.
>   *
> @@ -2039,8 +2108,8 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
>  			return NULL;
>  		}
>  	}
> -	mlx5_flow_rxq_mark(dev, flow);
>  	TAILQ_INSERT_TAIL(list, flow, next);
> +	mlx5_flow_rxqs(dev);
>  	return flow;
>  }
>  
> @@ -2144,19 +2213,11 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
>  void
>  mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
>  {
> -	struct priv *priv = dev->data->dev_private;
>  	struct rte_flow *flow;
> -	unsigned int i;
> -	unsigned int idx;
>  
>  	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
>  		mlx5_flow_fate_remove(dev, flow);
> -	for (idx = 0, i = 0; idx != priv->rxqs_n; ++i) {
> -		if (!(*priv->rxqs)[idx])
> -			continue;
> -		(*priv->rxqs)[idx]->mark = 0;
> -		++idx;
> -	}
> +	mlx5_flow_rxqs_clear(dev);
>  }
>  
>  /**
> @@ -2181,8 +2242,8 @@ mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
>  		ret = mlx5_flow_fate_apply(dev, flow, &error);
>  		if (ret < 0)
>  			goto error;
> -		mlx5_flow_rxq_mark(dev, flow);
>  	}
> +	mlx5_flow_rxqs(dev);
>  	return 0;
>  error:
>  	ret = rte_errno; /* Save rte_errno before cleanup. */
> diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> index bb67c32a6..57504ceb2 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.h
> +++ b/drivers/net/mlx5/mlx5_rxtx.h
> @@ -136,7 +136,6 @@ struct mlx5_rxq_ctrl {
>  	struct priv *priv; /* Back pointer to private data. */
>  	struct mlx5_rxq_data rxq; /* Data path structure. */
>  	unsigned int socket; /* CPU socket ID for allocations. */
> -	uint32_t tunnel_types[16]; /* Tunnel type counter. */
>  	unsigned int irq:1; /* Whether IRQ is enabled. */
>  	uint16_t idx; /* Queue index. */
>  };
> -- 
> 2.18.0
> 

[dpdk-dev] [PATCH v2 16/20] net/mlx5: add flow VXLAN item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 242 +++++++++++++++++++++++++++++------
 1 file changed, 202 insertions(+), 40 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index eedf0c461..daf5b9b5a 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -35,18 +35,45 @@
 extern const struct eth_dev_ops mlx5_dev_ops;
 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
-/* Pattern Layer bits. */
+/* Pattern outer Layer bits. */
 #define MLX5_FLOW_LAYER_OUTER_L2 (1u << 0)
 #define MLX5_FLOW_LAYER_OUTER_L3_IPV4 (1u << 1)
 #define MLX5_FLOW_LAYER_OUTER_L3_IPV6 (1u << 2)
 #define MLX5_FLOW_LAYER_OUTER_L4_UDP (1u << 3)
 #define MLX5_FLOW_LAYER_OUTER_L4_TCP (1u << 4)
 #define MLX5_FLOW_LAYER_OUTER_VLAN (1u << 5)
-/* Masks. */
+
+/* Pattern inner Layer bits. */
+#define MLX5_FLOW_LAYER_INNER_L2 (1u << 6)
+#define MLX5_FLOW_LAYER_INNER_L3_IPV4 (1u << 7)
+#define MLX5_FLOW_LAYER_INNER_L3_IPV6 (1u << 8)
+#define MLX5_FLOW_LAYER_INNER_L4_UDP (1u << 9)
+#define MLX5_FLOW_LAYER_INNER_L4_TCP (1u << 10)
+#define MLX5_FLOW_LAYER_INNER_VLAN (1u << 11)
+
+/* Pattern tunnel Layer bits. */
+#define MLX5_FLOW_LAYER_VXLAN (1u << 12)
+
+/* Outer Masks. */
 #define MLX5_FLOW_LAYER_OUTER_L3 \
 	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 #define MLX5_FLOW_LAYER_OUTER_L4 \
 	(MLX5_FLOW_LAYER_OUTER_L4_UDP | MLX5_FLOW_LAYER_OUTER_L4_TCP)
+#define MLX5_FLOW_LAYER_OUTER \
+	(MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_L3 | \
+	 MLX5_FLOW_LAYER_OUTER_L4)
+
+/* Tunnel masks. */
+#define MLX5_FLOW_LAYER_TUNNEL MLX5_FLOW_LAYER_VXLAN
+
+/* Inner Masks. */
+#define MLX5_FLOW_LAYER_INNER_L3 \
+	(MLX5_FLOW_LAYER_INNER_L3_IPV4 | MLX5_FLOW_LAYER_INNER_L3_IPV6)
+#define MLX5_FLOW_LAYER_INNER_L4 \
+	(MLX5_FLOW_LAYER_INNER_L4_UDP | MLX5_FLOW_LAYER_INNER_L4_TCP)
+#define MLX5_FLOW_LAYER_INNER \
+	(MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3 | \
+	 MLX5_FLOW_LAYER_INNER_L4)
 
 /* Action fate on the packet. */
 #define MLX5_FLOW_FATE_DROP (1u << 0)
@@ -406,10 +433,14 @@ mlx5_flow_print(struct rte_flow *flow __rte_unused)
 	LIST_FOREACH(verbs, &flow->verbs, next) {
 		uint32_t layers = flow->layers | verbs->layers;
 
-		fprintf(stdout, " layers: %s/%s/%s\n",
+		fprintf(stdout, " layers: %s/%s/%s/%s/%s/%s/%s\n",
 			layers & MLX5_FLOW_LAYER_OUTER_L2 ? "l2" : "-",
 			layers & MLX5_FLOW_LAYER_OUTER_L3 ? "l3" : "-",
-			layers & MLX5_FLOW_LAYER_OUTER_L4 ? "l4" : "-");
+			layers & MLX5_FLOW_LAYER_OUTER_L4 ? "l4" : "-",
+			layers & MLX5_FLOW_LAYER_TUNNEL ? "T" : "-",
+			layers & MLX5_FLOW_LAYER_INNER_L2 ? "l2" : "-",
+			layers & MLX5_FLOW_LAYER_INNER_L3 ? "l3" : "-",
+			layers & MLX5_FLOW_LAYER_INNER_L4 ? "l4" : "-");
 		if (verbs->attr) {
 			struct ibv_spec_header *hdr =
 				(struct ibv_spec_header *)verbs->specs;
@@ -634,16 +665,18 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 		.type = RTE_BE16(0xffff),
 	};
+	const uint32_t layers = mlx5_flow_layers(flow);
+	const int tunnel = !!(layers & MLX5_FLOW_LAYER_TUNNEL);
 	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 	struct ibv_flow_spec_eth eth = {
-		.type = IBV_FLOW_SPEC_ETH,
+		.type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
-	const uint32_t layers = mlx5_flow_layers(flow);
 	int ret;
 
 	if (!flow->expand) {
-		if (layers & MLX5_FLOW_LAYER_OUTER_L2)
+		if (layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+			      MLX5_FLOW_LAYER_OUTER_L2))
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  item,
@@ -658,7 +691,8 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 		if (ret)
 			return ret;
 	}
-	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L2);
+	mlx5_flow_layers_update(flow, tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+				MLX5_FLOW_LAYER_OUTER_L2);
 	if (size > flow_size)
 		return size;
 	if (spec) {
@@ -694,7 +728,7 @@ mlx5_flow_item_vlan_update(struct ibv_flow_attr *attr,
 			   struct ibv_flow_spec_eth *eth)
 {
 	unsigned int i;
-	enum ibv_flow_spec_type search = IBV_FLOW_SPEC_ETH;
+	const enum ibv_flow_spec_type search = eth->type;
 	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 
@@ -737,17 +771,20 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 		.tci = RTE_BE16(0x0fff),
 	};
 	unsigned int size = sizeof(struct ibv_flow_spec_eth);
-	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+	const uint32_t layers = mlx5_flow_layers(flow);
+	const int tunnel = !!(layers & MLX5_FLOW_LAYER_TUNNEL);
 	struct ibv_flow_spec_eth eth = {
-		.type = IBV_FLOW_SPEC_ETH,
+		.type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
-	const uint32_t lm = MLX5_FLOW_LAYER_OUTER_L3 |
-			MLX5_FLOW_LAYER_OUTER_L4;
-	const uint32_t vlanm = MLX5_FLOW_LAYER_OUTER_VLAN;
-	const uint32_t l2m = MLX5_FLOW_LAYER_OUTER_L2;
-	const uint32_t layers = mlx5_flow_layers(flow);
+	const uint32_t lm = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
+				      MLX5_FLOW_LAYER_INNER_L4) :
+		(MLX5_FLOW_LAYER_OUTER_L3 | MLX5_FLOW_LAYER_OUTER_L4);
+	const uint32_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
+		MLX5_FLOW_LAYER_OUTER_VLAN;
+	const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+		MLX5_FLOW_LAYER_OUTER_L2;
 
 	if (!flow->expand) {
 		if (layers & vlanm)
@@ -801,12 +838,17 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 		if (size <= flow_size)
 			mlx5_flow_spec_verbs_add(flow, &eth, size);
 	} else {
-		if (verbs->attr)
-			mlx5_flow_item_vlan_update(verbs->attr, &eth);
+		if (flow->cur_verbs)
+			mlx5_flow_item_vlan_update(flow->cur_verbs->attr,
+						   &eth);
 		size = 0; /**< Only an update is done in eth specification. */
 	}
-	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L2 |
-				MLX5_FLOW_LAYER_OUTER_VLAN);
+	mlx5_flow_layers_update(flow,
+				tunnel ?
+				(MLX5_FLOW_LAYER_INNER_L2 |
+				 MLX5_FLOW_LAYER_INNER_VLAN) :
+				(MLX5_FLOW_LAYER_OUTER_L2 |
+				 MLX5_FLOW_LAYER_OUTER_VLAN));
 	return size;
 }
 
@@ -840,22 +882,26 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 			.next_proto_id = 0xff,
 		},
 	};
+	const uint32_t layers = mlx5_flow_layers(flow);
+	const int tunnel = !!(layers & MLX5_FLOW_LAYER_TUNNEL);
 	unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 	struct ibv_flow_spec_ipv4_ext ipv4 = {
-		.type = IBV_FLOW_SPEC_IPV4_EXT,
+		.type = IBV_FLOW_SPEC_IPV4_EXT |
+			(tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
-	const uint32_t layers = mlx5_flow_layers(flow);
 
 	if (!flow->expand) {
-		if (layers & MLX5_FLOW_LAYER_OUTER_L3)
+		if (layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			      MLX5_FLOW_LAYER_OUTER_L3))
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  item,
 						  "multiple L3 layers not"
 						  " supported");
-		else if (layers & MLX5_FLOW_LAYER_OUTER_L4)
+		else if (layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+				   MLX5_FLOW_LAYER_OUTER_L4))
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  item,
@@ -870,7 +916,9 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 		if (ret < 0)
 			return ret;
 	}
-	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L3_IPV4);
+	mlx5_flow_layers_update(flow,
+				tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
+				MLX5_FLOW_LAYER_OUTER_L3_IPV4);
 	if (size > flow_size)
 		return size;
 	if (spec) {
@@ -931,22 +979,25 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 			.hop_limits = 0xff,
 		},
 	};
+	const uint32_t layers = mlx5_flow_layers(flow);
+	const int tunnel = !!(layers & MLX5_FLOW_LAYER_TUNNEL);
 	unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 	struct ibv_flow_spec_ipv6 ipv6 = {
-		.type = IBV_FLOW_SPEC_IPV6,
+		.type = IBV_FLOW_SPEC_IPV6 | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
-	const uint32_t layers = mlx5_flow_layers(flow);
 
 	if (!flow->expand) {
-		if (layers & MLX5_FLOW_LAYER_OUTER_L3)
+		if (layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			      MLX5_FLOW_LAYER_OUTER_L3))
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  item,
 						  "multiple L3 layers not"
 						  " supported");
-		else if (layers & MLX5_FLOW_LAYER_OUTER_L4)
+		else if (layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+				   MLX5_FLOW_LAYER_OUTER_L4))
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  item,
@@ -961,7 +1012,9 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 		if (ret < 0)
 			return ret;
 	}
-	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L3_IPV6);
+	mlx5_flow_layers_update(flow,
+				tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
+				MLX5_FLOW_LAYER_OUTER_L3_IPV6);
 	if (size > flow_size)
 		return size;
 	if (spec) {
@@ -1029,22 +1082,25 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 {
 	const struct rte_flow_item_udp *spec = item->spec;
 	const struct rte_flow_item_udp *mask = item->mask;
+	const uint32_t layers = mlx5_flow_layers(flow);
+	const int tunnel = !!(layers & MLX5_FLOW_LAYER_TUNNEL);
 	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 	struct ibv_flow_spec_tcp_udp udp = {
-		.type = IBV_FLOW_SPEC_UDP,
+		.type = IBV_FLOW_SPEC_UDP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
-	const uint32_t layers = mlx5_flow_layers(flow);
 
 	if (!flow->expand) {
-		if (!(layers & MLX5_FLOW_LAYER_OUTER_L3))
+		if (!(layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+				MLX5_FLOW_LAYER_OUTER_L3)))
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  item,
 						  "L3 is mandatory to filter"
 						  " on L4");
-		if (layers & MLX5_FLOW_LAYER_OUTER_L4)
+		if (layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+			      MLX5_FLOW_LAYER_OUTER_L4))
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  item,
@@ -1059,7 +1115,9 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 		if (ret < 0)
 			return ret;
 	}
-	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L4_UDP);
+	mlx5_flow_layers_update(flow,
+				tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
+				MLX5_FLOW_LAYER_OUTER_L4_UDP);
 	if (size > flow_size)
 		return size;
 	if (spec) {
@@ -1097,22 +1155,25 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 {
 	const struct rte_flow_item_tcp *spec = item->spec;
 	const struct rte_flow_item_tcp *mask = item->mask;
+	const uint32_t layers = mlx5_flow_layers(flow);
+	const int tunnel = !!(layers & MLX5_FLOW_LAYER_TUNNEL);
 	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 	struct ibv_flow_spec_tcp_udp tcp = {
-		.type = IBV_FLOW_SPEC_TCP,
+		.type = IBV_FLOW_SPEC_TCP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
-	const uint32_t layers = mlx5_flow_layers(flow);
 
 	if (!flow->expand) {
-		if (!(layers & MLX5_FLOW_LAYER_OUTER_L3))
+		if (!(layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+				MLX5_FLOW_LAYER_OUTER_L3)))
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  item,
 						  "L3 is mandatory to filter"
 						  " on L4");
-		if (layers & MLX5_FLOW_LAYER_OUTER_L4)
+		if (layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+			      MLX5_FLOW_LAYER_OUTER_L4))
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  item,
@@ -1127,7 +1188,9 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 		if (ret < 0)
 			return ret;
 	}
-	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_OUTER_L4_TCP);
+	mlx5_flow_layers_update(flow,
+				tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
+				MLX5_FLOW_LAYER_OUTER_L4_TCP);
 	if (size > flow_size)
 		return size;
 	if (spec) {
@@ -1143,6 +1206,102 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Validate VXLAN layer and possibly create the Verbs specification.
+ *
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param flow_size[in]
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
+		     const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_vxlan *spec = item->spec;
+	const struct rte_flow_item_vxlan *mask = item->mask;
+	const uint32_t layers = mlx5_flow_layers(flow);
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel vxlan = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+	int ret;
+	union vni {
+		uint32_t vlan_id;
+		uint8_t vni[4];
+	} id = { .vlan_id = 0, };
+
+	if (!flow->expand) {
+		if (layers & MLX5_FLOW_LAYER_TUNNEL)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "a tunnel is already"
+						  " present");
+		/*
+		 * Verify only UDPv4 is present as defined in
+		 * https://tools.ietf.org/html/rfc7348
+		 */
+		if (!(layers & MLX5_FLOW_LAYER_OUTER_L4_UDP))
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "no outer UDP layer found");
+		if (!mask)
+			mask = &rte_flow_item_vxlan_mask;
+		ret = mlx5_flow_item_validate
+			(item, (const uint8_t *)mask,
+			 (const uint8_t *)&rte_flow_item_vxlan_mask,
+			 sizeof(struct rte_flow_item_vxlan), error);
+		if (ret < 0)
+			return ret;
+	}
+	if (spec) {
+		memcpy(&id.vni[1], spec->vni, 3);
+		vxlan.val.tunnel_id = id.vlan_id;
+		memcpy(&id.vni[1], mask->vni, 3);
+		vxlan.mask.tunnel_id = id.vlan_id;
+		/* Remove unwanted bits from values. */
+		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
+	}
+	if (!flow->expand) {
+		/*
+		 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if
+		 * only this layer is defined in the Verbs specification it is
+		 * interpreted as wildcard and all packets will match this
+		 * rule, if it follows a full stack layer (ex: eth / ipv4 /
+		 * udp), all packets matching the layers before will also
+		 * match this rule.  To avoid such situation, VNI 0 is
+		 * currently refused.
+		 */
+		if (!vxlan.val.tunnel_id)
+			return rte_flow_error_set(error, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "VXLAN vni cannot be 0");
+		if (!(layers & MLX5_FLOW_LAYER_OUTER))
+			return rte_flow_error_set(error, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "VXLAN tunnel must be fully"
+						  " defined");
+	}
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &vxlan, size);
+	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_VXLAN);
+	flow->ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP;
+	return size;
+}
+
 /**
  * Validate items provided by the user.
  *
@@ -1191,6 +1350,9 @@ mlx5_flow_items(const struct rte_flow_item items[],
 		case RTE_FLOW_ITEM_TYPE_TCP:
 			ret = mlx5_flow_item_tcp(items, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN:
+			ret = mlx5_flow_item_vxlan(items, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 16/20] net/mlx5: add flow VXLAN item

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:48PM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
Acked-by: Yongseok Koh <yskoh@mellanox.com>

Thanks

[dpdk-dev] [PATCH v2 17/20] net/mlx5: add flow VXLAN-GPE item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 123 ++++++++++++++++++++++++++++++++++-
 1 file changed, 120 insertions(+), 3 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index daf5b9b5a..47c55b426 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -53,6 +53,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
 /* Pattern tunnel Layer bits. */
 #define MLX5_FLOW_LAYER_VXLAN (1u << 12)
+#define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
 
 /* Outer Masks. */
 #define MLX5_FLOW_LAYER_OUTER_L3 \
@@ -64,7 +65,8 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 	 MLX5_FLOW_LAYER_OUTER_L4)
 
 /* Tunnel masks. */
-#define MLX5_FLOW_LAYER_TUNNEL MLX5_FLOW_LAYER_VXLAN
+#define MLX5_FLOW_LAYER_TUNNEL \
+	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE)
 
 /* Inner Masks. */
 #define MLX5_FLOW_LAYER_INNER_L3 \
@@ -1302,9 +1304,118 @@ mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Validate VXLAN-GPE layer and possibly create the Verbs specification.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param flow_size[in]
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_vxlan_gpe(struct rte_eth_dev *dev,
+			 const struct rte_flow_item *item,
+			 struct rte_flow *flow, const size_t flow_size,
+			 struct rte_flow_error *error)
+{
+	const struct rte_flow_item_vxlan_gpe *spec = item->spec;
+	const struct rte_flow_item_vxlan_gpe *mask = item->mask;
+	const uint32_t layers = mlx5_flow_layers(flow);
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel vxlan_gpe = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+	int ret;
+	union vni {
+		uint32_t vlan_id;
+		uint8_t vni[4];
+	} id = { .vlan_id = 0, };
+
+	if (!((struct priv *)dev->data->dev_private)->config.l3_vxlan_en)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 VXLAN is not enabled by device"
+					  " parameter and/or not configured in"
+					  " firmware");
+	if (layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already present");
+	/*
+	 * Verify only UDPv4 is present as defined in
+	 * https://tools.ietf.org/html/rfc7348
+	 */
+	if (!(layers & MLX5_FLOW_LAYER_OUTER_L4_UDP))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "no outer UDP layer found");
+	if (!mask)
+		mask = &rte_flow_item_vxlan_gpe_mask;
+	ret = mlx5_flow_item_validate
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
+		 sizeof(struct rte_flow_item_vxlan_gpe), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		memcpy(&id.vni[1], spec->vni, 3);
+		vxlan_gpe.val.tunnel_id = id.vlan_id;
+		memcpy(&id.vni[1], mask->vni, 3);
+		vxlan_gpe.mask.tunnel_id = id.vlan_id;
+		if (spec->protocol)
+			return rte_flow_error_set
+				(error, EINVAL,
+				 RTE_FLOW_ERROR_TYPE_ITEM,
+				 item,
+				 "VxLAN-GPE protocol not supported");
+		/* Remove unwanted bits from values. */
+		vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
+	}
+	/*
+	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
+	 * layer is defined in the Verbs specification it is interpreted as
+	 * wildcard and all packets will match this rule, if it follows a full
+	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
+	 * before will also match this rule.  To avoid such situation, VNI 0
+	 * is currently refused.
+	 */
+	if (!vxlan_gpe.val.tunnel_id)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN-GPE vni cannot be 0");
+	if (!(layers & MLX5_FLOW_LAYER_OUTER))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN-GPE tunnel must be fully"
+					  " defined");
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &vxlan_gpe, size);
+	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_VXLAN_GPE);
+	flow->ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP;
+	return size;
+}
+
 /**
  * Validate items provided by the user.
  *
+ * @param dev
+ *   Pointer to Ethernet device.
  * @param items
  *   Pointer to flow items array.
  * @param flow
@@ -1319,7 +1430,8 @@ mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
  *   otherwise and rte_errno is set.
  */
 static int
-mlx5_flow_items(const struct rte_flow_item items[],
+mlx5_flow_items(struct rte_eth_dev *dev,
+		const struct rte_flow_item items[],
 		struct rte_flow *flow, const size_t flow_size,
 		struct rte_flow_error *error)
 {
@@ -1353,6 +1465,10 @@ mlx5_flow_items(const struct rte_flow_item items[],
 		case RTE_FLOW_ITEM_TYPE_VXLAN:
 			ret = mlx5_flow_item_vxlan(items, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
+			ret = mlx5_flow_item_vxlan_gpe(dev, items, flow,
+						       remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
@@ -1962,7 +2078,8 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 				(void *)(flow->cur_verbs->attr + 1);
 		}
 		ret = mlx5_flow_items
-			((const struct rte_flow_item *)buf->patterns[i],
+			(dev,
+			 (const struct rte_flow_item *)buf->patterns[i],
 			 flow, remain, error);
 		if (ret < 0)
 			goto error;
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 17/20] net/mlx5: add flow VXLAN-GPE item

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:49PM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
>  drivers/net/mlx5/mlx5_flow.c | 123 ++++++++++++++++++++++++++++++++++-
>  1 file changed, 120 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index daf5b9b5a..47c55b426 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -53,6 +53,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
>  
>  /* Pattern tunnel Layer bits. */
>  #define MLX5_FLOW_LAYER_VXLAN (1u << 12)
> +#define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
>  
>  /* Outer Masks. */
>  #define MLX5_FLOW_LAYER_OUTER_L3 \
> @@ -64,7 +65,8 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
>  	 MLX5_FLOW_LAYER_OUTER_L4)
>  
>  /* Tunnel masks. */
> -#define MLX5_FLOW_LAYER_TUNNEL MLX5_FLOW_LAYER_VXLAN
> +#define MLX5_FLOW_LAYER_TUNNEL \
> +	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE)
>  
>  /* Inner Masks. */
>  #define MLX5_FLOW_LAYER_INNER_L3 \
> @@ -1302,9 +1304,118 @@ mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
>  	return size;
>  }
>  
> +/**
> + * Validate VXLAN-GPE layer and possibly create the Verbs specification.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + * @param item[in]
> + *   Item specification.
> + * @param flow[in, out]
> + *   Pointer to flow structure.
> + * @param flow_size[in]
> + *   Size in bytes of the available space for to store the flow information.
> + * @param error
> + *   Pointer to error structure.
> + *
> + * @return
> + *   size in bytes necessary for the conversion, a negative errno value
> + *   otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_item_vxlan_gpe(struct rte_eth_dev *dev,
> +			 const struct rte_flow_item *item,
> +			 struct rte_flow *flow, const size_t flow_size,
> +			 struct rte_flow_error *error)

It is almost same as mlx5_flow_item_vxlan() except for checking
priv->config.l3_vxlan_en. One more difference I noticed is that it doesn't check
flow->exapnd on validation. Why is that? If that's a mistake, isn't it better to
make the common code shareable?

> +{
> +	const struct rte_flow_item_vxlan_gpe *spec = item->spec;
> +	const struct rte_flow_item_vxlan_gpe *mask = item->mask;
> +	const uint32_t layers = mlx5_flow_layers(flow);
> +	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
> +	struct ibv_flow_spec_tunnel vxlan_gpe = {
> +		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
> +		.size = size,
> +	};
> +	int ret;
> +	union vni {
> +		uint32_t vlan_id;
> +		uint8_t vni[4];
> +	} id = { .vlan_id = 0, };
> +
> +	if (!((struct priv *)dev->data->dev_private)->config.l3_vxlan_en)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "L3 VXLAN is not enabled by device"
> +					  " parameter and/or not configured in"
> +					  " firmware");
> +	if (layers & MLX5_FLOW_LAYER_TUNNEL)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "a tunnel is already present");
> +	/*
> +	 * Verify only UDPv4 is present as defined in
> +	 * https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Ftools.ietf.org%2Fhtml%2Frfc7348&data=02%7C01%7Cyskoh%40mellanox.com%7C15334e781d3e4dec8aec08d5dc3fbf72%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C0%7C636657088715446919&sdata=LwT6lKV%2B59%2F839S8M2wZPFeQwzxSDr8cW3e6zBTX4TA%3D&reserved=0
> +	 */
> +	if (!(layers & MLX5_FLOW_LAYER_OUTER_L4_UDP))
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "no outer UDP layer found");
> +	if (!mask)
> +		mask = &rte_flow_item_vxlan_gpe_mask;
> +	ret = mlx5_flow_item_validate
> +		(item, (const uint8_t *)mask,
> +		 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
> +		 sizeof(struct rte_flow_item_vxlan_gpe), error);
> +	if (ret < 0)
> +		return ret;
> +	if (spec) {
> +		memcpy(&id.vni[1], spec->vni, 3);
> +		vxlan_gpe.val.tunnel_id = id.vlan_id;
> +		memcpy(&id.vni[1], mask->vni, 3);
> +		vxlan_gpe.mask.tunnel_id = id.vlan_id;
> +		if (spec->protocol)
> +			return rte_flow_error_set
> +				(error, EINVAL,
> +				 RTE_FLOW_ERROR_TYPE_ITEM,
> +				 item,
> +				 "VxLAN-GPE protocol not supported");
> +		/* Remove unwanted bits from values. */
> +		vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
> +	}
> +	/*
> +	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
> +	 * layer is defined in the Verbs specification it is interpreted as
> +	 * wildcard and all packets will match this rule, if it follows a full
> +	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
> +	 * before will also match this rule.  To avoid such situation, VNI 0
> +	 * is currently refused.
> +	 */
> +	if (!vxlan_gpe.val.tunnel_id)
> +		return rte_flow_error_set(error, EINVAL,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "VXLAN-GPE vni cannot be 0");
> +	if (!(layers & MLX5_FLOW_LAYER_OUTER))
> +		return rte_flow_error_set(error, EINVAL,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "VXLAN-GPE tunnel must be fully"
> +					  " defined");
> +	if (size <= flow_size)
> +		mlx5_flow_spec_verbs_add(flow, &vxlan_gpe, size);
> +	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_VXLAN_GPE);
> +	flow->ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP;
> +	return size;
> +}
> +
>  /**
>   * Validate items provided by the user.
>   *
> + * @param dev
> + *   Pointer to Ethernet device.
>   * @param items
>   *   Pointer to flow items array.
>   * @param flow
> @@ -1319,7 +1430,8 @@ mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
>   *   otherwise and rte_errno is set.
>   */
>  static int
> -mlx5_flow_items(const struct rte_flow_item items[],
> +mlx5_flow_items(struct rte_eth_dev *dev,
> +		const struct rte_flow_item items[],
>  		struct rte_flow *flow, const size_t flow_size,
>  		struct rte_flow_error *error)
>  {
> @@ -1353,6 +1465,10 @@ mlx5_flow_items(const struct rte_flow_item items[],
>  		case RTE_FLOW_ITEM_TYPE_VXLAN:
>  			ret = mlx5_flow_item_vxlan(items, flow, remain, error);
>  			break;
> +		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
> +			ret = mlx5_flow_item_vxlan_gpe(dev, items, flow,
> +						       remain, error);
> +			break;
>  		default:
>  			return rte_flow_error_set(error, ENOTSUP,
>  						  RTE_FLOW_ERROR_TYPE_ITEM,
> @@ -1962,7 +2078,8 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
>  				(void *)(flow->cur_verbs->attr + 1);
>  		}
>  		ret = mlx5_flow_items
> -			((const struct rte_flow_item *)buf->patterns[i],
> +			(dev,
> +			 (const struct rte_flow_item *)buf->patterns[i],
>  			 flow, remain, error);
>  		if (ret < 0)
>  			goto error;
> -- 
> 2.18.0
> 

Re: [dpdk-dev] [PATCH v2 17/20] net/mlx5: add flow VXLAN-GPE item

[Nélio Laranjeiro]

On Fri, Jul 06, 2018 at 04:23:26PM -0700, Yongseok Koh wrote:
> On Wed, Jun 27, 2018 at 05:07:49PM +0200, Nelio Laranjeiro wrote:
> > Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> > ---
> >  drivers/net/mlx5/mlx5_flow.c | 123 ++++++++++++++++++++++++++++++++++-
> >  1 file changed, 120 insertions(+), 3 deletions(-)
> > 
>[...]  
> > +/**
> > + * Validate VXLAN-GPE layer and possibly create the Verbs specification.
> > + *
> > + * @param dev
> > + *   Pointer to Ethernet device.
> > + * @param item[in]
> > + *   Item specification.
> > + * @param flow[in, out]
> > + *   Pointer to flow structure.
> > + * @param flow_size[in]
> > + *   Size in bytes of the available space for to store the flow information.
> > + * @param error
> > + *   Pointer to error structure.
> > + *
> > + * @return
> > + *   size in bytes necessary for the conversion, a negative errno value
> > + *   otherwise and rte_errno is set.
> > + */
> > +static int
> > +mlx5_flow_item_vxlan_gpe(struct rte_eth_dev *dev,
> > +			 const struct rte_flow_item *item,
> > +			 struct rte_flow *flow, const size_t flow_size,
> > +			 struct rte_flow_error *error)
> 
> It is almost same as mlx5_flow_item_vxlan() except for checking
> priv->config.l3_vxlan_en. One more difference I noticed is that it doesn't check
> flow->exapnd on validation. Why is that? If that's a mistake, isn't it better to
> make the common code shareable?
>[...]

The GPE version needs:

 - l3_vxlan_en
 - set its own tunnel bit (as in this case the following layer may
   directly be an L3)

Indeed there are some common code (as for the TCP/UDP) but sharing it
will be more difficult to read and fix in case of bugs.

In addition if this RFC [1] is fully dropped it will be easier to remove
the dedicated code when the ITEM in the API will also be removed, it may
not be from Mellanox PMD team but from anyone proposing the drop.  The
chances he breaks anything if the code is shared among several items is
high.  It is better to have a single function per item/action according
to the API directly.

Thanks,

[1] https://datatracker.ietf.org/doc/draft-quinn-vxlan-gpe/

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH v2 18/20] net/mlx5: add flow GRE item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 191 ++++++++++++++++++++++++++++++++++-
 1 file changed, 186 insertions(+), 5 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 47c55b426..636aaabe8 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -54,6 +54,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* Pattern tunnel Layer bits. */
 #define MLX5_FLOW_LAYER_VXLAN (1u << 12)
 #define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
+#define MLX5_FLOW_LAYER_GRE (1u << 14)
 
 /* Outer Masks. */
 #define MLX5_FLOW_LAYER_OUTER_L3 \
@@ -66,7 +67,8 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
 /* Tunnel masks. */
 #define MLX5_FLOW_LAYER_TUNNEL \
-	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE)
+	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE | \
+	 MLX5_FLOW_LAYER_GRE)
 
 /* Inner Masks. */
 #define MLX5_FLOW_LAYER_INNER_L3 \
@@ -215,6 +217,9 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 	},
 };
 
+/* Tunnel protocol values. */
+#define MLX5_PROTOCOL_GRE 47
+
 /** Handles information leading to a drop fate. */
 struct mlx5_flow_verbs {
 	LIST_ENTRY(mlx5_flow_verbs) next;
@@ -1005,12 +1010,23 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 						  item,
 						  "L3 cannot follow an L4"
 						  " layer");
+		/*
+		 * IPv6 is not recognised by the NIC inside a GRE tunnel.
+		 * Such support has to be disabled as the rule will be
+		 * accepted.  Tested with Mellanox OFED 4.3-3.0.2.1
+		 */
+		if (tunnel && layers & MLX5_FLOW_LAYER_GRE)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "IPv6 inside a GRE tunnel is"
+						  " not recognised.");
 		if (!mask)
 			mask = &rte_flow_item_ipv6_mask;
-		ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
-					      (const uint8_t *)&nic_mask,
-					      sizeof(struct rte_flow_item_ipv6),
-					      error);
+		ret = mlx5_flow_item_validate
+			(item, (const uint8_t *)mask,
+			 (const uint8_t *)&nic_mask,
+			 sizeof(struct rte_flow_item_ipv6), error);
 		if (ret < 0)
 			return ret;
 	}
@@ -1411,6 +1427,168 @@ mlx5_flow_item_vxlan_gpe(struct rte_eth_dev *dev,
 	return size;
 }
 
+/**
+ * Update the protocol in Verbs IPv4 spec.
+ *
+ * @param attr[in, out]
+ *   Pointer to Verbs attributes structure.
+ * @param protocol[in]
+ *   Protocol value to set if none is present in the specification.
+ */
+static void
+mlx5_flow_item_gre_ipv4_protocol_update(struct ibv_flow_attr *attr,
+					uint8_t protocol)
+{
+	unsigned int i;
+	const enum ibv_flow_spec_type search = IBV_FLOW_SPEC_IPV4_EXT;
+	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
+		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
+
+	if (!attr)
+		return;
+	for (i = 0; i != attr->num_of_specs; ++i) {
+		if (hdr->type == search) {
+			struct ibv_flow_spec_ipv4_ext *ip =
+				(struct ibv_flow_spec_ipv4_ext *)hdr;
+
+			if (!ip->val.proto) {
+				ip->val.proto = protocol;
+				ip->mask.proto = 0xff;
+			}
+			break;
+		}
+		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
+	}
+}
+
+/**
+ * Update the protocol in Verbs IPv6 spec.
+ *
+ * @param attr[in, out]
+ *   Pointer to Verbs attributes structure.
+ * @param protocol[in]
+ *   Protocol value to set if none is present in the specification.
+ */
+static void
+mlx5_flow_item_gre_ipv6_protocol_update(struct ibv_flow_attr *attr,
+					uint8_t protocol)
+{
+	unsigned int i;
+	const enum ibv_flow_spec_type search = IBV_FLOW_SPEC_IPV6;
+	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
+		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
+
+	if (!attr)
+		return;
+	for (i = 0; i != attr->num_of_specs; ++i) {
+		if (hdr->type == search) {
+			struct ibv_flow_spec_ipv6 *ip =
+				(struct ibv_flow_spec_ipv6 *)hdr;
+
+			if (!ip->val.next_hdr) {
+				ip->val.next_hdr = protocol;
+				ip->mask.next_hdr = 0xff;
+			}
+			break;
+		}
+		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
+	}
+}
+
+/**
+ * Validate GRE layer and possibly create the Verbs specification.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param flow_size[in]
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_gre(const struct rte_flow_item *item,
+		   struct rte_flow *flow, const size_t flow_size,
+		   struct rte_flow_error *error)
+{
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+	const struct rte_flow_item_gre *spec = item->spec;
+	const struct rte_flow_item_gre *mask = item->mask;
+	const uint32_t layers = mlx5_flow_layers(flow);
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	unsigned int size = sizeof(struct ibv_flow_spec_gre);
+	struct ibv_flow_spec_gre tunnel = {
+		.type = IBV_FLOW_SPEC_GRE,
+		.size = size,
+	};
+#else
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel tunnel = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+#endif
+	int ret;
+
+	if (layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already present");
+	if (!(layers & MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 Layer is missing");
+	if (!mask)
+		mask = &rte_flow_item_gre_mask;
+	ret = mlx5_flow_item_validate
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_gre_mask,
+		 sizeof(struct rte_flow_item_gre), error);
+	if (ret < 0)
+		return ret;
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	if (spec) {
+		tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
+		tunnel.val.protocol = spec->protocol;
+		tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
+		tunnel.mask.protocol = mask->protocol;
+		/* Remove unwanted bits from values. */
+		tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
+		tunnel.val.protocol &= tunnel.mask.protocol;
+		tunnel.val.key &= tunnel.mask.key;
+	}
+#else
+	if (spec && (spec->protocol & mask->protocol))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "without MPLS support the"
+					  " specification cannot be used for"
+					  " filtering");
+#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
+	if (size <= flow_size) {
+		if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
+			mlx5_flow_item_gre_ipv4_protocol_update
+				(verbs->attr, MLX5_PROTOCOL_GRE);
+		else
+			mlx5_flow_item_gre_ipv6_protocol_update
+				(verbs->attr, MLX5_PROTOCOL_GRE);
+		mlx5_flow_spec_verbs_add(flow, &tunnel, size);
+	}
+	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_GRE);
+	flow->ptype = RTE_PTYPE_TUNNEL_GRE;
+	return size;
+}
+
 /**
  * Validate items provided by the user.
  *
@@ -1469,6 +1647,9 @@ mlx5_flow_items(struct rte_eth_dev *dev,
 			ret = mlx5_flow_item_vxlan_gpe(dev, items, flow,
 						       remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_GRE:
+			ret = mlx5_flow_item_gre(items, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 18/20] net/mlx5: add flow GRE item

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:50PM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
>  drivers/net/mlx5/mlx5_flow.c | 191 ++++++++++++++++++++++++++++++++++-
>  1 file changed, 186 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index 47c55b426..636aaabe8 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -54,6 +54,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
>  /* Pattern tunnel Layer bits. */
>  #define MLX5_FLOW_LAYER_VXLAN (1u << 12)
>  #define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
> +#define MLX5_FLOW_LAYER_GRE (1u << 14)
>  
>  /* Outer Masks. */
>  #define MLX5_FLOW_LAYER_OUTER_L3 \
> @@ -66,7 +67,8 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
>  
>  /* Tunnel masks. */
>  #define MLX5_FLOW_LAYER_TUNNEL \
> -	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE)
> +	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE | \
> +	 MLX5_FLOW_LAYER_GRE)
>  
>  /* Inner Masks. */
>  #define MLX5_FLOW_LAYER_INNER_L3 \
> @@ -215,6 +217,9 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
>  	},
>  };
>  
> +/* Tunnel protocol values. */
> +#define MLX5_PROTOCOL_GRE 47

How about using IPPROTO_GRE instead?

> +
>  /** Handles information leading to a drop fate. */
>  struct mlx5_flow_verbs {
>  	LIST_ENTRY(mlx5_flow_verbs) next;
> @@ -1005,12 +1010,23 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
>  						  item,
>  						  "L3 cannot follow an L4"
>  						  " layer");
> +		/*
> +		 * IPv6 is not recognised by the NIC inside a GRE tunnel.
> +		 * Such support has to be disabled as the rule will be
> +		 * accepted.  Tested with Mellanox OFED 4.3-3.0.2.1
> +		 */

This comment doesn't look appropriate. Do you think it is a bug of OFED/FW,
which can be fixed? Or, is it a HW erratum? Let's talk offline.

> +		if (tunnel && layers & MLX5_FLOW_LAYER_GRE)
> +			return rte_flow_error_set(error, ENOTSUP,
> +						  RTE_FLOW_ERROR_TYPE_ITEM,
> +						  item,
> +						  "IPv6 inside a GRE tunnel is"
> +						  " not recognised.");
>  		if (!mask)
>  			mask = &rte_flow_item_ipv6_mask;
> -		ret = mlx5_flow_item_validate(item, (const uint8_t *)mask,
> -					      (const uint8_t *)&nic_mask,
> -					      sizeof(struct rte_flow_item_ipv6),
> -					      error);
> +		ret = mlx5_flow_item_validate
> +			(item, (const uint8_t *)mask,
> +			 (const uint8_t *)&nic_mask,
> +			 sizeof(struct rte_flow_item_ipv6), error);
>  		if (ret < 0)
>  			return ret;
>  	}
> @@ -1411,6 +1427,168 @@ mlx5_flow_item_vxlan_gpe(struct rte_eth_dev *dev,
>  	return size;
>  }
>  
> +/**
> + * Update the protocol in Verbs IPv4 spec.
> + *
> + * @param attr[in, out]
> + *   Pointer to Verbs attributes structure.
> + * @param protocol[in]
> + *   Protocol value to set if none is present in the specification.
> + */
> +static void
> +mlx5_flow_item_gre_ipv4_protocol_update(struct ibv_flow_attr *attr,
> +					uint8_t protocol)
> +{
> +	unsigned int i;
> +	const enum ibv_flow_spec_type search = IBV_FLOW_SPEC_IPV4_EXT;
> +	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
> +		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
> +
> +	if (!attr)
> +		return;
> +	for (i = 0; i != attr->num_of_specs; ++i) {
> +		if (hdr->type == search) {
> +			struct ibv_flow_spec_ipv4_ext *ip =
> +				(struct ibv_flow_spec_ipv4_ext *)hdr;
> +
> +			if (!ip->val.proto) {
> +				ip->val.proto = protocol;
> +				ip->mask.proto = 0xff;
> +			}
> +			break;
> +		}
> +		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
> +	}
> +}
> +
> +/**
> + * Update the protocol in Verbs IPv6 spec.
> + *
> + * @param attr[in, out]
> + *   Pointer to Verbs attributes structure.
> + * @param protocol[in]
> + *   Protocol value to set if none is present in the specification.
> + */
> +static void
> +mlx5_flow_item_gre_ipv6_protocol_update(struct ibv_flow_attr *attr,
> +					uint8_t protocol)

How about consolidating the two funcs -
mlx5_flow_item_gre_ipv6_protocol_update() and
mlx5_flow_item_gre_ipv4_protocol_update()? There are many things in common.

> +{
> +	unsigned int i;
> +	const enum ibv_flow_spec_type search = IBV_FLOW_SPEC_IPV6;
> +	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
> +		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
> +
> +	if (!attr)
> +		return;
> +	for (i = 0; i != attr->num_of_specs; ++i) {
> +		if (hdr->type == search) {
> +			struct ibv_flow_spec_ipv6 *ip =
> +				(struct ibv_flow_spec_ipv6 *)hdr;
> +
> +			if (!ip->val.next_hdr) {

What if protocol in IP header does have wrong value other than 47 (IPPROTO_GRE)?
Shouldn't we have a validation check for it in mlx5_flow_item_gre()?

> +				ip->val.next_hdr = protocol;
> +				ip->mask.next_hdr = 0xff;
> +			}
> +			break;
> +		}
> +		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
> +	}
> +}
> +
> +/**
> + * Validate GRE layer and possibly create the Verbs specification.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + * @param item[in]
> + *   Item specification.
> + * @param flow[in, out]
> + *   Pointer to flow structure.
> + * @param flow_size[in]
> + *   Size in bytes of the available space for to store the flow information.
> + * @param error
> + *   Pointer to error structure.
> + *
> + * @return
> + *   size in bytes necessary for the conversion, a negative errno value
> + *   otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_item_gre(const struct rte_flow_item *item,
> +		   struct rte_flow *flow, const size_t flow_size,
> +		   struct rte_flow_error *error)
> +{
> +	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
> +	const struct rte_flow_item_gre *spec = item->spec;
> +	const struct rte_flow_item_gre *mask = item->mask;
> +	const uint32_t layers = mlx5_flow_layers(flow);
> +#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
> +	unsigned int size = sizeof(struct ibv_flow_spec_gre);
> +	struct ibv_flow_spec_gre tunnel = {
> +		.type = IBV_FLOW_SPEC_GRE,
> +		.size = size,
> +	};
> +#else
> +	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
> +	struct ibv_flow_spec_tunnel tunnel = {
> +		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
> +		.size = size,
> +	};
> +#endif
> +	int ret;
> +
> +	if (layers & MLX5_FLOW_LAYER_TUNNEL)
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "a tunnel is already present");
> +	if (!(layers & MLX5_FLOW_LAYER_OUTER_L3))
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "L3 Layer is missing");
> +	if (!mask)
> +		mask = &rte_flow_item_gre_mask;
> +	ret = mlx5_flow_item_validate
> +		(item, (const uint8_t *)mask,
> +		 (const uint8_t *)&rte_flow_item_gre_mask,
> +		 sizeof(struct rte_flow_item_gre), error);
> +	if (ret < 0)
> +		return ret;
> +#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
> +	if (spec) {
> +		tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
> +		tunnel.val.protocol = spec->protocol;
> +		tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
> +		tunnel.mask.protocol = mask->protocol;
> +		/* Remove unwanted bits from values. */
> +		tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
> +		tunnel.val.protocol &= tunnel.mask.protocol;
> +		tunnel.val.key &= tunnel.mask.key;
> +	}
> +#else
> +	if (spec && (spec->protocol & mask->protocol))
> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "without MPLS support the"
> +					  " specification cannot be used for"
> +					  " filtering");
> +#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
> +	if (size <= flow_size) {
> +		if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
> +			mlx5_flow_item_gre_ipv4_protocol_update
> +				(verbs->attr, MLX5_PROTOCOL_GRE);
> +		else
> +			mlx5_flow_item_gre_ipv6_protocol_update
> +				(verbs->attr, MLX5_PROTOCOL_GRE);
> +		mlx5_flow_spec_verbs_add(flow, &tunnel, size);
> +	}
> +	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_GRE);
> +	flow->ptype = RTE_PTYPE_TUNNEL_GRE;
> +	return size;
> +}
> +
>  /**
>   * Validate items provided by the user.
>   *
> @@ -1469,6 +1647,9 @@ mlx5_flow_items(struct rte_eth_dev *dev,
>  			ret = mlx5_flow_item_vxlan_gpe(dev, items, flow,
>  						       remain, error);
>  			break;
> +		case RTE_FLOW_ITEM_TYPE_GRE:
> +			ret = mlx5_flow_item_gre(items, flow, remain, error);
> +			break;
>  		default:
>  			return rte_flow_error_set(error, ENOTSUP,
>  						  RTE_FLOW_ERROR_TYPE_ITEM,
> -- 
> 2.18.0
> 

Re: [dpdk-dev] [PATCH v2 18/20] net/mlx5: add flow GRE item

[Nélio Laranjeiro]

Hi Yongseok,

Only discussing here question, other comments are address, as I don't
have any objection I'll make the modification for them.

On Fri, Jul 06, 2018 at 04:46:11PM -0700, Yongseok Koh wrote:
>[...] 
> > +
> >  /** Handles information leading to a drop fate. */
> >  struct mlx5_flow_verbs {
> >  	LIST_ENTRY(mlx5_flow_verbs) next;
> > @@ -1005,12 +1010,23 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
> >  						  item,
> >  						  "L3 cannot follow an L4"
> >  						  " layer");
> > +		/*
> > +		 * IPv6 is not recognised by the NIC inside a GRE tunnel.
> > +		 * Such support has to be disabled as the rule will be
> > +		 * accepted.  Tested with Mellanox OFED 4.3-3.0.2.1
> > +		 */
> 
> This comment doesn't look appropriate. Do you think it is a bug of OFED/FW,
> which can be fixed? Or, is it a HW erratum? Let's talk offline.

By the time it was as this Mellanox OFED was the latest GA 4.3-3.0.2.1,
this is no more the case today as it cannot be downloaded anymore. A
verification is still necessary.  If the issue is not present anymore
I'll remove the comment with the test.

>[...] 
> > +{
> > +	unsigned int i;
> > +	const enum ibv_flow_spec_type search = IBV_FLOW_SPEC_IPV6;
> > +	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
> > +		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
> > +
> > +	if (!attr)
> > +		return;
> > +	for (i = 0; i != attr->num_of_specs; ++i) {
> > +		if (hdr->type == search) {
> > +			struct ibv_flow_spec_ipv6 *ip =
> > +				(struct ibv_flow_spec_ipv6 *)hdr;
> > +
> > +			if (!ip->val.next_hdr) {
> 
> What if protocol in IP header does have wrong value other than 47 (IPPROTO_GRE)?
> Shouldn't we have a validation check for it in mlx5_flow_item_gre()?
>[...]

Already added, the same issue occurs also with UDP/TCP.  If the user
uses some protocol it must match the following layer, otherwise its
request won't be respected which is a bug.

Thanks,

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH v2 19/20] net/mlx5: add flow MPLS item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 77 +++++++++++++++++++++++++++++++++++-
 1 file changed, 76 insertions(+), 1 deletion(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 636aaabe8..7aa4e6ed5 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -55,6 +55,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_FLOW_LAYER_VXLAN (1u << 12)
 #define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
 #define MLX5_FLOW_LAYER_GRE (1u << 14)
+#define MLX5_FLOW_LAYER_MPLS (1u << 15)
 
 /* Outer Masks. */
 #define MLX5_FLOW_LAYER_OUTER_L3 \
@@ -68,7 +69,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* Tunnel masks. */
 #define MLX5_FLOW_LAYER_TUNNEL \
 	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE | \
-	 MLX5_FLOW_LAYER_GRE)
+	 MLX5_FLOW_LAYER_GRE | MLX5_FLOW_LAYER_MPLS)
 
 /* Inner Masks. */
 #define MLX5_FLOW_LAYER_INNER_L3 \
@@ -1589,6 +1590,77 @@ mlx5_flow_item_gre(const struct rte_flow_item *item,
 	return size;
 }
 
+/**
+ * Validate MPLS layer and possibly create the Verbs specification.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param item[in]
+ *   Item specification.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param flow_size[in]
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_mpls(const struct rte_flow_item *item __rte_unused,
+		    struct rte_flow *flow __rte_unused,
+		    const size_t flow_size __rte_unused,
+		    struct rte_flow_error *error)
+{
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	const struct rte_flow_item_mpls *spec = item->spec;
+	const struct rte_flow_item_mpls *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_mpls);
+	struct ibv_flow_spec_mpls mpls = {
+		.type = IBV_FLOW_SPEC_MPLS,
+		.size = size,
+	};
+	const uint32_t layers = mlx5_flow_layers(flow);
+	int ret;
+
+	if (layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already"
+					  " present");
+	if (!mask)
+		mask = &rte_flow_item_mpls_mask;
+	ret = mlx5_flow_item_validate
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_mpls_mask,
+		 sizeof(struct rte_flow_item_mpls), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
+		memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
+		/* Remove unwanted bits from values.  */
+		mpls.val.label &= mpls.mask.label;
+	}
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &mpls, size);
+	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_MPLS);
+	if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
+		flow->ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE | RTE_PTYPE_L4_UDP;
+	else
+		flow->ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE;
+	return size;
+#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_ITEM,
+				  item,
+				  "MPLS is not supported by Verbs, please"
+				  " update.");
+}
+
 /**
  * Validate items provided by the user.
  *
@@ -1650,6 +1722,9 @@ mlx5_flow_items(struct rte_eth_dev *dev,
 		case RTE_FLOW_ITEM_TYPE_GRE:
 			ret = mlx5_flow_item_gre(items, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_MPLS:
+			ret = mlx5_flow_item_mpls(items, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 19/20] net/mlx5: add flow MPLS item

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:51PM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
>  drivers/net/mlx5/mlx5_flow.c | 77 +++++++++++++++++++++++++++++++++++-
>  1 file changed, 76 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index 636aaabe8..7aa4e6ed5 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -55,6 +55,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
>  #define MLX5_FLOW_LAYER_VXLAN (1u << 12)
>  #define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
>  #define MLX5_FLOW_LAYER_GRE (1u << 14)
> +#define MLX5_FLOW_LAYER_MPLS (1u << 15)
>  
>  /* Outer Masks. */
>  #define MLX5_FLOW_LAYER_OUTER_L3 \
> @@ -68,7 +69,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
>  /* Tunnel masks. */
>  #define MLX5_FLOW_LAYER_TUNNEL \
>  	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE | \
> -	 MLX5_FLOW_LAYER_GRE)
> +	 MLX5_FLOW_LAYER_GRE | MLX5_FLOW_LAYER_MPLS)
>  
>  /* Inner Masks. */
>  #define MLX5_FLOW_LAYER_INNER_L3 \
> @@ -1589,6 +1590,77 @@ mlx5_flow_item_gre(const struct rte_flow_item *item,
>  	return size;
>  }
>  
> +/**
> + * Validate MPLS layer and possibly create the Verbs specification.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + * @param item[in]
> + *   Item specification.
> + * @param flow[in, out]
> + *   Pointer to flow structure.
> + * @param flow_size[in]
> + *   Size in bytes of the available space for to store the flow information.
> + * @param error
> + *   Pointer to error structure.
> + *
> + * @return
> + *   size in bytes necessary for the conversion, a negative errno value
> + *   otherwise and rte_errno is set.
> + */
> +static int
> +mlx5_flow_item_mpls(const struct rte_flow_item *item __rte_unused,
> +		    struct rte_flow *flow __rte_unused,
> +		    const size_t flow_size __rte_unused,
> +		    struct rte_flow_error *error)
> +{
> +#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
> +	const struct rte_flow_item_mpls *spec = item->spec;
> +	const struct rte_flow_item_mpls *mask = item->mask;
> +	unsigned int size = sizeof(struct ibv_flow_spec_mpls);
> +	struct ibv_flow_spec_mpls mpls = {
> +		.type = IBV_FLOW_SPEC_MPLS,
> +		.size = size,
> +	};
> +	const uint32_t layers = mlx5_flow_layers(flow);
> +	int ret;
> +
> +	if (layers & MLX5_FLOW_LAYER_TUNNEL)

Just curious, no need to check flow->exapnd?

> +		return rte_flow_error_set(error, ENOTSUP,
> +					  RTE_FLOW_ERROR_TYPE_ITEM,
> +					  item,
> +					  "a tunnel is already"
> +					  " present");
> +	if (!mask)
> +		mask = &rte_flow_item_mpls_mask;
> +	ret = mlx5_flow_item_validate
> +		(item, (const uint8_t *)mask,
> +		 (const uint8_t *)&rte_flow_item_mpls_mask,
> +		 sizeof(struct rte_flow_item_mpls), error);
> +	if (ret < 0)
> +		return ret;
> +	if (spec) {
> +		memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
> +		memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
> +		/* Remove unwanted bits from values.  */
> +		mpls.val.label &= mpls.mask.label;
> +	}
> +	if (size <= flow_size)

Is it guaranteed flow->cur_verbs isn't null if size fits? Could be obvious but
just want to make sure.

> +		mlx5_flow_spec_verbs_add(flow, &mpls, size);
> +	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_MPLS);
> +	if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
> +		flow->ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE | RTE_PTYPE_L4_UDP;
> +	else
> +		flow->ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE;
> +	return size;
> +#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
> +	return rte_flow_error_set(error, ENOTSUP,
> +				  RTE_FLOW_ERROR_TYPE_ITEM,
> +				  item,
> +				  "MPLS is not supported by Verbs, please"
> +				  " update.");
> +}
> +
>  /**
>   * Validate items provided by the user.
>   *
> @@ -1650,6 +1722,9 @@ mlx5_flow_items(struct rte_eth_dev *dev,
>  		case RTE_FLOW_ITEM_TYPE_GRE:
>  			ret = mlx5_flow_item_gre(items, flow, remain, error);
>  			break;

#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT

> +		case RTE_FLOW_ITEM_TYPE_MPLS:
> +			ret = mlx5_flow_item_mpls(items, flow, remain, error);
> +			break;

#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */

How about this?

>  		default:
>  			return rte_flow_error_set(error, ENOTSUP,
>  						  RTE_FLOW_ERROR_TYPE_ITEM,
> -- 
> 2.18.0
> 

Re: [dpdk-dev] [PATCH v2 19/20] net/mlx5: add flow MPLS item

[Nélio Laranjeiro]

On Fri, Jul 06, 2018 at 05:11:31PM -0700, Yongseok Koh wrote:
> On Wed, Jun 27, 2018 at 05:07:51PM +0200, Nelio Laranjeiro wrote:
> > Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> > ---
>[...]
> > +	if (spec) {
> > +		memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
> > +		memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
> > +		/* Remove unwanted bits from values.  */
> > +		mpls.val.label &= mpls.mask.label;
> > +	}
> > +	if (size <= flow_size)
> 
> Is it guaranteed flow->cur_verbs isn't null if size fits? Could be obvious but
> just want to make sure.

Yes it is.

> > +		mlx5_flow_spec_verbs_add(flow, &mpls, size);
> > +	mlx5_flow_layers_update(flow, MLX5_FLOW_LAYER_MPLS);
> > +	if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
> > +		flow->ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE | RTE_PTYPE_L4_UDP;
> > +	else
> > +		flow->ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE;
> > +	return size;
> > +#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
> > +	return rte_flow_error_set(error, ENOTSUP,
> > +				  RTE_FLOW_ERROR_TYPE_ITEM,
> > +				  item,
> > +				  "MPLS is not supported by Verbs, please"
> > +				  " update.");
> > +}
> > +
> >  /**
> >   * Validate items provided by the user.
> >   *
> > @@ -1650,6 +1722,9 @@ mlx5_flow_items(struct rte_eth_dev *dev,
> >  		case RTE_FLOW_ITEM_TYPE_GRE:
> >  			ret = mlx5_flow_item_gre(items, flow, remain, error);
> >  			break;
> 
> #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
> 
> > +		case RTE_FLOW_ITEM_TYPE_MPLS:
> > +			ret = mlx5_flow_item_mpls(items, flow, remain, error);
> > +			break;
> 
> #endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
> 
> How about this?
>[...]

It adds another couple of #ifdef #endif and the final output won't help
much the user, having an error "MPLS is not updated by Verbs, please
update" will help more than "item not supported".

Regards,

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH v2 20/20] net/mlx5: add count flow action

[Nelio Laranjeiro]

This is only supported by Mellanox OFED.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5.h      |   2 +
 drivers/net/mlx5/mlx5_flow.c | 238 +++++++++++++++++++++++++++++++++++
 2 files changed, 240 insertions(+)

diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index ed8c1c9a2..1d8e156c8 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -181,6 +181,8 @@ struct priv {
 	struct mlx5_drop drop; /* Flow drop queues. */
 	struct mlx5_flows flows; /* RTE Flow rules. */
 	struct mlx5_flows ctrl_flows; /* Control flow rules. */
+	LIST_HEAD(counters, mlx5_flow_counter) flow_counters;
+	/* Flow counters. */
 	struct {
 		uint32_t dev_gen; /* Generation number to flush local caches. */
 		rte_rwlock_t rwlock; /* MR Lock. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 7aa4e6ed5..9241855be 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -88,6 +88,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* Modify a packet. */
 #define MLX5_FLOW_MOD_FLAG (1u << 0)
 #define MLX5_FLOW_MOD_MARK (1u << 1)
+#define MLX5_FLOW_MOD_COUNT (1u << 2)
 
 /* Priority reserved for default flows. */
 #define MLX5_FLOW_PRIO_RSVD ((uint32_t)-1)
@@ -239,6 +240,17 @@ struct mlx5_flow_verbs {
 	uint64_t hash_fields; /**< Verbs hash Rx queue hash fields. */
 };
 
+/* Counters information. */
+struct mlx5_flow_counter {
+	LIST_ENTRY(mlx5_flow_counter) next; /**< Pointer to the next counter. */
+	uint32_t shared:1; /**< Share counter ID with other flow rules. */
+	uint32_t ref_cnt:31; /**< Reference counter. */
+	uint32_t id; /**< Counter ID. */
+	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
+	uint64_t hits; /**< Number of packets matched by the rule. */
+	uint64_t bytes; /**< Number of bytes matched by the rule. */
+};
+
 /* Flow structure. */
 struct rte_flow {
 	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
@@ -253,6 +265,7 @@ struct rte_flow {
 	LIST_HEAD(verbs, mlx5_flow_verbs) verbs; /**< Verbs flows list. */
 	struct mlx5_flow_verbs *cur_verbs;
 	/**< Current Verbs flow structure being filled. */
+	struct mlx5_flow_counter *counter; /**< Holds Verbs flow counter. */
 	struct rte_flow_action_rss rss;/**< RSS context. */
 	uint32_t ptype;
 	/**< Store tunnel packet type data to store in Rx queue. */
@@ -266,6 +279,7 @@ static const struct rte_flow_ops mlx5_flow_ops = {
 	.destroy = mlx5_flow_destroy,
 	.flush = mlx5_flow_flush,
 	.isolate = mlx5_flow_isolate,
+	.query = mlx5_flow_query,
 };
 
 /* Convert FDIR request to Generic flow. */
@@ -407,6 +421,81 @@ mlx5_flow_priority(struct rte_eth_dev *dev, uint32_t priority,
 	return priority;
 }
 
+/**
+ * Get a flow counter.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param id
+ *   Counter identifier.
+ *
+ * @return
+ *   A pointer to the counter, NULL otherwise and rte_errno is set.
+ */
+static struct mlx5_flow_counter *
+mlx5_flow_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_flow_counter *cnt;
+
+	LIST_FOREACH(cnt, &priv->flow_counters, next) {
+		if (cnt->shared != shared)
+			continue;
+		if (cnt->id != id)
+			continue;
+		cnt->ref_cnt++;
+		return cnt;
+	}
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+
+	struct mlx5_flow_counter tmpl = {
+		.shared = shared,
+		.id = id,
+		.cs = mlx5_glue->create_counter_set
+			(priv->ctx,
+			 &(struct ibv_counter_set_init_attr){
+				 .counter_set_id = id,
+			 }),
+		.hits = 0,
+		.bytes = 0,
+	};
+
+	if (!tmpl.cs) {
+		rte_errno = errno;
+		return NULL;
+	}
+	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
+	if (!cnt) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	*cnt = tmpl;
+	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
+	return cnt;
+#endif
+	rte_errno = ENOTSUP;
+	return NULL;
+}
+
+/**
+ * Release a flow counter.
+ *
+ * @param id
+ *   Counter identifier.
+ *
+ * @return
+ *   A pointer to the counter, NULL otherwise and rte_errno is set.
+ */
+static void
+mlx5_flow_counter_release(struct mlx5_flow_counter *counter)
+{
+	if (--counter->ref_cnt == 0) {
+		claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
+		LIST_REMOVE(counter, next);
+		rte_free(counter);
+	}
+}
+
 /**
  * Flow debug purpose function only available when
  * CONFIG_RTE_LIBRTE_MLX5_DEBUG=y
@@ -2169,6 +2258,65 @@ mlx5_flow_action_mark(const struct rte_flow_action *actions,
 	return size;
 }
 
+/**
+ * Validate action count provided by the user.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param actions
+ *   Pointer to flow actions array.
+ * @param flow
+ *   Pointer to the rte_flow structure.
+ * @param flow_size[in]
+ *   Size in bytes of the available space for to store the flow information.
+ * @param error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   size in bytes necessary for the conversion, a negative errno value
+ *   otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_action_count(struct rte_eth_dev *dev,
+		       const struct rte_flow_action *actions,
+		       struct rte_flow *flow,
+		       const size_t flow_size __rte_unused,
+		       struct rte_flow_error *error)
+{
+	const struct rte_flow_action_count *count = actions->conf;
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
+	struct ibv_flow_spec_counter_action counter = {
+		.type = IBV_FLOW_SPEC_ACTION_COUNT,
+		.size = size,
+	};
+#endif
+
+	if (!flow->counter) {
+		flow->counter = mlx5_flow_counter_new(dev, count->shared,
+						      count->id);
+		if (!flow->counter)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "cannot get counter"
+						  " context.");
+	}
+	if (!((struct priv *)dev->data->dev_private)->config.flow_counter_en)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  actions,
+					  "flow counters are not supported.");
+	flow->modifier |= MLX5_FLOW_MOD_COUNT;
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	counter.counter_set_handle = flow->counter->cs->handle;
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &counter, size);
+	return size;
+#endif
+	return 0;
+}
+
 /**
  * Validate actions provided by the user.
  *
@@ -2228,6 +2376,10 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 		case RTE_FLOW_ACTION_TYPE_RSS:
 			ret = mlx5_flow_action_rss(dev, actions, flow, error);
 			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = mlx5_flow_action_count(dev, actions, flow, remain,
+						     error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -2417,6 +2569,10 @@ mlx5_flow_fate_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
 			verbs->hrxq = NULL;
 		}
 	}
+	if (flow->counter) {
+		mlx5_flow_counter_release(flow->counter);
+		flow->counter = NULL;
+	}
 }
 
 /**
@@ -2974,6 +3130,88 @@ mlx5_flow_isolate(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/**
+ * Query flow counter.
+ *
+ * @param flow
+ *   Pointer to the flow.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_query_count(struct rte_flow *flow __rte_unused,
+		      void *data __rte_unused,
+		      struct rte_flow_error *error)
+{
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	struct rte_flow_query_count *qc = data;
+	uint64_t counters[2] = {0, 0};
+	struct ibv_query_counter_set_attr query_cs_attr = {
+		.cs = flow->counter->cs,
+		.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
+	};
+	struct ibv_counter_set_data query_out = {
+		.out = counters,
+		.outlen = 2 * sizeof(uint64_t),
+	};
+	int err = mlx5_glue->query_counter_set(&query_cs_attr, &query_out);
+
+	if (err)
+		return rte_flow_error_set(error, err,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "cannot read counter");
+	qc->hits_set = 1;
+	qc->bytes_set = 1;
+	qc->hits = counters[0] - flow->counter->hits;
+	qc->bytes = counters[1] - flow->counter->bytes;
+	if (qc->reset) {
+		flow->counter->hits = counters[0];
+		flow->counter->bytes = counters[1];
+	}
+	return 0;
+#endif
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "counters are not available");
+}
+
+/**
+ * Query a flows.
+ *
+ * @see rte_flow_query()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
+		struct rte_flow *flow,
+		const struct rte_flow_action *actions,
+		void *data,
+		struct rte_flow_error *error)
+{
+	int ret = 0;
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = mlx5_flow_query_count(flow, data, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+		if (ret < 0)
+			return ret;
+	}
+	return 0;
+}
+
 /**
  * Convert a flow director filter to a generic flow.
  *
-- 
2.18.0

Re: [dpdk-dev] [PATCH v2 20/20] net/mlx5: add count flow action

[Yongseok Koh]

On Wed, Jun 27, 2018 at 05:07:52PM +0200, Nelio Laranjeiro wrote:
> This is only supported by Mellanox OFED.
> 
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
Acked-by: Yongseok Koh <yskoh@mellanox.com>

Thanks

[dpdk-dev] [PATCH v3 00/21] net/mlx5: flow rework

[Nelio Laranjeiro]

Re-work flow engine to support port redirection actions through TC.

This first series depends on [1] which is implemented in commit 
"net/mlx5: support inner RSS computation" and on [2].
Next series will bring the port redirection as announced[3].

[1] https://mails.dpdk.org/archives/dev/2018-July/107378.html
[2] https://mails.dpdk.org/archives/dev/2018-June/104192.html
[3] https://mails.dpdk.org/archives/dev/2018-May/103043.html

Changes in v3:

- remove redundant parameters in drop queues internal API.
- simplify the RSS expansion by only adding missing items in the pattern.
- document all functions.

Nelio Laranjeiro (21):
  net/mlx5: remove flow support
  net/mlx5: handle drop queues as regular queues
  net/mlx5: replace verbs priorities by flow
  net/mlx5: support flow Ethernet item along with drop action
  net/mlx5: add flow queue action
  net/mlx5: add flow stop/start
  net/mlx5: add flow VLAN item
  net/mlx5: add flow IPv4 item
  net/mlx5: add flow IPv6 item
  net/mlx5: add flow UDP item
  net/mlx5: add flow TCP item
  net/mlx5: add mark/flag flow action
  net/mlx5: use a macro for the RSS key size
  net/mlx5: add RSS flow action
  net/mlx5: remove useless arguments in hrxq API
  net/mlx5: support inner RSS computation
  net/mlx5: add flow VXLAN item
  net/mlx5: add flow VXLAN-GPE item
  net/mlx5: add flow GRE item
  net/mlx5: add flow MPLS item
  net/mlx5: add count flow action

 drivers/net/mlx5/mlx5.c        |   22 +-
 drivers/net/mlx5/mlx5.h        |   18 +-
 drivers/net/mlx5/mlx5_ethdev.c |   14 +-
 drivers/net/mlx5/mlx5_flow.c   | 4815 ++++++++++++++++----------------
 drivers/net/mlx5/mlx5_prm.h    |    3 +
 drivers/net/mlx5/mlx5_rss.c    |    7 +-
 drivers/net/mlx5/mlx5_rxq.c    |  281 +-
 drivers/net/mlx5/mlx5_rxtx.h   |   21 +-
 8 files changed, 2632 insertions(+), 2549 deletions(-)

-- 
2.18.0

[dpdk-dev] [PATCH v3 01/21] net/mlx5: remove flow support

[Nelio Laranjeiro]

This start a series to re-work the flow engine in mlx5 to easily support
flow conversion to Verbs or TC.  This is necessary to handle both regular
flows and representors flows.

As the full file needs to be clean-up to re-write all items/actions
processing, this patch starts to disable the regular code and only let the
PMD to start in isolated mode.

After this patch flow API will not be usable.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 3095 +---------------------------------
 drivers/net/mlx5/mlx5_rxtx.h |    1 -
 2 files changed, 80 insertions(+), 3016 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 45207d70e..a45cb06e1 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -31,2406 +31,49 @@
 #include "mlx5_prm.h"
 #include "mlx5_glue.h"
 
-/* Flow priority for control plane flows. */
-#define MLX5_CTRL_FLOW_PRIORITY 1
-
-/* Internet Protocol versions. */
-#define MLX5_IPV4 4
-#define MLX5_IPV6 6
-#define MLX5_GRE 47
-
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-struct ibv_flow_spec_counter_action {
-	int dummy;
-};
-#endif
-
-/* Dev ops structure defined in mlx5.c */
-extern const struct eth_dev_ops mlx5_dev_ops;
-extern const struct eth_dev_ops mlx5_dev_ops_isolate;
-
-/** Structure give to the conversion functions. */
-struct mlx5_flow_data {
-	struct rte_eth_dev *dev; /** Ethernet device. */
-	struct mlx5_flow_parse *parser; /** Parser context. */
-	struct rte_flow_error *error; /** Error context. */
-};
-
-static int
-mlx5_flow_create_eth(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vlan(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_ipv4(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_ipv6(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_udp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_tcp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vxlan(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vxlan_gpe(const struct rte_flow_item *item,
-			   const void *default_mask,
-			   struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_gre(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_mpls(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-struct mlx5_flow_parse;
-
-static void
-mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
-		      unsigned int size);
-
-static int
-mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
-
-static int
-mlx5_flow_create_count(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser);
-
-/* Hash RX queue types. */
-enum hash_rxq_type {
-	HASH_RXQ_TCPV4,
-	HASH_RXQ_UDPV4,
-	HASH_RXQ_IPV4,
-	HASH_RXQ_TCPV6,
-	HASH_RXQ_UDPV6,
-	HASH_RXQ_IPV6,
-	HASH_RXQ_ETH,
-	HASH_RXQ_TUNNEL,
-};
-
-/* Initialization data for hash RX queue. */
-struct hash_rxq_init {
-	uint64_t hash_fields; /* Fields that participate in the hash. */
-	uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
-	unsigned int flow_priority; /* Flow priority to use. */
-	unsigned int ip_version; /* Internet protocol. */
-};
-
-/* Initialization data for hash RX queues. */
-const struct hash_rxq_init hash_rxq_init[] = {
-	[HASH_RXQ_TCPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4 |
-				IBV_RX_HASH_SRC_PORT_TCP |
-				IBV_RX_HASH_DST_PORT_TCP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV4,
-	},
-	[HASH_RXQ_UDPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4 |
-				IBV_RX_HASH_SRC_PORT_UDP |
-				IBV_RX_HASH_DST_PORT_UDP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV4,
-	},
-	[HASH_RXQ_IPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4),
-		.dpdk_rss_hf = (ETH_RSS_IPV4 |
-				ETH_RSS_FRAG_IPV4),
-		.flow_priority = 1,
-		.ip_version = MLX5_IPV4,
-	},
-	[HASH_RXQ_TCPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6 |
-				IBV_RX_HASH_SRC_PORT_TCP |
-				IBV_RX_HASH_DST_PORT_TCP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV6,
-	},
-	[HASH_RXQ_UDPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6 |
-				IBV_RX_HASH_SRC_PORT_UDP |
-				IBV_RX_HASH_DST_PORT_UDP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV6,
-	},
-	[HASH_RXQ_IPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6),
-		.dpdk_rss_hf = (ETH_RSS_IPV6 |
-				ETH_RSS_FRAG_IPV6),
-		.flow_priority = 1,
-		.ip_version = MLX5_IPV6,
-	},
-	[HASH_RXQ_ETH] = {
-		.hash_fields = 0,
-		.dpdk_rss_hf = 0,
-		.flow_priority = 2,
-	},
-};
-
-/* Number of entries in hash_rxq_init[]. */
-const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
-
-/** Structure for holding counter stats. */
-struct mlx5_flow_counter_stats {
-	uint64_t hits; /**< Number of packets matched by the rule. */
-	uint64_t bytes; /**< Number of bytes matched by the rule. */
-};
-
-/** Structure for Drop queue. */
-struct mlx5_hrxq_drop {
-	struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
-	struct ibv_qp *qp; /**< Verbs queue pair. */
-	struct ibv_wq *wq; /**< Verbs work queue. */
-	struct ibv_cq *cq; /**< Verbs completion queue. */
-};
-
-/* Flows structures. */
-struct mlx5_flow {
-	uint64_t hash_fields; /**< Fields that participate in the hash. */
-	struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
-	struct ibv_flow *ibv_flow; /**< Verbs flow. */
-	struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
-};
-
-/* Drop flows structures. */
-struct mlx5_flow_drop {
-	struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
-	struct ibv_flow *ibv_flow; /**< Verbs flow. */
-};
-
-struct rte_flow {
-	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
-	uint32_t mark:1; /**< Set if the flow is marked. */
-	uint32_t drop:1; /**< Drop queue. */
-	struct rte_flow_action_rss rss_conf; /**< RSS configuration */
-	uint16_t (*queues)[]; /**< Queues indexes to use. */
-	uint8_t rss_key[40]; /**< copy of the RSS key. */
-	uint32_t tunnel; /**< Tunnel type of RTE_PTYPE_TUNNEL_XXX. */
-	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
-	struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
-	struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
-	/**< Flow with Rx queue. */
-};
-
-/** Static initializer for items. */
-#define ITEMS(...) \
-	(const enum rte_flow_item_type []){ \
-		__VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
-	}
-
-#define IS_TUNNEL(type) ( \
-	(type) == RTE_FLOW_ITEM_TYPE_VXLAN || \
-	(type) == RTE_FLOW_ITEM_TYPE_VXLAN_GPE || \
-	(type) == RTE_FLOW_ITEM_TYPE_GRE || \
-	(type) == RTE_FLOW_ITEM_TYPE_MPLS)
-
-const uint32_t flow_ptype[] = {
-	[RTE_FLOW_ITEM_TYPE_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN,
-	[RTE_FLOW_ITEM_TYPE_VXLAN_GPE] = RTE_PTYPE_TUNNEL_VXLAN_GPE,
-	[RTE_FLOW_ITEM_TYPE_GRE] = RTE_PTYPE_TUNNEL_GRE,
-	[RTE_FLOW_ITEM_TYPE_MPLS] = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
-};
-
-#define PTYPE_IDX(t) ((RTE_PTYPE_TUNNEL_MASK & (t)) >> 12)
-
-const uint32_t ptype_ext[] = {
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] = RTE_PTYPE_TUNNEL_VXLAN |
-					      RTE_PTYPE_L4_UDP,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN_GPE)]	= RTE_PTYPE_TUNNEL_VXLAN_GPE |
-						  RTE_PTYPE_L4_UDP,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_GRE)] = RTE_PTYPE_TUNNEL_GRE,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_GRE)] =
-		RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_UDP)] =
-		RTE_PTYPE_TUNNEL_MPLS_IN_GRE | RTE_PTYPE_L4_UDP,
-};
-
-/** Structure to generate a simple graph of layers supported by the NIC. */
-struct mlx5_flow_items {
-	/** List of possible actions for these items. */
-	const enum rte_flow_action_type *const actions;
-	/** Bit-masks corresponding to the possibilities for the item. */
-	const void *mask;
-	/**
-	 * Default bit-masks to use when item->mask is not provided. When
-	 * \default_mask is also NULL, the full supported bit-mask (\mask) is
-	 * used instead.
-	 */
-	const void *default_mask;
-	/** Bit-masks size in bytes. */
-	const unsigned int mask_sz;
-	/**
-	 * Conversion function from rte_flow to NIC specific flow.
-	 *
-	 * @param item
-	 *   rte_flow item to convert.
-	 * @param default_mask
-	 *   Default bit-masks to use when item->mask is not provided.
-	 * @param data
-	 *   Internal structure to store the conversion.
-	 *
-	 * @return
-	 *   0 on success, a negative errno value otherwise and rte_errno is
-	 *   set.
-	 */
-	int (*convert)(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data);
-	/** Size in bytes of the destination structure. */
-	const unsigned int dst_sz;
-	/** List of possible following items.  */
-	const enum rte_flow_item_type *const items;
-};
-
-/** Valid action for this PMD. */
-static const enum rte_flow_action_type valid_actions[] = {
-	RTE_FLOW_ACTION_TYPE_DROP,
-	RTE_FLOW_ACTION_TYPE_QUEUE,
-	RTE_FLOW_ACTION_TYPE_MARK,
-	RTE_FLOW_ACTION_TYPE_FLAG,
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	RTE_FLOW_ACTION_TYPE_COUNT,
-#endif
-	RTE_FLOW_ACTION_TYPE_END,
-};
-
-/** Graph of supported items and associated actions. */
-static const struct mlx5_flow_items mlx5_flow_items[] = {
-	[RTE_FLOW_ITEM_TYPE_END] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_VXLAN,
-			       RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
-			       RTE_FLOW_ITEM_TYPE_GRE),
-	},
-	[RTE_FLOW_ITEM_TYPE_ETH] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_eth){
-			.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
-			.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
-			.type = -1,
-		},
-		.default_mask = &rte_flow_item_eth_mask,
-		.mask_sz = sizeof(struct rte_flow_item_eth),
-		.convert = mlx5_flow_create_eth,
-		.dst_sz = sizeof(struct ibv_flow_spec_eth),
-	},
-	[RTE_FLOW_ITEM_TYPE_VLAN] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vlan){
-			.tci = -1,
-			.inner_type = -1,
-		},
-		.default_mask = &rte_flow_item_vlan_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vlan),
-		.convert = mlx5_flow_create_vlan,
-		.dst_sz = 0,
-	},
-	[RTE_FLOW_ITEM_TYPE_IPV4] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
-			       RTE_FLOW_ITEM_TYPE_TCP,
-			       RTE_FLOW_ITEM_TYPE_GRE),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_ipv4){
-			.hdr = {
-				.src_addr = -1,
-				.dst_addr = -1,
-				.type_of_service = -1,
-				.next_proto_id = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_ipv4_mask,
-		.mask_sz = sizeof(struct rte_flow_item_ipv4),
-		.convert = mlx5_flow_create_ipv4,
-		.dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
-	},
-	[RTE_FLOW_ITEM_TYPE_IPV6] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
-			       RTE_FLOW_ITEM_TYPE_TCP,
-			       RTE_FLOW_ITEM_TYPE_GRE),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_ipv6){
-			.hdr = {
-				.src_addr = {
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-				},
-				.dst_addr = {
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-				},
-				.vtc_flow = -1,
-				.proto = -1,
-				.hop_limits = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_ipv6_mask,
-		.mask_sz = sizeof(struct rte_flow_item_ipv6),
-		.convert = mlx5_flow_create_ipv6,
-		.dst_sz = sizeof(struct ibv_flow_spec_ipv6),
-	},
-	[RTE_FLOW_ITEM_TYPE_UDP] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN,
-			       RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
-			       RTE_FLOW_ITEM_TYPE_MPLS),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_udp){
-			.hdr = {
-				.src_port = -1,
-				.dst_port = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_udp_mask,
-		.mask_sz = sizeof(struct rte_flow_item_udp),
-		.convert = mlx5_flow_create_udp,
-		.dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
-	},
-	[RTE_FLOW_ITEM_TYPE_TCP] = {
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_tcp){
-			.hdr = {
-				.src_port = -1,
-				.dst_port = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_tcp_mask,
-		.mask_sz = sizeof(struct rte_flow_item_tcp),
-		.convert = mlx5_flow_create_tcp,
-		.dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
-	},
-	[RTE_FLOW_ITEM_TYPE_GRE] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6,
-			       RTE_FLOW_ITEM_TYPE_MPLS),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_gre){
-			.protocol = -1,
-		},
-		.default_mask = &rte_flow_item_gre_mask,
-		.mask_sz = sizeof(struct rte_flow_item_gre),
-		.convert = mlx5_flow_create_gre,
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-		.dst_sz = sizeof(struct ibv_flow_spec_gre),
-#else
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
-#endif
-	},
-	[RTE_FLOW_ITEM_TYPE_MPLS] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_mpls){
-			.label_tc_s = "\xff\xff\xf0",
-		},
-		.default_mask = &rte_flow_item_mpls_mask,
-		.mask_sz = sizeof(struct rte_flow_item_mpls),
-		.convert = mlx5_flow_create_mpls,
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-		.dst_sz = sizeof(struct ibv_flow_spec_mpls),
-#endif
-	},
-	[RTE_FLOW_ITEM_TYPE_VXLAN] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4, /* For L3 VXLAN. */
-			       RTE_FLOW_ITEM_TYPE_IPV6), /* For L3 VXLAN. */
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vxlan){
-			.vni = "\xff\xff\xff",
-		},
-		.default_mask = &rte_flow_item_vxlan_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vxlan),
-		.convert = mlx5_flow_create_vxlan,
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
-	},
-	[RTE_FLOW_ITEM_TYPE_VXLAN_GPE] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vxlan_gpe){
-			.vni = "\xff\xff\xff",
-		},
-		.default_mask = &rte_flow_item_vxlan_gpe_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vxlan_gpe),
-		.convert = mlx5_flow_create_vxlan_gpe,
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
-	},
-};
-
-/** Structure to pass to the conversion function. */
-struct mlx5_flow_parse {
-	uint32_t inner; /**< Verbs value, set once tunnel is encountered. */
-	uint32_t create:1;
-	/**< Whether resources should remain after a validate. */
-	uint32_t drop:1; /**< Target is a drop queue. */
-	uint32_t mark:1; /**< Mark is present in the flow. */
-	uint32_t count:1; /**< Count is present in the flow. */
-	uint32_t mark_id; /**< Mark identifier. */
-	struct rte_flow_action_rss rss_conf; /**< RSS configuration */
-	uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
-	uint8_t rss_key[40]; /**< copy of the RSS key. */
-	enum hash_rxq_type layer; /**< Last pattern layer detected. */
-	enum hash_rxq_type out_layer; /**< Last outer pattern layer detected. */
-	uint32_t tunnel; /**< Tunnel type of RTE_PTYPE_TUNNEL_XXX. */
-	struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
-	struct {
-		struct ibv_flow_attr *ibv_attr;
-		/**< Pointer to Verbs attributes. */
-		unsigned int offset;
-		/**< Current position or total size of the attribute. */
-		uint64_t hash_fields; /**< Verbs hash fields. */
-	} queue[RTE_DIM(hash_rxq_init)];
-};
-
-static const struct rte_flow_ops mlx5_flow_ops = {
-	.validate = mlx5_flow_validate,
-	.create = mlx5_flow_create,
-	.destroy = mlx5_flow_destroy,
-	.flush = mlx5_flow_flush,
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	.query = mlx5_flow_query,
-#else
-	.query = NULL,
-#endif
-	.isolate = mlx5_flow_isolate,
-};
-
-/* Convert FDIR request to Generic flow. */
-struct mlx5_fdir {
-	struct rte_flow_attr attr;
-	struct rte_flow_action actions[2];
-	struct rte_flow_item items[4];
-	struct rte_flow_item_eth l2;
-	struct rte_flow_item_eth l2_mask;
-	union {
-		struct rte_flow_item_ipv4 ipv4;
-		struct rte_flow_item_ipv6 ipv6;
-	} l3;
-	union {
-		struct rte_flow_item_ipv4 ipv4;
-		struct rte_flow_item_ipv6 ipv6;
-	} l3_mask;
-	union {
-		struct rte_flow_item_udp udp;
-		struct rte_flow_item_tcp tcp;
-	} l4;
-	union {
-		struct rte_flow_item_udp udp;
-		struct rte_flow_item_tcp tcp;
-	} l4_mask;
-	struct rte_flow_action_queue queue;
-};
-
-/* Verbs specification header. */
-struct ibv_spec_header {
-	enum ibv_flow_spec_type type;
-	uint16_t size;
-};
-
-/**
- * Check item is fully supported by the NIC matching capability.
- *
- * @param item[in]
- *   Item specification.
- * @param mask[in]
- *   Bit-masks covering supported fields to compare with spec, last and mask in
- *   \item.
- * @param size
- *   Bit-Mask size in bytes.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_item_validate(const struct rte_flow_item *item,
-			const uint8_t *mask, unsigned int size)
-{
-	unsigned int i;
-	const uint8_t *spec = item->spec;
-	const uint8_t *last = item->last;
-	const uint8_t *m = item->mask ? item->mask : mask;
-
-	if (!spec && (item->mask || last))
-		goto error;
-	if (!spec)
-		return 0;
-	/*
-	 * Single-pass check to make sure that:
-	 * - item->mask is supported, no bits are set outside mask.
-	 * - Both masked item->spec and item->last are equal (no range
-	 *   supported).
-	 */
-	for (i = 0; i < size; i++) {
-		if (!m[i])
-			continue;
-		if ((m[i] | mask[i]) != mask[i])
-			goto error;
-		if (last && ((spec[i] & m[i]) != (last[i] & m[i])))
-			goto error;
-	}
-	return 0;
-error:
-	rte_errno = ENOTSUP;
-	return -rte_errno;
-}
-
-/**
- * Extract attribute to the parser.
- *
- * @param[in] attr
- *   Flow rule attributes.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_attributes(const struct rte_flow_attr *attr,
-			     struct rte_flow_error *error)
-{
-	if (attr->group) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
-				   NULL,
-				   "groups are not supported");
-		return -rte_errno;
-	}
-	if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
-				   NULL,
-				   "priorities are not supported");
-		return -rte_errno;
-	}
-	if (attr->egress) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
-				   NULL,
-				   "egress is not supported");
-		return -rte_errno;
-	}
-	if (attr->transfer) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
-				   NULL,
-				   "transfer is not supported");
-		return -rte_errno;
-	}
-	if (!attr->ingress) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
-				   NULL,
-				   "only ingress is supported");
-		return -rte_errno;
-	}
-	return 0;
-}
-
-/**
- * Extract actions request to the parser.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[in] actions
- *   Associated actions (list terminated by the END action).
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_actions(struct rte_eth_dev *dev,
-			  const struct rte_flow_action actions[],
-			  struct rte_flow_error *error,
-			  struct mlx5_flow_parse *parser)
-{
-	enum { FATE = 1, MARK = 2, COUNT = 4, };
-	uint32_t overlap = 0;
-	struct priv *priv = dev->data->dev_private;
-
-	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
-		if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
-			continue;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			parser->drop = 1;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
-			const struct rte_flow_action_queue *queue =
-				(const struct rte_flow_action_queue *)
-				actions->conf;
-
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			if (!queue || (queue->index > (priv->rxqs_n - 1)))
-				goto exit_action_not_supported;
-			parser->queues[0] = queue->index;
-			parser->rss_conf = (struct rte_flow_action_rss){
-				.queue_num = 1,
-				.queue = parser->queues,
-			};
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
-			const struct rte_flow_action_rss *rss =
-				(const struct rte_flow_action_rss *)
-				actions->conf;
-			const uint8_t *rss_key;
-			uint32_t rss_key_len;
-			uint16_t n;
-
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			if (rss->func &&
-			    rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "the only supported RSS hash"
-						   " function is Toeplitz");
-				return -rte_errno;
-			}
-#ifndef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-			if (parser->rss_conf.level > 1) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "a nonzero RSS encapsulation"
-						   " level is not supported");
-				return -rte_errno;
-			}
-#endif
-			if (parser->rss_conf.level > 2) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "RSS encapsulation level"
-						   " > 1 is not supported");
-				return -rte_errno;
-			}
-			if (rss->types & MLX5_RSS_HF_MASK) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "unsupported RSS type"
-						   " requested");
-				return -rte_errno;
-			}
-			if (rss->key_len) {
-				rss_key_len = rss->key_len;
-				rss_key = rss->key;
-			} else {
-				rss_key_len = rss_hash_default_key_len;
-				rss_key = rss_hash_default_key;
-			}
-			if (rss_key_len != RTE_DIM(parser->rss_key)) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "RSS hash key must be"
-						   " exactly 40 bytes long");
-				return -rte_errno;
-			}
-			if (!rss->queue_num) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "no valid queues");
-				return -rte_errno;
-			}
-			if (rss->queue_num > RTE_DIM(parser->queues)) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "too many queues for RSS"
-						   " context");
-				return -rte_errno;
-			}
-			for (n = 0; n < rss->queue_num; ++n) {
-				if (rss->queue[n] >= priv->rxqs_n) {
-					rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "queue id > number of"
-						   " queues");
-					return -rte_errno;
-				}
-			}
-			parser->rss_conf = (struct rte_flow_action_rss){
-				.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
-				.level = rss->level ? rss->level : 1,
-				.types = rss->types,
-				.key_len = rss_key_len,
-				.queue_num = rss->queue_num,
-				.key = memcpy(parser->rss_key, rss_key,
-					      sizeof(*rss_key) * rss_key_len),
-				.queue = memcpy(parser->queues, rss->queue,
-						sizeof(*rss->queue) *
-						rss->queue_num),
-			};
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
-			const struct rte_flow_action_mark *mark =
-				(const struct rte_flow_action_mark *)
-				actions->conf;
-
-			if (overlap & MARK)
-				goto exit_action_overlap;
-			overlap |= MARK;
-			if (!mark) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "mark must be defined");
-				return -rte_errno;
-			} else if (mark->id >= MLX5_FLOW_MARK_MAX) {
-				rte_flow_error_set(error, ENOTSUP,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "mark must be between 0"
-						   " and 16777199");
-				return -rte_errno;
-			}
-			parser->mark = 1;
-			parser->mark_id = mark->id;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
-			if (overlap & MARK)
-				goto exit_action_overlap;
-			overlap |= MARK;
-			parser->mark = 1;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
-			   priv->config.flow_counter_en) {
-			if (overlap & COUNT)
-				goto exit_action_overlap;
-			overlap |= COUNT;
-			parser->count = 1;
-		} else {
-			goto exit_action_not_supported;
-		}
-	}
-	/* When fate is unknown, drop traffic. */
-	if (!(overlap & FATE))
-		parser->drop = 1;
-	if (parser->drop && parser->mark)
-		parser->mark = 0;
-	if (!parser->rss_conf.queue_num && !parser->drop) {
-		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "no valid action");
-		return -rte_errno;
-	}
-	return 0;
-exit_action_not_supported:
-	rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
-			   actions, "action not supported");
-	return -rte_errno;
-exit_action_overlap:
-	rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
-			   actions, "overlapping actions are not supported");
-	return -rte_errno;
-}
-
-/**
- * Validate items.
- *
- * @param[in] items
- *   Pattern specification (list terminated by the END pattern item).
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_items_validate(struct rte_eth_dev *dev,
-				 const struct rte_flow_item items[],
-				 struct rte_flow_error *error,
-				 struct mlx5_flow_parse *parser)
-{
-	struct priv *priv = dev->data->dev_private;
-	const struct mlx5_flow_items *cur_item = mlx5_flow_items;
-	unsigned int i;
-	unsigned int last_voids = 0;
-	int ret = 0;
-
-	/* Initialise the offsets to start after verbs attribute. */
-	for (i = 0; i != hash_rxq_init_n; ++i)
-		parser->queue[i].offset = sizeof(struct ibv_flow_attr);
-	for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
-		const struct mlx5_flow_items *token = NULL;
-		unsigned int n;
-
-		if (items->type == RTE_FLOW_ITEM_TYPE_VOID) {
-			last_voids++;
-			continue;
-		}
-		for (i = 0;
-		     cur_item->items &&
-		     cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
-		     ++i) {
-			if (cur_item->items[i] == items->type) {
-				token = &mlx5_flow_items[items->type];
-				break;
-			}
-		}
-		if (!token) {
-			ret = -ENOTSUP;
-			goto exit_item_not_supported;
-		}
-		cur_item = token;
-		ret = mlx5_flow_item_validate(items,
-					      (const uint8_t *)cur_item->mask,
-					      cur_item->mask_sz);
-		if (ret)
-			goto exit_item_not_supported;
-		if (IS_TUNNEL(items->type)) {
-			if (parser->tunnel &&
-			    !((items - last_voids - 1)->type ==
-			      RTE_FLOW_ITEM_TYPE_GRE && items->type ==
-			      RTE_FLOW_ITEM_TYPE_MPLS)) {
-				rte_flow_error_set(error, ENOTSUP,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   items,
-						   "Cannot recognize multiple"
-						   " tunnel encapsulations.");
-				return -rte_errno;
-			}
-			if (items->type == RTE_FLOW_ITEM_TYPE_MPLS &&
-			    !priv->config.mpls_en) {
-				rte_flow_error_set(error, ENOTSUP,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   items,
-						   "MPLS not supported or"
-						   " disabled in firmware"
-						   " configuration.");
-				return -rte_errno;
-			}
-			if (!priv->config.tunnel_en &&
-			    parser->rss_conf.level > 1) {
-				rte_flow_error_set(error, ENOTSUP,
-					RTE_FLOW_ERROR_TYPE_ITEM,
-					items,
-					"RSS on tunnel is not supported");
-				return -rte_errno;
-			}
-			parser->inner = IBV_FLOW_SPEC_INNER;
-			parser->tunnel = flow_ptype[items->type];
-		}
-		if (parser->drop) {
-			parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
-		} else {
-			for (n = 0; n != hash_rxq_init_n; ++n)
-				parser->queue[n].offset += cur_item->dst_sz;
-		}
-		last_voids = 0;
-	}
-	if (parser->drop) {
-		parser->queue[HASH_RXQ_ETH].offset +=
-			sizeof(struct ibv_flow_spec_action_drop);
-	}
-	if (parser->mark) {
-		for (i = 0; i != hash_rxq_init_n; ++i)
-			parser->queue[i].offset +=
-				sizeof(struct ibv_flow_spec_action_tag);
-	}
-	if (parser->count) {
-		unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
-
-		for (i = 0; i != hash_rxq_init_n; ++i)
-			parser->queue[i].offset += size;
-	}
-	return 0;
-exit_item_not_supported:
-	return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
-				  items, "item not supported");
-}
-
-/**
- * Allocate memory space to store verbs flow attributes.
- *
- * @param[in] size
- *   Amount of byte to allocate.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   A verbs flow attribute on success, NULL otherwise and rte_errno is set.
- */
-static struct ibv_flow_attr *
-mlx5_flow_convert_allocate(unsigned int size, struct rte_flow_error *error)
-{
-	struct ibv_flow_attr *ibv_attr;
-
-	ibv_attr = rte_calloc(__func__, 1, size, 0);
-	if (!ibv_attr) {
-		rte_flow_error_set(error, ENOMEM,
-				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				   NULL,
-				   "cannot allocate verbs spec attributes");
-		return NULL;
-	}
-	return ibv_attr;
-}
-
-/**
- * Make inner packet matching with an higher priority from the non Inner
- * matching.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[in, out] parser
- *   Internal parser structure.
- * @param attr
- *   User flow attribute.
- */
-static void
-mlx5_flow_update_priority(struct rte_eth_dev *dev,
-			  struct mlx5_flow_parse *parser,
-			  const struct rte_flow_attr *attr)
-{
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-	uint16_t priority;
-
-	/*			8 priorities	>= 16 priorities
-	 * Control flow:	4-7		8-15
-	 * User normal flow:	1-3		4-7
-	 * User tunnel flow:	0-2		0-3
-	 */
-	priority = attr->priority * MLX5_VERBS_FLOW_PRIO_8;
-	if (priv->config.max_verbs_prio == MLX5_VERBS_FLOW_PRIO_8)
-		priority /= 2;
-	/*
-	 * Lower non-tunnel flow Verbs priority 1 if only support 8 Verbs
-	 * priorities, lower 4 otherwise.
-	 */
-	if (!parser->inner) {
-		if (priv->config.max_verbs_prio == MLX5_VERBS_FLOW_PRIO_8)
-			priority += 1;
-		else
-			priority += MLX5_VERBS_FLOW_PRIO_8 / 2;
-	}
-	if (parser->drop) {
-		parser->queue[HASH_RXQ_ETH].ibv_attr->priority = priority +
-				hash_rxq_init[HASH_RXQ_ETH].flow_priority;
-		return;
-	}
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		parser->queue[i].ibv_attr->priority = priority +
-				hash_rxq_init[i].flow_priority;
-	}
-}
-
-/**
- * Finalise verbs flow attributes.
- *
- * @param[in, out] parser
- *   Internal parser structure.
- */
-static void
-mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser)
-{
-	unsigned int i;
-	uint32_t inner = parser->inner;
-
-	/* Don't create extra flows for outer RSS. */
-	if (parser->tunnel && parser->rss_conf.level < 2)
-		return;
-	/*
-	 * Fill missing layers in verbs specifications, or compute the correct
-	 * offset to allocate the memory space for the attributes and
-	 * specifications.
-	 */
-	for (i = 0; i != hash_rxq_init_n - 1; ++i) {
-		union {
-			struct ibv_flow_spec_ipv4_ext ipv4;
-			struct ibv_flow_spec_ipv6 ipv6;
-			struct ibv_flow_spec_tcp_udp udp_tcp;
-			struct ibv_flow_spec_eth eth;
-		} specs;
-		void *dst;
-		uint16_t size;
-
-		if (i == parser->layer)
-			continue;
-		if (parser->layer == HASH_RXQ_ETH ||
-		    parser->layer == HASH_RXQ_TUNNEL) {
-			if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
-				size = sizeof(struct ibv_flow_spec_ipv4_ext);
-				specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
-					.type = inner | IBV_FLOW_SPEC_IPV4_EXT,
-					.size = size,
-				};
-			} else {
-				size = sizeof(struct ibv_flow_spec_ipv6);
-				specs.ipv6 = (struct ibv_flow_spec_ipv6){
-					.type = inner | IBV_FLOW_SPEC_IPV6,
-					.size = size,
-				};
-			}
-			if (parser->queue[i].ibv_attr) {
-				dst = (void *)((uintptr_t)
-					       parser->queue[i].ibv_attr +
-					       parser->queue[i].offset);
-				memcpy(dst, &specs, size);
-				++parser->queue[i].ibv_attr->num_of_specs;
-			}
-			parser->queue[i].offset += size;
-		}
-		if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
-		    (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
-			size = sizeof(struct ibv_flow_spec_tcp_udp);
-			specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
-				.type = inner | ((i == HASH_RXQ_UDPV4 ||
-					  i == HASH_RXQ_UDPV6) ?
-					 IBV_FLOW_SPEC_UDP :
-					 IBV_FLOW_SPEC_TCP),
-				.size = size,
-			};
-			if (parser->queue[i].ibv_attr) {
-				dst = (void *)((uintptr_t)
-					       parser->queue[i].ibv_attr +
-					       parser->queue[i].offset);
-				memcpy(dst, &specs, size);
-				++parser->queue[i].ibv_attr->num_of_specs;
-			}
-			parser->queue[i].offset += size;
-		}
-	}
-}
-
-/**
- * Update flows according to pattern and RSS hash fields.
- *
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_rss(struct mlx5_flow_parse *parser)
-{
-	unsigned int i;
-	enum hash_rxq_type start;
-	enum hash_rxq_type layer;
-	int outer = parser->tunnel && parser->rss_conf.level < 2;
-	uint64_t rss = parser->rss_conf.types;
-
-	layer = outer ? parser->out_layer : parser->layer;
-	if (layer == HASH_RXQ_TUNNEL)
-		layer = HASH_RXQ_ETH;
-	if (outer) {
-		/* Only one hash type for outer RSS. */
-		if (rss && layer == HASH_RXQ_ETH) {
-			start = HASH_RXQ_TCPV4;
-		} else if (rss && layer != HASH_RXQ_ETH &&
-			   !(rss & hash_rxq_init[layer].dpdk_rss_hf)) {
-			/* If RSS not match L4 pattern, try L3 RSS. */
-			if (layer < HASH_RXQ_IPV4)
-				layer = HASH_RXQ_IPV4;
-			else if (layer > HASH_RXQ_IPV4 && layer < HASH_RXQ_IPV6)
-				layer = HASH_RXQ_IPV6;
-			start = layer;
-		} else {
-			start = layer;
-		}
-		/* Scan first valid hash type. */
-		for (i = start; rss && i <= layer; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-			if (hash_rxq_init[i].dpdk_rss_hf & rss)
-				break;
-		}
-		if (rss && i <= layer)
-			parser->queue[layer].hash_fields =
-					hash_rxq_init[i].hash_fields;
-		/* Trim unused hash types. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (parser->queue[i].ibv_attr && i != layer) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			}
-		}
-	} else {
-		/* Expand for inner or normal RSS. */
-		if (rss && (layer == HASH_RXQ_ETH || layer == HASH_RXQ_IPV4))
-			start = HASH_RXQ_TCPV4;
-		else if (rss && layer == HASH_RXQ_IPV6)
-			start = HASH_RXQ_TCPV6;
-		else
-			start = layer;
-		/* For L4 pattern, try L3 RSS if no L4 RSS. */
-		/* Trim unused hash types. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-			if (i < start || i > layer) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-				continue;
-			}
-			if (!rss)
-				continue;
-			if (hash_rxq_init[i].dpdk_rss_hf & rss) {
-				parser->queue[i].hash_fields =
-						hash_rxq_init[i].hash_fields;
-			} else if (i != layer) {
-				/* Remove unused RSS expansion. */
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			} else if (layer < HASH_RXQ_IPV4 &&
-				   (hash_rxq_init[HASH_RXQ_IPV4].dpdk_rss_hf &
-				    rss)) {
-				/* Allow IPv4 RSS on L4 pattern. */
-				parser->queue[i].hash_fields =
-					hash_rxq_init[HASH_RXQ_IPV4]
-						.hash_fields;
-			} else if (i > HASH_RXQ_IPV4 && i < HASH_RXQ_IPV6 &&
-				   (hash_rxq_init[HASH_RXQ_IPV6].dpdk_rss_hf &
-				    rss)) {
-				/* Allow IPv4 RSS on L4 pattern. */
-				parser->queue[i].hash_fields =
-					hash_rxq_init[HASH_RXQ_IPV6]
-						.hash_fields;
-			}
-		}
-	}
-	return 0;
-}
-
-/**
- * Validate and convert a flow supported by the NIC.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[in] attr
- *   Flow rule attributes.
- * @param[in] pattern
- *   Pattern specification (list terminated by the END pattern item).
- * @param[in] actions
- *   Associated actions (list terminated by the END action).
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert(struct rte_eth_dev *dev,
-		  const struct rte_flow_attr *attr,
-		  const struct rte_flow_item items[],
-		  const struct rte_flow_action actions[],
-		  struct rte_flow_error *error,
-		  struct mlx5_flow_parse *parser)
-{
-	const struct mlx5_flow_items *cur_item = mlx5_flow_items;
-	unsigned int i;
-	int ret;
-
-	/* First step. Validate the attributes, items and actions. */
-	*parser = (struct mlx5_flow_parse){
-		.create = parser->create,
-		.layer = HASH_RXQ_ETH,
-		.mark_id = MLX5_FLOW_MARK_DEFAULT,
-	};
-	ret = mlx5_flow_convert_attributes(attr, error);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert_actions(dev, actions, error, parser);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert_items_validate(dev, items, error, parser);
-	if (ret)
-		return ret;
-	mlx5_flow_convert_finalise(parser);
-	/*
-	 * Second step.
-	 * Allocate the memory space to store verbs specifications.
-	 */
-	if (parser->drop) {
-		unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
-
-		parser->queue[HASH_RXQ_ETH].ibv_attr =
-			mlx5_flow_convert_allocate(offset, error);
-		if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
-			goto exit_enomem;
-		parser->queue[HASH_RXQ_ETH].offset =
-			sizeof(struct ibv_flow_attr);
-	} else {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			unsigned int offset;
-
-			offset = parser->queue[i].offset;
-			parser->queue[i].ibv_attr =
-				mlx5_flow_convert_allocate(offset, error);
-			if (!parser->queue[i].ibv_attr)
-				goto exit_enomem;
-			parser->queue[i].offset = sizeof(struct ibv_flow_attr);
-		}
-	}
-	/* Third step. Conversion parse, fill the specifications. */
-	parser->inner = 0;
-	parser->tunnel = 0;
-	parser->layer = HASH_RXQ_ETH;
-	for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
-		struct mlx5_flow_data data = {
-			.dev = dev,
-			.parser = parser,
-			.error = error,
-		};
-
-		if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
-			continue;
-		cur_item = &mlx5_flow_items[items->type];
-		ret = cur_item->convert(items,
-					(cur_item->default_mask ?
-					 cur_item->default_mask :
-					 cur_item->mask),
-					 &data);
-		if (ret)
-			goto exit_free;
-	}
-	if (!parser->drop) {
-		/* RSS check, remove unused hash types. */
-		ret = mlx5_flow_convert_rss(parser);
-		if (ret)
-			goto exit_free;
-		/* Complete missing specification. */
-		mlx5_flow_convert_finalise(parser);
-	}
-	mlx5_flow_update_priority(dev, parser, attr);
-	if (parser->mark)
-		mlx5_flow_create_flag_mark(parser, parser->mark_id);
-	if (parser->count && parser->create) {
-		mlx5_flow_create_count(dev, parser);
-		if (!parser->cs)
-			goto exit_count_error;
-	}
-exit_free:
-	/* Only verification is expected, all resources should be released. */
-	if (!parser->create) {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (parser->queue[i].ibv_attr) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			}
-		}
-	}
-	return ret;
-exit_enomem:
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser->queue[i].ibv_attr) {
-			rte_free(parser->queue[i].ibv_attr);
-			parser->queue[i].ibv_attr = NULL;
-		}
-	}
-	rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL, "cannot allocate verbs spec attributes");
-	return -rte_errno;
-exit_count_error:
-	rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL, "cannot create counter");
-	return -rte_errno;
-}
-
-/**
- * Copy the specification created into the flow.
- *
- * @param parser
- *   Internal parser structure.
- * @param src
- *   Create specification.
- * @param size
- *   Size in bytes of the specification to copy.
- */
-static void
-mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
-		      unsigned int size)
-{
-	unsigned int i;
-	void *dst;
-
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset);
-		memcpy(dst, src, size);
-		++parser->queue[i].ibv_attr->num_of_specs;
-		parser->queue[i].offset += size;
-	}
-}
-
-/**
- * Convert Ethernet item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_eth(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_eth *spec = item->spec;
-	const struct rte_flow_item_eth *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
-	struct ibv_flow_spec_eth eth = {
-		.type = parser->inner | IBV_FLOW_SPEC_ETH,
-		.size = eth_size,
-	};
-
-	parser->layer = HASH_RXQ_ETH;
-	if (spec) {
-		unsigned int i;
-
-		if (!mask)
-			mask = default_mask;
-		memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
-		memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
-		eth.val.ether_type = spec->type;
-		memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
-		memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
-		eth.mask.ether_type = mask->type;
-		/* Remove unwanted bits from values. */
-		for (i = 0; i < ETHER_ADDR_LEN; ++i) {
-			eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
-			eth.val.src_mac[i] &= eth.mask.src_mac[i];
-		}
-		eth.val.ether_type &= eth.mask.ether_type;
-	}
-	mlx5_flow_create_copy(parser, &eth, eth_size);
-	return 0;
-}
-
-/**
- * Convert VLAN item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_vlan(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_vlan *spec = item->spec;
-	const struct rte_flow_item_vlan *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	struct ibv_flow_spec_eth *eth;
-	const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
-	const char *msg = "VLAN cannot be empty";
-
-	if (spec) {
-		unsigned int i;
-		if (!mask)
-			mask = default_mask;
-
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-
-			eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				       parser->queue[i].offset - eth_size);
-			eth->val.vlan_tag = spec->tci;
-			eth->mask.vlan_tag = mask->tci;
-			eth->val.vlan_tag &= eth->mask.vlan_tag;
-			/*
-			 * From verbs perspective an empty VLAN is equivalent
-			 * to a packet without VLAN layer.
-			 */
-			if (!eth->mask.vlan_tag)
-				goto error;
-			/* Outer TPID cannot be matched. */
-			if (eth->mask.ether_type) {
-				msg = "VLAN TPID matching is not supported";
-				goto error;
-			}
-			eth->val.ether_type = spec->inner_type;
-			eth->mask.ether_type = mask->inner_type;
-			eth->val.ether_type &= eth->mask.ether_type;
-		}
-		return 0;
-	}
-error:
-	return rte_flow_error_set(data->error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
-				  item, msg);
-}
-
-/**
- * Convert IPv4 item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_ipv4(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-	struct priv *priv = data->dev->data->dev_private;
-	const struct rte_flow_item_ipv4 *spec = item->spec;
-	const struct rte_flow_item_ipv4 *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
-	struct ibv_flow_spec_ipv4_ext ipv4 = {
-		.type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
-		.size = ipv4_size,
-	};
-
-	if (parser->layer == HASH_RXQ_TUNNEL &&
-	    parser->tunnel == ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] &&
-	    !priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	parser->layer = HASH_RXQ_IPV4;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
-			.src_ip = spec->hdr.src_addr,
-			.dst_ip = spec->hdr.dst_addr,
-			.proto = spec->hdr.next_proto_id,
-			.tos = spec->hdr.type_of_service,
-		};
-		ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
-			.src_ip = mask->hdr.src_addr,
-			.dst_ip = mask->hdr.dst_addr,
-			.proto = mask->hdr.next_proto_id,
-			.tos = mask->hdr.type_of_service,
-		};
-		/* Remove unwanted bits from values. */
-		ipv4.val.src_ip &= ipv4.mask.src_ip;
-		ipv4.val.dst_ip &= ipv4.mask.dst_ip;
-		ipv4.val.proto &= ipv4.mask.proto;
-		ipv4.val.tos &= ipv4.mask.tos;
-	}
-	mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
-	return 0;
-}
-
-/**
- * Convert IPv6 item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_ipv6(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-	struct priv *priv = data->dev->data->dev_private;
-	const struct rte_flow_item_ipv6 *spec = item->spec;
-	const struct rte_flow_item_ipv6 *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
-	struct ibv_flow_spec_ipv6 ipv6 = {
-		.type = parser->inner | IBV_FLOW_SPEC_IPV6,
-		.size = ipv6_size,
-	};
-
-	if (parser->layer == HASH_RXQ_TUNNEL &&
-	    parser->tunnel == ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] &&
-	    !priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	parser->layer = HASH_RXQ_IPV6;
-	if (spec) {
-		unsigned int i;
-		uint32_t vtc_flow_val;
-		uint32_t vtc_flow_mask;
-
-		if (!mask)
-			mask = default_mask;
-		memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
-		       RTE_DIM(ipv6.val.src_ip));
-		memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
-		       RTE_DIM(ipv6.val.dst_ip));
-		memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
-		       RTE_DIM(ipv6.mask.src_ip));
-		memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
-		       RTE_DIM(ipv6.mask.dst_ip));
-		vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
-		vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
-		ipv6.val.flow_label =
-			rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
-					 IPV6_HDR_FL_SHIFT);
-		ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
-					 IPV6_HDR_TC_SHIFT;
-		ipv6.val.next_hdr = spec->hdr.proto;
-		ipv6.val.hop_limit = spec->hdr.hop_limits;
-		ipv6.mask.flow_label =
-			rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
-					 IPV6_HDR_FL_SHIFT);
-		ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
-					  IPV6_HDR_TC_SHIFT;
-		ipv6.mask.next_hdr = mask->hdr.proto;
-		ipv6.mask.hop_limit = mask->hdr.hop_limits;
-		/* Remove unwanted bits from values. */
-		for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
-			ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
-			ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
-		}
-		ipv6.val.flow_label &= ipv6.mask.flow_label;
-		ipv6.val.traffic_class &= ipv6.mask.traffic_class;
-		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
-		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
-	}
-	mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
-	return 0;
-}
-
-/**
- * Convert UDP item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_udp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_udp *spec = item->spec;
-	const struct rte_flow_item_udp *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
-	struct ibv_flow_spec_tcp_udp udp = {
-		.type = parser->inner | IBV_FLOW_SPEC_UDP,
-		.size = udp_size,
-	};
-
-	if (parser->layer == HASH_RXQ_IPV4)
-		parser->layer = HASH_RXQ_UDPV4;
-	else
-		parser->layer = HASH_RXQ_UDPV6;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		udp.val.dst_port = spec->hdr.dst_port;
-		udp.val.src_port = spec->hdr.src_port;
-		udp.mask.dst_port = mask->hdr.dst_port;
-		udp.mask.src_port = mask->hdr.src_port;
-		/* Remove unwanted bits from values. */
-		udp.val.src_port &= udp.mask.src_port;
-		udp.val.dst_port &= udp.mask.dst_port;
-	}
-	mlx5_flow_create_copy(parser, &udp, udp_size);
-	return 0;
-}
-
-/**
- * Convert TCP item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_tcp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_tcp *spec = item->spec;
-	const struct rte_flow_item_tcp *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
-	struct ibv_flow_spec_tcp_udp tcp = {
-		.type = parser->inner | IBV_FLOW_SPEC_TCP,
-		.size = tcp_size,
-	};
-
-	if (parser->layer == HASH_RXQ_IPV4)
-		parser->layer = HASH_RXQ_TCPV4;
-	else
-		parser->layer = HASH_RXQ_TCPV6;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		tcp.val.dst_port = spec->hdr.dst_port;
-		tcp.val.src_port = spec->hdr.src_port;
-		tcp.mask.dst_port = mask->hdr.dst_port;
-		tcp.mask.src_port = mask->hdr.src_port;
-		/* Remove unwanted bits from values. */
-		tcp.val.src_port &= tcp.mask.src_port;
-		tcp.val.dst_port &= tcp.mask.dst_port;
-	}
-	mlx5_flow_create_copy(parser, &tcp, tcp_size);
-	return 0;
-}
-
-/**
- * Convert VXLAN item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_vxlan(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_vxlan *spec = item->spec;
-	const struct rte_flow_item_vxlan *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel vxlan = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
-	union vni {
-		uint32_t vlan_id;
-		uint8_t vni[4];
-	} id;
-
-	id.vni[0] = 0;
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		memcpy(&id.vni[1], spec->vni, 3);
-		vxlan.val.tunnel_id = id.vlan_id;
-		memcpy(&id.vni[1], mask->vni, 3);
-		vxlan.mask.tunnel_id = id.vlan_id;
-		/* Remove unwanted bits from values. */
-		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
-	}
-	/*
-	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
-	 * layer is defined in the Verbs specification it is interpreted as
-	 * wildcard and all packets will match this rule, if it follows a full
-	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
-	 * before will also match this rule.
-	 * To avoid such situation, VNI 0 is currently refused.
-	 */
-	/* Only allow tunnel w/o tunnel id pattern after proper outer spec. */
-	if (parser->out_layer == HASH_RXQ_ETH && !vxlan.val.tunnel_id)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "VxLAN vni cannot be 0");
-	mlx5_flow_create_copy(parser, &vxlan, size);
-	return 0;
-}
-
-/**
- * Convert VXLAN-GPE item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_vxlan_gpe(const struct rte_flow_item *item,
-			   const void *default_mask,
-			   struct mlx5_flow_data *data)
-{
-	struct priv *priv = data->dev->data->dev_private;
-	const struct rte_flow_item_vxlan_gpe *spec = item->spec;
-	const struct rte_flow_item_vxlan_gpe *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel vxlan = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
-	union vni {
-		uint32_t vlan_id;
-		uint8_t vni[4];
-	} id;
-
-	if (!priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	id.vni[0] = 0;
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN_GPE)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		memcpy(&id.vni[1], spec->vni, 3);
-		vxlan.val.tunnel_id = id.vlan_id;
-		memcpy(&id.vni[1], mask->vni, 3);
-		vxlan.mask.tunnel_id = id.vlan_id;
-		if (spec->protocol)
-			return rte_flow_error_set(data->error, EINVAL,
-						  RTE_FLOW_ERROR_TYPE_ITEM,
-						  item,
-						  "VxLAN-GPE protocol not"
-						  " supported");
-		/* Remove unwanted bits from values. */
-		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
-	}
-	/*
-	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
-	 * layer is defined in the Verbs specification it is interpreted as
-	 * wildcard and all packets will match this rule, if it follows a full
-	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
-	 * before will also match this rule.
-	 * To avoid such situation, VNI 0 is currently refused.
-	 */
-	/* Only allow tunnel w/o tunnel id pattern after proper outer spec. */
-	if (parser->out_layer == HASH_RXQ_ETH && !vxlan.val.tunnel_id)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "VxLAN-GPE vni cannot be 0");
-	mlx5_flow_create_copy(parser, &vxlan, size);
-	return 0;
-}
-
-/**
- * Convert GRE item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_gre(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	struct mlx5_flow_parse *parser = data->parser;
-#ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	(void)default_mask;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel tunnel = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
-#else
-	const struct rte_flow_item_gre *spec = item->spec;
-	const struct rte_flow_item_gre *mask = item->mask;
-	unsigned int size = sizeof(struct ibv_flow_spec_gre);
-	struct ibv_flow_spec_gre tunnel = {
-		.type = parser->inner | IBV_FLOW_SPEC_GRE,
-		.size = size,
-	};
-#endif
-	struct ibv_flow_spec_ipv4_ext *ipv4;
-	struct ibv_flow_spec_ipv6 *ipv6;
-	unsigned int i;
-
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_GRE)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
-		tunnel.val.protocol = spec->protocol;
-		tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
-		tunnel.mask.protocol = mask->protocol;
-		/* Remove unwanted bits from values. */
-		tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
-		tunnel.val.protocol &= tunnel.mask.protocol;
-		tunnel.val.key &= tunnel.mask.key;
-	}
-#endif
-	/* Update encapsulation IP layer protocol. */
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		if (parser->out_layer == HASH_RXQ_IPV4) {
-			ipv4 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset -
-				sizeof(struct ibv_flow_spec_ipv4_ext));
-			if (ipv4->mask.proto && ipv4->val.proto != MLX5_GRE)
-				break;
-			ipv4->val.proto = MLX5_GRE;
-			ipv4->mask.proto = 0xff;
-		} else if (parser->out_layer == HASH_RXQ_IPV6) {
-			ipv6 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset -
-				sizeof(struct ibv_flow_spec_ipv6));
-			if (ipv6->mask.next_hdr &&
-			    ipv6->val.next_hdr != MLX5_GRE)
-				break;
-			ipv6->val.next_hdr = MLX5_GRE;
-			ipv6->mask.next_hdr = 0xff;
-		}
-	}
-	if (i != hash_rxq_init_n)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "IP protocol of GRE must be 47");
-	mlx5_flow_create_copy(parser, &tunnel, size);
-	return 0;
-}
-
-/**
- * Convert MPLS item to Verbs specification.
- * MPLS tunnel types currently supported are MPLS-in-GRE and MPLS-in-UDP.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_mpls(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-#ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	(void)default_mask;
-	return rte_flow_error_set(data->error, ENOTSUP,
-				  RTE_FLOW_ERROR_TYPE_ITEM,
-				  item,
-				  "MPLS is not supported by driver");
-#else
-	const struct rte_flow_item_mpls *spec = item->spec;
-	const struct rte_flow_item_mpls *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int size = sizeof(struct ibv_flow_spec_mpls);
-	struct ibv_flow_spec_mpls mpls = {
-		.type = IBV_FLOW_SPEC_MPLS,
-		.size = size,
-	};
-
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	if (parser->layer == HASH_RXQ_UDPV4 ||
-	    parser->layer == HASH_RXQ_UDPV6) {
-		parser->tunnel =
-			ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_UDP)];
-		parser->out_layer = parser->layer;
-	} else {
-		parser->tunnel =
-			ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_GRE)];
-		/* parser->out_layer stays as in GRE out_layer. */
-	}
-	parser->layer = HASH_RXQ_TUNNEL;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		/*
-		 * The verbs label field includes the entire MPLS header:
-		 * bits 0:19 - label value field.
-		 * bits 20:22 - traffic class field.
-		 * bits 23 - bottom of stack bit.
-		 * bits 24:31 - ttl field.
-		 */
-		mpls.val.label = *(const uint32_t *)spec;
-		mpls.mask.label = *(const uint32_t *)mask;
-		/* Remove unwanted bits from values. */
-		mpls.val.label &= mpls.mask.label;
-	}
-	mlx5_flow_create_copy(parser, &mpls, size);
-	return 0;
-#endif
-}
-
-/**
- * Convert mark/flag action to Verbs specification.
- *
- * @param parser
- *   Internal parser structure.
- * @param mark_id
- *   Mark identifier.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
-{
-	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
-	struct ibv_flow_spec_action_tag tag = {
-		.type = IBV_FLOW_SPEC_ACTION_TAG,
-		.size = size,
-		.tag_id = mlx5_flow_mark_set(mark_id),
-	};
-
-	assert(parser->mark);
-	mlx5_flow_create_copy(parser, &tag, size);
-	return 0;
-}
-
-/**
- * Convert count action to Verbs specification.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Pointer to MLX5 flow parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused,
-		       struct mlx5_flow_parse *parser __rte_unused)
-{
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	struct priv *priv = dev->data->dev_private;
-	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
-	struct ibv_counter_set_init_attr init_attr = {0};
-	struct ibv_flow_spec_counter_action counter = {
-		.type = IBV_FLOW_SPEC_ACTION_COUNT,
-		.size = size,
-		.counter_set_handle = 0,
-	};
-
-	init_attr.counter_set_id = 0;
-	parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr);
-	if (!parser->cs) {
-		rte_errno = EINVAL;
-		return -rte_errno;
-	}
-	counter.counter_set_handle = parser->cs->handle;
-	mlx5_flow_create_copy(parser, &counter, size);
-#endif
-	return 0;
-}
-
-/**
- * Complete flow rule creation with a drop queue.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev,
-				   struct mlx5_flow_parse *parser,
-				   struct rte_flow *flow,
-				   struct rte_flow_error *error)
-{
-	struct priv *priv = dev->data->dev_private;
-	struct ibv_flow_spec_action_drop *drop;
-	unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
-
-	assert(priv->pd);
-	assert(priv->ctx);
-	flow->drop = 1;
-	drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr +
-			parser->queue[HASH_RXQ_ETH].offset);
-	*drop = (struct ibv_flow_spec_action_drop){
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-			.size = size,
-	};
-	++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs;
-	parser->queue[HASH_RXQ_ETH].offset += size;
-	flow->frxq[HASH_RXQ_ETH].ibv_attr =
-		parser->queue[HASH_RXQ_ETH].ibv_attr;
-	if (parser->count)
-		flow->cs = parser->cs;
-	if (!dev->data->dev_started)
-		return 0;
-	parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
-	flow->frxq[HASH_RXQ_ETH].ibv_flow =
-		mlx5_glue->create_flow(priv->flow_drop_queue->qp,
-				       flow->frxq[HASH_RXQ_ETH].ibv_attr);
-	if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-		rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "flow rule creation failure");
-		goto error;
-	}
-	return 0;
-error:
-	assert(flow);
-	if (flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-		claim_zero(mlx5_glue->destroy_flow
-			   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-		flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
-	}
-	if (flow->frxq[HASH_RXQ_ETH].ibv_attr) {
-		rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
-		flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL;
-	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-		parser->cs = NULL;
-	}
-	return -rte_errno;
-}
-
-/**
- * Create hash Rx queues when RSS is enabled.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev,
-				  struct mlx5_flow_parse *parser,
-				  struct rte_flow *flow,
-				  struct rte_flow_error *error)
-{
-	unsigned int i;
-
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
-		parser->queue[i].ibv_attr = NULL;
-		flow->frxq[i].hash_fields = parser->queue[i].hash_fields;
-		if (!dev->data->dev_started)
-			continue;
-		flow->frxq[i].hrxq =
-			mlx5_hrxq_get(dev,
-				      parser->rss_conf.key,
-				      parser->rss_conf.key_len,
-				      flow->frxq[i].hash_fields,
-				      parser->rss_conf.queue,
-				      parser->rss_conf.queue_num,
-				      parser->tunnel,
-				      parser->rss_conf.level);
-		if (flow->frxq[i].hrxq)
-			continue;
-		flow->frxq[i].hrxq =
-			mlx5_hrxq_new(dev,
-				      parser->rss_conf.key,
-				      parser->rss_conf.key_len,
-				      flow->frxq[i].hash_fields,
-				      parser->rss_conf.queue,
-				      parser->rss_conf.queue_num,
-				      parser->tunnel,
-				      parser->rss_conf.level);
-		if (!flow->frxq[i].hrxq) {
-			return rte_flow_error_set(error, ENOMEM,
-						  RTE_FLOW_ERROR_TYPE_HANDLE,
-						  NULL,
-						  "cannot create hash rxq");
-		}
-	}
-	return 0;
-}
-
-/**
- * RXQ update after flow rule creation.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Pointer to the flow rule.
- */
-static void
-mlx5_flow_create_update_rxqs(struct rte_eth_dev *dev, struct rte_flow *flow)
-{
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-	unsigned int j;
-
-	if (!dev->data->dev_started)
-		return;
-	for (i = 0; i != flow->rss_conf.queue_num; ++i) {
-		struct mlx5_rxq_data *rxq_data = (*priv->rxqs)
-						 [(*flow->queues)[i]];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
-		uint8_t tunnel = PTYPE_IDX(flow->tunnel);
-
-		rxq_data->mark |= flow->mark;
-		if (!tunnel)
-			continue;
-		rxq_ctrl->tunnel_types[tunnel] += 1;
-		/* Clear tunnel type if more than one tunnel types set. */
-		for (j = 0; j != RTE_DIM(rxq_ctrl->tunnel_types); ++j) {
-			if (j == tunnel)
-				continue;
-			if (rxq_ctrl->tunnel_types[j] > 0) {
-				rxq_data->tunnel = 0;
-				break;
-			}
-		}
-		if (j == RTE_DIM(rxq_ctrl->tunnel_types))
-			rxq_data->tunnel = flow->tunnel;
-	}
-}
-
-/**
- * Dump flow hash RX queue detail.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Pointer to the rte_flow.
- * @param hrxq_idx
- *   Hash RX queue index.
- */
-static void
-mlx5_flow_dump(struct rte_eth_dev *dev __rte_unused,
-	       struct rte_flow *flow __rte_unused,
-	       unsigned int hrxq_idx __rte_unused)
-{
-#ifndef NDEBUG
-	uintptr_t spec_ptr;
-	uint16_t j;
-	char buf[256];
-	uint8_t off;
-	uint64_t extra_hash_fields = 0;
+/* Dev ops structure defined in mlx5.c */
+extern const struct eth_dev_ops mlx5_dev_ops;
+extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
-#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	if (flow->tunnel && flow->rss_conf.level > 1)
-		extra_hash_fields = (uint32_t)IBV_RX_HASH_INNER;
-#endif
-	spec_ptr = (uintptr_t)(flow->frxq[hrxq_idx].ibv_attr + 1);
-	for (j = 0, off = 0; j < flow->frxq[hrxq_idx].ibv_attr->num_of_specs;
-	     j++) {
-		struct ibv_flow_spec *spec = (void *)spec_ptr;
-		off += sprintf(buf + off, " %x(%hu)", spec->hdr.type,
-			       spec->hdr.size);
-		spec_ptr += spec->hdr.size;
-	}
-	DRV_LOG(DEBUG,
-		"port %u Verbs flow %p type %u: hrxq:%p qp:%p ind:%p,"
-		" hash:%" PRIx64 "/%u specs:%hhu(%hu), priority:%hu, type:%d,"
-		" flags:%x, comp_mask:%x specs:%s",
-		dev->data->port_id, (void *)flow, hrxq_idx,
-		(void *)flow->frxq[hrxq_idx].hrxq,
-		(void *)flow->frxq[hrxq_idx].hrxq->qp,
-		(void *)flow->frxq[hrxq_idx].hrxq->ind_table,
-		(flow->frxq[hrxq_idx].hash_fields | extra_hash_fields),
-		flow->rss_conf.queue_num,
-		flow->frxq[hrxq_idx].ibv_attr->num_of_specs,
-		flow->frxq[hrxq_idx].ibv_attr->size,
-		flow->frxq[hrxq_idx].ibv_attr->priority,
-		flow->frxq[hrxq_idx].ibv_attr->type,
-		flow->frxq[hrxq_idx].ibv_attr->flags,
-		flow->frxq[hrxq_idx].ibv_attr->comp_mask,
-		buf);
-#endif
-}
+struct rte_flow {
+	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
+};
 
-/**
- * Complete flow rule creation.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_action_queue(struct rte_eth_dev *dev,
-			      struct mlx5_flow_parse *parser,
-			      struct rte_flow *flow,
-			      struct rte_flow_error *error)
-{
-	struct priv *priv __rte_unused = dev->data->dev_private;
-	int ret;
-	unsigned int i;
-	unsigned int flows_n = 0;
+static const struct rte_flow_ops mlx5_flow_ops = {
+	.isolate = mlx5_flow_isolate,
+};
 
-	assert(priv->pd);
-	assert(priv->ctx);
-	assert(!parser->drop);
-	ret = mlx5_flow_create_action_queue_rss(dev, parser, flow, error);
-	if (ret)
-		goto error;
-	if (parser->count)
-		flow->cs = parser->cs;
-	if (!dev->data->dev_started)
-		return 0;
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!flow->frxq[i].hrxq)
-			continue;
-		flow->frxq[i].ibv_flow =
-			mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
-					       flow->frxq[i].ibv_attr);
-		mlx5_flow_dump(dev, flow, i);
-		if (!flow->frxq[i].ibv_flow) {
-			rte_flow_error_set(error, ENOMEM,
-					   RTE_FLOW_ERROR_TYPE_HANDLE,
-					   NULL, "flow rule creation failure");
-			goto error;
-		}
-		++flows_n;
-	}
-	if (!flows_n) {
-		rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "internal error in flow creation");
-		goto error;
-	}
-	mlx5_flow_create_update_rxqs(dev, flow);
-	return 0;
-error:
-	ret = rte_errno; /* Save rte_errno before cleanup. */
-	assert(flow);
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (flow->frxq[i].ibv_flow) {
-			struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
+/* Convert FDIR request to Generic flow. */
+struct mlx5_fdir {
+	struct rte_flow_attr attr;
+	struct rte_flow_action actions[2];
+	struct rte_flow_item items[4];
+	struct rte_flow_item_eth l2;
+	struct rte_flow_item_eth l2_mask;
+	union {
+		struct rte_flow_item_ipv4 ipv4;
+		struct rte_flow_item_ipv6 ipv6;
+	} l3;
+	union {
+		struct rte_flow_item_ipv4 ipv4;
+		struct rte_flow_item_ipv6 ipv6;
+	} l3_mask;
+	union {
+		struct rte_flow_item_udp udp;
+		struct rte_flow_item_tcp tcp;
+	} l4;
+	union {
+		struct rte_flow_item_udp udp;
+		struct rte_flow_item_tcp tcp;
+	} l4_mask;
+	struct rte_flow_action_queue queue;
+};
 
-			claim_zero(mlx5_glue->destroy_flow(ibv_flow));
-		}
-		if (flow->frxq[i].hrxq)
-			mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
-		if (flow->frxq[i].ibv_attr)
-			rte_free(flow->frxq[i].ibv_attr);
-	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-		parser->cs = NULL;
-	}
-	rte_errno = ret; /* Restore rte_errno. */
-	return -rte_errno;
-}
+/* Verbs specification header. */
+struct ibv_spec_header {
+	enum ibv_flow_spec_type type;
+	uint16_t size;
+};
 
 /**
  * Convert a flow.
@@ -2452,69 +95,17 @@ mlx5_flow_create_action_queue(struct rte_eth_dev *dev,
  *   A flow on success, NULL otherwise and rte_errno is set.
  */
 static struct rte_flow *
-mlx5_flow_list_create(struct rte_eth_dev *dev,
-		      struct mlx5_flows *list,
-		      const struct rte_flow_attr *attr,
-		      const struct rte_flow_item items[],
-		      const struct rte_flow_action actions[],
+mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
+		      struct mlx5_flows *list __rte_unused,
+		      const struct rte_flow_attr *attr __rte_unused,
+		      const struct rte_flow_item items[] __rte_unused,
+		      const struct rte_flow_action actions[] __rte_unused,
 		      struct rte_flow_error *error)
 {
-	struct mlx5_flow_parse parser = { .create = 1, };
-	struct rte_flow *flow = NULL;
-	unsigned int i;
-	int ret;
-
-	ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser);
-	if (ret)
-		goto exit;
-	flow = rte_calloc(__func__, 1,
-			  sizeof(*flow) +
-			  parser.rss_conf.queue_num * sizeof(uint16_t),
-			  0);
-	if (!flow) {
-		rte_flow_error_set(error, ENOMEM,
-				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				   NULL,
-				   "cannot allocate flow memory");
-		return NULL;
-	}
-	/* Copy configuration. */
-	flow->queues = (uint16_t (*)[])(flow + 1);
-	flow->tunnel = parser.tunnel;
-	flow->rss_conf = (struct rte_flow_action_rss){
-		.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
-		.level = parser.rss_conf.level,
-		.types = parser.rss_conf.types,
-		.key_len = parser.rss_conf.key_len,
-		.queue_num = parser.rss_conf.queue_num,
-		.key = memcpy(flow->rss_key, parser.rss_conf.key,
-			      sizeof(*parser.rss_conf.key) *
-			      parser.rss_conf.key_len),
-		.queue = memcpy(flow->queues, parser.rss_conf.queue,
-				sizeof(*parser.rss_conf.queue) *
-				parser.rss_conf.queue_num),
-	};
-	flow->mark = parser.mark;
-	/* finalise the flow. */
-	if (parser.drop)
-		ret = mlx5_flow_create_action_queue_drop(dev, &parser, flow,
-							 error);
-	else
-		ret = mlx5_flow_create_action_queue(dev, &parser, flow, error);
-	if (ret)
-		goto exit;
-	TAILQ_INSERT_TAIL(list, flow, next);
-	DRV_LOG(DEBUG, "port %u flow created %p", dev->data->port_id,
-		(void *)flow);
-	return flow;
-exit:
-	DRV_LOG(ERR, "port %u flow creation error: %s", dev->data->port_id,
-		error->message);
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser.queue[i].ibv_attr)
-			rte_free(parser.queue[i].ibv_attr);
-	}
-	rte_free(flow);
+	rte_flow_error_set(error, ENOTSUP,
+			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			   NULL,
+			   "action not supported");
 	return NULL;
 }
 
@@ -2525,15 +116,16 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
  * @see rte_flow_ops
  */
 int
-mlx5_flow_validate(struct rte_eth_dev *dev,
-		   const struct rte_flow_attr *attr,
-		   const struct rte_flow_item items[],
-		   const struct rte_flow_action actions[],
+mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
+		   const struct rte_flow_attr *attr __rte_unused,
+		   const struct rte_flow_item items[] __rte_unused,
+		   const struct rte_flow_action actions[] __rte_unused,
 		   struct rte_flow_error *error)
 {
-	struct mlx5_flow_parse parser = { .create = 0, };
-
-	return mlx5_flow_convert(dev, attr, items, actions, error, &parser);
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "action not supported");
 }
 
 /**
@@ -2543,16 +135,17 @@ mlx5_flow_validate(struct rte_eth_dev *dev,
  * @see rte_flow_ops
  */
 struct rte_flow *
-mlx5_flow_create(struct rte_eth_dev *dev,
-		 const struct rte_flow_attr *attr,
-		 const struct rte_flow_item items[],
-		 const struct rte_flow_action actions[],
+mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
+		 const struct rte_flow_attr *attr __rte_unused,
+		 const struct rte_flow_item items[] __rte_unused,
+		 const struct rte_flow_action actions[] __rte_unused,
 		 struct rte_flow_error *error)
 {
-	struct priv *priv = dev->data->dev_private;
-
-	return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions,
-				     error);
+	rte_flow_error_set(error, ENOTSUP,
+			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			   NULL,
+			   "action not supported");
+	return NULL;
 }
 
 /**
@@ -2566,99 +159,10 @@ mlx5_flow_create(struct rte_eth_dev *dev,
  *   Flow to destroy.
  */
 static void
-mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
-		       struct rte_flow *flow)
+mlx5_flow_list_destroy(struct rte_eth_dev *dev __rte_unused,
+		       struct mlx5_flows *list __rte_unused,
+		       struct rte_flow *flow __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-
-	if (flow->drop || !dev->data->dev_started)
-		goto free;
-	for (i = 0; flow->tunnel && i != flow->rss_conf.queue_num; ++i) {
-		/* Update queue tunnel type. */
-		struct mlx5_rxq_data *rxq_data = (*priv->rxqs)
-						 [(*flow->queues)[i]];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
-		uint8_t tunnel = PTYPE_IDX(flow->tunnel);
-
-		assert(rxq_ctrl->tunnel_types[tunnel] > 0);
-		rxq_ctrl->tunnel_types[tunnel] -= 1;
-		if (!rxq_ctrl->tunnel_types[tunnel]) {
-			/* Update tunnel type. */
-			uint8_t j;
-			uint8_t types = 0;
-			uint8_t last;
-
-			for (j = 0; j < RTE_DIM(rxq_ctrl->tunnel_types); j++)
-				if (rxq_ctrl->tunnel_types[j]) {
-					types += 1;
-					last = j;
-				}
-			/* Keep same if more than one tunnel types left. */
-			if (types == 1)
-				rxq_data->tunnel = ptype_ext[last];
-			else if (types == 0)
-				/* No tunnel type left. */
-				rxq_data->tunnel = 0;
-		}
-	}
-	for (i = 0; flow->mark && i != flow->rss_conf.queue_num; ++i) {
-		struct rte_flow *tmp;
-		int mark = 0;
-
-		/*
-		 * To remove the mark from the queue, the queue must not be
-		 * present in any other marked flow (RSS or not).
-		 */
-		TAILQ_FOREACH(tmp, list, next) {
-			unsigned int j;
-			uint16_t *tqs = NULL;
-			uint16_t tq_n = 0;
-
-			if (!tmp->mark)
-				continue;
-			for (j = 0; j != hash_rxq_init_n; ++j) {
-				if (!tmp->frxq[j].hrxq)
-					continue;
-				tqs = tmp->frxq[j].hrxq->ind_table->queues;
-				tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
-			}
-			if (!tq_n)
-				continue;
-			for (j = 0; (j != tq_n) && !mark; j++)
-				if (tqs[j] == (*flow->queues)[i])
-					mark = 1;
-		}
-		(*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
-	}
-free:
-	if (flow->drop) {
-		if (flow->frxq[HASH_RXQ_ETH].ibv_flow)
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-		rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
-	} else {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			struct mlx5_flow *frxq = &flow->frxq[i];
-
-			if (frxq->ibv_flow)
-				claim_zero(mlx5_glue->destroy_flow
-					   (frxq->ibv_flow));
-			if (frxq->hrxq)
-				mlx5_hrxq_release(dev, frxq->hrxq);
-			if (frxq->ibv_attr)
-				rte_free(frxq->ibv_attr);
-		}
-	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-	}
-	TAILQ_REMOVE(list, flow, next);
-	DRV_LOG(DEBUG, "port %u flow destroyed %p", dev->data->port_id,
-		(void *)flow);
-	rte_free(flow);
 }
 
 /**
@@ -2690,97 +194,9 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_flow_create_drop_queue(struct rte_eth_dev *dev)
+mlx5_flow_create_drop_queue(struct rte_eth_dev *dev __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_hrxq_drop *fdq = NULL;
-
-	assert(priv->pd);
-	assert(priv->ctx);
-	fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
-	if (!fdq) {
-		DRV_LOG(WARNING,
-			"port %u cannot allocate memory for drop queue",
-			dev->data->port_id);
-		rte_errno = ENOMEM;
-		return -rte_errno;
-	}
-	fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
-	if (!fdq->cq) {
-		DRV_LOG(WARNING, "port %u cannot allocate CQ for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->wq = mlx5_glue->create_wq
-		(priv->ctx,
-		 &(struct ibv_wq_init_attr){
-			.wq_type = IBV_WQT_RQ,
-			.max_wr = 1,
-			.max_sge = 1,
-			.pd = priv->pd,
-			.cq = fdq->cq,
-		 });
-	if (!fdq->wq) {
-		DRV_LOG(WARNING, "port %u cannot allocate WQ for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->ind_table = mlx5_glue->create_rwq_ind_table
-		(priv->ctx,
-		 &(struct ibv_rwq_ind_table_init_attr){
-			.log_ind_tbl_size = 0,
-			.ind_tbl = &fdq->wq,
-			.comp_mask = 0,
-		 });
-	if (!fdq->ind_table) {
-		DRV_LOG(WARNING,
-			"port %u cannot allocate indirection table for drop"
-			" queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->qp = mlx5_glue->create_qp_ex
-		(priv->ctx,
-		 &(struct ibv_qp_init_attr_ex){
-			.qp_type = IBV_QPT_RAW_PACKET,
-			.comp_mask =
-				IBV_QP_INIT_ATTR_PD |
-				IBV_QP_INIT_ATTR_IND_TABLE |
-				IBV_QP_INIT_ATTR_RX_HASH,
-			.rx_hash_conf = (struct ibv_rx_hash_conf){
-				.rx_hash_function =
-					IBV_RX_HASH_FUNC_TOEPLITZ,
-				.rx_hash_key_len = rss_hash_default_key_len,
-				.rx_hash_key = rss_hash_default_key,
-				.rx_hash_fields_mask = 0,
-				},
-			.rwq_ind_tbl = fdq->ind_table,
-			.pd = priv->pd
-		 });
-	if (!fdq->qp) {
-		DRV_LOG(WARNING, "port %u cannot allocate QP for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	priv->flow_drop_queue = fdq;
 	return 0;
-error:
-	if (fdq->qp)
-		claim_zero(mlx5_glue->destroy_qp(fdq->qp));
-	if (fdq->ind_table)
-		claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
-	if (fdq->wq)
-		claim_zero(mlx5_glue->destroy_wq(fdq->wq));
-	if (fdq->cq)
-		claim_zero(mlx5_glue->destroy_cq(fdq->cq));
-	if (fdq)
-		rte_free(fdq);
-	priv->flow_drop_queue = NULL;
-	return -rte_errno;
 }
 
 /**
@@ -2790,23 +206,8 @@ mlx5_flow_create_drop_queue(struct rte_eth_dev *dev)
  *   Pointer to Ethernet device.
  */
 void
-mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
+mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
-
-	if (!fdq)
-		return;
-	if (fdq->qp)
-		claim_zero(mlx5_glue->destroy_qp(fdq->qp));
-	if (fdq->ind_table)
-		claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
-	if (fdq->wq)
-		claim_zero(mlx5_glue->destroy_wq(fdq->wq));
-	if (fdq->cq)
-		claim_zero(mlx5_glue->destroy_cq(fdq->cq));
-	rte_free(fdq);
-	priv->flow_drop_queue = NULL;
 }
 
 /**
@@ -2818,70 +219,9 @@ mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
  *   Pointer to a TAILQ flow list.
  */
 void
-mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
+mlx5_flow_stop(struct rte_eth_dev *dev __rte_unused,
+	       struct mlx5_flows *list __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct rte_flow *flow;
-	unsigned int i;
-
-	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
-		struct mlx5_ind_table_ibv *ind_tbl = NULL;
-
-		if (flow->drop) {
-			if (!flow->frxq[HASH_RXQ_ETH].ibv_flow)
-				continue;
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-			flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
-			DRV_LOG(DEBUG, "port %u flow %p removed",
-				dev->data->port_id, (void *)flow);
-			/* Next flow. */
-			continue;
-		}
-		/* Verify the flow has not already been cleaned. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_flow)
-				continue;
-			/*
-			 * Indirection table may be necessary to remove the
-			 * flags in the Rx queues.
-			 * This helps to speed-up the process by avoiding
-			 * another loop.
-			 */
-			ind_tbl = flow->frxq[i].hrxq->ind_table;
-			break;
-		}
-		if (i == hash_rxq_init_n)
-			return;
-		if (flow->mark) {
-			assert(ind_tbl);
-			for (i = 0; i != ind_tbl->queues_n; ++i)
-				(*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
-		}
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_flow)
-				continue;
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[i].ibv_flow));
-			flow->frxq[i].ibv_flow = NULL;
-			mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
-			flow->frxq[i].hrxq = NULL;
-		}
-		DRV_LOG(DEBUG, "port %u flow %p removed", dev->data->port_id,
-			(void *)flow);
-	}
-	/* Cleanup Rx queue tunnel info. */
-	for (i = 0; i != priv->rxqs_n; ++i) {
-		struct mlx5_rxq_data *q = (*priv->rxqs)[i];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(q, struct mlx5_rxq_ctrl, rxq);
-
-		if (!q)
-			continue;
-		memset((void *)rxq_ctrl->tunnel_types, 0,
-		       sizeof(rxq_ctrl->tunnel_types));
-		q->tunnel = 0;
-	}
 }
 
 /**
@@ -2896,76 +236,9 @@ mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
+mlx5_flow_start(struct rte_eth_dev *dev __rte_unused,
+		struct mlx5_flows *list __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct rte_flow *flow;
-
-	TAILQ_FOREACH(flow, list, next) {
-		unsigned int i;
-
-		if (flow->drop) {
-			flow->frxq[HASH_RXQ_ETH].ibv_flow =
-				mlx5_glue->create_flow
-				(priv->flow_drop_queue->qp,
-				 flow->frxq[HASH_RXQ_ETH].ibv_attr);
-			if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p cannot be applied",
-					dev->data->port_id, (void *)flow);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-			DRV_LOG(DEBUG, "port %u flow %p applied",
-				dev->data->port_id, (void *)flow);
-			/* Next flow. */
-			continue;
-		}
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_attr)
-				continue;
-			flow->frxq[i].hrxq =
-				mlx5_hrxq_get(dev, flow->rss_conf.key,
-					      flow->rss_conf.key_len,
-					      flow->frxq[i].hash_fields,
-					      flow->rss_conf.queue,
-					      flow->rss_conf.queue_num,
-					      flow->tunnel,
-					      flow->rss_conf.level);
-			if (flow->frxq[i].hrxq)
-				goto flow_create;
-			flow->frxq[i].hrxq =
-				mlx5_hrxq_new(dev, flow->rss_conf.key,
-					      flow->rss_conf.key_len,
-					      flow->frxq[i].hash_fields,
-					      flow->rss_conf.queue,
-					      flow->rss_conf.queue_num,
-					      flow->tunnel,
-					      flow->rss_conf.level);
-			if (!flow->frxq[i].hrxq) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p cannot create hash"
-					" rxq",
-					dev->data->port_id, (void *)flow);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-flow_create:
-			mlx5_flow_dump(dev, flow, i);
-			flow->frxq[i].ibv_flow =
-				mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
-						       flow->frxq[i].ibv_attr);
-			if (!flow->frxq[i].ibv_flow) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p type %u cannot be"
-					" applied",
-					dev->data->port_id, (void *)flow, i);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-		}
-		mlx5_flow_create_update_rxqs(dev, flow);
-	}
 	return 0;
 }
 
@@ -3019,7 +292,6 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
-		.priority = MLX5_CTRL_FLOW_PRIORITY,
 	};
 	struct rte_flow_item items[] = {
 		{
@@ -3129,83 +401,6 @@ mlx5_flow_flush(struct rte_eth_dev *dev,
 	return 0;
 }
 
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-/**
- * Query flow counter.
- *
- * @param cs
- *   the counter set.
- * @param counter_value
- *   returned data from the counter.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_query_count(struct ibv_counter_set *cs,
-		      struct mlx5_flow_counter_stats *counter_stats,
-		      struct rte_flow_query_count *query_count,
-		      struct rte_flow_error *error)
-{
-	uint64_t counters[2];
-	struct ibv_query_counter_set_attr query_cs_attr = {
-		.cs = cs,
-		.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
-	};
-	struct ibv_counter_set_data query_out = {
-		.out = counters,
-		.outlen = 2 * sizeof(uint64_t),
-	};
-	int err = mlx5_glue->query_counter_set(&query_cs_attr, &query_out);
-
-	if (err)
-		return rte_flow_error_set(error, err,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "cannot read counter");
-	query_count->hits_set = 1;
-	query_count->bytes_set = 1;
-	query_count->hits = counters[0] - counter_stats->hits;
-	query_count->bytes = counters[1] - counter_stats->bytes;
-	if (query_count->reset) {
-		counter_stats->hits = counters[0];
-		counter_stats->bytes = counters[1];
-	}
-	return 0;
-}
-
-/**
- * Query a flows.
- *
- * @see rte_flow_query()
- * @see rte_flow_ops
- */
-int
-mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
-		struct rte_flow *flow,
-		const struct rte_flow_action *action __rte_unused,
-		void *data,
-		struct rte_flow_error *error)
-{
-	if (flow->cs) {
-		int ret;
-
-		ret = mlx5_flow_query_count(flow->cs,
-					    &flow->counter_stats,
-					    (struct rte_flow_query_count *)data,
-					    error);
-		if (ret)
-			return ret;
-	} else {
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "no counter found for flow");
-	}
-	return 0;
-}
-#endif
-
 /**
  * Isolated mode.
  *
@@ -3445,18 +640,11 @@ mlx5_fdir_filter_add(struct rte_eth_dev *dev,
 			.type = 0,
 		},
 	};
-	struct mlx5_flow_parse parser = {
-		.layer = HASH_RXQ_ETH,
-	};
 	struct rte_flow_error error;
 	struct rte_flow *flow;
 	int ret;
 
 	ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
-				attributes.actions, &error, &parser);
 	if (ret)
 		return ret;
 	flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr,
@@ -3482,96 +670,11 @@ mlx5_fdir_filter_add(struct rte_eth_dev *dev,
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx5_fdir_filter_delete(struct rte_eth_dev *dev,
-			const struct rte_eth_fdir_filter *fdir_filter)
+mlx5_fdir_filter_delete(struct rte_eth_dev *dev __rte_unused,
+			const struct rte_eth_fdir_filter *fdir_filter
+			__rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_fdir attributes = {
-		.attr.group = 0,
-	};
-	struct mlx5_flow_parse parser = {
-		.create = 1,
-		.layer = HASH_RXQ_ETH,
-	};
-	struct rte_flow_error error;
-	struct rte_flow *flow;
-	unsigned int i;
-	int ret;
-
-	ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
-				attributes.actions, &error, &parser);
-	if (ret)
-		goto exit;
-	/*
-	 * Special case for drop action which is only set in the
-	 * specifications when the flow is created.  In this situation the
-	 * drop specification is missing.
-	 */
-	if (parser.drop) {
-		struct ibv_flow_spec_action_drop *drop;
-
-		drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr +
-				parser.queue[HASH_RXQ_ETH].offset);
-		*drop = (struct ibv_flow_spec_action_drop){
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-			.size = sizeof(struct ibv_flow_spec_action_drop),
-		};
-		parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++;
-	}
-	TAILQ_FOREACH(flow, &priv->flows, next) {
-		struct ibv_flow_attr *attr;
-		struct ibv_spec_header *attr_h;
-		void *spec;
-		struct ibv_flow_attr *flow_attr;
-		struct ibv_spec_header *flow_h;
-		void *flow_spec;
-		unsigned int specs_n;
-		unsigned int queue_id = parser.drop ? HASH_RXQ_ETH :
-						      parser.layer;
-
-		attr = parser.queue[queue_id].ibv_attr;
-		flow_attr = flow->frxq[queue_id].ibv_attr;
-		/* Compare first the attributes. */
-		if (!flow_attr ||
-		    memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
-			continue;
-		if (attr->num_of_specs == 0)
-			continue;
-		spec = (void *)((uintptr_t)attr +
-				sizeof(struct ibv_flow_attr));
-		flow_spec = (void *)((uintptr_t)flow_attr +
-				     sizeof(struct ibv_flow_attr));
-		specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
-		for (i = 0; i != specs_n; ++i) {
-			attr_h = spec;
-			flow_h = flow_spec;
-			if (memcmp(spec, flow_spec,
-				   RTE_MIN(attr_h->size, flow_h->size)))
-				goto wrong_flow;
-			spec = (void *)((uintptr_t)spec + attr_h->size);
-			flow_spec = (void *)((uintptr_t)flow_spec +
-					     flow_h->size);
-		}
-		/* At this point, the flow match. */
-		break;
-wrong_flow:
-		/* The flow does not match. */
-		continue;
-	}
-	if (flow)
-		mlx5_flow_list_destroy(dev, &priv->flows, flow);
-exit:
-	if (ret)
-		ret = rte_errno; /* Save rte_errno before cleanup. */
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser.queue[i].ibv_attr)
-			rte_free(parser.queue[i].ibv_attr);
-	}
-	if (ret)
-		rte_errno = ret; /* Restore rte_errno. */
+	rte_errno = ENOTSUP;
 	return -rte_errno;
 }
 
@@ -3738,45 +841,7 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
  *   number of supported Verbs flow priority.
  */
 unsigned int
-mlx5_get_max_verbs_prio(struct rte_eth_dev *dev)
+mlx5_get_max_verbs_prio(struct rte_eth_dev *dev __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	unsigned int verb_priorities = MLX5_VERBS_FLOW_PRIO_8;
-	struct {
-		struct ibv_flow_attr attr;
-		struct ibv_flow_spec_eth eth;
-		struct ibv_flow_spec_action_drop drop;
-	} flow_attr = {
-		.attr = {
-			.num_of_specs = 2,
-		},
-		.eth = {
-			.type = IBV_FLOW_SPEC_ETH,
-			.size = sizeof(struct ibv_flow_spec_eth),
-		},
-		.drop = {
-			.size = sizeof(struct ibv_flow_spec_action_drop),
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-		},
-	};
-	struct ibv_flow *flow;
-
-	do {
-		flow_attr.attr.priority = verb_priorities - 1;
-		flow = mlx5_glue->create_flow(priv->flow_drop_queue->qp,
-					      &flow_attr.attr);
-		if (flow) {
-			claim_zero(mlx5_glue->destroy_flow(flow));
-			/* Try more priorities. */
-			verb_priorities *= 2;
-		} else {
-			/* Failed, restore last right number. */
-			verb_priorities /= 2;
-			break;
-		}
-	} while (1);
-	DRV_LOG(DEBUG, "port %u Verbs flow priorities: %d,"
-		" user flow priorities: %d",
-		dev->data->port_id, verb_priorities, MLX5_CTRL_FLOW_PRIORITY);
-	return verb_priorities;
+	return 8;
 }
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 0007be08b..97b4d9eb6 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -136,7 +136,6 @@ struct mlx5_rxq_ctrl {
 	struct priv *priv; /* Back pointer to private data. */
 	struct mlx5_rxq_data rxq; /* Data path structure. */
 	unsigned int socket; /* CPU socket ID for allocations. */
-	uint32_t tunnel_types[16]; /* Tunnel type counter. */
 	unsigned int irq:1; /* Whether IRQ is enabled. */
 	uint16_t idx; /* Queue index. */
 };
-- 
2.18.0

[dpdk-dev] [PATCH v3 02/21] net/mlx5: handle drop queues as regular queues

[Nelio Laranjeiro]

Drop queues are essentially used in flows due to Verbs API, the
information if the fate of the flow is a drop or not is already present
in the flow.  Due to this, drop queues can be fully mapped on regular
queues.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5.c      |  24 ++--
 drivers/net/mlx5/mlx5.h      |  14 ++-
 drivers/net/mlx5/mlx5_flow.c |  94 +++++++-------
 drivers/net/mlx5/mlx5_rxq.c  | 232 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_rxtx.h |   6 +
 5 files changed, 308 insertions(+), 62 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index df7f39844..e9780ac8f 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -261,7 +261,6 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 		priv->txqs_n = 0;
 		priv->txqs = NULL;
 	}
-	mlx5_flow_delete_drop_queue(dev);
 	mlx5_mprq_free_mp(dev);
 	mlx5_mr_release(dev);
 	if (priv->pd != NULL) {
@@ -1139,22 +1138,15 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	mlx5_link_update(eth_dev, 0);
 	/* Store device configuration on private structure. */
 	priv->config = config;
-	/* Create drop queue. */
-	err = mlx5_flow_create_drop_queue(eth_dev);
-	if (err) {
-		DRV_LOG(ERR, "port %u drop queue allocation failed: %s",
-			eth_dev->data->port_id, strerror(rte_errno));
-		err = rte_errno;
-		goto error;
-	}
 	/* Supported Verbs flow priority number detection. */
-	if (verb_priorities == 0)
-		verb_priorities = mlx5_get_max_verbs_prio(eth_dev);
-	if (verb_priorities < MLX5_VERBS_FLOW_PRIO_8) {
-		DRV_LOG(ERR, "port %u wrong Verbs flow priorities: %u",
-			eth_dev->data->port_id, verb_priorities);
-		err = ENOTSUP;
-		goto error;
+	if (verb_priorities == 0) {
+		err = mlx5_verbs_max_prio(eth_dev);
+		if (err < 0) {
+			DRV_LOG(ERR, "port %u wrong Verbs flow priorities",
+				eth_dev->data->port_id);
+			goto error;
+		}
+		verb_priorities = err;
 	}
 	priv->config.max_verbs_prio = verb_priorities;
 	/*
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index cc01310e0..227429848 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -139,9 +139,6 @@ enum mlx5_verbs_alloc_type {
 	MLX5_VERBS_ALLOC_TYPE_RX_QUEUE,
 };
 
-/* 8 Verbs priorities. */
-#define MLX5_VERBS_FLOW_PRIO_8 8
-
 /**
  * Verbs allocator needs a context to know in the callback which kind of
  * resources it is allocating.
@@ -153,6 +150,12 @@ struct mlx5_verbs_alloc_ctx {
 
 LIST_HEAD(mlx5_mr_list, mlx5_mr);
 
+/* Flow drop context necessary due to Verbs API. */
+struct mlx5_drop {
+	struct mlx5_hrxq *hrxq; /* Hash Rx queue queue. */
+	struct mlx5_rxq_ibv *rxq; /* Verbs Rx queue. */
+};
+
 struct priv {
 	LIST_ENTRY(priv) mem_event_cb; /* Called by memory event callback. */
 	struct rte_eth_dev_data *dev_data;  /* Pointer to device data. */
@@ -182,7 +185,7 @@ struct priv {
 	struct rte_intr_handle intr_handle; /* Interrupt handler. */
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
-	struct mlx5_hrxq_drop *flow_drop_queue; /* Flow drop queue. */
+	struct mlx5_drop drop_queue; /* Flow drop queues. */
 	struct mlx5_flows flows; /* RTE Flow rules. */
 	struct mlx5_flows ctrl_flows; /* Control flow rules. */
 	struct {
@@ -314,7 +317,8 @@ int mlx5_traffic_restart(struct rte_eth_dev *dev);
 
 /* mlx5_flow.c */
 
-unsigned int mlx5_get_max_verbs_prio(struct rte_eth_dev *dev);
+int mlx5_verbs_max_prio(struct rte_eth_dev *dev);
+void mlx5_flow_print(struct rte_flow *flow);
 int mlx5_flow_validate(struct rte_eth_dev *dev,
 		       const struct rte_flow_attr *attr,
 		       const struct rte_flow_item items[],
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index a45cb06e1..5e325be37 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -75,6 +75,58 @@ struct ibv_spec_header {
 	uint16_t size;
 };
 
+ /**
+  * Get the maximum number of priority available.
+  *
+  * @param[in] dev
+  *   Pointer to Ethernet device.
+  *
+  * @return
+  *   number of supported Verbs flow priority on success, a negative errno
+  *   value otherwise and rte_errno is set.
+  */
+int
+mlx5_verbs_max_prio(struct rte_eth_dev *dev)
+{
+	struct {
+		struct ibv_flow_attr attr;
+		struct ibv_flow_spec_eth eth;
+		struct ibv_flow_spec_action_drop drop;
+	} flow_attr = {
+		.attr = {
+			.num_of_specs = 2,
+		},
+		.eth = {
+			.type = IBV_FLOW_SPEC_ETH,
+			.size = sizeof(struct ibv_flow_spec_eth),
+		},
+		.drop = {
+			.size = sizeof(struct ibv_flow_spec_action_drop),
+			.type = IBV_FLOW_SPEC_ACTION_DROP,
+		},
+	};
+	struct ibv_flow *flow;
+	uint32_t verb_priorities;
+	struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
+
+	if (!drop) {
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	for (verb_priorities = 0; 1; verb_priorities++) {
+		flow_attr.attr.priority = verb_priorities;
+		flow = mlx5_glue->create_flow(drop->qp,
+					      &flow_attr.attr);
+		if (!flow)
+			break;
+		claim_zero(mlx5_glue->destroy_flow(flow));
+	}
+	mlx5_hrxq_drop_release(dev);
+	DRV_LOG(INFO, "port %u flow maximum priority: %d",
+		dev->data->port_id, verb_priorities);
+	return verb_priorities;
+}
+
 /**
  * Convert a flow.
  *
@@ -184,32 +236,6 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
 	}
 }
 
-/**
- * Create drop queue.
- *
- * @param dev
- *   Pointer to Ethernet device.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-int
-mlx5_flow_create_drop_queue(struct rte_eth_dev *dev __rte_unused)
-{
-	return 0;
-}
-
-/**
- * Delete drop queue.
- *
- * @param dev
- *   Pointer to Ethernet device.
- */
-void
-mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev __rte_unused)
-{
-}
-
 /**
  * Remove all flows.
  *
@@ -292,6 +318,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
+		.priority = priv->config.max_verbs_prio - 1,
 	};
 	struct rte_flow_item items[] = {
 		{
@@ -830,18 +857,3 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
 	}
 	return 0;
 }
-
-/**
- * Detect number of Verbs flow priorities supported.
- *
- * @param dev
- *   Pointer to Ethernet device.
- *
- * @return
- *   number of supported Verbs flow priority.
- */
-unsigned int
-mlx5_get_max_verbs_prio(struct rte_eth_dev *dev __rte_unused)
-{
-	return 8;
-}
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index fd0df177e..d960daa43 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1957,3 +1957,235 @@ mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev)
 	}
 	return ret;
 }
+
+/**
+ * Create a drop Rx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_rxq_ibv *
+mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct ibv_cq *cq;
+	struct ibv_wq *wq = NULL;
+	struct mlx5_rxq_ibv *rxq;
+
+	if (priv->drop_queue.rxq)
+		return priv->drop_queue.rxq;
+	cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
+	if (!cq) {
+		DEBUG("port %u cannot allocate CQ for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	wq = mlx5_glue->create_wq(priv->ctx,
+		 &(struct ibv_wq_init_attr){
+			.wq_type = IBV_WQT_RQ,
+			.max_wr = 1,
+			.max_sge = 1,
+			.pd = priv->pd,
+			.cq = cq,
+		 });
+	if (!wq) {
+		DEBUG("port %u cannot allocate WQ for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	rxq = rte_calloc(__func__, 1, sizeof(*rxq), 0);
+	if (!rxq) {
+		DEBUG("port %u cannot allocate drop Rx queue memory",
+		      dev->data->port_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	rxq->cq = cq;
+	rxq->wq = wq;
+	priv->drop_queue.rxq = rxq;
+	return rxq;
+error:
+	if (wq)
+		claim_zero(mlx5_glue->destroy_wq(wq));
+	if (cq)
+		claim_zero(mlx5_glue->destroy_cq(cq));
+	return NULL;
+}
+
+/**
+ * Release a drop Rx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+void
+mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_ibv *rxq = priv->drop_queue.rxq;
+
+	if (rxq->wq)
+		claim_zero(mlx5_glue->destroy_wq(rxq->wq));
+	if (rxq->cq)
+		claim_zero(mlx5_glue->destroy_cq(rxq->cq));
+	rte_free(rxq);
+	priv->drop_queue.rxq = NULL;
+}
+
+/**
+ * Create a drop indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_ind_table_ibv *
+mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	struct mlx5_rxq_ibv *rxq;
+	struct mlx5_ind_table_ibv tmpl;
+
+	rxq = mlx5_rxq_ibv_drop_new(dev);
+	if (!rxq)
+		return NULL;
+	tmpl.ind_table = mlx5_glue->create_rwq_ind_table
+		(priv->ctx,
+		 &(struct ibv_rwq_ind_table_init_attr){
+			.log_ind_tbl_size = 0,
+			.ind_tbl = &rxq->wq,
+			.comp_mask = 0,
+		 });
+	if (!tmpl.ind_table) {
+		DEBUG("port %u cannot allocate indirection table for drop"
+		      " queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	ind_tbl = rte_calloc(__func__, 1, sizeof(*ind_tbl), 0);
+	if (!ind_tbl) {
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	ind_tbl->ind_table = tmpl.ind_table;
+	return ind_tbl;
+error:
+	mlx5_rxq_ibv_drop_release(dev);
+	return NULL;
+}
+
+/**
+ * Release a drop indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl = priv->drop_queue.hrxq->ind_table;
+
+	claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl->ind_table));
+	mlx5_rxq_ibv_drop_release(dev);
+	rte_free(ind_tbl);
+	priv->drop_queue.hrxq->ind_table = NULL;
+}
+
+/**
+ * Create a drop Rx Hash queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_hrxq *
+mlx5_hrxq_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	struct ibv_qp *qp;
+	struct mlx5_hrxq *hrxq;
+
+	if (priv->drop_queue.hrxq) {
+		rte_atomic32_inc(&priv->drop_queue.hrxq->refcnt);
+		return priv->drop_queue.hrxq;
+	}
+	ind_tbl = mlx5_ind_table_ibv_drop_new(dev);
+	if (!ind_tbl)
+		return NULL;
+	qp = mlx5_glue->create_qp_ex(priv->ctx,
+		 &(struct ibv_qp_init_attr_ex){
+			.qp_type = IBV_QPT_RAW_PACKET,
+			.comp_mask =
+				IBV_QP_INIT_ATTR_PD |
+				IBV_QP_INIT_ATTR_IND_TABLE |
+				IBV_QP_INIT_ATTR_RX_HASH,
+			.rx_hash_conf = (struct ibv_rx_hash_conf){
+				.rx_hash_function =
+					IBV_RX_HASH_FUNC_TOEPLITZ,
+				.rx_hash_key_len = rss_hash_default_key_len,
+				.rx_hash_key = rss_hash_default_key,
+				.rx_hash_fields_mask = 0,
+				},
+			.rwq_ind_tbl = ind_tbl->ind_table,
+			.pd = priv->pd
+		 });
+	if (!qp) {
+		DEBUG("port %u cannot allocate QP for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	hrxq = rte_calloc(__func__, 1, sizeof(*hrxq), 0);
+	if (!hrxq) {
+		DRV_LOG(WARNING,
+			"port %u cannot allocate memory for drop queue",
+			dev->data->port_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	hrxq->ind_table = ind_tbl;
+	hrxq->qp = qp;
+	priv->drop_queue.hrxq = hrxq;
+	rte_atomic32_set(&hrxq->refcnt, 1);
+	return hrxq;
+error:
+	if (ind_tbl)
+		mlx5_ind_table_ibv_drop_release(dev);
+	return NULL;
+}
+
+/**
+ * Release a drop hash Rx queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_hrxq_drop_release(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq;
+
+	if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
+		claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
+		mlx5_ind_table_ibv_drop_release(dev);
+		rte_free(hrxq);
+		priv->drop_queue.hrxq = NULL;
+	}
+}
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 97b4d9eb6..708fdd4fa 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -244,6 +244,8 @@ struct mlx5_rxq_ibv *mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
 struct mlx5_rxq_ibv *mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
 int mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv);
 int mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv);
+struct mlx5_rxq_ibv *mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev);
+void mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev);
 int mlx5_rxq_ibv_verify(struct rte_eth_dev *dev);
 struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx,
 				   uint16_t desc, unsigned int socket,
@@ -264,6 +266,8 @@ struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_get(struct rte_eth_dev *dev,
 int mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
 			       struct mlx5_ind_table_ibv *ind_tbl);
 int mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev);
+void mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev);
 struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev,
 				const uint8_t *rss_key, uint32_t rss_key_len,
 				uint64_t hash_fields,
@@ -276,6 +280,8 @@ struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev,
 				uint32_t tunnel, uint32_t rss_level);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
 int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev);
+void mlx5_hrxq_drop_release(struct rte_eth_dev *dev);
 uint64_t mlx5_get_rx_port_offloads(void);
 uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
 
-- 
2.18.0

[dpdk-dev] [PATCH v3 03/21] net/mlx5: replace verbs priorities by flow

[Nelio Laranjeiro]

Previous work introduce verbs priorities, whereas the PMD is making
translation between Flow priority into Verbs.  Rename this to make more
sense on what the PMD has to translate.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5.c      | 15 ++++-----------
 drivers/net/mlx5/mlx5.h      |  4 ++--
 drivers/net/mlx5/mlx5_flow.c | 24 ++++++++++++------------
 3 files changed, 18 insertions(+), 25 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index e9780ac8f..74248f098 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -717,7 +717,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	unsigned int tunnel_en = 0;
 	unsigned int mpls_en = 0;
 	unsigned int swp = 0;
-	unsigned int verb_priorities = 0;
 	unsigned int mprq = 0;
 	unsigned int mprq_min_stride_size_n = 0;
 	unsigned int mprq_max_stride_size_n = 0;
@@ -1139,16 +1138,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	/* Store device configuration on private structure. */
 	priv->config = config;
 	/* Supported Verbs flow priority number detection. */
-	if (verb_priorities == 0) {
-		err = mlx5_verbs_max_prio(eth_dev);
-		if (err < 0) {
-			DRV_LOG(ERR, "port %u wrong Verbs flow priorities",
-				eth_dev->data->port_id);
-			goto error;
-		}
-		verb_priorities = err;
-	}
-	priv->config.max_verbs_prio = verb_priorities;
+	err = mlx5_flow_discover_priorities(eth_dev);
+	if (err < 0)
+		goto error;
+	priv->config.flow_prio = err;
 	/*
 	 * Once the device is added to the list of memory event
 	 * callback, its global MR cache table cannot be expanded
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 227429848..9949cd3fa 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -122,7 +122,7 @@ struct mlx5_dev_config {
 		unsigned int min_rxqs_num;
 		/* Rx queue count threshold to enable MPRQ. */
 	} mprq; /* Configurations for Multi-Packet RQ. */
-	unsigned int max_verbs_prio; /* Number of Verb flow priorities. */
+	unsigned int flow_prio; /* Number of flow priorities. */
 	unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
 	unsigned int ind_table_max_size; /* Maximum indirection table size. */
 	int txq_inline; /* Maximum packet size for inlining. */
@@ -317,7 +317,7 @@ int mlx5_traffic_restart(struct rte_eth_dev *dev);
 
 /* mlx5_flow.c */
 
-int mlx5_verbs_max_prio(struct rte_eth_dev *dev);
+int mlx5_flow_discover_priorities(struct rte_eth_dev *dev);
 void mlx5_flow_print(struct rte_flow *flow);
 int mlx5_flow_validate(struct rte_eth_dev *dev,
 		       const struct rte_flow_attr *attr,
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 5e325be37..8fdc6d7bb 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -76,17 +76,17 @@ struct ibv_spec_header {
 };
 
  /**
-  * Get the maximum number of priority available.
+  * Discover the maximum number of priority available.
   *
   * @param[in] dev
   *   Pointer to Ethernet device.
   *
   * @return
-  *   number of supported Verbs flow priority on success, a negative errno
-  *   value otherwise and rte_errno is set.
+  *   number of supported flow priority on success, a negative errno value
+  *   otherwise and rte_errno is set.
   */
 int
-mlx5_verbs_max_prio(struct rte_eth_dev *dev)
+mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 {
 	struct {
 		struct ibv_flow_attr attr;
@@ -106,25 +106,25 @@ mlx5_verbs_max_prio(struct rte_eth_dev *dev)
 		},
 	};
 	struct ibv_flow *flow;
-	uint32_t verb_priorities;
 	struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
+	uint16_t vprio[] = { 8, 16 };
+	int i;
 
 	if (!drop) {
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	for (verb_priorities = 0; 1; verb_priorities++) {
-		flow_attr.attr.priority = verb_priorities;
-		flow = mlx5_glue->create_flow(drop->qp,
-					      &flow_attr.attr);
+	for (i = 0; i != RTE_DIM(vprio); i++) {
+		flow_attr.attr.priority = vprio[i] - 1;
+		flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
 		if (!flow)
 			break;
 		claim_zero(mlx5_glue->destroy_flow(flow));
 	}
 	mlx5_hrxq_drop_release(dev);
 	DRV_LOG(INFO, "port %u flow maximum priority: %d",
-		dev->data->port_id, verb_priorities);
-	return verb_priorities;
+		dev->data->port_id, vprio[i - 1]);
+	return vprio[i - 1];
 }
 
 /**
@@ -318,7 +318,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
-		.priority = priv->config.max_verbs_prio - 1,
+		.priority = priv->config.flow_prio - 1,
 	};
 	struct rte_flow_item items[] = {
 		{
-- 
2.18.0

[dpdk-dev] [PATCH v3 04/21] net/mlx5: support flow Ethernet item along with drop action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5.c      |   1 +
 drivers/net/mlx5/mlx5_flow.c | 664 +++++++++++++++++++++++++++++++++--
 2 files changed, 627 insertions(+), 38 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 74248f098..6d3421fae 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -242,6 +242,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	/* In case mlx5_dev_stop() has not been called. */
 	mlx5_dev_interrupt_handler_uninstall(dev);
 	mlx5_traffic_disable(dev);
+	mlx5_flow_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
 	dev->rx_pkt_burst = removed_rx_burst;
 	dev->tx_pkt_burst = removed_tx_burst;
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 8fdc6d7bb..036a8d440 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -35,11 +35,50 @@
 extern const struct eth_dev_ops mlx5_dev_ops;
 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
+/* Pattern Layer bits. */
+#define MLX5_FLOW_LAYER_OUTER_L2 (1u << 0)
+#define MLX5_FLOW_LAYER_OUTER_L3_IPV4 (1u << 1)
+#define MLX5_FLOW_LAYER_OUTER_L3_IPV6 (1u << 2)
+#define MLX5_FLOW_LAYER_OUTER_L4_UDP (1u << 3)
+#define MLX5_FLOW_LAYER_OUTER_L4_TCP (1u << 4)
+#define MLX5_FLOW_LAYER_OUTER_VLAN (1u << 5)
+/* Masks. */
+#define MLX5_FLOW_LAYER_OUTER_L3 \
+	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
+#define MLX5_FLOW_LAYER_OUTER_L4 \
+	(MLX5_FLOW_LAYER_OUTER_L4_UDP | MLX5_FLOW_LAYER_OUTER_L4_TCP)
+
+/* Actions that modify the fate of matching traffic. */
+#define MLX5_FLOW_FATE_DROP (1u << 0)
+
+/** Handles information leading to a drop fate. */
+struct mlx5_flow_verbs {
+	unsigned int size; /**< Size of the attribute. */
+	struct {
+		struct ibv_flow_attr *attr;
+		/**< Pointer to the Specification buffer. */
+		uint8_t *specs; /**< Pointer to the specifications. */
+	};
+	struct ibv_flow *flow; /**< Verbs flow pointer. */
+	struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */
+};
+
+/* Flow structure. */
 struct rte_flow {
 	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
+	struct rte_flow_attr attributes; /**< User flow attribute. */
+	uint32_t layers;
+	/**< Bit-fields of present layers see MLX5_FLOW_LAYER_*. */
+	uint32_t fate;
+	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
+	struct mlx5_flow_verbs verbs; /* Verbs drop flow. */
 };
 
 static const struct rte_flow_ops mlx5_flow_ops = {
+	.validate = mlx5_flow_validate,
+	.create = mlx5_flow_create,
+	.destroy = mlx5_flow_destroy,
+	.flush = mlx5_flow_flush,
 	.isolate = mlx5_flow_isolate,
 };
 
@@ -128,13 +167,415 @@ mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 }
 
 /**
- * Convert a flow.
+ * Verify the @p attributes will be correctly understood by the NIC and store
+ * them in the @p flow if everything is correct.
  *
- * @param dev
+ * @param[in] dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to a TAILQ flow list.
- * @param[in] attr
+ * @param[in] attributes
+ *   Pointer to flow attributes
+ * @param[in, out] flow
+ *   Pointer to the rte_flow structure.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_attributes(struct rte_eth_dev *dev,
+		     const struct rte_flow_attr *attributes,
+		     struct rte_flow *flow,
+		     struct rte_flow_error *error)
+{
+	uint32_t priority_max =
+		((struct priv *)dev->data->dev_private)->config.flow_prio;
+
+	if (attributes->group)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+					  NULL,
+					  "groups is not supported");
+	if (attributes->priority >= priority_max)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+					  NULL,
+					  "priority out of range");
+	if (attributes->egress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+					  NULL,
+					  "egress is not supported");
+	if (attributes->transfer)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
+					  NULL,
+					  "transfer is not supported");
+	if (!attributes->ingress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+					  NULL,
+					  "ingress attribute is mandatory");
+	flow->attributes = *attributes;
+	return 0;
+}
+
+/**
+ * Verify the @p item specifications (spec, last, mask) are compatible with the
+ * NIC capabilities.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in] mask
+ *   @p item->mask or flow default bit-masks.
+ * @param[in] nic_mask
+ *   Bit-masks covering supported fields by the NIC to compare with user mask.
+ * @param[in] size
+ *   Bit-masks size in bytes.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_acceptable(const struct rte_flow_item *item,
+			  const uint8_t *mask,
+			  const uint8_t *nic_mask,
+			  unsigned int size,
+			  struct rte_flow_error *error)
+{
+	unsigned int i;
+
+	assert(nic_mask);
+	for (i = 0; i < size; ++i)
+		if ((nic_mask[i] | mask[i]) != nic_mask[i])
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "mask enables non supported"
+						  " bits");
+	if (!item->spec && (item->mask || item->last))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "mask/last without a spec is not"
+					  " supported");
+	if (item->spec && item->last) {
+		uint8_t spec[size];
+		uint8_t last[size];
+		unsigned int i;
+		int ret;
+
+		for (i = 0; i < size; ++i) {
+			spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
+			last[i] = ((const uint8_t *)item->last)[i] & mask[i];
+		}
+		ret = memcmp(spec, last, size);
+		if (ret != 0)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "range is not supported");
+	}
+	return 0;
+}
+
+/**
+ * Add a verbs specification into @p flow.
+ *
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] src
+ *   Create specification.
+ * @param[in] size
+ *   Size in bytes of the specification to copy.
+ */
+static void
+mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
+{
+	if (flow->verbs.specs) {
+		void *dst;
+
+		dst = (void *)(flow->verbs.specs + flow->verbs.size);
+		memcpy(dst, src, size);
+		++flow->verbs.attr->num_of_specs;
+	}
+	flow->verbs.size += size;
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_eth *spec = item->spec;
+	const struct rte_flow_item_eth *mask = item->mask;
+	const struct rte_flow_item_eth nic_mask = {
+		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.type = RTE_BE16(0xffff),
+	};
+	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	struct ibv_flow_spec_eth eth = {
+		.type = IBV_FLOW_SPEC_ETH,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L2)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L2 layers already configured");
+	if (!mask)
+		mask = &rte_flow_item_eth_mask;
+	ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
+					(const uint8_t *)&nic_mask,
+					sizeof(struct rte_flow_item_eth),
+					error);
+	if (ret)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		unsigned int i;
+
+		memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
+		memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
+		eth.val.ether_type = spec->type;
+		memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
+		memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
+		eth.mask.ether_type = mask->type;
+		/* Remove unwanted bits from values. */
+		for (i = 0; i < ETHER_ADDR_LEN; ++i) {
+			eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
+			eth.val.src_mac[i] &= eth.mask.src_mac[i];
+		}
+		eth.val.ether_type &= eth.mask.ether_type;
+	}
+	mlx5_flow_spec_verbs_add(flow, &eth, size);
+	return size;
+}
+
+/**
+ * Convert the @p pattern into a Verbs specifications after ensuring the NIC
+ * will understand and process it correctly.
+ * The conversion is performed item per item, each of them is written into
+ * the @p flow if its size is lesser or equal to @p flow_size.
+ * Validation and memory consumption computation are still performed until the
+ * end of @p pattern, unless an error is encountered.
+ *
+ * @param[in] pattern
+ *   Flow pattern.
+ * @param[in, out] flow
+ *   Pointer to the rte_flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small some
+ *   garbage may be present.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @pattern  has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_items(const struct rte_flow_item pattern[],
+		struct rte_flow *flow, const size_t flow_size,
+		struct rte_flow_error *error)
+{
+	int remain = flow_size;
+	size_t size = 0;
+
+	for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
+		int ret = 0;
+
+		switch (pattern->type) {
+		case RTE_FLOW_ITEM_TYPE_VOID:
+			break;
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			ret = mlx5_flow_item_eth(pattern, flow, remain, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  pattern,
+						  "item not supported");
+		}
+		if (ret < 0)
+			return ret;
+		if (remain > ret)
+			remain -= ret;
+		else
+			remain = 0;
+		size += ret;
+	}
+	if (!flow->layers) {
+		const struct rte_flow_item item = {
+			.type = RTE_FLOW_ITEM_TYPE_ETH,
+		};
+
+		return mlx5_flow_item_eth(&item, flow, flow_size, error);
+	}
+	return size;
+}
+
+/**
+ * Convert the @p action into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_drop(const struct rte_flow_action *action,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
+{
+	unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
+	struct ibv_flow_spec_action_drop drop = {
+			.type = IBV_FLOW_SPEC_ACTION_DROP,
+			.size = size,
+	};
+
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "multiple fate actions are not"
+					  " supported");
+	if (size < flow_size)
+		mlx5_flow_spec_verbs_add(flow, &drop, size);
+	flow->fate |= MLX5_FLOW_FATE_DROP;
+	return size;
+}
+
+/**
+ * Convert the @p action into @p flow after ensuring the NIC will understand
+ * and process it correctly.
+ * The conversion is performed action per action, each of them is written into
+ * the @p flow if its size is lesser or equal to @p flow_size.
+ * Validation and memory consumption computation are still performed until the
+ * end of @p action, unless an error is encountered.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device structure.
+ * @param[in] actions
+ *   Pointer to flow actions array.
+ * @param[in, out] flow
+ *   Pointer to the rte_flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small some
+ *   garbage may be present.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p actions has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_actions(struct rte_eth_dev *dev __rte_unused,
+		  const struct rte_flow_action actions[],
+		  struct rte_flow *flow, const size_t flow_size,
+		  struct rte_flow_error *error)
+{
+	size_t size = 0;
+	int remain = flow_size;
+	int ret = 0;
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_DROP:
+			ret = mlx5_flow_action_drop(actions, flow, remain,
+						    error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+		if (ret < 0)
+			return ret;
+		if (remain > ret)
+			remain -= ret;
+		else
+			remain = 0;
+		size += ret;
+	}
+	if (!flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "no fate action found");
+	return size;
+}
+
+/**
+ * Convert the @p attributes, @p pattern, @p action, into an flow for the NIC
+ * after ensuring the NIC will understand and process it correctly.
+ * The conversion is only performed item/action per item/action, each of
+ * them is written into the @p flow if its size is lesser or equal to @p
+ * flow_size.
+ * Validation and memory consumption computation are still performed until the
+ * end, unless an error is encountered.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small some
+ *   garbage may be present.
+ * @param[in] attributes
  *   Flow rule attributes.
  * @param[in] pattern
  *   Pattern specification (list terminated by the END pattern item).
@@ -144,21 +585,42 @@ mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
  *   Perform verbose error reporting if not NULL.
  *
  * @return
- *   A flow on success, NULL otherwise and rte_errno is set.
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the flow has fully been converted and
+ *   can be applied, otherwise another call with this returned memory size
+ *   should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
  */
-static struct rte_flow *
-mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
-		      struct mlx5_flows *list __rte_unused,
-		      const struct rte_flow_attr *attr __rte_unused,
-		      const struct rte_flow_item items[] __rte_unused,
-		      const struct rte_flow_action actions[] __rte_unused,
-		      struct rte_flow_error *error)
+static int
+mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
+		const size_t flow_size,
+		const struct rte_flow_attr *attributes,
+		const struct rte_flow_item pattern[],
+		const struct rte_flow_action actions[],
+		struct rte_flow_error *error)
 {
-	rte_flow_error_set(error, ENOTSUP,
-			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL,
-			   "action not supported");
-	return NULL;
+	struct rte_flow local_flow = { .layers = 0, };
+	size_t size = sizeof(*flow) + sizeof(struct ibv_flow_attr);
+	int remain = (flow_size > size) ? flow_size - size : 0;
+	int ret;
+
+	if (!remain)
+		flow = &local_flow;
+	ret = mlx5_flow_attributes(dev, attributes, flow, error);
+	if (ret < 0)
+		return ret;
+	ret = mlx5_flow_items(pattern, flow, remain, error);
+	if (ret < 0)
+		return ret;
+	size += ret;
+	remain = (flow_size > size) ? flow_size - size : 0;
+	ret = mlx5_flow_actions(dev, actions, flow, remain, error);
+	if (ret < 0)
+		return ret;
+	size += ret;
+	if (size <= flow_size)
+		flow->verbs.attr->priority = flow->attributes.priority;
+	return size;
 }
 
 /**
@@ -168,16 +630,142 @@ mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
  * @see rte_flow_ops
  */
 int
-mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
-		   const struct rte_flow_attr *attr __rte_unused,
-		   const struct rte_flow_item items[] __rte_unused,
-		   const struct rte_flow_action actions[] __rte_unused,
+mlx5_flow_validate(struct rte_eth_dev *dev,
+		   const struct rte_flow_attr *attr,
+		   const struct rte_flow_item items[],
+		   const struct rte_flow_action actions[],
 		   struct rte_flow_error *error)
 {
-	return rte_flow_error_set(error, ENOTSUP,
-				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				  NULL,
-				  "action not supported");
+	int ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+
+	if (ret < 0)
+		return ret;
+	return 0;
+}
+
+/**
+ * Remove the flow.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ */
+static void
+mlx5_flow_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	if (flow->fate & MLX5_FLOW_FATE_DROP) {
+		if (flow->verbs.flow) {
+			claim_zero(mlx5_glue->destroy_flow(flow->verbs.flow));
+			flow->verbs.flow = NULL;
+		}
+	}
+	if (flow->verbs.hrxq) {
+		mlx5_hrxq_drop_release(dev);
+		flow->verbs.hrxq = NULL;
+	}
+}
+
+/**
+ * Apply the flow.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
+		struct rte_flow_error *error)
+{
+	flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
+	if (!flow->verbs.hrxq)
+		return rte_flow_error_set
+			(error, errno,
+			 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			 NULL,
+			 "cannot allocate Drop queue");
+	flow->verbs.flow =
+		mlx5_glue->create_flow(flow->verbs.hrxq->qp, flow->verbs.attr);
+	if (!flow->verbs.flow) {
+		mlx5_hrxq_drop_release(dev);
+		flow->verbs.hrxq = NULL;
+		return rte_flow_error_set(error, errno,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "kernel module refuses to create"
+					  " flow");
+	}
+	return 0;
+}
+
+/**
+ * Create a flow and add it to @p list.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param list
+ *   Pointer to a TAILQ flow list.
+ * @param[in] attr
+ *   Flow rule attributes.
+ * @param[in] items
+ *   Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ *   Associated actions (list terminated by the END action).
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   A flow on success, NULL otherwise and rte_errno is set.
+ */
+static struct rte_flow *
+mlx5_flow_list_create(struct rte_eth_dev *dev,
+		      struct mlx5_flows *list,
+		      const struct rte_flow_attr *attr,
+		      const struct rte_flow_item items[],
+		      const struct rte_flow_action actions[],
+		      struct rte_flow_error *error)
+{
+	struct rte_flow *flow;
+	size_t size;
+	int ret;
+
+	ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+	if (ret < 0)
+		return NULL;
+	size = ret;
+	flow = rte_zmalloc(__func__, size, 0);
+	if (!flow) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "cannot allocate memory");
+		return NULL;
+	}
+	flow->verbs.attr = (struct ibv_flow_attr *)(flow + 1);
+	flow->verbs.specs = (uint8_t *)(flow->verbs.attr + 1);
+	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
+	if (ret < 0)
+		goto error;
+	assert((size_t)ret == size);
+	if (dev->data->dev_started) {
+		ret = mlx5_flow_apply(dev, flow, error);
+		if (ret < 0)
+			goto error;
+	}
+	TAILQ_INSERT_TAIL(list, flow, next);
+	return flow;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	mlx5_flow_remove(dev, flow);
+	rte_free(flow);
+	rte_errno = ret; /* Restore rte_errno. */
+	return NULL;
 }
 
 /**
@@ -187,17 +775,15 @@ mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
  * @see rte_flow_ops
  */
 struct rte_flow *
-mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
-		 const struct rte_flow_attr *attr __rte_unused,
-		 const struct rte_flow_item items[] __rte_unused,
-		 const struct rte_flow_action actions[] __rte_unused,
+mlx5_flow_create(struct rte_eth_dev *dev,
+		 const struct rte_flow_attr *attr,
+		 const struct rte_flow_item items[],
+		 const struct rte_flow_action actions[],
 		 struct rte_flow_error *error)
 {
-	rte_flow_error_set(error, ENOTSUP,
-			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL,
-			   "action not supported");
-	return NULL;
+	return mlx5_flow_list_create
+		(dev, &((struct priv *)dev->data->dev_private)->flows,
+		 attr, items, actions, error);
 }
 
 /**
@@ -211,10 +797,12 @@ mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
  *   Flow to destroy.
  */
 static void
-mlx5_flow_list_destroy(struct rte_eth_dev *dev __rte_unused,
-		       struct mlx5_flows *list __rte_unused,
-		       struct rte_flow *flow __rte_unused)
+mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
+		       struct rte_flow *flow)
 {
+	mlx5_flow_remove(dev, flow);
+	TAILQ_REMOVE(list, flow, next);
+	rte_free(flow);
 }
 
 /**
-- 
2.18.0

[dpdk-dev] [PATCH v3 05/21] net/mlx5: add flow queue action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 97 ++++++++++++++++++++++++++++++++----
 1 file changed, 86 insertions(+), 11 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 036a8d440..6041a4573 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -50,6 +50,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
 /* Actions that modify the fate of matching traffic. */
 #define MLX5_FLOW_FATE_DROP (1u << 0)
+#define MLX5_FLOW_FATE_QUEUE (1u << 1)
 
 /** Handles information leading to a drop fate. */
 struct mlx5_flow_verbs {
@@ -71,7 +72,8 @@ struct rte_flow {
 	/**< Bit-fields of present layers see MLX5_FLOW_LAYER_*. */
 	uint32_t fate;
 	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
-	struct mlx5_flow_verbs verbs; /* Verbs drop flow. */
+	struct mlx5_flow_verbs verbs; /* Verbs flow. */
+	uint16_t queue; /**< Destination queue to redirect traffic to. */
 };
 
 static const struct rte_flow_ops mlx5_flow_ops = {
@@ -492,6 +494,52 @@ mlx5_flow_action_drop(const struct rte_flow_action *action,
 	return size;
 }
 
+/**
+ * Convert the @p action into @p flow after ensuring the NIC will understand
+ * and process it correctly.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device structure.
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_action_queue(struct rte_eth_dev *dev,
+		       const struct rte_flow_action *action,
+		       struct rte_flow *flow,
+		       struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_action_queue *queue = action->conf;
+
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "multiple fate actions are not"
+					  " supported");
+	if (queue->index >= priv->rxqs_n)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &queue->index,
+					  "queue index out of range");
+	if (!(*priv->rxqs)[queue->index])
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &queue->index,
+					  "queue is not configured");
+	flow->queue = queue->index;
+	flow->fate |= MLX5_FLOW_FATE_QUEUE;
+	return 0;
+}
+
 /**
  * Convert the @p action into @p flow after ensuring the NIC will understand
  * and process it correctly.
@@ -520,7 +568,7 @@ mlx5_flow_action_drop(const struct rte_flow_action *action,
  *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_actions(struct rte_eth_dev *dev __rte_unused,
+mlx5_flow_actions(struct rte_eth_dev *dev,
 		  const struct rte_flow_action actions[],
 		  struct rte_flow *flow, const size_t flow_size,
 		  struct rte_flow_error *error)
@@ -537,6 +585,9 @@ mlx5_flow_actions(struct rte_eth_dev *dev __rte_unused,
 			ret = mlx5_flow_action_drop(actions, flow, remain,
 						    error);
 			break;
+		case RTE_FLOW_ACTION_TYPE_QUEUE:
+			ret = mlx5_flow_action_queue(dev, actions, flow, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -661,7 +712,10 @@ mlx5_flow_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
 		}
 	}
 	if (flow->verbs.hrxq) {
-		mlx5_hrxq_drop_release(dev);
+		if (flow->fate & MLX5_FLOW_FATE_DROP)
+			mlx5_hrxq_drop_release(dev);
+		else if (flow->fate & MLX5_FLOW_FATE_QUEUE)
+			mlx5_hrxq_release(dev, flow->verbs.hrxq);
 		flow->verbs.hrxq = NULL;
 	}
 }
@@ -683,17 +737,38 @@ static int
 mlx5_flow_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 		struct rte_flow_error *error)
 {
-	flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
-	if (!flow->verbs.hrxq)
-		return rte_flow_error_set
-			(error, errno,
-			 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			 NULL,
-			 "cannot allocate Drop queue");
+	if (flow->fate & MLX5_FLOW_FATE_DROP) {
+		flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
+		if (!flow->verbs.hrxq)
+			return rte_flow_error_set
+				(error, errno,
+				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				 NULL,
+				 "cannot allocate Drop queue");
+	} else if (flow->fate & MLX5_FLOW_FATE_QUEUE) {
+		struct mlx5_hrxq *hrxq;
+
+		hrxq = mlx5_hrxq_get(dev, rss_hash_default_key,
+				     rss_hash_default_key_len, 0,
+				     &flow->queue, 1, 0, 0);
+		if (!hrxq)
+			hrxq = mlx5_hrxq_new(dev, rss_hash_default_key,
+					     rss_hash_default_key_len, 0,
+					     &flow->queue, 1, 0, 0);
+		if (!hrxq)
+			return rte_flow_error_set(error, rte_errno,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"cannot create flow");
+		flow->verbs.hrxq = hrxq;
+	}
 	flow->verbs.flow =
 		mlx5_glue->create_flow(flow->verbs.hrxq->qp, flow->verbs.attr);
 	if (!flow->verbs.flow) {
-		mlx5_hrxq_drop_release(dev);
+		if (flow->fate & MLX5_FLOW_FATE_DROP)
+			mlx5_hrxq_drop_release(dev);
+		else
+			mlx5_hrxq_release(dev, flow->verbs.hrxq);
 		flow->verbs.hrxq = NULL;
 		return rte_flow_error_set(error, errno,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-- 
2.18.0

[dpdk-dev] [PATCH v3 06/21] net/mlx5: add flow stop/start

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 6041a4573..77f1bd5cc 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -908,9 +908,12 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
  *   Pointer to a TAILQ flow list.
  */
 void
-mlx5_flow_stop(struct rte_eth_dev *dev __rte_unused,
-	       struct mlx5_flows *list __rte_unused)
+mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
 {
+	struct rte_flow *flow;
+
+	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
+		mlx5_flow_remove(dev, flow);
 }
 
 /**
@@ -925,10 +928,23 @@ mlx5_flow_stop(struct rte_eth_dev *dev __rte_unused,
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_flow_start(struct rte_eth_dev *dev __rte_unused,
-		struct mlx5_flows *list __rte_unused)
+mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
 {
+	struct rte_flow *flow;
+	struct rte_flow_error error;
+	int ret = 0;
+
+	TAILQ_FOREACH(flow, list, next) {
+		ret = mlx5_flow_apply(dev, flow, &error);
+		if (ret < 0)
+			goto error;
+	}
 	return 0;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	mlx5_flow_stop(dev, list);
+	rte_errno = ret; /* Restore rte_errno. */
+	return -rte_errno;
 }
 
 /**
-- 
2.18.0

[dpdk-dev] [PATCH v3 07/21] net/mlx5: add flow VLAN item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 127 +++++++++++++++++++++++++++++++++++
 1 file changed, 127 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 77f1bd5cc..659979283 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -382,6 +382,130 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Update the VLAN tag in the Verbs Ethernet specification.
+ *
+ * @param[in, out] attr
+ *   Pointer to Verbs attributes structure.
+ * @param[in] eth
+ *   Verbs structure containing the VLAN information to copy.
+ */
+static void
+mlx5_flow_item_vlan_update(struct ibv_flow_attr *attr,
+			   struct ibv_flow_spec_eth *eth)
+{
+	unsigned int i;
+	enum ibv_flow_spec_type search = IBV_FLOW_SPEC_ETH;
+	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
+		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
+
+	for (i = 0; i != attr->num_of_specs; ++i) {
+		if (hdr->type == search) {
+			struct ibv_flow_spec_eth *e =
+				(struct ibv_flow_spec_eth *)hdr;
+
+			e->val.vlan_tag = eth->val.vlan_tag;
+			e->mask.vlan_tag = eth->mask.vlan_tag;
+			e->val.ether_type = eth->val.ether_type;
+			e->mask.ether_type = eth->mask.ether_type;
+			break;
+		}
+		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
+	}
+}
+
+/**
+ * Convert the @p item into @p flow (or by updating the already present
+ * Ethernet Verbs) specification after ensuring the NIC will understand and
+ * process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_vlan *spec = item->spec;
+	const struct rte_flow_item_vlan *mask = item->mask;
+	const struct rte_flow_item_vlan nic_mask = {
+		.tci = RTE_BE16(0x0fff),
+		.inner_type = RTE_BE16(0xffff),
+	};
+	unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	struct ibv_flow_spec_eth eth = {
+		.type = IBV_FLOW_SPEC_ETH,
+		.size = size,
+	};
+	int ret;
+	const uint32_t l34m = MLX5_FLOW_LAYER_OUTER_L3 |
+			MLX5_FLOW_LAYER_OUTER_L4;
+	const uint32_t vlanm = MLX5_FLOW_LAYER_OUTER_VLAN;
+	const uint32_t l2m = MLX5_FLOW_LAYER_OUTER_L2;
+
+	if (flow->layers & vlanm)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VLAN layer already configured");
+	else if ((flow->layers & l34m) != 0)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L2 layer cannot follow L3/L4 layer");
+	if (!mask)
+		mask = &rte_flow_item_vlan_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&nic_mask,
+		 sizeof(struct rte_flow_item_vlan), error);
+	if (ret)
+		return ret;
+	if (spec) {
+		eth.val.vlan_tag = spec->tci;
+		eth.mask.vlan_tag = mask->tci;
+		eth.val.vlan_tag &= eth.mask.vlan_tag;
+		eth.val.ether_type = spec->inner_type;
+		eth.mask.ether_type = mask->inner_type;
+		eth.val.ether_type &= eth.mask.ether_type;
+	}
+	/*
+	 * From verbs perspective an empty VLAN is equivalent
+	 * to a packet without VLAN layer.
+	 */
+	if (!eth.mask.vlan_tag)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+					  item->spec,
+					  "VLAN cannot be empty");
+	if (!(flow->layers & l2m)) {
+		if (size <= flow_size)
+			mlx5_flow_spec_verbs_add(flow, &eth, size);
+	} else {
+		if (flow->verbs.attr)
+			mlx5_flow_item_vlan_update(flow->verbs.attr, &eth);
+		size = 0; /* Only an update is done in eth specification. */
+	}
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2 |
+		MLX5_FLOW_LAYER_OUTER_VLAN;
+	return size;
+}
+
 /**
  * Convert the @p pattern into a Verbs specifications after ensuring the NIC
  * will understand and process it correctly.
@@ -424,6 +548,9 @@ mlx5_flow_items(const struct rte_flow_item pattern[],
 		case RTE_FLOW_ITEM_TYPE_ETH:
 			ret = mlx5_flow_item_eth(pattern, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			ret = mlx5_flow_item_vlan(pattern, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

[dpdk-dev] [PATCH v3 08/21] net/mlx5: add flow IPv4 item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 90 ++++++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 659979283..c05b8498d 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -506,6 +506,93 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_ipv4 *spec = item->spec;
+	const struct rte_flow_item_ipv4 *mask = item->mask;
+	const struct rte_flow_item_ipv4 nic_mask = {
+		.hdr = {
+			.src_addr = RTE_BE32(0xffffffff),
+			.dst_addr = RTE_BE32(0xffffffff),
+			.type_of_service = 0xff,
+			.next_proto_id = 0xff,
+		},
+	};
+	unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
+	struct ibv_flow_spec_ipv4_ext ipv4 = {
+		.type = IBV_FLOW_SPEC_IPV4_EXT,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "multiple L3 layers not supported");
+	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 cannot follow an L4 layer.");
+	if (!mask)
+		mask = &rte_flow_item_ipv4_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&nic_mask,
+		 sizeof(struct rte_flow_item_ipv4), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
+			.src_ip = spec->hdr.src_addr,
+			.dst_ip = spec->hdr.dst_addr,
+			.proto = spec->hdr.next_proto_id,
+			.tos = spec->hdr.type_of_service,
+		};
+		ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
+			.src_ip = mask->hdr.src_addr,
+			.dst_ip = mask->hdr.dst_addr,
+			.proto = mask->hdr.next_proto_id,
+			.tos = mask->hdr.type_of_service,
+		};
+		/* Remove unwanted bits from values. */
+		ipv4.val.src_ip &= ipv4.mask.src_ip;
+		ipv4.val.dst_ip &= ipv4.mask.dst_ip;
+		ipv4.val.proto &= ipv4.mask.proto;
+		ipv4.val.tos &= ipv4.mask.tos;
+	}
+	mlx5_flow_spec_verbs_add(flow, &ipv4, size);
+	return size;
+}
+
 /**
  * Convert the @p pattern into a Verbs specifications after ensuring the NIC
  * will understand and process it correctly.
@@ -551,6 +638,9 @@ mlx5_flow_items(const struct rte_flow_item pattern[],
 		case RTE_FLOW_ITEM_TYPE_VLAN:
 			ret = mlx5_flow_item_vlan(pattern, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			ret = mlx5_flow_item_ipv4(pattern, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

[dpdk-dev] [PATCH v3 09/21] net/mlx5: add flow IPv6 item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 115 +++++++++++++++++++++++++++++++++++
 1 file changed, 115 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index c05b8498d..513f70d40 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -593,6 +593,118 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_ipv6 *spec = item->spec;
+	const struct rte_flow_item_ipv6 *mask = item->mask;
+	const struct rte_flow_item_ipv6 nic_mask = {
+		.hdr = {
+			.src_addr =
+				"\xff\xff\xff\xff\xff\xff\xff\xff"
+				"\xff\xff\xff\xff\xff\xff\xff\xff",
+			.dst_addr =
+				"\xff\xff\xff\xff\xff\xff\xff\xff"
+				"\xff\xff\xff\xff\xff\xff\xff\xff",
+			.vtc_flow = RTE_BE32(0xffffffff),
+			.proto = 0xff,
+			.hop_limits = 0xff,
+		},
+	};
+	unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
+	struct ibv_flow_spec_ipv6 ipv6 = {
+		.type = IBV_FLOW_SPEC_IPV6,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "multiple L3 layers not supported");
+	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 cannot follow an L4 layer.");
+	if (!mask)
+		mask = &rte_flow_item_ipv6_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&nic_mask,
+		 sizeof(struct rte_flow_item_ipv6), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		unsigned int i;
+		uint32_t vtc_flow_val;
+		uint32_t vtc_flow_mask;
+
+		memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
+		       RTE_DIM(ipv6.val.src_ip));
+		memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
+		       RTE_DIM(ipv6.val.dst_ip));
+		memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
+		       RTE_DIM(ipv6.mask.src_ip));
+		memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
+		       RTE_DIM(ipv6.mask.dst_ip));
+		vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
+		vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
+		ipv6.val.flow_label =
+			rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
+					 IPV6_HDR_FL_SHIFT);
+		ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
+					 IPV6_HDR_TC_SHIFT;
+		ipv6.val.next_hdr = spec->hdr.proto;
+		ipv6.val.hop_limit = spec->hdr.hop_limits;
+		ipv6.mask.flow_label =
+			rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
+					 IPV6_HDR_FL_SHIFT);
+		ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
+					  IPV6_HDR_TC_SHIFT;
+		ipv6.mask.next_hdr = mask->hdr.proto;
+		ipv6.mask.hop_limit = mask->hdr.hop_limits;
+		/* Remove unwanted bits from values. */
+		for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
+			ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
+			ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
+		}
+		ipv6.val.flow_label &= ipv6.mask.flow_label;
+		ipv6.val.traffic_class &= ipv6.mask.traffic_class;
+		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
+		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
+	}
+	mlx5_flow_spec_verbs_add(flow, &ipv6, size);
+	return size;
+}
+
 /**
  * Convert the @p pattern into a Verbs specifications after ensuring the NIC
  * will understand and process it correctly.
@@ -641,6 +753,9 @@ mlx5_flow_items(const struct rte_flow_item pattern[],
 		case RTE_FLOW_ITEM_TYPE_IPV4:
 			ret = mlx5_flow_item_ipv4(pattern, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			ret = mlx5_flow_item_ipv6(pattern, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

[dpdk-dev] [PATCH v3 10/21] net/mlx5: add flow UDP item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 97 +++++++++++++++++++++++++++++++++---
 1 file changed, 91 insertions(+), 6 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 513f70d40..0096ed8a2 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -52,6 +52,9 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_FLOW_FATE_DROP (1u << 0)
 #define MLX5_FLOW_FATE_QUEUE (1u << 1)
 
+/* possible L3 layers protocols filtering. */
+#define MLX5_IP_PROTOCOL_UDP 17
+
 /** Handles information leading to a drop fate. */
 struct mlx5_flow_verbs {
 	unsigned int size; /**< Size of the attribute. */
@@ -68,10 +71,12 @@ struct mlx5_flow_verbs {
 struct rte_flow {
 	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
 	struct rte_flow_attr attributes; /**< User flow attribute. */
+	uint32_t l3_protocol_en:1; /**< Protocol filtering requested. */
 	uint32_t layers;
 	/**< Bit-fields of present layers see MLX5_FLOW_LAYER_*. */
 	uint32_t fate;
 	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
+	uint8_t l3_protocol; /**< valid when l3_protocol_en is set. */
 	struct mlx5_flow_verbs verbs; /* Verbs flow. */
 	uint16_t queue; /**< Destination queue to redirect traffic to. */
 };
@@ -568,8 +573,6 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 	if (ret < 0)
 		return ret;
 	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
-	if (size > flow_size)
-		return size;
 	if (spec) {
 		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 			.src_ip = spec->hdr.src_addr,
@@ -589,7 +592,10 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 		ipv4.val.proto &= ipv4.mask.proto;
 		ipv4.val.tos &= ipv4.mask.tos;
 	}
-	mlx5_flow_spec_verbs_add(flow, &ipv4, size);
+	flow->l3_protocol_en = !!ipv4.mask.proto;
+	flow->l3_protocol = ipv4.val.proto;
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &ipv4, size);
 	return size;
 }
 
@@ -660,8 +666,6 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 	if (ret < 0)
 		return ret;
 	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
-	if (size > flow_size)
-		return size;
 	if (spec) {
 		unsigned int i;
 		uint32_t vtc_flow_val;
@@ -701,7 +705,85 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
 	}
-	mlx5_flow_spec_verbs_add(flow, &ipv6, size);
+	flow->l3_protocol_en = !!ipv6.mask.next_hdr;
+	flow->l3_protocol = ipv6.val.next_hdr;
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &ipv6, size);
+	return size;
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_udp *spec = item->spec;
+	const struct rte_flow_item_udp *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
+	struct ibv_flow_spec_tcp_udp udp = {
+		.type = IBV_FLOW_SPEC_UDP,
+		.size = size,
+	};
+	int ret;
+
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 is mandatory to filter on L4");
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L4 layer is already present");
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_UDP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with UDP layer");
+	if (!mask)
+		mask = &rte_flow_item_udp_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_udp_mask,
+		 sizeof(struct rte_flow_item_udp), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		udp.val.dst_port = spec->hdr.dst_port;
+		udp.val.src_port = spec->hdr.src_port;
+		udp.mask.dst_port = mask->hdr.dst_port;
+		udp.mask.src_port = mask->hdr.src_port;
+		/* Remove unwanted bits from values. */
+		udp.val.src_port &= udp.mask.src_port;
+		udp.val.dst_port &= udp.mask.dst_port;
+	}
+	mlx5_flow_spec_verbs_add(flow, &udp, size);
 	return size;
 }
 
@@ -756,6 +838,9 @@ mlx5_flow_items(const struct rte_flow_item pattern[],
 		case RTE_FLOW_ITEM_TYPE_IPV6:
 			ret = mlx5_flow_item_ipv6(pattern, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			ret = mlx5_flow_item_udp(pattern, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

[dpdk-dev] [PATCH v3 11/21] net/mlx5: add flow TCP item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 79 ++++++++++++++++++++++++++++++++++++
 1 file changed, 79 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 0096ed8a2..f646eee01 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -53,6 +53,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_FLOW_FATE_QUEUE (1u << 1)
 
 /* possible L3 layers protocols filtering. */
+#define MLX5_IP_PROTOCOL_TCP 6
 #define MLX5_IP_PROTOCOL_UDP 17
 
 /** Handles information leading to a drop fate. */
@@ -787,6 +788,81 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_tcp *spec = item->spec;
+	const struct rte_flow_item_tcp *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
+	struct ibv_flow_spec_tcp_udp tcp = {
+		.type = IBV_FLOW_SPEC_TCP,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_TCP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with TCP layer");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 is mandatory to filter on L4");
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L4 layer is already present");
+	if (!mask)
+		mask = &rte_flow_item_tcp_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_tcp_mask,
+		 sizeof(struct rte_flow_item_tcp), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		tcp.val.dst_port = spec->hdr.dst_port;
+		tcp.val.src_port = spec->hdr.src_port;
+		tcp.mask.dst_port = mask->hdr.dst_port;
+		tcp.mask.src_port = mask->hdr.src_port;
+		/* Remove unwanted bits from values. */
+		tcp.val.src_port &= tcp.mask.src_port;
+		tcp.val.dst_port &= tcp.mask.dst_port;
+	}
+	mlx5_flow_spec_verbs_add(flow, &tcp, size);
+	return size;
+}
+
 /**
  * Convert the @p pattern into a Verbs specifications after ensuring the NIC
  * will understand and process it correctly.
@@ -841,6 +917,9 @@ mlx5_flow_items(const struct rte_flow_item pattern[],
 		case RTE_FLOW_ITEM_TYPE_UDP:
 			ret = mlx5_flow_item_udp(pattern, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			ret = mlx5_flow_item_tcp(pattern, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

[dpdk-dev] [PATCH v3 12/21] net/mlx5: add mark/flag flow action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 252 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_rxtx.h |   1 +
 2 files changed, 253 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index f646eee01..1280db486 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -52,6 +52,10 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_FLOW_FATE_DROP (1u << 0)
 #define MLX5_FLOW_FATE_QUEUE (1u << 1)
 
+/* Modify a packet. */
+#define MLX5_FLOW_MOD_FLAG (1u << 0)
+#define MLX5_FLOW_MOD_MARK (1u << 1)
+
 /* possible L3 layers protocols filtering. */
 #define MLX5_IP_PROTOCOL_TCP 6
 #define MLX5_IP_PROTOCOL_UDP 17
@@ -75,6 +79,8 @@ struct rte_flow {
 	uint32_t l3_protocol_en:1; /**< Protocol filtering requested. */
 	uint32_t layers;
 	/**< Bit-fields of present layers see MLX5_FLOW_LAYER_*. */
+	uint32_t modifier;
+	/**< Bit-fields of present modifier see MLX5_FLOW_MOD_*. */
 	uint32_t fate;
 	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
 	uint8_t l3_protocol; /**< valid when l3_protocol_en is set. */
@@ -984,6 +990,12 @@ mlx5_flow_action_drop(const struct rte_flow_action *action,
 					  action,
 					  "multiple fate actions are not"
 					  " supported");
+	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "drop is not compatible with"
+					  " flag/mark action");
 	if (size < flow_size)
 		mlx5_flow_spec_verbs_add(flow, &drop, size);
 	flow->fate |= MLX5_FLOW_FATE_DROP;
@@ -1036,6 +1048,161 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/**
+ * Convert the @p action into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_flag(const struct rte_flow_action *action,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
+{
+	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
+	struct ibv_flow_spec_action_tag tag = {
+		.type = IBV_FLOW_SPEC_ACTION_TAG,
+		.size = size,
+		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
+	};
+
+	if (flow->modifier & MLX5_FLOW_MOD_FLAG)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "flag action already present");
+	if (flow->fate & MLX5_FLOW_FATE_DROP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "flag is not compatible with drop"
+					  " action");
+	if (flow->modifier & MLX5_FLOW_MOD_MARK)
+		return 0;
+	flow->modifier |= MLX5_FLOW_MOD_FLAG;
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &tag, size);
+	return size;
+}
+
+/**
+ * Update verbs specification to modify the flag to mark.
+ *
+ * @param[in, out] flow
+ *   Pointer to the rte_flow structure.
+ * @param[in] mark_id
+ *   Mark identifier to replace the flag.
+ */
+static void
+mlx5_flow_verbs_mark_update(struct rte_flow *flow, uint32_t mark_id)
+{
+	struct ibv_spec_header *hdr;
+	int i;
+
+	/* Update Verbs specification. */
+	hdr = (struct ibv_spec_header *)flow->verbs.specs;
+	if (!hdr)
+		return;
+	for (i = 0; i != flow->verbs.attr->num_of_specs; ++i) {
+		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
+			struct ibv_flow_spec_action_tag *t =
+				(struct ibv_flow_spec_action_tag *)hdr;
+
+			t->tag_id = mlx5_flow_mark_set(mark_id);
+		}
+		hdr = (struct ibv_spec_header *)((uintptr_t)hdr + hdr->size);
+	}
+}
+
+/**
+ * Convert the @p action into @p flow (or by updating the already present
+ * Flag Verbs specification) after ensuring the NIC will understand and
+ * process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_mark(const struct rte_flow_action *action,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
+{
+	const struct rte_flow_action_mark *mark = action->conf;
+	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
+	struct ibv_flow_spec_action_tag tag = {
+		.type = IBV_FLOW_SPEC_ACTION_TAG,
+		.size = size,
+	};
+
+	if (!mark)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "configuration cannot be null");
+	if (mark->id >= MLX5_FLOW_MARK_MAX)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &mark->id,
+					  "mark id must in 0 <= id < "
+					  RTE_STR(MLX5_FLOW_MARK_MAX));
+	if (flow->modifier & MLX5_FLOW_MOD_MARK)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "mark action already present");
+	if (flow->fate & MLX5_FLOW_FATE_DROP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "mark is not compatible with drop"
+					  " action");
+	if (flow->modifier & MLX5_FLOW_MOD_FLAG) {
+		mlx5_flow_verbs_mark_update(flow, mark->id);
+		size = 0; /**< Only an update is done in the specification. */
+	} else {
+		tag.tag_id = mlx5_flow_mark_set(mark->id);
+		if (size <= flow_size) {
+			tag.tag_id = mlx5_flow_mark_set(mark->id);
+			mlx5_flow_spec_verbs_add(flow, &tag, size);
+		}
+	}
+	flow->modifier |= MLX5_FLOW_MOD_MARK;
+	return size;
+}
+
 /**
  * Convert the @p action into @p flow after ensuring the NIC will understand
  * and process it correctly.
@@ -1077,6 +1244,14 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 		switch (actions->type) {
 		case RTE_FLOW_ACTION_TYPE_VOID:
 			break;
+		case RTE_FLOW_ACTION_TYPE_FLAG:
+			ret = mlx5_flow_action_flag(actions, flow, remain,
+						    error);
+			break;
+		case RTE_FLOW_ACTION_TYPE_MARK:
+			ret = mlx5_flow_action_mark(actions, flow, remain,
+						    error);
+			break;
 		case RTE_FLOW_ACTION_TYPE_DROP:
 			ret = mlx5_flow_action_drop(actions, flow, remain,
 						    error);
@@ -1170,6 +1345,79 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Mark the Rx queues mark flag if the flow has a mark or flag modifier.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in] flow
+ *   Pointer to flow structure.
+ */
+static void
+mlx5_flow_rxq_mark_set(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK)) {
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of((*priv->rxqs)[flow->queue],
+				     struct mlx5_rxq_ctrl, rxq);
+
+		rxq_ctrl->rxq.mark = 1;
+		rxq_ctrl->flow_mark_n++;
+	}
+}
+
+/**
+ * Clear the Rx queue mark associated with the @p flow if no other flow uses
+ * it with a mark request.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[in] flow
+ *   Pointer to the flow.
+ */
+static void
+mlx5_flow_rxq_mark_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK)) {
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of((*priv->rxqs)[flow->queue],
+				     struct mlx5_rxq_ctrl, rxq);
+
+		rxq_ctrl->flow_mark_n--;
+		rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
+	}
+}
+
+/**
+ * Clear the mark bit in all Rx queues.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+static void
+mlx5_flow_rxq_mark_clear(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+	unsigned int idx;
+
+	for (idx = 0, i = 0; idx != priv->rxqs_n; ++i) {
+		struct mlx5_rxq_ctrl *rxq_ctrl;
+
+		if (!(*priv->rxqs)[idx])
+			continue;
+		rxq_ctrl = container_of((*priv->rxqs)[idx],
+					struct mlx5_rxq_ctrl, rxq);
+		rxq_ctrl->flow_mark_n = 0;
+		rxq_ctrl->rxq.mark = 0;
+		++idx;
+	}
+}
+
 /**
  * Validate a flow supported by the NIC.
  *
@@ -1329,6 +1577,7 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
 		if (ret < 0)
 			goto error;
 	}
+	mlx5_flow_rxq_mark_set(dev, flow);
 	TAILQ_INSERT_TAIL(list, flow, next);
 	return flow;
 error:
@@ -1373,6 +1622,7 @@ mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
 {
 	mlx5_flow_remove(dev, flow);
 	TAILQ_REMOVE(list, flow, next);
+	mlx5_flow_rxq_mark_trim(dev, flow);
 	rte_free(flow);
 }
 
@@ -1410,6 +1660,7 @@ mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
 
 	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
 		mlx5_flow_remove(dev, flow);
+	mlx5_flow_rxq_mark_clear(dev);
 }
 
 /**
@@ -1434,6 +1685,7 @@ mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
 		ret = mlx5_flow_apply(dev, flow, &error);
 		if (ret < 0)
 			goto error;
+		mlx5_flow_rxq_mark_set(dev, flow);
 	}
 	return 0;
 error:
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 708fdd4fa..99b769d8a 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -138,6 +138,7 @@ struct mlx5_rxq_ctrl {
 	unsigned int socket; /* CPU socket ID for allocations. */
 	unsigned int irq:1; /* Whether IRQ is enabled. */
 	uint16_t idx; /* Queue index. */
+	uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */
 };
 
 /* Indirection table. */
-- 
2.18.0

[dpdk-dev] [PATCH v3 13/21] net/mlx5: use a macro for the RSS key size

[Nelio Laranjeiro]

ConnectX 4-5 support only 40 bytes of RSS key, using a compiled size
hash key is not necessary.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_ethdev.c | 14 +++++++-------
 drivers/net/mlx5/mlx5_flow.c   |  4 ++--
 drivers/net/mlx5/mlx5_prm.h    |  3 +++
 drivers/net/mlx5/mlx5_rss.c    |  7 ++++---
 drivers/net/mlx5/mlx5_rxq.c    | 12 +++++++-----
 drivers/net/mlx5/mlx5_rxtx.h   |  1 -
 6 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 05f66f7b6..6e44d5ff0 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -377,15 +377,15 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
 
 	if (use_app_rss_key &&
 	    (dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len !=
-	     rss_hash_default_key_len)) {
-		DRV_LOG(ERR, "port %u RSS key len must be %zu Bytes long",
-			dev->data->port_id, rss_hash_default_key_len);
+	     MLX5_RSS_HASH_KEY_LEN)) {
+		DRV_LOG(ERR, "port %u RSS key len must be %s Bytes long",
+			dev->data->port_id, RTE_STR(MLX5_RSS_HASH_KEY_LEN));
 		rte_errno = EINVAL;
 		return -rte_errno;
 	}
 	priv->rss_conf.rss_key =
 		rte_realloc(priv->rss_conf.rss_key,
-			    rss_hash_default_key_len, 0);
+			    MLX5_RSS_HASH_KEY_LEN, 0);
 	if (!priv->rss_conf.rss_key) {
 		DRV_LOG(ERR, "port %u cannot allocate RSS hash key memory (%u)",
 			dev->data->port_id, rxqs_n);
@@ -396,8 +396,8 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
 	       use_app_rss_key ?
 	       dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key :
 	       rss_hash_default_key,
-	       rss_hash_default_key_len);
-	priv->rss_conf.rss_key_len = rss_hash_default_key_len;
+	       MLX5_RSS_HASH_KEY_LEN);
+	priv->rss_conf.rss_key_len = MLX5_RSS_HASH_KEY_LEN;
 	priv->rss_conf.rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
 	priv->rxqs = (void *)dev->data->rx_queues;
 	priv->txqs = (void *)dev->data->tx_queues;
@@ -515,7 +515,7 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 		info->if_index = if_nametoindex(ifname);
 	info->reta_size = priv->reta_idx_n ?
 		priv->reta_idx_n : config->ind_table_max_size;
-	info->hash_key_size = rss_hash_default_key_len;
+	info->hash_key_size = MLX5_RSS_HASH_KEY_LEN;
 	info->speed_capa = priv->link_speed_capa;
 	info->flow_type_rss_offloads = ~MLX5_RSS_HF_MASK;
 	mlx5_set_default_params(dev, info);
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 1280db486..77483bd1f 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -1493,11 +1493,11 @@ mlx5_flow_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 		struct mlx5_hrxq *hrxq;
 
 		hrxq = mlx5_hrxq_get(dev, rss_hash_default_key,
-				     rss_hash_default_key_len, 0,
+				     MLX5_RSS_HASH_KEY_LEN, 0,
 				     &flow->queue, 1, 0, 0);
 		if (!hrxq)
 			hrxq = mlx5_hrxq_new(dev, rss_hash_default_key,
-					     rss_hash_default_key_len, 0,
+					     MLX5_RSS_HASH_KEY_LEN, 0,
 					     &flow->queue, 1, 0, 0);
 		if (!hrxq)
 			return rte_flow_error_set(error, rte_errno,
diff --git a/drivers/net/mlx5/mlx5_prm.h b/drivers/net/mlx5/mlx5_prm.h
index f9fae1e50..0870d32fd 100644
--- a/drivers/net/mlx5/mlx5_prm.h
+++ b/drivers/net/mlx5/mlx5_prm.h
@@ -21,6 +21,9 @@
 #include <rte_vect.h>
 #include "mlx5_autoconf.h"
 
+/* RSS hash key size. */
+#define MLX5_RSS_HASH_KEY_LEN 40
+
 /* Get CQE owner bit. */
 #define MLX5_CQE_OWNER(op_own) ((op_own) & MLX5_CQE_OWNER_MASK)
 
diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c
index d69b4c09e..b95778a8c 100644
--- a/drivers/net/mlx5/mlx5_rss.c
+++ b/drivers/net/mlx5/mlx5_rss.c
@@ -50,10 +50,11 @@ mlx5_rss_hash_update(struct rte_eth_dev *dev,
 		return -rte_errno;
 	}
 	if (rss_conf->rss_key && rss_conf->rss_key_len) {
-		if (rss_conf->rss_key_len != rss_hash_default_key_len) {
+		if (rss_conf->rss_key_len != MLX5_RSS_HASH_KEY_LEN) {
 			DRV_LOG(ERR,
-				"port %u RSS key len must be %zu Bytes long",
-				dev->data->port_id, rss_hash_default_key_len);
+				"port %u RSS key len must be %s Bytes long",
+				dev->data->port_id,
+				RTE_STR(MLX5_RSS_HASH_KEY_LEN));
 			rte_errno = EINVAL;
 			return -rte_errno;
 		}
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index d960daa43..d50b82c69 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -52,7 +52,9 @@ uint8_t rss_hash_default_key[] = {
 };
 
 /* Length of the default RSS hash key. */
-const size_t rss_hash_default_key_len = sizeof(rss_hash_default_key);
+static_assert(MLX5_RSS_HASH_KEY_LEN ==
+	      (unsigned int)sizeof(rss_hash_default_key),
+	      "wrong RSS default key size.");
 
 /**
  * Check whether Multi-Packet RQ can be enabled for the device.
@@ -1771,7 +1773,7 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 		return NULL;
 	}
 	if (!rss_key_len) {
-		rss_key_len = rss_hash_default_key_len;
+		rss_key_len = MLX5_RSS_HASH_KEY_LEN;
 		rss_key = rss_hash_default_key;
 	}
 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
@@ -1791,7 +1793,7 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 			.rx_hash_conf = (struct ibv_rx_hash_conf){
 				.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ,
 				.rx_hash_key_len = rss_key_len ? rss_key_len :
-						   rss_hash_default_key_len,
+						   MLX5_RSS_HASH_KEY_LEN,
 				.rx_hash_key = rss_key ?
 					       (void *)(uintptr_t)rss_key :
 					       rss_hash_default_key,
@@ -1815,7 +1817,7 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 			.rx_hash_conf = (struct ibv_rx_hash_conf){
 				.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ,
 				.rx_hash_key_len = rss_key_len ? rss_key_len :
-						   rss_hash_default_key_len,
+						   MLX5_RSS_HASH_KEY_LEN,
 				.rx_hash_key = rss_key ?
 					       (void *)(uintptr_t)rss_key :
 					       rss_hash_default_key,
@@ -2138,7 +2140,7 @@ mlx5_hrxq_drop_new(struct rte_eth_dev *dev)
 			.rx_hash_conf = (struct ibv_rx_hash_conf){
 				.rx_hash_function =
 					IBV_RX_HASH_FUNC_TOEPLITZ,
-				.rx_hash_key_len = rss_hash_default_key_len,
+				.rx_hash_key_len = MLX5_RSS_HASH_KEY_LEN,
 				.rx_hash_key = rss_hash_default_key,
 				.rx_hash_fields_mask = 0,
 				},
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 99b769d8a..59e374d8d 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -225,7 +225,6 @@ struct mlx5_txq_ctrl {
 /* mlx5_rxq.c */
 
 extern uint8_t rss_hash_default_key[];
-extern const size_t rss_hash_default_key_len;
 
 int mlx5_check_mprq_support(struct rte_eth_dev *dev);
 int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq);
-- 
2.18.0

[dpdk-dev] [PATCH v3 14/21] net/mlx5: add RSS flow action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 682 +++++++++++++++++++++++++++--------
 1 file changed, 537 insertions(+), 145 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 77483bd1f..f098e727a 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -51,6 +51,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* Actions that modify the fate of matching traffic. */
 #define MLX5_FLOW_FATE_DROP (1u << 0)
 #define MLX5_FLOW_FATE_QUEUE (1u << 1)
+#define MLX5_FLOW_FATE_RSS (1u << 2)
 
 /* Modify a packet. */
 #define MLX5_FLOW_MOD_FLAG (1u << 0)
@@ -60,8 +61,68 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_IP_PROTOCOL_TCP 6
 #define MLX5_IP_PROTOCOL_UDP 17
 
+/* Priority reserved for default flows. */
+#define MLX5_FLOW_PRIO_RSVD ((uint32_t)-1)
+
+enum mlx5_expansion {
+	MLX5_EXPANSION_ROOT,
+	MLX5_EXPANSION_ETH,
+	MLX5_EXPANSION_IPV4,
+	MLX5_EXPANSION_IPV4_UDP,
+	MLX5_EXPANSION_IPV4_TCP,
+	MLX5_EXPANSION_IPV6,
+	MLX5_EXPANSION_IPV6_UDP,
+	MLX5_EXPANSION_IPV6_TCP,
+};
+
+/** Supported expansion of items. */
+static const struct rte_flow_expand_node mlx5_support_expansion[] = {
+	[MLX5_EXPANSION_ROOT] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
+						 MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_END,
+	},
+	[MLX5_EXPANSION_ETH] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+	},
+	[MLX5_EXPANSION_IPV4] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
+						 MLX5_EXPANSION_IPV4_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV4,
+		.rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+			ETH_RSS_NONFRAG_IPV4_OTHER,
+	},
+	[MLX5_EXPANSION_IPV4_UDP] = {
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
+	},
+	[MLX5_EXPANSION_IPV4_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
+	},
+	[MLX5_EXPANSION_IPV6] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
+						 MLX5_EXPANSION_IPV6_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV6,
+		.rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
+			ETH_RSS_NONFRAG_IPV6_OTHER,
+	},
+	[MLX5_EXPANSION_IPV6_UDP] = {
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
+	},
+	[MLX5_EXPANSION_IPV6_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
+	},
+};
+
 /** Handles information leading to a drop fate. */
 struct mlx5_flow_verbs {
+	LIST_ENTRY(mlx5_flow_verbs) next;
 	unsigned int size; /**< Size of the attribute. */
 	struct {
 		struct ibv_flow_attr *attr;
@@ -70,6 +131,7 @@ struct mlx5_flow_verbs {
 	};
 	struct ibv_flow *flow; /**< Verbs flow pointer. */
 	struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */
+	uint64_t hash_fields; /**< Verbs hash Rx queue hash fields. */
 };
 
 /* Flow structure. */
@@ -84,8 +146,12 @@ struct rte_flow {
 	uint32_t fate;
 	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
 	uint8_t l3_protocol; /**< valid when l3_protocol_en is set. */
-	struct mlx5_flow_verbs verbs; /* Verbs flow. */
-	uint16_t queue; /**< Destination queue to redirect traffic to. */
+	LIST_HEAD(verbs, mlx5_flow_verbs) verbs; /**< Verbs flows list. */
+	struct mlx5_flow_verbs *cur_verbs;
+	/**< Current Verbs flow structure being filled. */
+	struct rte_flow_action_rss rss;/**< RSS context. */
+	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
+	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
 };
 
 static const struct rte_flow_ops mlx5_flow_ops = {
@@ -128,16 +194,38 @@ struct ibv_spec_header {
 	uint16_t size;
 };
 
- /**
-  * Discover the maximum number of priority available.
-  *
-  * @param[in] dev
-  *   Pointer to Ethernet device.
-  *
-  * @return
-  *   number of supported flow priority on success, a negative errno value
-  *   otherwise and rte_errno is set.
-  */
+/*
+ * Number of sub priorities.
+ * For each kind of pattern matching i.e. L2, L3, L4 to have a correct
+ * matching on the NIC (firmware dependent) L4 most have the higher priority
+ * followed by L3 and ending with L2.
+ */
+#define MLX5_PRIORITY_MAP_L2 2
+#define MLX5_PRIORITY_MAP_L3 1
+#define MLX5_PRIORITY_MAP_L4 0
+#define MLX5_PRIORITY_MAP_MAX 3
+
+/* Map of Verbs to Flow priority with 8 Verbs priorities. */
+static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
+	{ 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
+};
+
+/* Map of Verbs to Flow priority with 16 Verbs priorities. */
+static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
+	{ 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
+	{ 9, 10, 11 }, { 12, 13, 14 },
+};
+
+/**
+ * Discover the maximum number of priority available.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   number of supported flow priority on success, a negative errno
+ *   value otherwise and rte_errno is set.
+ */
 int
 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 {
@@ -162,6 +250,7 @@ mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 	struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
 	uint16_t vprio[] = { 8, 16 };
 	int i;
+	int priority = 0;
 
 	if (!drop) {
 		rte_errno = ENOTSUP;
@@ -173,11 +262,52 @@ mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 		if (!flow)
 			break;
 		claim_zero(mlx5_glue->destroy_flow(flow));
+		priority = vprio[i];
+	}
+	switch (priority) {
+	case 8:
+		priority = RTE_DIM(priority_map_3);
+		break;
+	case 16:
+		priority = RTE_DIM(priority_map_5);
+		break;
+	default:
+		rte_errno = ENOTSUP;
+		DRV_LOG(ERR,
+			"port %u verbs maximum priority: %d expected 8/16",
+			dev->data->port_id, vprio[i]);
+		return -rte_errno;
 	}
 	mlx5_hrxq_drop_release(dev);
 	DRV_LOG(INFO, "port %u flow maximum priority: %d",
-		dev->data->port_id, vprio[i - 1]);
-	return vprio[i - 1];
+		dev->data->port_id, priority);
+	return priority;
+}
+
+/**
+ * Adjust flow priority.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param flow
+ *   Pointer to an rte flow.
+ */
+static void
+mlx5_flow_adjust_priority(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct priv *priv = dev->data->dev_private;
+	uint32_t priority = flow->attributes.priority;
+	uint32_t subpriority = flow->cur_verbs->attr->priority;
+
+	switch (priv->config.flow_prio) {
+	case RTE_DIM(priority_map_3):
+		priority = priority_map_3[priority][subpriority];
+		break;
+	case RTE_DIM(priority_map_5):
+		priority = priority_map_5[priority][subpriority];
+		break;
+	}
+	flow->cur_verbs->attr->priority = priority;
 }
 
 /**
@@ -203,14 +333,15 @@ mlx5_flow_attributes(struct rte_eth_dev *dev,
 		     struct rte_flow_error *error)
 {
 	uint32_t priority_max =
-		((struct priv *)dev->data->dev_private)->config.flow_prio;
+		((struct priv *)dev->data->dev_private)->config.flow_prio - 1;
 
 	if (attributes->group)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 					  NULL,
 					  "groups is not supported");
-	if (attributes->priority >= priority_max)
+	if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
+	    attributes->priority >= priority_max)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 					  NULL,
@@ -231,6 +362,8 @@ mlx5_flow_attributes(struct rte_eth_dev *dev,
 					  NULL,
 					  "ingress attribute is mandatory");
 	flow->attributes = *attributes;
+	if (attributes->priority == MLX5_FLOW_PRIO_RSVD)
+		flow->attributes.priority = priority_max;
 	return 0;
 }
 
@@ -296,7 +429,7 @@ mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 }
 
 /**
- * Add a verbs specification into @p flow.
+ * Add a verbs item specification into @p flow.
  *
  * @param[in, out] flow
  *   Pointer to flow structure.
@@ -308,14 +441,16 @@ mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 static void
 mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
 {
-	if (flow->verbs.specs) {
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+
+	if (verbs->specs) {
 		void *dst;
 
-		dst = (void *)(flow->verbs.specs + flow->verbs.size);
+		dst = (void *)(verbs->specs + verbs->size);
 		memcpy(dst, src, size);
-		++flow->verbs.attr->num_of_specs;
+		++verbs->attr->num_of_specs;
 	}
-	flow->verbs.size += size;
+	verbs->size += size;
 }
 
 /**
@@ -390,6 +525,7 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 		}
 		eth.val.ether_type &= eth.mask.ether_type;
 	}
+	flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
 	mlx5_flow_spec_verbs_add(flow, &eth, size);
 	return size;
 }
@@ -460,6 +596,7 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 		.inner_type = RTE_BE16(0xffff),
 	};
 	unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 	struct ibv_flow_spec_eth eth = {
 		.type = IBV_FLOW_SPEC_ETH,
 		.size = size,
@@ -506,15 +643,16 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 					  item->spec,
 					  "VLAN cannot be empty");
 	if (!(flow->layers & l2m)) {
-		if (size <= flow_size)
+		if (size <= flow_size) {
+			flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
 			mlx5_flow_spec_verbs_add(flow, &eth, size);
+		}
 	} else {
-		if (flow->verbs.attr)
-			mlx5_flow_item_vlan_update(flow->verbs.attr, &eth);
+		if (verbs->attr)
+			mlx5_flow_item_vlan_update(verbs->attr, &eth);
 		size = 0; /* Only an update is done in eth specification. */
 	}
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2 |
-		MLX5_FLOW_LAYER_OUTER_VLAN;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_VLAN;
 	return size;
 }
 
@@ -601,8 +739,18 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 	}
 	flow->l3_protocol_en = !!ipv4.mask.proto;
 	flow->l3_protocol = ipv4.val.proto;
-	if (size <= flow_size)
+	if (size <= flow_size) {
+		uint64_t hash_fields = IBV_RX_HASH_SRC_IPV4 |
+			IBV_RX_HASH_DST_IPV4;
+
+		if (!(flow->rss.types &
+		      (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+		       ETH_RSS_NONFRAG_IPV4_OTHER)))
+			hash_fields = 0;
+		flow->cur_verbs->hash_fields |= hash_fields;
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L3;
 		mlx5_flow_spec_verbs_add(flow, &ipv4, size);
+	}
 	return size;
 }
 
@@ -714,8 +862,17 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 	}
 	flow->l3_protocol_en = !!ipv6.mask.next_hdr;
 	flow->l3_protocol = ipv6.val.next_hdr;
-	if (size <= flow_size)
+	if (size <= flow_size) {
+		uint64_t hash_fields = IBV_RX_HASH_SRC_IPV6 |
+			IBV_RX_HASH_DST_IPV6;
+
+		if (!(flow->rss.types &
+		      (ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER)))
+			hash_fields = 0;
+		flow->cur_verbs->hash_fields |= hash_fields;
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L3;
 		mlx5_flow_spec_verbs_add(flow, &ipv6, size);
+	}
 	return size;
 }
 
@@ -754,22 +911,24 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 	};
 	int ret;
 
-	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_UDP)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
-					  "L3 is mandatory to filter on L4");
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+					  "protocol filtering not compatible"
+					  " with UDP layer");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
-					  "L4 layer is already present");
-	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_UDP)
+					  "L3 is mandatory to filter"
+					  " on L4");
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
-					  "protocol filtering not compatible"
-					  " with UDP layer");
+					  "L4 layer is already"
+					  " present");
 	if (!mask)
 		mask = &rte_flow_item_udp_mask;
 	ret = mlx5_flow_item_acceptable
@@ -779,8 +938,6 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 	if (ret < 0)
 		return ret;
 	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
-	if (size > flow_size)
-		return size;
 	if (spec) {
 		udp.val.dst_port = spec->hdr.dst_port;
 		udp.val.src_port = spec->hdr.src_port;
@@ -790,7 +947,16 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 		udp.val.src_port &= udp.mask.src_port;
 		udp.val.dst_port &= udp.mask.dst_port;
 	}
-	mlx5_flow_spec_verbs_add(flow, &udp, size);
+	if (size <= flow_size) {
+		uint64_t hash_fields = IBV_RX_HASH_SRC_PORT_UDP |
+			IBV_RX_HASH_DST_PORT_UDP;
+
+		if (!(flow->rss.types & ETH_RSS_UDP))
+			hash_fields = 0;
+		flow->cur_verbs->hash_fields |= hash_fields;
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L4;
+		mlx5_flow_spec_verbs_add(flow, &udp, size);
+	}
 	return size;
 }
 
@@ -854,8 +1020,6 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 	if (ret < 0)
 		return ret;
 	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
-	if (size > flow_size)
-		return size;
 	if (spec) {
 		tcp.val.dst_port = spec->hdr.dst_port;
 		tcp.val.src_port = spec->hdr.src_port;
@@ -865,7 +1029,16 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 		tcp.val.src_port &= tcp.mask.src_port;
 		tcp.val.dst_port &= tcp.mask.dst_port;
 	}
-	mlx5_flow_spec_verbs_add(flow, &tcp, size);
+	if (size <= flow_size) {
+		uint64_t hash_fields = IBV_RX_HASH_SRC_PORT_TCP |
+			IBV_RX_HASH_DST_PORT_TCP;
+
+		if (!(flow->rss.types & ETH_RSS_TCP))
+			hash_fields = 0;
+		flow->cur_verbs->hash_fields |= hash_fields;
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L4;
+		mlx5_flow_spec_verbs_add(flow, &tcp, size);
+	}
 	return size;
 }
 
@@ -1043,11 +1216,95 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 					  &queue->index,
 					  "queue is not configured");
-	flow->queue = queue->index;
+	if (flow->queue)
+		(*flow->queue)[0] = queue->index;
+	flow->rss.queue_num = 1;
 	flow->fate |= MLX5_FLOW_FATE_QUEUE;
 	return 0;
 }
 
+/**
+ * Ensure the @p action will be understood and used correctly by the  NIC.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param action[in]
+ *   Pointer to flow actions array.
+ * @param flow[in, out]
+ *   Pointer to the rte_flow structure.
+ * @param error[in, out]
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success @p flow->queue array and @p flow->rss are filled and valid.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_rss(struct rte_eth_dev *dev,
+		     const struct rte_flow_action *action,
+		     struct rte_flow *flow,
+		     struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_action_rss *rss = action->conf;
+	unsigned int i;
+
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "multiple fate actions are not"
+					  " supported");
+	if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
+	    rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->func,
+					  "RSS hash function not supported");
+	if (rss->level > 1)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->level,
+					  "tunnel RSS is not supported");
+	if (rss->key_len < MLX5_RSS_HASH_KEY_LEN)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->key_len,
+					  "RSS hash key too small");
+	if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->key_len,
+					  "RSS hash key too large");
+	if (rss->queue_num > priv->config.ind_table_max_size)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->queue_num,
+					  "number of queues too large");
+	if (rss->types & MLX5_RSS_HF_MASK)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->types,
+					  "some RSS protocols are not"
+					  " supported");
+	for (i = 0; i != rss->queue_num; ++i) {
+		if (!(*priv->rxqs)[rss->queue[i]])
+			return rte_flow_error_set
+				(error, EINVAL,
+				 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+				 &rss->queue[i],
+				 "queue is not configured");
+	}
+	if (flow->queue)
+		memcpy((*flow->queue), rss->queue,
+		       rss->queue_num * sizeof(uint16_t));
+	flow->rss.queue_num = rss->queue_num;
+	memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
+	flow->rss.types = rss->types;
+	flow->fate |= MLX5_FLOW_FATE_RSS;
+	return 0;
+}
+
 /**
  * Convert the @p action into a Verbs specification after ensuring the NIC
  * will understand and process it correctly.
@@ -1082,6 +1339,7 @@ mlx5_flow_action_flag(const struct rte_flow_action *action,
 		.size = size,
 		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
 	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 
 	if (flow->modifier & MLX5_FLOW_MOD_FLAG)
 		return rte_flow_error_set(error, ENOTSUP,
@@ -1095,32 +1353,34 @@ mlx5_flow_action_flag(const struct rte_flow_action *action,
 					  "flag is not compatible with drop"
 					  " action");
 	if (flow->modifier & MLX5_FLOW_MOD_MARK)
-		return 0;
-	flow->modifier |= MLX5_FLOW_MOD_FLAG;
-	if (size <= flow_size)
+		size = 0;
+	else if (size <= flow_size && verbs)
 		mlx5_flow_spec_verbs_add(flow, &tag, size);
+	flow->modifier |= MLX5_FLOW_MOD_FLAG;
 	return size;
 }
 
 /**
  * Update verbs specification to modify the flag to mark.
  *
- * @param[in, out] flow
- *   Pointer to the rte_flow structure.
+ * @param[in, out] verbs
+ *   Pointer to the mlx5_flow_verbs structure.
  * @param[in] mark_id
  *   Mark identifier to replace the flag.
  */
 static void
-mlx5_flow_verbs_mark_update(struct rte_flow *flow, uint32_t mark_id)
+mlx5_flow_verbs_mark_update(struct mlx5_flow_verbs *verbs, uint32_t mark_id)
 {
 	struct ibv_spec_header *hdr;
 	int i;
 
+	if (!verbs)
+		return;
 	/* Update Verbs specification. */
-	hdr = (struct ibv_spec_header *)flow->verbs.specs;
+	hdr = (struct ibv_spec_header *)verbs->specs;
 	if (!hdr)
 		return;
-	for (i = 0; i != flow->verbs.attr->num_of_specs; ++i) {
+	for (i = 0; i != verbs->attr->num_of_specs; ++i) {
 		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
 			struct ibv_flow_spec_action_tag *t =
 				(struct ibv_flow_spec_action_tag *)hdr;
@@ -1166,6 +1426,7 @@ mlx5_flow_action_mark(const struct rte_flow_action *action,
 		.type = IBV_FLOW_SPEC_ACTION_TAG,
 		.size = size,
 	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 
 	if (!mark)
 		return rte_flow_error_set(error, EINVAL,
@@ -1190,14 +1451,11 @@ mlx5_flow_action_mark(const struct rte_flow_action *action,
 					  "mark is not compatible with drop"
 					  " action");
 	if (flow->modifier & MLX5_FLOW_MOD_FLAG) {
-		mlx5_flow_verbs_mark_update(flow, mark->id);
-		size = 0; /**< Only an update is done in the specification. */
-	} else {
+		mlx5_flow_verbs_mark_update(verbs, mark->id);
+		size = 0;
+	} else if (size <= flow_size) {
 		tag.tag_id = mlx5_flow_mark_set(mark->id);
-		if (size <= flow_size) {
-			tag.tag_id = mlx5_flow_mark_set(mark->id);
-			mlx5_flow_spec_verbs_add(flow, &tag, size);
-		}
+		mlx5_flow_spec_verbs_add(flow, &tag, size);
 	}
 	flow->modifier |= MLX5_FLOW_MOD_MARK;
 	return size;
@@ -1259,6 +1517,9 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 		case RTE_FLOW_ACTION_TYPE_QUEUE:
 			ret = mlx5_flow_action_queue(dev, actions, flow, error);
 			break;
+		case RTE_FLOW_ACTION_TYPE_RSS:
+			ret = mlx5_flow_action_rss(dev, actions, flow, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -1322,26 +1583,122 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 		struct rte_flow_error *error)
 {
 	struct rte_flow local_flow = { .layers = 0, };
-	size_t size = sizeof(*flow) + sizeof(struct ibv_flow_attr);
-	int remain = (flow_size > size) ? flow_size - size : 0;
+	size_t size = sizeof(*flow);
+	union {
+		struct rte_flow_expand_rss buf;
+		uint8_t buffer[2048];
+	} expand_buffer;
+	struct rte_flow_expand_rss *buf = &expand_buffer.buf;
+	struct mlx5_flow_verbs *original_verbs = NULL;
+	size_t original_verbs_size = 0;
+	uint32_t original_layers = 0;
 	int ret;
+	uint32_t i;
 
-	if (!remain)
+	if (size > flow_size)
 		flow = &local_flow;
 	ret = mlx5_flow_attributes(dev, attributes, flow, error);
 	if (ret < 0)
 		return ret;
-	ret = mlx5_flow_items(pattern, flow, remain, error);
-	if (ret < 0)
-		return ret;
-	size += ret;
-	remain = (flow_size > size) ? flow_size - size : 0;
-	ret = mlx5_flow_actions(dev, actions, flow, remain, error);
+	ret = mlx5_flow_actions(dev, actions, &local_flow, 0, error);
 	if (ret < 0)
 		return ret;
-	size += ret;
+	if (local_flow.rss.types) {
+		ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
+					  pattern, local_flow.rss.types,
+					  mlx5_support_expansion,
+					  MLX5_EXPANSION_ROOT);
+		assert(ret > 0 &&
+		       (unsigned int)ret < sizeof(expand_buffer.buffer));
+	} else {
+		buf->entries = 1;
+		buf->entry[0].pattern = (void *)(uintptr_t)pattern;
+	}
+	size += RTE_ALIGN_CEIL(local_flow.rss.queue_num * sizeof(uint16_t),
+			       sizeof(void *));
 	if (size <= flow_size)
-		flow->verbs.attr->priority = flow->attributes.priority;
+		flow->queue = (void *)(flow + 1);
+	LIST_INIT(&flow->verbs);
+	flow->layers = 0;
+	flow->modifier = 0;
+	flow->fate = 0;
+	for (i = 0; i != buf->entries; ++i) {
+		size_t off = size;
+		size_t off2;
+		int pattern_start_idx = 0;
+
+		flow->layers = original_layers;
+		size += sizeof(struct ibv_flow_attr) +
+			sizeof(struct mlx5_flow_verbs);
+		off2 = size;
+		if (size < flow_size) {
+			flow->cur_verbs = (void *)((uintptr_t)flow + off);
+			flow->cur_verbs->attr = (void *)(flow->cur_verbs + 1);
+			flow->cur_verbs->specs =
+				(void *)(flow->cur_verbs->attr + 1);
+		}
+		/* First iteration convert the pattern into Verbs. */
+		if (i == 0) {
+			/* Actions don't need to be converted several time. */
+			ret = mlx5_flow_actions(dev, actions, flow,
+						(size < flow_size) ?
+						flow_size - size : 0,
+						error);
+			if (ret < 0)
+				return ret;
+			size += ret;
+		} else {
+			/*
+			 * Next iteration means the pattern has already been
+			 * converted and an expansion is necessary to match
+			 * the user RSS request.  For that only the expanded
+			 * items will be converted, the common part with the
+			 * user pattern are just copied into the next buffer
+			 * zone.
+			 */
+			const struct rte_flow_item *item = pattern;
+
+			size += original_verbs_size;
+			if (size < flow_size) {
+				rte_memcpy(flow->cur_verbs->attr,
+					   original_verbs->attr,
+					   original_verbs_size +
+					   sizeof(struct ibv_flow_attr));
+				flow->cur_verbs->size = original_verbs_size;
+			}
+			if (pattern->type == RTE_FLOW_ITEM_TYPE_END) {
+				pattern_start_idx++;
+			} else {
+				for (item = pattern;
+				     item->type != RTE_FLOW_ITEM_TYPE_END;
+				     ++item)
+					pattern_start_idx++;
+			}
+		}
+		ret = mlx5_flow_items
+			((const struct rte_flow_item *)
+			 &buf->entry[i].pattern[pattern_start_idx],
+			 flow,
+			 (size < flow_size) ? flow_size - size : 0, error);
+		if (ret < 0)
+			return ret;
+		size += ret;
+		if (size <= flow_size) {
+			mlx5_flow_adjust_priority(dev, flow);
+			LIST_INSERT_HEAD(&flow->verbs, flow->cur_verbs, next);
+		}
+		/*
+		 * Keep a pointer of the first verbs conversion and the layers
+		 * it has encountered.
+		 */
+		if (i == 0) {
+			original_verbs = flow->cur_verbs;
+			original_verbs_size = size - off2;
+			original_layers = flow->layers;
+		}
+	}
+	/* Restore the origin layers in the flow. */
+	flow->layers = original_layers;
 	return size;
 }
 
@@ -1359,12 +1716,17 @@ mlx5_flow_rxq_mark_set(struct rte_eth_dev *dev, struct rte_flow *flow)
 	struct priv *priv = dev->data->dev_private;
 
 	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK)) {
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of((*priv->rxqs)[flow->queue],
-				     struct mlx5_rxq_ctrl, rxq);
+		unsigned int i;
+
+		for (i = 0; i != flow->rss.queue_num; ++i) {
+			int idx = (*flow->queue)[i];
+			struct mlx5_rxq_ctrl *rxq_ctrl =
+				container_of((*priv->rxqs)[idx],
+					     struct mlx5_rxq_ctrl, rxq);
 
-		rxq_ctrl->rxq.mark = 1;
-		rxq_ctrl->flow_mark_n++;
+			rxq_ctrl->rxq.mark = 1;
+			rxq_ctrl->flow_mark_n++;
+		}
 	}
 }
 
@@ -1383,12 +1745,17 @@ mlx5_flow_rxq_mark_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
 	struct priv *priv = dev->data->dev_private;
 
 	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK)) {
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of((*priv->rxqs)[flow->queue],
-				     struct mlx5_rxq_ctrl, rxq);
+		unsigned int i;
+
+		for (i = 0; i != flow->rss.queue_num; ++i) {
+			int idx = (*flow->queue)[i];
+			struct mlx5_rxq_ctrl *rxq_ctrl =
+				container_of((*priv->rxqs)[idx],
+					     struct mlx5_rxq_ctrl, rxq);
 
-		rxq_ctrl->flow_mark_n--;
-		rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
+			rxq_ctrl->flow_mark_n--;
+			rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
+		}
 	}
 }
 
@@ -1449,18 +1816,20 @@ mlx5_flow_validate(struct rte_eth_dev *dev,
 static void
 mlx5_flow_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
-	if (flow->fate & MLX5_FLOW_FATE_DROP) {
-		if (flow->verbs.flow) {
-			claim_zero(mlx5_glue->destroy_flow(flow->verbs.flow));
-			flow->verbs.flow = NULL;
+	struct mlx5_flow_verbs *verbs;
+
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (verbs->flow) {
+			claim_zero(mlx5_glue->destroy_flow(verbs->flow));
+			verbs->flow = NULL;
+		}
+		if (verbs->hrxq) {
+			if (flow->fate & MLX5_FLOW_FATE_DROP)
+				mlx5_hrxq_drop_release(dev);
+			else
+				mlx5_hrxq_release(dev, verbs->hrxq);
+			verbs->hrxq = NULL;
 		}
-	}
-	if (flow->verbs.hrxq) {
-		if (flow->fate & MLX5_FLOW_FATE_DROP)
-			mlx5_hrxq_drop_release(dev);
-		else if (flow->fate & MLX5_FLOW_FATE_QUEUE)
-			mlx5_hrxq_release(dev, flow->verbs.hrxq);
-		flow->verbs.hrxq = NULL;
 	}
 }
 
@@ -1481,46 +1850,68 @@ static int
 mlx5_flow_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 		struct rte_flow_error *error)
 {
-	if (flow->fate & MLX5_FLOW_FATE_DROP) {
-		flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
-		if (!flow->verbs.hrxq)
-			return rte_flow_error_set
-				(error, errno,
-				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				 NULL,
-				 "cannot allocate Drop queue");
-	} else if (flow->fate & MLX5_FLOW_FATE_QUEUE) {
-		struct mlx5_hrxq *hrxq;
-
-		hrxq = mlx5_hrxq_get(dev, rss_hash_default_key,
-				     MLX5_RSS_HASH_KEY_LEN, 0,
-				     &flow->queue, 1, 0, 0);
-		if (!hrxq)
-			hrxq = mlx5_hrxq_new(dev, rss_hash_default_key,
-					     MLX5_RSS_HASH_KEY_LEN, 0,
-					     &flow->queue, 1, 0, 0);
-		if (!hrxq)
-			return rte_flow_error_set(error, rte_errno,
-					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					NULL,
-					"cannot create flow");
-		flow->verbs.hrxq = hrxq;
-	}
-	flow->verbs.flow =
-		mlx5_glue->create_flow(flow->verbs.hrxq->qp, flow->verbs.attr);
-	if (!flow->verbs.flow) {
-		if (flow->fate & MLX5_FLOW_FATE_DROP)
-			mlx5_hrxq_drop_release(dev);
-		else
-			mlx5_hrxq_release(dev, flow->verbs.hrxq);
-		flow->verbs.hrxq = NULL;
-		return rte_flow_error_set(error, errno,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "kernel module refuses to create"
-					  " flow");
+	struct mlx5_flow_verbs *verbs;
+	int err;
+
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (flow->fate & MLX5_FLOW_FATE_DROP) {
+			verbs->hrxq = mlx5_hrxq_drop_new(dev);
+			if (!verbs->hrxq) {
+				rte_flow_error_set
+					(error, errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					 NULL,
+					 "cannot get drop hash queue");
+				goto error;
+			}
+		} else {
+			struct mlx5_hrxq *hrxq;
+
+			hrxq = mlx5_hrxq_get(dev, flow->key,
+					     MLX5_RSS_HASH_KEY_LEN,
+					     verbs->hash_fields,
+					     (*flow->queue),
+					     flow->rss.queue_num, 0, 0);
+			if (!hrxq)
+				hrxq = mlx5_hrxq_new(dev, flow->key,
+						     MLX5_RSS_HASH_KEY_LEN,
+						     verbs->hash_fields,
+						     (*flow->queue),
+						     flow->rss.queue_num, 0, 0);
+			if (!hrxq) {
+				rte_flow_error_set
+					(error, rte_errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					 NULL,
+					 "cannot get hash queue");
+				goto error;
+			}
+			verbs->hrxq = hrxq;
+		}
+		verbs->flow =
+			mlx5_glue->create_flow(verbs->hrxq->qp, verbs->attr);
+		if (!verbs->flow) {
+			rte_flow_error_set(error, errno,
+					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					   NULL,
+					   "hardware refuses to create flow");
+			goto error;
+		}
 	}
 	return 0;
+error:
+	err = rte_errno; /* Save rte_errno before cleanup. */
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (verbs->hrxq) {
+			if (flow->fate & MLX5_FLOW_FATE_DROP)
+				mlx5_hrxq_drop_release(dev);
+			else
+				mlx5_hrxq_release(dev, verbs->hrxq);
+			verbs->hrxq = NULL;
+		}
+	}
+	rte_errno = err; /* Restore rte_errno. */
+	return -rte_errno;
 }
 
 /**
@@ -1550,42 +1941,43 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
 		      const struct rte_flow_action actions[],
 		      struct rte_flow_error *error)
 {
-	struct rte_flow *flow;
-	size_t size;
+	struct rte_flow *flow = NULL;
+	size_t size = 0;
 	int ret;
 
-	ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
 	if (ret < 0)
 		return NULL;
 	size = ret;
-	flow = rte_zmalloc(__func__, size, 0);
+	flow = rte_calloc(__func__, 1, size, 0);
 	if (!flow) {
 		rte_flow_error_set(error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL,
-				   "cannot allocate memory");
+				   "not enough memory to create flow");
 		return NULL;
 	}
-	flow->verbs.attr = (struct ibv_flow_attr *)(flow + 1);
-	flow->verbs.specs = (uint8_t *)(flow->verbs.attr + 1);
 	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
-	if (ret < 0)
-		goto error;
+	if (ret < 0) {
+		rte_free(flow);
+		return NULL;
+	}
 	assert((size_t)ret == size);
 	if (dev->data->dev_started) {
 		ret = mlx5_flow_apply(dev, flow, error);
-		if (ret < 0)
-			goto error;
+		if (ret < 0) {
+			ret = rte_errno; /* Save rte_errno before cleanup. */
+			if (flow) {
+				mlx5_flow_remove(dev, flow);
+				rte_free(flow);
+			}
+			rte_errno = ret; /* Restore rte_errno. */
+			return NULL;
+		}
 	}
 	mlx5_flow_rxq_mark_set(dev, flow);
 	TAILQ_INSERT_TAIL(list, flow, next);
 	return flow;
-error:
-	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_remove(dev, flow);
-	rte_free(flow);
-	rte_errno = ret; /* Restore rte_errno. */
-	return NULL;
 }
 
 /**
@@ -1745,7 +2137,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
-		.priority = priv->config.flow_prio - 1,
+		.priority = MLX5_FLOW_PRIO_RSVD,
 	};
 	struct rte_flow_item items[] = {
 		{
-- 
2.18.0

Re: [dpdk-dev] [PATCH v3 14/21] net/mlx5: add RSS flow action

[Yongseok Koh]

On Wed, Jul 11, 2018 at 09:22:47AM +0200, Nelio Laranjeiro wrote:
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
> ---
>  drivers/net/mlx5/mlx5_flow.c | 682 +++++++++++++++++++++++++++--------
>  1 file changed, 537 insertions(+), 145 deletions(-)
> 
[...]
> @@ -1322,26 +1583,122 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
>  		struct rte_flow_error *error)
>  {
>  	struct rte_flow local_flow = { .layers = 0, };
> -	size_t size = sizeof(*flow) + sizeof(struct ibv_flow_attr);
> -	int remain = (flow_size > size) ? flow_size - size : 0;
> +	size_t size = sizeof(*flow);
> +	union {
> +		struct rte_flow_expand_rss buf;
> +		uint8_t buffer[2048];
> +	} expand_buffer;
> +	struct rte_flow_expand_rss *buf = &expand_buffer.buf;
> +	struct mlx5_flow_verbs *original_verbs = NULL;
> +	size_t original_verbs_size = 0;
> +	uint32_t original_layers = 0;
>  	int ret;
> +	uint32_t i;
>  
> -	if (!remain)
> +	if (size > flow_size)
>  		flow = &local_flow;
>  	ret = mlx5_flow_attributes(dev, attributes, flow, error);
>  	if (ret < 0)
>  		return ret;
> -	ret = mlx5_flow_items(pattern, flow, remain, error);
> -	if (ret < 0)
> -		return ret;
> -	size += ret;
> -	remain = (flow_size > size) ? flow_size - size : 0;
> -	ret = mlx5_flow_actions(dev, actions, flow, remain, error);
> +	ret = mlx5_flow_actions(dev, actions, &local_flow, 0, error);
>  	if (ret < 0)
>  		return ret;
> -	size += ret;
> +	if (local_flow.rss.types) {
> +		ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
> +					  pattern, local_flow.rss.types,
> +					  mlx5_support_expansion,
> +					  MLX5_EXPANSION_ROOT);
> +		assert(ret > 0 &&
> +		       (unsigned int)ret < sizeof(expand_buffer.buffer));
> +	} else {
> +		buf->entries = 1;
> +		buf->entry[0].pattern = (void *)(uintptr_t)pattern;
> +	}
> +	size += RTE_ALIGN_CEIL(local_flow.rss.queue_num * sizeof(uint16_t),
> +			       sizeof(void *));
>  	if (size <= flow_size)
> -		flow->verbs.attr->priority = flow->attributes.priority;
> +		flow->queue = (void *)(flow + 1);
> +	LIST_INIT(&flow->verbs);
> +	flow->layers = 0;
> +	flow->modifier = 0;
> +	flow->fate = 0;
> +	for (i = 0; i != buf->entries; ++i) {
> +		size_t off = size;
> +		size_t off2;
> +		int pattern_start_idx = 0;
> +
> +		flow->layers = original_layers;
> +		size += sizeof(struct ibv_flow_attr) +
> +			sizeof(struct mlx5_flow_verbs);
> +		off2 = size;
> +		if (size < flow_size) {
> +			flow->cur_verbs = (void *)((uintptr_t)flow + off);
> +			flow->cur_verbs->attr = (void *)(flow->cur_verbs + 1);
> +			flow->cur_verbs->specs =
> +				(void *)(flow->cur_verbs->attr + 1);
> +		}
> +		/* First iteration convert the pattern into Verbs. */
> +		if (i == 0) {
> +			/* Actions don't need to be converted several time. */
> +			ret = mlx5_flow_actions(dev, actions, flow,
> +						(size < flow_size) ?
> +						flow_size - size : 0,
> +						error);
> +			if (ret < 0)
> +				return ret;
> +			size += ret;
> +		} else {
> +			/*
> +			 * Next iteration means the pattern has already been
> +			 * converted and an expansion is necessary to match
> +			 * the user RSS request.  For that only the expanded
> +			 * items will be converted, the common part with the
> +			 * user pattern are just copied into the next buffer
> +			 * zone.
> +			 */
> +			const struct rte_flow_item *item = pattern;
> +
> +			size += original_verbs_size;
> +			if (size < flow_size) {
> +				rte_memcpy(flow->cur_verbs->attr,
> +					   original_verbs->attr,
> +					   original_verbs_size +
> +					   sizeof(struct ibv_flow_attr));
> +				flow->cur_verbs->size = original_verbs_size;
> +			}
> +			if (pattern->type == RTE_FLOW_ITEM_TYPE_END) {
> +				pattern_start_idx++;
> +			} else {
> +				for (item = pattern;
> +				     item->type != RTE_FLOW_ITEM_TYPE_END;
> +				     ++item)
> +					pattern_start_idx++;
> +			}

Small nit.
Can't this be run once when? As pattern isn't changing, it might be run redundant
from i > 1. And how about expanded_pattern_idx instead of pattern_start_idx?

Acked-by: Yongseok Koh <yskoh@mellanox.com>

Thanks

> +		}
> +		ret = mlx5_flow_items
> +			((const struct rte_flow_item *)
> +			 &buf->entry[i].pattern[pattern_start_idx],
> +			 flow,
> +			 (size < flow_size) ? flow_size - size : 0, error);
> +		if (ret < 0)
> +			return ret;
> +		size += ret;
> +		if (size <= flow_size) {
> +			mlx5_flow_adjust_priority(dev, flow);
> +			LIST_INSERT_HEAD(&flow->verbs, flow->cur_verbs, next);
> +		}
> +		/*
> +		 * Keep a pointer of the first verbs conversion and the layers
> +		 * it has encountered.
> +		 */
> +		if (i == 0) {
> +			original_verbs = flow->cur_verbs;
> +			original_verbs_size = size - off2;
> +			original_layers = flow->layers;
> +		}
> +	}
> +	/* Restore the origin layers in the flow. */
> +	flow->layers = original_layers;
>  	return size;
>  }
>  
> @@ -1359,12 +1716,17 @@ mlx5_flow_rxq_mark_set(struct rte_eth_dev *dev, struct rte_flow *flow)
>  	struct priv *priv = dev->data->dev_private;
>  
>  	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK)) {
> -		struct mlx5_rxq_ctrl *rxq_ctrl =
> -			container_of((*priv->rxqs)[flow->queue],
> -				     struct mlx5_rxq_ctrl, rxq);
> +		unsigned int i;
> +
> +		for (i = 0; i != flow->rss.queue_num; ++i) {
> +			int idx = (*flow->queue)[i];
> +			struct mlx5_rxq_ctrl *rxq_ctrl =
> +				container_of((*priv->rxqs)[idx],
> +					     struct mlx5_rxq_ctrl, rxq);
>  
> -		rxq_ctrl->rxq.mark = 1;
> -		rxq_ctrl->flow_mark_n++;
> +			rxq_ctrl->rxq.mark = 1;
> +			rxq_ctrl->flow_mark_n++;
> +		}
>  	}
>  }
>  
> @@ -1383,12 +1745,17 @@ mlx5_flow_rxq_mark_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
>  	struct priv *priv = dev->data->dev_private;
>  
>  	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK)) {
> -		struct mlx5_rxq_ctrl *rxq_ctrl =
> -			container_of((*priv->rxqs)[flow->queue],
> -				     struct mlx5_rxq_ctrl, rxq);
> +		unsigned int i;
> +
> +		for (i = 0; i != flow->rss.queue_num; ++i) {
> +			int idx = (*flow->queue)[i];
> +			struct mlx5_rxq_ctrl *rxq_ctrl =
> +				container_of((*priv->rxqs)[idx],
> +					     struct mlx5_rxq_ctrl, rxq);
>  
> -		rxq_ctrl->flow_mark_n--;
> -		rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
> +			rxq_ctrl->flow_mark_n--;
> +			rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
> +		}
>  	}
>  }
>  
> @@ -1449,18 +1816,20 @@ mlx5_flow_validate(struct rte_eth_dev *dev,
>  static void
>  mlx5_flow_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
>  {
> -	if (flow->fate & MLX5_FLOW_FATE_DROP) {
> -		if (flow->verbs.flow) {
> -			claim_zero(mlx5_glue->destroy_flow(flow->verbs.flow));
> -			flow->verbs.flow = NULL;
> +	struct mlx5_flow_verbs *verbs;
> +
> +	LIST_FOREACH(verbs, &flow->verbs, next) {
> +		if (verbs->flow) {
> +			claim_zero(mlx5_glue->destroy_flow(verbs->flow));
> +			verbs->flow = NULL;
> +		}
> +		if (verbs->hrxq) {
> +			if (flow->fate & MLX5_FLOW_FATE_DROP)
> +				mlx5_hrxq_drop_release(dev);
> +			else
> +				mlx5_hrxq_release(dev, verbs->hrxq);
> +			verbs->hrxq = NULL;
>  		}
> -	}
> -	if (flow->verbs.hrxq) {
> -		if (flow->fate & MLX5_FLOW_FATE_DROP)
> -			mlx5_hrxq_drop_release(dev);
> -		else if (flow->fate & MLX5_FLOW_FATE_QUEUE)
> -			mlx5_hrxq_release(dev, flow->verbs.hrxq);
> -		flow->verbs.hrxq = NULL;
>  	}
>  }
>  
> @@ -1481,46 +1850,68 @@ static int
>  mlx5_flow_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
>  		struct rte_flow_error *error)
>  {
> -	if (flow->fate & MLX5_FLOW_FATE_DROP) {
> -		flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
> -		if (!flow->verbs.hrxq)
> -			return rte_flow_error_set
> -				(error, errno,
> -				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> -				 NULL,
> -				 "cannot allocate Drop queue");
> -	} else if (flow->fate & MLX5_FLOW_FATE_QUEUE) {
> -		struct mlx5_hrxq *hrxq;
> -
> -		hrxq = mlx5_hrxq_get(dev, rss_hash_default_key,
> -				     MLX5_RSS_HASH_KEY_LEN, 0,
> -				     &flow->queue, 1, 0, 0);
> -		if (!hrxq)
> -			hrxq = mlx5_hrxq_new(dev, rss_hash_default_key,
> -					     MLX5_RSS_HASH_KEY_LEN, 0,
> -					     &flow->queue, 1, 0, 0);
> -		if (!hrxq)
> -			return rte_flow_error_set(error, rte_errno,
> -					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> -					NULL,
> -					"cannot create flow");
> -		flow->verbs.hrxq = hrxq;
> -	}
> -	flow->verbs.flow =
> -		mlx5_glue->create_flow(flow->verbs.hrxq->qp, flow->verbs.attr);
> -	if (!flow->verbs.flow) {
> -		if (flow->fate & MLX5_FLOW_FATE_DROP)
> -			mlx5_hrxq_drop_release(dev);
> -		else
> -			mlx5_hrxq_release(dev, flow->verbs.hrxq);
> -		flow->verbs.hrxq = NULL;
> -		return rte_flow_error_set(error, errno,
> -					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> -					  NULL,
> -					  "kernel module refuses to create"
> -					  " flow");
> +	struct mlx5_flow_verbs *verbs;
> +	int err;
> +
> +	LIST_FOREACH(verbs, &flow->verbs, next) {
> +		if (flow->fate & MLX5_FLOW_FATE_DROP) {
> +			verbs->hrxq = mlx5_hrxq_drop_new(dev);
> +			if (!verbs->hrxq) {
> +				rte_flow_error_set
> +					(error, errno,
> +					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +					 NULL,
> +					 "cannot get drop hash queue");
> +				goto error;
> +			}
> +		} else {
> +			struct mlx5_hrxq *hrxq;
> +
> +			hrxq = mlx5_hrxq_get(dev, flow->key,
> +					     MLX5_RSS_HASH_KEY_LEN,
> +					     verbs->hash_fields,
> +					     (*flow->queue),
> +					     flow->rss.queue_num, 0, 0);
> +			if (!hrxq)
> +				hrxq = mlx5_hrxq_new(dev, flow->key,
> +						     MLX5_RSS_HASH_KEY_LEN,
> +						     verbs->hash_fields,
> +						     (*flow->queue),
> +						     flow->rss.queue_num, 0, 0);
> +			if (!hrxq) {
> +				rte_flow_error_set
> +					(error, rte_errno,
> +					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +					 NULL,
> +					 "cannot get hash queue");
> +				goto error;
> +			}
> +			verbs->hrxq = hrxq;
> +		}
> +		verbs->flow =
> +			mlx5_glue->create_flow(verbs->hrxq->qp, verbs->attr);
> +		if (!verbs->flow) {
> +			rte_flow_error_set(error, errno,
> +					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +					   NULL,
> +					   "hardware refuses to create flow");
> +			goto error;
> +		}
>  	}
>  	return 0;
> +error:
> +	err = rte_errno; /* Save rte_errno before cleanup. */
> +	LIST_FOREACH(verbs, &flow->verbs, next) {
> +		if (verbs->hrxq) {
> +			if (flow->fate & MLX5_FLOW_FATE_DROP)
> +				mlx5_hrxq_drop_release(dev);
> +			else
> +				mlx5_hrxq_release(dev, verbs->hrxq);
> +			verbs->hrxq = NULL;
> +		}
> +	}
> +	rte_errno = err; /* Restore rte_errno. */
> +	return -rte_errno;
>  }
>  
>  /**
> @@ -1550,42 +1941,43 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
>  		      const struct rte_flow_action actions[],
>  		      struct rte_flow_error *error)
>  {
> -	struct rte_flow *flow;
> -	size_t size;
> +	struct rte_flow *flow = NULL;
> +	size_t size = 0;
>  	int ret;
>  
> -	ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
> +	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
>  	if (ret < 0)
>  		return NULL;
>  	size = ret;
> -	flow = rte_zmalloc(__func__, size, 0);
> +	flow = rte_calloc(__func__, 1, size, 0);
>  	if (!flow) {
>  		rte_flow_error_set(error, ENOMEM,
>  				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
>  				   NULL,
> -				   "cannot allocate memory");
> +				   "not enough memory to create flow");
>  		return NULL;
>  	}
> -	flow->verbs.attr = (struct ibv_flow_attr *)(flow + 1);
> -	flow->verbs.specs = (uint8_t *)(flow->verbs.attr + 1);
>  	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
> -	if (ret < 0)
> -		goto error;
> +	if (ret < 0) {
> +		rte_free(flow);
> +		return NULL;
> +	}
>  	assert((size_t)ret == size);
>  	if (dev->data->dev_started) {
>  		ret = mlx5_flow_apply(dev, flow, error);
> -		if (ret < 0)
> -			goto error;
> +		if (ret < 0) {
> +			ret = rte_errno; /* Save rte_errno before cleanup. */
> +			if (flow) {
> +				mlx5_flow_remove(dev, flow);
> +				rte_free(flow);
> +			}
> +			rte_errno = ret; /* Restore rte_errno. */
> +			return NULL;
> +		}
>  	}
>  	mlx5_flow_rxq_mark_set(dev, flow);
>  	TAILQ_INSERT_TAIL(list, flow, next);
>  	return flow;
> -error:
> -	ret = rte_errno; /* Save rte_errno before cleanup. */
> -	mlx5_flow_remove(dev, flow);
> -	rte_free(flow);
> -	rte_errno = ret; /* Restore rte_errno. */
> -	return NULL;
>  }
>  
>  /**
> @@ -1745,7 +2137,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
>  	struct priv *priv = dev->data->dev_private;
>  	const struct rte_flow_attr attr = {
>  		.ingress = 1,
> -		.priority = priv->config.flow_prio - 1,
> +		.priority = MLX5_FLOW_PRIO_RSVD,
>  	};
>  	struct rte_flow_item items[] = {
>  		{
> -- 
> 2.18.0
> 

[dpdk-dev] [PATCH v3 15/21] net/mlx5: remove useless arguments in hrxq API

[Nelio Laranjeiro]

RSS level is necessary to had a bit in the hash_fields which is already
provided in this API, for the tunnel, it is necessary to request such
queue to compute the checksum on the inner most, this last one should
always be activated.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c |  4 ++--
 drivers/net/mlx5/mlx5_rxq.c  | 39 +++++++++---------------------------
 drivers/net/mlx5/mlx5_rxtx.h |  8 ++------
 3 files changed, 13 insertions(+), 38 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index f098e727a..f2acac1f5 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -1871,13 +1871,13 @@ mlx5_flow_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 					     MLX5_RSS_HASH_KEY_LEN,
 					     verbs->hash_fields,
 					     (*flow->queue),
-					     flow->rss.queue_num, 0, 0);
+					     flow->rss.queue_num);
 			if (!hrxq)
 				hrxq = mlx5_hrxq_new(dev, flow->key,
 						     MLX5_RSS_HASH_KEY_LEN,
 						     verbs->hash_fields,
 						     (*flow->queue),
-						     flow->rss.queue_num, 0, 0);
+						     flow->rss.queue_num);
 			if (!hrxq) {
 				rte_flow_error_set
 					(error, rte_errno,
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index d50b82c69..071740b6d 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1740,10 +1740,6 @@ mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev)
  *   first queue index will be taken for the indirection table.
  * @param queues_n
  *   Number of queues.
- * @param tunnel
- *   Tunnel type, implies tunnel offloading like inner checksum if available.
- * @param rss_level
- *   RSS hash on tunnel level.
  *
  * @return
  *   The Verbs object initialised, NULL otherwise and rte_errno is set.
@@ -1752,17 +1748,13 @@ struct mlx5_hrxq *
 mlx5_hrxq_new(struct rte_eth_dev *dev,
 	      const uint8_t *rss_key, uint32_t rss_key_len,
 	      uint64_t hash_fields,
-	      const uint16_t *queues, uint32_t queues_n,
-	      uint32_t tunnel, uint32_t rss_level)
+	      const uint16_t *queues, uint32_t queues_n)
 {
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
 	struct mlx5_ind_table_ibv *ind_tbl;
 	struct ibv_qp *qp;
 	int err;
-#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	struct mlx5dv_qp_init_attr qp_init_attr = {0};
-#endif
 
 	queues_n = hash_fields ? queues_n : 1;
 	ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n);
@@ -1777,11 +1769,6 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 		rss_key = rss_hash_default_key;
 	}
 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	if (tunnel) {
-		qp_init_attr.comp_mask =
-				MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
-		qp_init_attr.create_flags = MLX5DV_QP_CREATE_TUNNEL_OFFLOADS;
-	}
 	qp = mlx5_glue->dv_create_qp
 		(priv->ctx,
 		 &(struct ibv_qp_init_attr_ex){
@@ -1797,14 +1784,17 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 				.rx_hash_key = rss_key ?
 					       (void *)(uintptr_t)rss_key :
 					       rss_hash_default_key,
-				.rx_hash_fields_mask = hash_fields |
-					(tunnel && rss_level > 1 ?
-					(uint32_t)IBV_RX_HASH_INNER : 0),
+				.rx_hash_fields_mask = hash_fields,
 			},
 			.rwq_ind_tbl = ind_tbl->ind_table,
 			.pd = priv->pd,
 		 },
-		 &qp_init_attr);
+		 &(struct mlx5dv_qp_init_attr){
+			.comp_mask = (hash_fields & IBV_RX_HASH_INNER) ?
+				 MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS :
+				 0,
+			.create_flags = MLX5DV_QP_CREATE_TUNNEL_OFFLOADS,
+		 });
 #else
 	qp = mlx5_glue->create_qp_ex
 		(priv->ctx,
@@ -1838,8 +1828,6 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 	hrxq->qp = qp;
 	hrxq->rss_key_len = rss_key_len;
 	hrxq->hash_fields = hash_fields;
-	hrxq->tunnel = tunnel;
-	hrxq->rss_level = rss_level;
 	memcpy(hrxq->rss_key, rss_key, rss_key_len);
 	rte_atomic32_inc(&hrxq->refcnt);
 	LIST_INSERT_HEAD(&priv->hrxqs, hrxq, next);
@@ -1865,10 +1853,6 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
  *   first queue index will be taken for the indirection table.
  * @param queues_n
  *   Number of queues.
- * @param tunnel
- *   Tunnel type, implies tunnel offloading like inner checksum if available.
- * @param rss_level
- *   RSS hash on tunnel level
  *
  * @return
  *   An hash Rx queue on success.
@@ -1877,8 +1861,7 @@ struct mlx5_hrxq *
 mlx5_hrxq_get(struct rte_eth_dev *dev,
 	      const uint8_t *rss_key, uint32_t rss_key_len,
 	      uint64_t hash_fields,
-	      const uint16_t *queues, uint32_t queues_n,
-	      uint32_t tunnel, uint32_t rss_level)
+	      const uint16_t *queues, uint32_t queues_n)
 {
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
@@ -1893,10 +1876,6 @@ mlx5_hrxq_get(struct rte_eth_dev *dev,
 			continue;
 		if (hrxq->hash_fields != hash_fields)
 			continue;
-		if (hrxq->tunnel != tunnel)
-			continue;
-		if (hrxq->rss_level != rss_level)
-			continue;
 		ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n);
 		if (!ind_tbl)
 			continue;
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 59e374d8d..808118e50 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -157,8 +157,6 @@ struct mlx5_hrxq {
 	struct mlx5_ind_table_ibv *ind_table; /* Indirection table. */
 	struct ibv_qp *qp; /* Verbs queue pair. */
 	uint64_t hash_fields; /* Verbs Hash fields. */
-	uint32_t tunnel; /* Tunnel type. */
-	uint32_t rss_level; /* RSS on tunnel level. */
 	uint32_t rss_key_len; /* Hash key length in bytes. */
 	uint8_t rss_key[]; /* Hash key. */
 };
@@ -271,13 +269,11 @@ void mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev);
 struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev,
 				const uint8_t *rss_key, uint32_t rss_key_len,
 				uint64_t hash_fields,
-				const uint16_t *queues, uint32_t queues_n,
-				uint32_t tunnel, uint32_t rss_level);
+				const uint16_t *queues, uint32_t queues_n);
 struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev,
 				const uint8_t *rss_key, uint32_t rss_key_len,
 				uint64_t hash_fields,
-				const uint16_t *queues, uint32_t queues_n,
-				uint32_t tunnel, uint32_t rss_level);
+				const uint16_t *queues, uint32_t queues_n);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
 int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev);
 struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev);
-- 
2.18.0

[dpdk-dev] [PATCH v3 16/21] net/mlx5: support inner RSS computation

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 241 ++++++++++++++++++++++++++---------
 1 file changed, 180 insertions(+), 61 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index f2acac1f5..edceb17ed 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -35,18 +35,42 @@
 extern const struct eth_dev_ops mlx5_dev_ops;
 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
-/* Pattern Layer bits. */
+/* Pattern outer Layer bits. */
 #define MLX5_FLOW_LAYER_OUTER_L2 (1u << 0)
 #define MLX5_FLOW_LAYER_OUTER_L3_IPV4 (1u << 1)
 #define MLX5_FLOW_LAYER_OUTER_L3_IPV6 (1u << 2)
 #define MLX5_FLOW_LAYER_OUTER_L4_UDP (1u << 3)
 #define MLX5_FLOW_LAYER_OUTER_L4_TCP (1u << 4)
 #define MLX5_FLOW_LAYER_OUTER_VLAN (1u << 5)
-/* Masks. */
+
+/* Pattern inner Layer bits. */
+#define MLX5_FLOW_LAYER_INNER_L2 (1u << 6)
+#define MLX5_FLOW_LAYER_INNER_L3_IPV4 (1u << 7)
+#define MLX5_FLOW_LAYER_INNER_L3_IPV6 (1u << 8)
+#define MLX5_FLOW_LAYER_INNER_L4_UDP (1u << 9)
+#define MLX5_FLOW_LAYER_INNER_L4_TCP (1u << 10)
+#define MLX5_FLOW_LAYER_INNER_VLAN (1u << 11)
+
+/* Outer Masks. */
 #define MLX5_FLOW_LAYER_OUTER_L3 \
 	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 #define MLX5_FLOW_LAYER_OUTER_L4 \
 	(MLX5_FLOW_LAYER_OUTER_L4_UDP | MLX5_FLOW_LAYER_OUTER_L4_TCP)
+#define MLX5_FLOW_LAYER_OUTER \
+	(MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_L3 | \
+	 MLX5_FLOW_LAYER_OUTER_L4)
+
+/* Tunnel Masks. */
+#define MLX5_FLOW_LAYER_TUNNEL 0
+
+/* Inner Masks. */
+#define MLX5_FLOW_LAYER_INNER_L3 \
+	(MLX5_FLOW_LAYER_INNER_L3_IPV4 | MLX5_FLOW_LAYER_INNER_L3_IPV6)
+#define MLX5_FLOW_LAYER_INNER_L4 \
+	(MLX5_FLOW_LAYER_INNER_L4_UDP | MLX5_FLOW_LAYER_INNER_L4_TCP)
+#define MLX5_FLOW_LAYER_INNER \
+	(MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3 | \
+	 MLX5_FLOW_LAYER_INNER_L4)
 
 /* Actions that modify the fate of matching traffic. */
 #define MLX5_FLOW_FATE_DROP (1u << 0)
@@ -66,6 +90,14 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
 enum mlx5_expansion {
 	MLX5_EXPANSION_ROOT,
+	MLX5_EXPANSION_ROOT_OUTER,
+	MLX5_EXPANSION_OUTER_ETH,
+	MLX5_EXPANSION_OUTER_IPV4,
+	MLX5_EXPANSION_OUTER_IPV4_UDP,
+	MLX5_EXPANSION_OUTER_IPV4_TCP,
+	MLX5_EXPANSION_OUTER_IPV6,
+	MLX5_EXPANSION_OUTER_IPV6_UDP,
+	MLX5_EXPANSION_OUTER_IPV6_TCP,
 	MLX5_EXPANSION_ETH,
 	MLX5_EXPANSION_IPV4,
 	MLX5_EXPANSION_IPV4_UDP,
@@ -83,6 +115,50 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 						 MLX5_EXPANSION_IPV6),
 		.type = RTE_FLOW_ITEM_TYPE_END,
 	},
+	[MLX5_EXPANSION_ROOT_OUTER] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
+						 MLX5_EXPANSION_OUTER_IPV4,
+						 MLX5_EXPANSION_OUTER_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_END,
+	},
+	[MLX5_EXPANSION_OUTER_ETH] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
+						 MLX5_EXPANSION_OUTER_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+		.rss_types = 0,
+	},
+	[MLX5_EXPANSION_OUTER_IPV4] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT
+			(MLX5_EXPANSION_OUTER_IPV4_UDP,
+			 MLX5_EXPANSION_OUTER_IPV4_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV4,
+		.rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+			ETH_RSS_NONFRAG_IPV4_OTHER,
+	},
+	[MLX5_EXPANSION_OUTER_IPV4_UDP] = {
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
+	},
+	[MLX5_EXPANSION_OUTER_IPV4_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
+	},
+	[MLX5_EXPANSION_OUTER_IPV6] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT
+			(MLX5_EXPANSION_OUTER_IPV6_UDP,
+			 MLX5_EXPANSION_OUTER_IPV6_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV6,
+		.rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
+			ETH_RSS_NONFRAG_IPV6_OTHER,
+	},
+	[MLX5_EXPANSION_OUTER_IPV6_UDP] = {
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
+	},
+	[MLX5_EXPANSION_OUTER_IPV6_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
+	},
 	[MLX5_EXPANSION_ETH] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 						 MLX5_EXPANSION_IPV6),
@@ -453,6 +529,32 @@ mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
 	verbs->size += size;
 }
 
+/**
+ * Adjust verbs hash fields according to the @p flow information.
+ *
+ * @param[in, out] flow.
+ *   Pointer to flow structure.
+ * @param[in] tunnel
+ *   1 when the hash field is for a tunnel item.
+ * @param[in] layer_types
+ *   ETH_RSS_* types.
+ * @param[in] hash_fields
+ *   Item hash fields.
+ */
+static void
+mlx5_flow_verbs_hashfields_adjust(struct rte_flow *flow, int tunnel,
+				  uint32_t layer_types, uint64_t hash_fields)
+{
+	hash_fields |= (tunnel ? IBV_RX_HASH_INNER : 0);
+	if (!(flow->rss.types & layer_types))
+		hash_fields = 0;
+	if (flow->rss.level == 2 && !tunnel)
+		hash_fields = 0;
+	else if (flow->rss.level < 2 && tunnel)
+		hash_fields = 0;
+	flow->cur_verbs->hash_fields |= hash_fields;
+}
+
 /**
  * Convert the @p item into a Verbs specification after ensuring the NIC
  * will understand and process it correctly.
@@ -486,14 +588,16 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 		.type = RTE_BE16(0xffff),
 	};
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 	struct ibv_flow_spec_eth eth = {
-		.type = IBV_FLOW_SPEC_ETH,
+		.type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
 
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L2)
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+			    MLX5_FLOW_LAYER_OUTER_L2))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -506,7 +610,8 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 					error);
 	if (ret)
 		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+		MLX5_FLOW_LAYER_OUTER_L2;
 	if (size > flow_size)
 		return size;
 	if (spec) {
@@ -543,7 +648,7 @@ mlx5_flow_item_vlan_update(struct ibv_flow_attr *attr,
 			   struct ibv_flow_spec_eth *eth)
 {
 	unsigned int i;
-	enum ibv_flow_spec_type search = IBV_FLOW_SPEC_ETH;
+	const enum ibv_flow_spec_type search = eth->type;
 	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 
@@ -596,16 +701,19 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 		.inner_type = RTE_BE16(0xffff),
 	};
 	unsigned int size = sizeof(struct ibv_flow_spec_eth);
-	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 	struct ibv_flow_spec_eth eth = {
-		.type = IBV_FLOW_SPEC_ETH,
+		.type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
-	const uint32_t l34m = MLX5_FLOW_LAYER_OUTER_L3 |
-			MLX5_FLOW_LAYER_OUTER_L4;
-	const uint32_t vlanm = MLX5_FLOW_LAYER_OUTER_VLAN;
-	const uint32_t l2m = MLX5_FLOW_LAYER_OUTER_L2;
+	const uint32_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
+					MLX5_FLOW_LAYER_INNER_L4) :
+		(MLX5_FLOW_LAYER_OUTER_L3 | MLX5_FLOW_LAYER_OUTER_L4);
+	const uint32_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
+		MLX5_FLOW_LAYER_OUTER_VLAN;
+	const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+		MLX5_FLOW_LAYER_OUTER_L2;
 
 	if (flow->layers & vlanm)
 		return rte_flow_error_set(error, ENOTSUP,
@@ -648,11 +756,14 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 			mlx5_flow_spec_verbs_add(flow, &eth, size);
 		}
 	} else {
-		if (verbs->attr)
-			mlx5_flow_item_vlan_update(verbs->attr, &eth);
+		if (flow->cur_verbs)
+			mlx5_flow_item_vlan_update(flow->cur_verbs->attr,
+						   &eth);
 		size = 0; /* Only an update is done in eth specification. */
 	}
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_VLAN;
+	flow->layers |= tunnel ?
+		(MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_VLAN) :
+		(MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_VLAN);
 	return size;
 }
 
@@ -692,19 +803,23 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 			.next_proto_id = 0xff,
 		},
 	};
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 	unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 	struct ibv_flow_spec_ipv4_ext ipv4 = {
-		.type = IBV_FLOW_SPEC_IPV4_EXT,
+		.type = IBV_FLOW_SPEC_IPV4_EXT |
+			(tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
 
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			    MLX5_FLOW_LAYER_OUTER_L3))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
 					  "multiple L3 layers not supported");
-	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+	else if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+				 MLX5_FLOW_LAYER_OUTER_L4))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -717,7 +832,8 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 		 sizeof(struct rte_flow_item_ipv4), error);
 	if (ret < 0)
 		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
+		MLX5_FLOW_LAYER_OUTER_L3_IPV4;
 	if (spec) {
 		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 			.src_ip = spec->hdr.src_addr,
@@ -740,14 +856,11 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 	flow->l3_protocol_en = !!ipv4.mask.proto;
 	flow->l3_protocol = ipv4.val.proto;
 	if (size <= flow_size) {
-		uint64_t hash_fields = IBV_RX_HASH_SRC_IPV4 |
-			IBV_RX_HASH_DST_IPV4;
-
-		if (!(flow->rss.types &
-		      (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
-		       ETH_RSS_NONFRAG_IPV4_OTHER)))
-			hash_fields = 0;
-		flow->cur_verbs->hash_fields |= hash_fields;
+		mlx5_flow_verbs_hashfields_adjust
+			(flow, tunnel,
+			 (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+			  ETH_RSS_NONFRAG_IPV4_OTHER),
+			 (IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4));
 		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L3;
 		mlx5_flow_spec_verbs_add(flow, &ipv4, size);
 	}
@@ -795,19 +908,22 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 			.hop_limits = 0xff,
 		},
 	};
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 	unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 	struct ibv_flow_spec_ipv6 ipv6 = {
-		.type = IBV_FLOW_SPEC_IPV6,
+		.type = IBV_FLOW_SPEC_IPV6 | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
 
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			    MLX5_FLOW_LAYER_OUTER_L3))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
 					  "multiple L3 layers not supported");
-	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+	else if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+				 MLX5_FLOW_LAYER_OUTER_L4))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -820,7 +936,8 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 		 sizeof(struct rte_flow_item_ipv6), error);
 	if (ret < 0)
 		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
+		MLX5_FLOW_LAYER_OUTER_L3_IPV6;
 	if (spec) {
 		unsigned int i;
 		uint32_t vtc_flow_val;
@@ -863,13 +980,10 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 	flow->l3_protocol_en = !!ipv6.mask.next_hdr;
 	flow->l3_protocol = ipv6.val.next_hdr;
 	if (size <= flow_size) {
-		uint64_t hash_fields = IBV_RX_HASH_SRC_IPV6 |
-			IBV_RX_HASH_DST_IPV6;
-
-		if (!(flow->rss.types &
-		      (ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER)))
-			hash_fields = 0;
-		flow->cur_verbs->hash_fields |= hash_fields;
+		mlx5_flow_verbs_hashfields_adjust
+			(flow, tunnel,
+			 (ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER),
+			 (IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6));
 		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L3;
 		mlx5_flow_spec_verbs_add(flow, &ipv6, size);
 	}
@@ -904,9 +1018,10 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 {
 	const struct rte_flow_item_udp *spec = item->spec;
 	const struct rte_flow_item_udp *mask = item->mask;
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 	struct ibv_flow_spec_tcp_udp udp = {
-		.type = IBV_FLOW_SPEC_UDP,
+		.type = IBV_FLOW_SPEC_UDP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
@@ -917,13 +1032,15 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 					  item,
 					  "protocol filtering not compatible"
 					  " with UDP layer");
-	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+	if (!(flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			      MLX5_FLOW_LAYER_OUTER_L3)))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
 					  "L3 is mandatory to filter"
 					  " on L4");
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+			    MLX5_FLOW_LAYER_OUTER_L4))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -937,7 +1054,8 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 		 sizeof(struct rte_flow_item_udp), error);
 	if (ret < 0)
 		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
+		MLX5_FLOW_LAYER_OUTER_L4_UDP;
 	if (spec) {
 		udp.val.dst_port = spec->hdr.dst_port;
 		udp.val.src_port = spec->hdr.src_port;
@@ -948,12 +1066,9 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 		udp.val.dst_port &= udp.mask.dst_port;
 	}
 	if (size <= flow_size) {
-		uint64_t hash_fields = IBV_RX_HASH_SRC_PORT_UDP |
-			IBV_RX_HASH_DST_PORT_UDP;
-
-		if (!(flow->rss.types & ETH_RSS_UDP))
-			hash_fields = 0;
-		flow->cur_verbs->hash_fields |= hash_fields;
+		mlx5_flow_verbs_hashfields_adjust(flow, tunnel, ETH_RSS_UDP,
+						  (IBV_RX_HASH_SRC_PORT_UDP |
+						   IBV_RX_HASH_DST_PORT_UDP));
 		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L4;
 		mlx5_flow_spec_verbs_add(flow, &udp, size);
 	}
@@ -988,9 +1103,10 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 {
 	const struct rte_flow_item_tcp *spec = item->spec;
 	const struct rte_flow_item_tcp *mask = item->mask;
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 	struct ibv_flow_spec_tcp_udp tcp = {
-		.type = IBV_FLOW_SPEC_TCP,
+		.type = IBV_FLOW_SPEC_TCP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
@@ -1001,12 +1117,14 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 					  item,
 					  "protocol filtering not compatible"
 					  " with TCP layer");
-	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+	if (!(flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			      MLX5_FLOW_LAYER_OUTER_L3)))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
 					  "L3 is mandatory to filter on L4");
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+			    MLX5_FLOW_LAYER_OUTER_L4))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -1019,7 +1137,8 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 		 sizeof(struct rte_flow_item_tcp), error);
 	if (ret < 0)
 		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
+	flow->layers |=  tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
+		MLX5_FLOW_LAYER_OUTER_L4_TCP;
 	if (spec) {
 		tcp.val.dst_port = spec->hdr.dst_port;
 		tcp.val.src_port = spec->hdr.src_port;
@@ -1030,12 +1149,9 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 		tcp.val.dst_port &= tcp.mask.dst_port;
 	}
 	if (size <= flow_size) {
-		uint64_t hash_fields = IBV_RX_HASH_SRC_PORT_TCP |
-			IBV_RX_HASH_DST_PORT_TCP;
-
-		if (!(flow->rss.types & ETH_RSS_TCP))
-			hash_fields = 0;
-		flow->cur_verbs->hash_fields |= hash_fields;
+		mlx5_flow_verbs_hashfields_adjust(flow, tunnel, ETH_RSS_TCP,
+						  (IBV_RX_HASH_SRC_PORT_TCP |
+						   IBV_RX_HASH_DST_PORT_TCP));
 		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L4;
 		mlx5_flow_spec_verbs_add(flow, &tcp, size);
 	}
@@ -1261,7 +1377,7 @@ mlx5_flow_action_rss(struct rte_eth_dev *dev,
 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 					  &rss->func,
 					  "RSS hash function not supported");
-	if (rss->level > 1)
+	if (rss->level > 2)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 					  &rss->level,
@@ -1301,6 +1417,7 @@ mlx5_flow_action_rss(struct rte_eth_dev *dev,
 	flow->rss.queue_num = rss->queue_num;
 	memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
 	flow->rss.types = rss->types;
+	flow->rss.level = rss->level;
 	flow->fate |= MLX5_FLOW_FATE_RSS;
 	return 0;
 }
@@ -1607,7 +1724,9 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 		ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
 					  pattern, local_flow.rss.types,
 					  mlx5_support_expansion,
-					  MLX5_EXPANSION_ROOT);
+					  local_flow.rss.level < 2 ?
+					  MLX5_EXPANSION_ROOT :
+					  MLX5_EXPANSION_ROOT_OUTER);
 		assert(ret > 0 &&
 		       (unsigned int)ret < sizeof(expand_buffer.buffer));
 	} else {
@@ -1975,8 +2094,8 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
 			return NULL;
 		}
 	}
-	mlx5_flow_rxq_mark_set(dev, flow);
 	TAILQ_INSERT_TAIL(list, flow, next);
+	mlx5_flow_rxq_mark_set(dev, flow);
 	return flow;
 }
 
-- 
2.18.0

[dpdk-dev] [PATCH v3 17/21] net/mlx5: add flow VXLAN item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 190 +++++++++++++++++++++++++++++------
 drivers/net/mlx5/mlx5_rxtx.h |   1 +
 2 files changed, 163 insertions(+), 28 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index edceb17ed..95507059e 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -51,6 +51,9 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_FLOW_LAYER_INNER_L4_TCP (1u << 10)
 #define MLX5_FLOW_LAYER_INNER_VLAN (1u << 11)
 
+/* Pattern tunnel Layer bits. */
+#define MLX5_FLOW_LAYER_VXLAN (1u << 12)
+
 /* Outer Masks. */
 #define MLX5_FLOW_LAYER_OUTER_L3 \
 	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
@@ -61,7 +64,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 	 MLX5_FLOW_LAYER_OUTER_L4)
 
 /* Tunnel Masks. */
-#define MLX5_FLOW_LAYER_TUNNEL 0
+#define MLX5_FLOW_LAYER_TUNNEL MLX5_FLOW_LAYER_VXLAN
 
 /* Inner Masks. */
 #define MLX5_FLOW_LAYER_INNER_L3 \
@@ -98,6 +101,7 @@ enum mlx5_expansion {
 	MLX5_EXPANSION_OUTER_IPV6,
 	MLX5_EXPANSION_OUTER_IPV6_UDP,
 	MLX5_EXPANSION_OUTER_IPV6_TCP,
+	MLX5_EXPANSION_VXLAN,
 	MLX5_EXPANSION_ETH,
 	MLX5_EXPANSION_IPV4,
 	MLX5_EXPANSION_IPV4_UDP,
@@ -136,6 +140,7 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 			ETH_RSS_NONFRAG_IPV4_OTHER,
 	},
 	[MLX5_EXPANSION_OUTER_IPV4_UDP] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN),
 		.type = RTE_FLOW_ITEM_TYPE_UDP,
 		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 	},
@@ -152,6 +157,7 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 			ETH_RSS_NONFRAG_IPV6_OTHER,
 	},
 	[MLX5_EXPANSION_OUTER_IPV6_UDP] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN),
 		.type = RTE_FLOW_ITEM_TYPE_UDP,
 		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 	},
@@ -159,6 +165,10 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 		.type = RTE_FLOW_ITEM_TYPE_TCP,
 		.rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 	},
+	[MLX5_EXPANSION_VXLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
+		.type = RTE_FLOW_ITEM_TYPE_VXLAN,
+	},
 	[MLX5_EXPANSION_ETH] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 						 MLX5_EXPANSION_IPV6),
@@ -226,6 +236,8 @@ struct rte_flow {
 	struct mlx5_flow_verbs *cur_verbs;
 	/**< Current Verbs flow structure being filled. */
 	struct rte_flow_action_rss rss;/**< RSS context. */
+	uint32_t tunnel_ptype;
+	/**< Store tunnel packet type data to store in Rx queue. */
 	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
 	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
 };
@@ -1158,6 +1170,103 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
+		     const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_vxlan *spec = item->spec;
+	const struct rte_flow_item_vxlan *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel vxlan = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+	int ret;
+	union vni {
+		uint32_t vlan_id;
+		uint8_t vni[4];
+	} id = { .vlan_id = 0, };
+
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already present");
+	/*
+	 * Verify only UDPv4 is present as defined in
+	 * https://tools.ietf.org/html/rfc7348
+	 */
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L4_UDP))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "no outer UDP layer found");
+	if (!mask)
+		mask = &rte_flow_item_vxlan_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_vxlan_mask,
+		 sizeof(struct rte_flow_item_vxlan), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		memcpy(&id.vni[1], spec->vni, 3);
+		vxlan.val.tunnel_id = id.vlan_id;
+		memcpy(&id.vni[1], mask->vni, 3);
+		vxlan.mask.tunnel_id = id.vlan_id;
+		/* Remove unwanted bits from values. */
+		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
+	}
+	/*
+	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if
+	 * only this layer is defined in the Verbs specification it is
+	 * interpreted as wildcard and all packets will match this
+	 * rule, if it follows a full stack layer (ex: eth / ipv4 /
+	 * udp), all packets matching the layers before will also
+	 * match this rule.  To avoid such situation, VNI 0 is
+	 * currently refused.
+	 */
+	if (!vxlan.val.tunnel_id)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN vni cannot be 0");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN tunnel must be fully defined");
+	if (size <= flow_size) {
+		mlx5_flow_spec_verbs_add(flow, &vxlan, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_VXLAN;
+	flow->tunnel_ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP;
+	return size;
+}
+
 /**
  * Convert the @p pattern into a Verbs specifications after ensuring the NIC
  * will understand and process it correctly.
@@ -1215,6 +1324,10 @@ mlx5_flow_items(const struct rte_flow_item pattern[],
 		case RTE_FLOW_ITEM_TYPE_TCP:
 			ret = mlx5_flow_item_tcp(pattern, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN:
+			ret = mlx5_flow_item_vxlan(pattern, flow, remain,
+						   error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
@@ -1822,7 +1935,7 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 }
 
 /**
- * Mark the Rx queues mark flag if the flow has a mark or flag modifier.
+ * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the flow.
  *
  * @param[in] dev
  *   Pointer to Ethernet device.
@@ -1830,28 +1943,34 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
  *   Pointer to flow structure.
  */
 static void
-mlx5_flow_rxq_mark_set(struct rte_eth_dev *dev, struct rte_flow *flow)
+mlx5_flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
 	struct priv *priv = dev->data->dev_private;
+	const int mark = !!(flow->modifier &
+			    (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK));
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int i;
 
-	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK)) {
-		unsigned int i;
-
-		for (i = 0; i != flow->rss.queue_num; ++i) {
-			int idx = (*flow->queue)[i];
-			struct mlx5_rxq_ctrl *rxq_ctrl =
-				container_of((*priv->rxqs)[idx],
-					     struct mlx5_rxq_ctrl, rxq);
+	for (i = 0; i != flow->rss.queue_num; ++i) {
+		int idx = (*flow->queue)[i];
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of((*priv->rxqs)[idx],
+				     struct mlx5_rxq_ctrl, rxq);
 
+		if (mark) {
 			rxq_ctrl->rxq.mark = 1;
 			rxq_ctrl->flow_mark_n++;
 		}
+		if (tunnel) {
+			rxq_ctrl->rxq.tunnel = flow->tunnel_ptype;
+			rxq_ctrl->flow_vxlan_n++;
+		}
 	}
 }
 
 /**
- * Clear the Rx queue mark associated with the @p flow if no other flow uses
- * it with a mark request.
+ * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
+ * @p flow if no other flow uses it with the same kind of request.
  *
  * @param dev
  *   Pointer to Ethernet device.
@@ -1859,33 +1978,41 @@ mlx5_flow_rxq_mark_set(struct rte_eth_dev *dev, struct rte_flow *flow)
  *   Pointer to the flow.
  */
 static void
-mlx5_flow_rxq_mark_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
+mlx5_flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
 	struct priv *priv = dev->data->dev_private;
+	const int mark = !!(flow->modifier &
+			    (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK));
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int i;
 
-	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK)) {
-		unsigned int i;
-
-		for (i = 0; i != flow->rss.queue_num; ++i) {
-			int idx = (*flow->queue)[i];
-			struct mlx5_rxq_ctrl *rxq_ctrl =
-				container_of((*priv->rxqs)[idx],
-					     struct mlx5_rxq_ctrl, rxq);
+	assert(dev->data->dev_started);
+	for (i = 0; i != flow->rss.queue_num; ++i) {
+		int idx = (*flow->queue)[i];
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of((*priv->rxqs)[idx],
+				     struct mlx5_rxq_ctrl, rxq);
 
+		if (mark) {
 			rxq_ctrl->flow_mark_n--;
 			rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
 		}
+		if (tunnel) {
+			rxq_ctrl->flow_vxlan_n++;
+			if (!rxq_ctrl->flow_vxlan_n)
+				rxq_ctrl->rxq.tunnel = 0;
+		}
 	}
 }
 
 /**
- * Clear the mark bit in all Rx queues.
+ * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
  *
  * @param dev
  *   Pointer to Ethernet device.
  */
 static void
-mlx5_flow_rxq_mark_clear(struct rte_eth_dev *dev)
+mlx5_flow_rxq_flags_clear(struct rte_eth_dev *dev)
 {
 	struct priv *priv = dev->data->dev_private;
 	unsigned int i;
@@ -1900,6 +2027,8 @@ mlx5_flow_rxq_mark_clear(struct rte_eth_dev *dev)
 					struct mlx5_rxq_ctrl, rxq);
 		rxq_ctrl->flow_mark_n = 0;
 		rxq_ctrl->rxq.mark = 0;
+		rxq_ctrl->flow_vxlan_n = 0;
+		rxq_ctrl->rxq.tunnel = 0;
 		++idx;
 	}
 }
@@ -2095,7 +2224,7 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
 		}
 	}
 	TAILQ_INSERT_TAIL(list, flow, next);
-	mlx5_flow_rxq_mark_set(dev, flow);
+	mlx5_flow_rxq_flags_set(dev, flow);
 	return flow;
 }
 
@@ -2133,7 +2262,12 @@ mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
 {
 	mlx5_flow_remove(dev, flow);
 	TAILQ_REMOVE(list, flow, next);
-	mlx5_flow_rxq_mark_trim(dev, flow);
+	/*
+	 * Update RX queue flags only if port is started, otherwise it is
+	 * already clean.
+	 */
+	if (dev->data->dev_started)
+		mlx5_flow_rxq_flags_trim(dev, flow);
 	rte_free(flow);
 }
 
@@ -2171,7 +2305,7 @@ mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
 
 	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
 		mlx5_flow_remove(dev, flow);
-	mlx5_flow_rxq_mark_clear(dev);
+	mlx5_flow_rxq_flags_clear(dev);
 }
 
 /**
@@ -2196,7 +2330,7 @@ mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
 		ret = mlx5_flow_apply(dev, flow, &error);
 		if (ret < 0)
 			goto error;
-		mlx5_flow_rxq_mark_set(dev, flow);
+		mlx5_flow_rxq_flags_set(dev, flow);
 	}
 	return 0;
 error:
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 808118e50..ae9b564dd 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -139,6 +139,7 @@ struct mlx5_rxq_ctrl {
 	unsigned int irq:1; /* Whether IRQ is enabled. */
 	uint16_t idx; /* Queue index. */
 	uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */
+	uint32_t flow_vxlan_n; /* Number of VXLAN flows using this queue. */
 };
 
 /* Indirection table. */
-- 
2.18.0

[dpdk-dev] [PATCH v3 18/21] net/mlx5: add flow VXLAN-GPE item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 219 ++++++++++++++++++++++++++++++++---
 drivers/net/mlx5/mlx5_rxtx.h |   5 +-
 2 files changed, 209 insertions(+), 15 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 95507059e..e06df0eb5 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -53,6 +53,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
 /* Pattern tunnel Layer bits. */
 #define MLX5_FLOW_LAYER_VXLAN (1u << 12)
+#define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
 
 /* Outer Masks. */
 #define MLX5_FLOW_LAYER_OUTER_L3 \
@@ -64,7 +65,8 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 	 MLX5_FLOW_LAYER_OUTER_L4)
 
 /* Tunnel Masks. */
-#define MLX5_FLOW_LAYER_TUNNEL MLX5_FLOW_LAYER_VXLAN
+#define MLX5_FLOW_LAYER_TUNNEL \
+	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE)
 
 /* Inner Masks. */
 #define MLX5_FLOW_LAYER_INNER_L3 \
@@ -102,6 +104,7 @@ enum mlx5_expansion {
 	MLX5_EXPANSION_OUTER_IPV6_UDP,
 	MLX5_EXPANSION_OUTER_IPV6_TCP,
 	MLX5_EXPANSION_VXLAN,
+	MLX5_EXPANSION_VXLAN_GPE,
 	MLX5_EXPANSION_ETH,
 	MLX5_EXPANSION_IPV4,
 	MLX5_EXPANSION_IPV4_UDP,
@@ -140,7 +143,8 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 			ETH_RSS_NONFRAG_IPV4_OTHER,
 	},
 	[MLX5_EXPANSION_OUTER_IPV4_UDP] = {
-		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN),
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
+						 MLX5_EXPANSION_VXLAN_GPE),
 		.type = RTE_FLOW_ITEM_TYPE_UDP,
 		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 	},
@@ -157,7 +161,8 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 			ETH_RSS_NONFRAG_IPV6_OTHER,
 	},
 	[MLX5_EXPANSION_OUTER_IPV6_UDP] = {
-		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN),
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
+						 MLX5_EXPANSION_VXLAN_GPE),
 		.type = RTE_FLOW_ITEM_TYPE_UDP,
 		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 	},
@@ -169,6 +174,12 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 		.type = RTE_FLOW_ITEM_TYPE_VXLAN,
 	},
+	[MLX5_EXPANSION_VXLAN_GPE] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
+						 MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
+	},
 	[MLX5_EXPANSION_ETH] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 						 MLX5_EXPANSION_IPV6),
@@ -236,8 +247,6 @@ struct rte_flow {
 	struct mlx5_flow_verbs *cur_verbs;
 	/**< Current Verbs flow structure being filled. */
 	struct rte_flow_action_rss rss;/**< RSS context. */
-	uint32_t tunnel_ptype;
-	/**< Store tunnel packet type data to store in Rx queue. */
 	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
 	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
 };
@@ -304,6 +313,23 @@ static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
 	{ 9, 10, 11 }, { 12, 13, 14 },
 };
 
+/* Tunnel information. */
+struct mlx5_flow_tunnel_info {
+	uint32_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
+	uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
+};
+
+static struct mlx5_flow_tunnel_info tunnels_info[] = {
+	{
+		.tunnel = MLX5_FLOW_LAYER_VXLAN,
+		.ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
+	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
+		.ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
+	},
+};
+
 /**
  * Discover the maximum number of priority available.
  *
@@ -1263,7 +1289,119 @@ mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
 		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
 	}
 	flow->layers |= MLX5_FLOW_LAYER_VXLAN;
-	flow->tunnel_ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP;
+	return size;
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_vxlan_gpe(struct rte_eth_dev *dev,
+			 const struct rte_flow_item *item,
+			 struct rte_flow *flow, const size_t flow_size,
+			 struct rte_flow_error *error)
+{
+	const struct rte_flow_item_vxlan_gpe *spec = item->spec;
+	const struct rte_flow_item_vxlan_gpe *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel vxlan_gpe = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+	int ret;
+	union vni {
+		uint32_t vlan_id;
+		uint8_t vni[4];
+	} id = { .vlan_id = 0, };
+
+	if (!((struct priv *)dev->data->dev_private)->config.l3_vxlan_en)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 VXLAN is not enabled by device"
+					  " parameter and/or not configured in"
+					  " firmware");
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already present");
+	/*
+	 * Verify only UDPv4 is present as defined in
+	 * https://tools.ietf.org/html/rfc7348
+	 */
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L4_UDP))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "no outer UDP layer found");
+	if (!mask)
+		mask = &rte_flow_item_vxlan_gpe_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
+		 sizeof(struct rte_flow_item_vxlan_gpe), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		memcpy(&id.vni[1], spec->vni, 3);
+		vxlan_gpe.val.tunnel_id = id.vlan_id;
+		memcpy(&id.vni[1], mask->vni, 3);
+		vxlan_gpe.mask.tunnel_id = id.vlan_id;
+		if (spec->protocol)
+			return rte_flow_error_set
+				(error, EINVAL,
+				 RTE_FLOW_ERROR_TYPE_ITEM,
+				 item,
+				 "VxLAN-GPE protocol not supported");
+		/* Remove unwanted bits from values. */
+		vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
+	}
+	/*
+	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
+	 * layer is defined in the Verbs specification it is interpreted as
+	 * wildcard and all packets will match this rule, if it follows a full
+	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
+	 * before will also match this rule.  To avoid such situation, VNI 0
+	 * is currently refused.
+	 */
+	if (!vxlan_gpe.val.tunnel_id)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN-GPE vni cannot be 0");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN-GPE tunnel must be fully"
+					  " defined");
+	if (size <= flow_size) {
+		mlx5_flow_spec_verbs_add(flow, &vxlan_gpe, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_VXLAN_GPE;
 	return size;
 }
 
@@ -1293,7 +1431,8 @@ mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
  *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_items(const struct rte_flow_item pattern[],
+mlx5_flow_items(struct rte_eth_dev *dev,
+		const struct rte_flow_item pattern[],
 		struct rte_flow *flow, const size_t flow_size,
 		struct rte_flow_error *error)
 {
@@ -1328,6 +1467,10 @@ mlx5_flow_items(const struct rte_flow_item pattern[],
 			ret = mlx5_flow_item_vxlan(pattern, flow, remain,
 						   error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
+			ret = mlx5_flow_item_vxlan_gpe(dev, pattern, flow,
+						       remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
@@ -1908,7 +2051,8 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 			}
 		}
 		ret = mlx5_flow_items
-			((const struct rte_flow_item *)
+			(dev,
+			 (const struct rte_flow_item *)
 			 &buf->entry[i].pattern[pattern_start_idx],
 			 flow,
 			 (size < flow_size) ? flow_size - size : 0, error);
@@ -1934,6 +2078,34 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
+ * if several tunnel rules are used on this queue, the tunnel ptype will be
+ * cleared.
+ *
+ * @param rxq_ctrl
+ *   Rx queue to update.
+ */
+static void
+mlx5_flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
+{
+	unsigned int i;
+	uint32_t tunnel_ptype = 0;
+
+	/* Look up for the ptype to use. */
+	for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
+		if (!rxq_ctrl->flow_tunnels_n[i])
+			continue;
+		if (!tunnel_ptype) {
+			tunnel_ptype = tunnels_info[i].ptype;
+		} else {
+			tunnel_ptype = 0;
+			break;
+		}
+	}
+	rxq_ctrl->rxq.tunnel = tunnel_ptype;
+}
+
 /**
  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the flow.
  *
@@ -1962,8 +2134,17 @@ mlx5_flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
 			rxq_ctrl->flow_mark_n++;
 		}
 		if (tunnel) {
-			rxq_ctrl->rxq.tunnel = flow->tunnel_ptype;
-			rxq_ctrl->flow_vxlan_n++;
+			unsigned int j;
+
+			/* Increase the counter matching the flow. */
+			for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
+				if ((tunnels_info[j].tunnel & flow->layers) ==
+				    tunnels_info[j].tunnel) {
+					rxq_ctrl->flow_tunnels_n[j]++;
+					break;
+				}
+			}
+			mlx5_flow_rxq_tunnel_ptype_update(rxq_ctrl);
 		}
 	}
 }
@@ -1998,9 +2179,17 @@ mlx5_flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
 			rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
 		}
 		if (tunnel) {
-			rxq_ctrl->flow_vxlan_n++;
-			if (!rxq_ctrl->flow_vxlan_n)
-				rxq_ctrl->rxq.tunnel = 0;
+			unsigned int j;
+
+			/* Decrease the counter matching the flow. */
+			for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
+				if ((tunnels_info[j].tunnel & flow->layers) ==
+				    tunnels_info[j].tunnel) {
+					rxq_ctrl->flow_tunnels_n[j]--;
+					break;
+				}
+			}
+			mlx5_flow_rxq_tunnel_ptype_update(rxq_ctrl);
 		}
 	}
 }
@@ -2020,6 +2209,7 @@ mlx5_flow_rxq_flags_clear(struct rte_eth_dev *dev)
 
 	for (idx = 0, i = 0; idx != priv->rxqs_n; ++i) {
 		struct mlx5_rxq_ctrl *rxq_ctrl;
+		unsigned int j;
 
 		if (!(*priv->rxqs)[idx])
 			continue;
@@ -2027,7 +2217,8 @@ mlx5_flow_rxq_flags_clear(struct rte_eth_dev *dev)
 					struct mlx5_rxq_ctrl, rxq);
 		rxq_ctrl->flow_mark_n = 0;
 		rxq_ctrl->rxq.mark = 0;
-		rxq_ctrl->flow_vxlan_n = 0;
+		for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
+			rxq_ctrl->flow_tunnels_n[j] = 0;
 		rxq_ctrl->rxq.tunnel = 0;
 		++idx;
 	}
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index ae9b564dd..e97f5766c 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -34,6 +34,9 @@
 #include "mlx5_defs.h"
 #include "mlx5_prm.h"
 
+/* Support tunnel matching. */
+#define MLX5_FLOW_TUNNEL 2
+
 struct mlx5_rxq_stats {
 	unsigned int idx; /**< Mapping index. */
 #ifdef MLX5_PMD_SOFT_COUNTERS
@@ -139,7 +142,7 @@ struct mlx5_rxq_ctrl {
 	unsigned int irq:1; /* Whether IRQ is enabled. */
 	uint16_t idx; /* Queue index. */
 	uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */
-	uint32_t flow_vxlan_n; /* Number of VXLAN flows using this queue. */
+	uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels counters. */
 };
 
 /* Indirection table. */
-- 
2.18.0

[dpdk-dev] [PATCH v3 19/21] net/mlx5: add flow GRE item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 193 ++++++++++++++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rxtx.h |   2 +-
 2 files changed, 192 insertions(+), 3 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index e06df0eb5..3412c856a 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -54,6 +54,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* Pattern tunnel Layer bits. */
 #define MLX5_FLOW_LAYER_VXLAN (1u << 12)
 #define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
+#define MLX5_FLOW_LAYER_GRE (1u << 14)
 
 /* Outer Masks. */
 #define MLX5_FLOW_LAYER_OUTER_L3 \
@@ -66,7 +67,8 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
 /* Tunnel Masks. */
 #define MLX5_FLOW_LAYER_TUNNEL \
-	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE)
+	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE | \
+	 MLX5_FLOW_LAYER_GRE)
 
 /* Inner Masks. */
 #define MLX5_FLOW_LAYER_INNER_L3 \
@@ -89,6 +91,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* possible L3 layers protocols filtering. */
 #define MLX5_IP_PROTOCOL_TCP 6
 #define MLX5_IP_PROTOCOL_UDP 17
+#define MLX5_IP_PROTOCOL_GRE 47
 
 /* Priority reserved for default flows. */
 #define MLX5_FLOW_PRIO_RSVD ((uint32_t)-1)
@@ -105,6 +108,7 @@ enum mlx5_expansion {
 	MLX5_EXPANSION_OUTER_IPV6_TCP,
 	MLX5_EXPANSION_VXLAN,
 	MLX5_EXPANSION_VXLAN_GPE,
+	MLX5_EXPANSION_GRE,
 	MLX5_EXPANSION_ETH,
 	MLX5_EXPANSION_IPV4,
 	MLX5_EXPANSION_IPV4_UDP,
@@ -137,7 +141,8 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 	[MLX5_EXPANSION_OUTER_IPV4] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT
 			(MLX5_EXPANSION_OUTER_IPV4_UDP,
-			 MLX5_EXPANSION_OUTER_IPV4_TCP),
+			 MLX5_EXPANSION_OUTER_IPV4_TCP,
+			 MLX5_EXPANSION_GRE),
 		.type = RTE_FLOW_ITEM_TYPE_IPV4,
 		.rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 			ETH_RSS_NONFRAG_IPV4_OTHER,
@@ -180,6 +185,10 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 						 MLX5_EXPANSION_IPV6),
 		.type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 	},
+	[MLX5_EXPANSION_GRE] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
+		.type = RTE_FLOW_ITEM_TYPE_GRE,
+	},
 	[MLX5_EXPANSION_ETH] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 						 MLX5_EXPANSION_IPV6),
@@ -328,6 +337,10 @@ static struct mlx5_flow_tunnel_info tunnels_info[] = {
 		.tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 		.ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_GRE,
+		.ptype = RTE_PTYPE_TUNNEL_GRE,
+	},
 };
 
 /**
@@ -966,6 +979,18 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
 					  "L3 cannot follow an L4 layer.");
+	/*
+	 * IPv6 is not recognised by the NIC inside a GRE tunnel.
+	 * Such support has to be disabled as the rule will be
+	 * accepted.  Issue reproduced with Mellanox OFED 4.3-3.0.2.1 and
+	 * Mellanox OFED 4.4-1.0.0.0.
+	 */
+	if (tunnel && flow->layers & MLX5_FLOW_LAYER_GRE)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "IPv6 inside a GRE tunnel is"
+					  " not recognised.");
 	if (!mask)
 		mask = &rte_flow_item_ipv6_mask;
 	ret = mlx5_flow_item_acceptable
@@ -1405,6 +1430,167 @@ mlx5_flow_item_vxlan_gpe(struct rte_eth_dev *dev,
 	return size;
 }
 
+/**
+ * Update the protocol in Verbs IPv4/IPv6 spec.
+ *
+ * @param[in, out] attr
+ *   Pointer to Verbs attributes structure.
+ * @param[in] search
+ *   Specification type to search in order to update the IP protocol.
+ * @param[in] protocol
+ *   Protocol value to set if none is present in the specification.
+ */
+static void
+mlx5_flow_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
+				      enum ibv_flow_spec_type search,
+				      uint8_t protocol)
+{
+	unsigned int i;
+	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
+		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
+
+	if (!attr)
+		return;
+	for (i = 0; i != attr->num_of_specs; ++i) {
+		if (hdr->type == search) {
+			union {
+				struct ibv_flow_spec_ipv4_ext *ipv4;
+				struct ibv_flow_spec_ipv6 *ipv6;
+			} ip;
+
+			switch (search) {
+			case IBV_FLOW_SPEC_IPV4_EXT:
+				ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
+				if (!ip.ipv4->val.proto) {
+					ip.ipv4->val.proto = protocol;
+					ip.ipv4->mask.proto = 0xff;
+				}
+				break;
+			case IBV_FLOW_SPEC_IPV6:
+				ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
+				if (!ip.ipv6->val.next_hdr) {
+					ip.ipv6->val.next_hdr = protocol;
+					ip.ipv6->mask.next_hdr = 0xff;
+				}
+				break;
+			default:
+				break;
+			}
+			break;
+		}
+		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
+	}
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * It will also update the previous L3 layer with the protocol value matching
+ * the GRE.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_gre(const struct rte_flow_item *item,
+		   struct rte_flow *flow, const size_t flow_size,
+		   struct rte_flow_error *error)
+{
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+	const struct rte_flow_item_gre *spec = item->spec;
+	const struct rte_flow_item_gre *mask = item->mask;
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	unsigned int size = sizeof(struct ibv_flow_spec_gre);
+	struct ibv_flow_spec_gre tunnel = {
+		.type = IBV_FLOW_SPEC_GRE,
+		.size = size,
+	};
+#else
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel tunnel = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+#endif
+	int ret;
+
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_GRE)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with this GRE layer");
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already present");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 Layer is missing");
+	if (!mask)
+		mask = &rte_flow_item_gre_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_gre_mask,
+		 sizeof(struct rte_flow_item_gre), error);
+	if (ret < 0)
+		return ret;
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	if (spec) {
+		tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
+		tunnel.val.protocol = spec->protocol;
+		tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
+		tunnel.mask.protocol = mask->protocol;
+		/* Remove unwanted bits from values. */
+		tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
+		tunnel.val.protocol &= tunnel.mask.protocol;
+		tunnel.val.key &= tunnel.mask.key;
+	}
+#else
+	if (spec && (spec->protocol & mask->protocol))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "without MPLS support the"
+					  " specification cannot be used for"
+					  " filtering");
+#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
+	if (size <= flow_size) {
+		if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
+			mlx5_flow_item_gre_ip_protocol_update
+				(verbs->attr, IBV_FLOW_SPEC_IPV4_EXT,
+				 MLX5_IP_PROTOCOL_GRE);
+		else
+			mlx5_flow_item_gre_ip_protocol_update
+				(verbs->attr, IBV_FLOW_SPEC_IPV6,
+				 MLX5_IP_PROTOCOL_GRE);
+		mlx5_flow_spec_verbs_add(flow, &tunnel, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_GRE;
+	return size;
+}
+
 /**
  * Convert the @p pattern into a Verbs specifications after ensuring the NIC
  * will understand and process it correctly.
@@ -1471,6 +1657,9 @@ mlx5_flow_items(struct rte_eth_dev *dev,
 			ret = mlx5_flow_item_vxlan_gpe(dev, pattern, flow,
 						       remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_GRE:
+			ret = mlx5_flow_item_gre(pattern, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index e97f5766c..53992e8db 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -35,7 +35,7 @@
 #include "mlx5_prm.h"
 
 /* Support tunnel matching. */
-#define MLX5_FLOW_TUNNEL 2
+#define MLX5_FLOW_TUNNEL 3
 
 struct mlx5_rxq_stats {
 	unsigned int idx; /**< Mapping index. */
-- 
2.18.0

[dpdk-dev] [PATCH v3 20/21] net/mlx5: add flow MPLS item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_flow.c | 102 ++++++++++++++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rxtx.h |   2 +-
 2 files changed, 101 insertions(+), 3 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 3412c856a..4ad581ed9 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -55,6 +55,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_FLOW_LAYER_VXLAN (1u << 12)
 #define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
 #define MLX5_FLOW_LAYER_GRE (1u << 14)
+#define MLX5_FLOW_LAYER_MPLS (1u << 15)
 
 /* Outer Masks. */
 #define MLX5_FLOW_LAYER_OUTER_L3 \
@@ -68,7 +69,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* Tunnel Masks. */
 #define MLX5_FLOW_LAYER_TUNNEL \
 	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE | \
-	 MLX5_FLOW_LAYER_GRE)
+	 MLX5_FLOW_LAYER_GRE | MLX5_FLOW_LAYER_MPLS)
 
 /* Inner Masks. */
 #define MLX5_FLOW_LAYER_INNER_L3 \
@@ -92,6 +93,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_IP_PROTOCOL_TCP 6
 #define MLX5_IP_PROTOCOL_UDP 17
 #define MLX5_IP_PROTOCOL_GRE 47
+#define MLX5_IP_PROTOCOL_MPLS 147
 
 /* Priority reserved for default flows. */
 #define MLX5_FLOW_PRIO_RSVD ((uint32_t)-1)
@@ -109,6 +111,7 @@ enum mlx5_expansion {
 	MLX5_EXPANSION_VXLAN,
 	MLX5_EXPANSION_VXLAN_GPE,
 	MLX5_EXPANSION_GRE,
+	MLX5_EXPANSION_MPLS,
 	MLX5_EXPANSION_ETH,
 	MLX5_EXPANSION_IPV4,
 	MLX5_EXPANSION_IPV4_UDP,
@@ -134,7 +137,8 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 	},
 	[MLX5_EXPANSION_OUTER_ETH] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
-						 MLX5_EXPANSION_OUTER_IPV6),
+						 MLX5_EXPANSION_OUTER_IPV6,
+						 MLX5_EXPANSION_MPLS),
 		.type = RTE_FLOW_ITEM_TYPE_ETH,
 		.rss_types = 0,
 	},
@@ -189,6 +193,11 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
 		.type = RTE_FLOW_ITEM_TYPE_GRE,
 	},
+	[MLX5_EXPANSION_MPLS] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_MPLS,
+	},
 	[MLX5_EXPANSION_ETH] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 						 MLX5_EXPANSION_IPV6),
@@ -341,6 +350,14 @@ static struct mlx5_flow_tunnel_info tunnels_info[] = {
 		.tunnel = MLX5_FLOW_LAYER_GRE,
 		.ptype = RTE_PTYPE_TUNNEL_GRE,
 	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
+		.ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE | RTE_PTYPE_L4_UDP,
+	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_MPLS,
+		.ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
+	},
 };
 
 /**
@@ -1591,6 +1608,84 @@ mlx5_flow_item_gre(const struct rte_flow_item *item,
 	return size;
 }
 
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_mpls(const struct rte_flow_item *item __rte_unused,
+		    struct rte_flow *flow __rte_unused,
+		    const size_t flow_size __rte_unused,
+		    struct rte_flow_error *error)
+{
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	const struct rte_flow_item_mpls *spec = item->spec;
+	const struct rte_flow_item_mpls *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_mpls);
+	struct ibv_flow_spec_mpls mpls = {
+		.type = IBV_FLOW_SPEC_MPLS,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_MPLS)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with MPLS layer");
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already"
+					  " present");
+	if (!mask)
+		mask = &rte_flow_item_mpls_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_mpls_mask,
+		 sizeof(struct rte_flow_item_mpls), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
+		memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
+		/* Remove unwanted bits from values.  */
+		mpls.val.label &= mpls.mask.label;
+	}
+	if (size <= flow_size) {
+		mlx5_flow_spec_verbs_add(flow, &mpls, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_MPLS;
+	return size;
+#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_ITEM,
+				  item,
+				  "MPLS is not supported by Verbs, please"
+				  " update.");
+}
+
 /**
  * Convert the @p pattern into a Verbs specifications after ensuring the NIC
  * will understand and process it correctly.
@@ -1660,6 +1755,9 @@ mlx5_flow_items(struct rte_eth_dev *dev,
 		case RTE_FLOW_ITEM_TYPE_GRE:
 			ret = mlx5_flow_item_gre(pattern, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_MPLS:
+			ret = mlx5_flow_item_mpls(pattern, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 53992e8db..a04a84fec 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -35,7 +35,7 @@
 #include "mlx5_prm.h"
 
 /* Support tunnel matching. */
-#define MLX5_FLOW_TUNNEL 3
+#define MLX5_FLOW_TUNNEL 5
 
 struct mlx5_rxq_stats {
 	unsigned int idx; /**< Mapping index. */
-- 
2.18.0

[dpdk-dev] [PATCH v3 21/21] net/mlx5: add count flow action

[Nelio Laranjeiro]

This is only supported by Mellanox OFED.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5.h      |   2 +
 drivers/net/mlx5/mlx5_flow.c | 242 +++++++++++++++++++++++++++++++++++
 2 files changed, 244 insertions(+)

diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 9949cd3fa..131be334c 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -188,6 +188,8 @@ struct priv {
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
 	struct mlx5_flows flows; /* RTE Flow rules. */
 	struct mlx5_flows ctrl_flows; /* Control flow rules. */
+	LIST_HEAD(counters, mlx5_flow_counter) flow_counters;
+	/* Flow counters. */
 	struct {
 		uint32_t dev_gen; /* Generation number to flush local caches. */
 		rte_rwlock_t rwlock; /* MR Lock. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 4ad581ed9..9ce4dddd7 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -88,6 +88,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* Modify a packet. */
 #define MLX5_FLOW_MOD_FLAG (1u << 0)
 #define MLX5_FLOW_MOD_MARK (1u << 1)
+#define MLX5_FLOW_MOD_COUNT (1u << 2)
 
 /* possible L3 layers protocols filtering. */
 #define MLX5_IP_PROTOCOL_TCP 6
@@ -249,6 +250,17 @@ struct mlx5_flow_verbs {
 	uint64_t hash_fields; /**< Verbs hash Rx queue hash fields. */
 };
 
+/* Counters information. */
+struct mlx5_flow_counter {
+	LIST_ENTRY(mlx5_flow_counter) next; /**< Pointer to the next counter. */
+	uint32_t shared:1; /**< Share counter ID with other flow rules. */
+	uint32_t ref_cnt:31; /**< Reference counter. */
+	uint32_t id; /**< Counter ID. */
+	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
+	uint64_t hits; /**< Number of packets matched by the rule. */
+	uint64_t bytes; /**< Number of bytes matched by the rule. */
+};
+
 /* Flow structure. */
 struct rte_flow {
 	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
@@ -264,6 +276,7 @@ struct rte_flow {
 	LIST_HEAD(verbs, mlx5_flow_verbs) verbs; /**< Verbs flows list. */
 	struct mlx5_flow_verbs *cur_verbs;
 	/**< Current Verbs flow structure being filled. */
+	struct mlx5_flow_counter *counter; /**< Holds Verbs flow counter. */
 	struct rte_flow_action_rss rss;/**< RSS context. */
 	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
 	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
@@ -275,6 +288,7 @@ static const struct rte_flow_ops mlx5_flow_ops = {
 	.destroy = mlx5_flow_destroy,
 	.flush = mlx5_flow_flush,
 	.isolate = mlx5_flow_isolate,
+	.query = mlx5_flow_query,
 };
 
 /* Convert FDIR request to Generic flow. */
@@ -454,6 +468,80 @@ mlx5_flow_adjust_priority(struct rte_eth_dev *dev, struct rte_flow *flow)
 	flow->cur_verbs->attr->priority = priority;
 }
 
+/**
+ * Get a flow counter.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in] shared
+ *   Indicate if this counter is shared with other flows.
+ * @param[in] id
+ *   Counter identifier.
+ *
+ * @return
+ *   A pointer to the counter, NULL otherwise and rte_errno is set.
+ */
+static struct mlx5_flow_counter *
+mlx5_flow_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_flow_counter *cnt;
+
+	LIST_FOREACH(cnt, &priv->flow_counters, next) {
+		if (cnt->shared != shared)
+			continue;
+		if (cnt->id != id)
+			continue;
+		cnt->ref_cnt++;
+		return cnt;
+	}
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+
+	struct mlx5_flow_counter tmpl = {
+		.shared = shared,
+		.id = id,
+		.cs = mlx5_glue->create_counter_set
+			(priv->ctx,
+			 &(struct ibv_counter_set_init_attr){
+				 .counter_set_id = id,
+			 }),
+		.hits = 0,
+		.bytes = 0,
+	};
+
+	if (!tmpl.cs) {
+		rte_errno = errno;
+		return NULL;
+	}
+	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
+	if (!cnt) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	*cnt = tmpl;
+	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
+	return cnt;
+#endif
+	rte_errno = ENOTSUP;
+	return NULL;
+}
+
+/**
+ * Release a flow counter.
+ *
+ * @param[in] counter
+ *   Pointer to the counter handler.
+ */
+static void
+mlx5_flow_counter_release(struct mlx5_flow_counter *counter)
+{
+	if (--counter->ref_cnt == 0) {
+		claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
+		LIST_REMOVE(counter, next);
+		rte_free(counter);
+	}
+}
+
 /**
  * Verify the @p attributes will be correctly understood by the NIC and store
  * them in the @p flow if everything is correct.
@@ -2121,6 +2209,70 @@ mlx5_flow_action_mark(const struct rte_flow_action *action,
 	return size;
 }
 
+/**
+ * Convert the @p action into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param action[in]
+ *   Action configuration.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param flow_size[in]
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param error[int, out]
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_count(struct rte_eth_dev *dev,
+		       const struct rte_flow_action *action,
+		       struct rte_flow *flow,
+		       const size_t flow_size __rte_unused,
+		       struct rte_flow_error *error)
+{
+	const struct rte_flow_action_count *count = action->conf;
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
+	struct ibv_flow_spec_counter_action counter = {
+		.type = IBV_FLOW_SPEC_ACTION_COUNT,
+		.size = size,
+	};
+#endif
+
+	if (!flow->counter) {
+		flow->counter = mlx5_flow_counter_new(dev, count->shared,
+						      count->id);
+		if (!flow->counter)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  action,
+						  "cannot get counter"
+						  " context.");
+	}
+	if (!((struct priv *)dev->data->dev_private)->config.flow_counter_en)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "flow counters are not supported.");
+	flow->modifier |= MLX5_FLOW_MOD_COUNT;
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	counter.counter_set_handle = flow->counter->cs->handle;
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &counter, size);
+	return size;
+#endif
+	return 0;
+}
+
 /**
  * Convert the @p action into @p flow after ensuring the NIC will understand
  * and process it correctly.
@@ -2180,6 +2332,10 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 		case RTE_FLOW_ACTION_TYPE_RSS:
 			ret = mlx5_flow_action_rss(dev, actions, flow, error);
 			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = mlx5_flow_action_count(dev, actions, flow, remain,
+						     error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -2557,6 +2713,10 @@ mlx5_flow_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
 			verbs->hrxq = NULL;
 		}
 	}
+	if (flow->counter) {
+		mlx5_flow_counter_release(flow->counter);
+		flow->counter = NULL;
+	}
 }
 
 /**
@@ -3006,6 +3166,88 @@ mlx5_flow_isolate(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/**
+ * Query flow counter.
+ *
+ * @param flow
+ *   Pointer to the flow.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_query_count(struct rte_flow *flow __rte_unused,
+		      void *data __rte_unused,
+		      struct rte_flow_error *error)
+{
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	struct rte_flow_query_count *qc = data;
+	uint64_t counters[2] = {0, 0};
+	struct ibv_query_counter_set_attr query_cs_attr = {
+		.cs = flow->counter->cs,
+		.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
+	};
+	struct ibv_counter_set_data query_out = {
+		.out = counters,
+		.outlen = 2 * sizeof(uint64_t),
+	};
+	int err = mlx5_glue->query_counter_set(&query_cs_attr, &query_out);
+
+	if (err)
+		return rte_flow_error_set(error, err,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "cannot read counter");
+	qc->hits_set = 1;
+	qc->bytes_set = 1;
+	qc->hits = counters[0] - flow->counter->hits;
+	qc->bytes = counters[1] - flow->counter->bytes;
+	if (qc->reset) {
+		flow->counter->hits = counters[0];
+		flow->counter->bytes = counters[1];
+	}
+	return 0;
+#endif
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "counters are not available");
+}
+
+/**
+ * Query a flows.
+ *
+ * @see rte_flow_query()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
+		struct rte_flow *flow,
+		const struct rte_flow_action *actions,
+		void *data,
+		struct rte_flow_error *error)
+{
+	int ret = 0;
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = mlx5_flow_query_count(flow, data, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+		if (ret < 0)
+			return ret;
+	}
+	return 0;
+}
+
 /**
  * Convert a flow director filter to a generic flow.
  *
-- 
2.18.0

Re: [dpdk-dev] [PATCH v3 00/21] net/mlx5: flow rework

[Yongseok Koh]

On Wed, Jul 11, 2018 at 09:22:33AM +0200, Nelio Laranjeiro wrote:
> Re-work flow engine to support port redirection actions through TC.
> 
> This first series depends on [1] which is implemented in commit 
> "net/mlx5: support inner RSS computation" and on [2].
> Next series will bring the port redirection as announced[3].
> 
> [1] https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fmails.dpdk.org%2Farchives%2Fdev%2F2018-July%2F107378.html&data=02%7C01%7Cyskoh%40mellanox.com%7C9bef604d078f40e6595908d5e6ff2bf0%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C0%7C636668905992302639&sdata=0qLUG%2BKS8RgVHtHxHusUc52VYm9dk00670It5xBBovk%3D&reserved=0
> [2] https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fmails.dpdk.org%2Farchives%2Fdev%2F2018-June%2F104192.html&data=02%7C01%7Cyskoh%40mellanox.com%7C9bef604d078f40e6595908d5e6ff2bf0%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C0%7C636668905992302639&sdata=WExSBvD31dcbmGg6sli7eNAIuUA4P15C0AeeLX3UgCg%3D&reserved=0
> [3] https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fmails.dpdk.org%2Farchives%2Fdev%2F2018-May%2F103043.html&data=02%7C01%7Cyskoh%40mellanox.com%7C9bef604d078f40e6595908d5e6ff2bf0%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C0%7C636668905992302639&sdata=NklC3wWN7wwWQTWeqxtiVq3uj4TXPcuwSH5z0a%2B4yOk%3D&reserved=0
> 
> Changes in v3:
> 
> - remove redundant parameters in drop queues internal API.
> - simplify the RSS expansion by only adding missing items in the pattern.
> - document all functions.
> 
> Nelio Laranjeiro (21):
>   net/mlx5: remove flow support
>   net/mlx5: handle drop queues as regular queues
>   net/mlx5: replace verbs priorities by flow
>   net/mlx5: support flow Ethernet item along with drop action
>   net/mlx5: add flow queue action
>   net/mlx5: add flow stop/start
>   net/mlx5: add flow VLAN item
>   net/mlx5: add flow IPv4 item
>   net/mlx5: add flow IPv6 item
>   net/mlx5: add flow UDP item
>   net/mlx5: add flow TCP item
>   net/mlx5: add mark/flag flow action
>   net/mlx5: use a macro for the RSS key size
>   net/mlx5: add RSS flow action
>   net/mlx5: remove useless arguments in hrxq API
>   net/mlx5: support inner RSS computation
>   net/mlx5: add flow VXLAN item
>   net/mlx5: add flow VXLAN-GPE item
>   net/mlx5: add flow GRE item
>   net/mlx5: add flow MPLS item
>   net/mlx5: add count flow action
> 
>  drivers/net/mlx5/mlx5.c        |   22 +-
>  drivers/net/mlx5/mlx5.h        |   18 +-
>  drivers/net/mlx5/mlx5_ethdev.c |   14 +-
>  drivers/net/mlx5/mlx5_flow.c   | 4815 ++++++++++++++++----------------
>  drivers/net/mlx5/mlx5_prm.h    |    3 +
>  drivers/net/mlx5/mlx5_rss.c    |    7 +-
>  drivers/net/mlx5/mlx5_rxq.c    |  281 +-
>  drivers/net/mlx5/mlx5_rxtx.h   |   21 +-
>  8 files changed, 2632 insertions(+), 2549 deletions(-)
> 
> -- 
For the series,

Acked-by: Yongseok Koh <yskoh@mellanox.com>

Thanks

[dpdk-dev] [PATCH v4 00/21] net/mlx5: flow rework

[Nelio Laranjeiro]

Re-work flow engine to support port redirection actions through TC.

This first series depends on [1] which is implemented in commit 
"net/mlx5: support inner RSS computation" and on [2].
Next series will bring the port redirection as announced[3].

[1] https://mails.dpdk.org/archives/dev/2018-July/107378.html
[2] https://mails.dpdk.org/archives/dev/2018-June/104192.html
[3] https://mails.dpdk.org/archives/dev/2018-May/103043.html

Changes in v4:

- fix compilation on redhat 7.5 without Mellanox OFED.
- avoid multiple pattern parsing for the expansion.

Changes in v3:

- remove redundant parameters in drop queues internal API.
- simplify the RSS expansion by only adding missing items in the pattern.
- document all functions.


Nelio Laranjeiro (21):
  net/mlx5: remove flow support
  net/mlx5: handle drop queues as regular queues
  net/mlx5: replace verbs priorities by flow
  net/mlx5: support flow Ethernet item along with drop action
  net/mlx5: add flow queue action
  net/mlx5: add flow stop/start
  net/mlx5: add flow VLAN item
  net/mlx5: add flow IPv4 item
  net/mlx5: add flow IPv6 item
  net/mlx5: add flow UDP item
  net/mlx5: add flow TCP item
  net/mlx5: add mark/flag flow action
  net/mlx5: use a macro for the RSS key size
  net/mlx5: add RSS flow action
  net/mlx5: remove useless arguments in hrxq API
  net/mlx5: support inner RSS computation
  net/mlx5: add flow VXLAN item
  net/mlx5: add flow VXLAN-GPE item
  net/mlx5: add flow GRE item
  net/mlx5: add flow MPLS item
  net/mlx5: add count flow action

 drivers/net/mlx5/mlx5.c        |   22 +-
 drivers/net/mlx5/mlx5.h        |   18 +-
 drivers/net/mlx5/mlx5_ethdev.c |   14 +-
 drivers/net/mlx5/mlx5_flow.c   | 4821 ++++++++++++++++----------------
 drivers/net/mlx5/mlx5_prm.h    |    3 +
 drivers/net/mlx5/mlx5_rss.c    |    7 +-
 drivers/net/mlx5/mlx5_rxq.c    |  281 +-
 drivers/net/mlx5/mlx5_rxtx.h   |   21 +-
 8 files changed, 2640 insertions(+), 2547 deletions(-)

-- 
2.18.0

[dpdk-dev] [PATCH v4 01/21] net/mlx5: remove flow support

[Nelio Laranjeiro]

This start a series to re-work the flow engine in mlx5 to easily support
flow conversion to Verbs or TC.  This is necessary to handle both regular
flows and representors flows.

As the full file needs to be clean-up to re-write all items/actions
processing, this patch starts to disable the regular code and only let the
PMD to start in isolated mode.

After this patch flow API will not be usable.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 3095 +---------------------------------
 drivers/net/mlx5/mlx5_rxtx.h |    1 -
 2 files changed, 80 insertions(+), 3016 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 45207d70e..a45cb06e1 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -31,2406 +31,49 @@
 #include "mlx5_prm.h"
 #include "mlx5_glue.h"
 
-/* Flow priority for control plane flows. */
-#define MLX5_CTRL_FLOW_PRIORITY 1
-
-/* Internet Protocol versions. */
-#define MLX5_IPV4 4
-#define MLX5_IPV6 6
-#define MLX5_GRE 47
-
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-struct ibv_flow_spec_counter_action {
-	int dummy;
-};
-#endif
-
-/* Dev ops structure defined in mlx5.c */
-extern const struct eth_dev_ops mlx5_dev_ops;
-extern const struct eth_dev_ops mlx5_dev_ops_isolate;
-
-/** Structure give to the conversion functions. */
-struct mlx5_flow_data {
-	struct rte_eth_dev *dev; /** Ethernet device. */
-	struct mlx5_flow_parse *parser; /** Parser context. */
-	struct rte_flow_error *error; /** Error context. */
-};
-
-static int
-mlx5_flow_create_eth(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vlan(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_ipv4(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_ipv6(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_udp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_tcp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vxlan(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_vxlan_gpe(const struct rte_flow_item *item,
-			   const void *default_mask,
-			   struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_gre(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data);
-
-static int
-mlx5_flow_create_mpls(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data);
-
-struct mlx5_flow_parse;
-
-static void
-mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
-		      unsigned int size);
-
-static int
-mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
-
-static int
-mlx5_flow_create_count(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser);
-
-/* Hash RX queue types. */
-enum hash_rxq_type {
-	HASH_RXQ_TCPV4,
-	HASH_RXQ_UDPV4,
-	HASH_RXQ_IPV4,
-	HASH_RXQ_TCPV6,
-	HASH_RXQ_UDPV6,
-	HASH_RXQ_IPV6,
-	HASH_RXQ_ETH,
-	HASH_RXQ_TUNNEL,
-};
-
-/* Initialization data for hash RX queue. */
-struct hash_rxq_init {
-	uint64_t hash_fields; /* Fields that participate in the hash. */
-	uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
-	unsigned int flow_priority; /* Flow priority to use. */
-	unsigned int ip_version; /* Internet protocol. */
-};
-
-/* Initialization data for hash RX queues. */
-const struct hash_rxq_init hash_rxq_init[] = {
-	[HASH_RXQ_TCPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4 |
-				IBV_RX_HASH_SRC_PORT_TCP |
-				IBV_RX_HASH_DST_PORT_TCP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV4,
-	},
-	[HASH_RXQ_UDPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4 |
-				IBV_RX_HASH_SRC_PORT_UDP |
-				IBV_RX_HASH_DST_PORT_UDP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV4,
-	},
-	[HASH_RXQ_IPV4] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV4 |
-				IBV_RX_HASH_DST_IPV4),
-		.dpdk_rss_hf = (ETH_RSS_IPV4 |
-				ETH_RSS_FRAG_IPV4),
-		.flow_priority = 1,
-		.ip_version = MLX5_IPV4,
-	},
-	[HASH_RXQ_TCPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6 |
-				IBV_RX_HASH_SRC_PORT_TCP |
-				IBV_RX_HASH_DST_PORT_TCP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV6,
-	},
-	[HASH_RXQ_UDPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6 |
-				IBV_RX_HASH_SRC_PORT_UDP |
-				IBV_RX_HASH_DST_PORT_UDP),
-		.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
-		.flow_priority = 0,
-		.ip_version = MLX5_IPV6,
-	},
-	[HASH_RXQ_IPV6] = {
-		.hash_fields = (IBV_RX_HASH_SRC_IPV6 |
-				IBV_RX_HASH_DST_IPV6),
-		.dpdk_rss_hf = (ETH_RSS_IPV6 |
-				ETH_RSS_FRAG_IPV6),
-		.flow_priority = 1,
-		.ip_version = MLX5_IPV6,
-	},
-	[HASH_RXQ_ETH] = {
-		.hash_fields = 0,
-		.dpdk_rss_hf = 0,
-		.flow_priority = 2,
-	},
-};
-
-/* Number of entries in hash_rxq_init[]. */
-const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
-
-/** Structure for holding counter stats. */
-struct mlx5_flow_counter_stats {
-	uint64_t hits; /**< Number of packets matched by the rule. */
-	uint64_t bytes; /**< Number of bytes matched by the rule. */
-};
-
-/** Structure for Drop queue. */
-struct mlx5_hrxq_drop {
-	struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
-	struct ibv_qp *qp; /**< Verbs queue pair. */
-	struct ibv_wq *wq; /**< Verbs work queue. */
-	struct ibv_cq *cq; /**< Verbs completion queue. */
-};
-
-/* Flows structures. */
-struct mlx5_flow {
-	uint64_t hash_fields; /**< Fields that participate in the hash. */
-	struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
-	struct ibv_flow *ibv_flow; /**< Verbs flow. */
-	struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
-};
-
-/* Drop flows structures. */
-struct mlx5_flow_drop {
-	struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
-	struct ibv_flow *ibv_flow; /**< Verbs flow. */
-};
-
-struct rte_flow {
-	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
-	uint32_t mark:1; /**< Set if the flow is marked. */
-	uint32_t drop:1; /**< Drop queue. */
-	struct rte_flow_action_rss rss_conf; /**< RSS configuration */
-	uint16_t (*queues)[]; /**< Queues indexes to use. */
-	uint8_t rss_key[40]; /**< copy of the RSS key. */
-	uint32_t tunnel; /**< Tunnel type of RTE_PTYPE_TUNNEL_XXX. */
-	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
-	struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
-	struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
-	/**< Flow with Rx queue. */
-};
-
-/** Static initializer for items. */
-#define ITEMS(...) \
-	(const enum rte_flow_item_type []){ \
-		__VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
-	}
-
-#define IS_TUNNEL(type) ( \
-	(type) == RTE_FLOW_ITEM_TYPE_VXLAN || \
-	(type) == RTE_FLOW_ITEM_TYPE_VXLAN_GPE || \
-	(type) == RTE_FLOW_ITEM_TYPE_GRE || \
-	(type) == RTE_FLOW_ITEM_TYPE_MPLS)
-
-const uint32_t flow_ptype[] = {
-	[RTE_FLOW_ITEM_TYPE_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN,
-	[RTE_FLOW_ITEM_TYPE_VXLAN_GPE] = RTE_PTYPE_TUNNEL_VXLAN_GPE,
-	[RTE_FLOW_ITEM_TYPE_GRE] = RTE_PTYPE_TUNNEL_GRE,
-	[RTE_FLOW_ITEM_TYPE_MPLS] = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
-};
-
-#define PTYPE_IDX(t) ((RTE_PTYPE_TUNNEL_MASK & (t)) >> 12)
-
-const uint32_t ptype_ext[] = {
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] = RTE_PTYPE_TUNNEL_VXLAN |
-					      RTE_PTYPE_L4_UDP,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN_GPE)]	= RTE_PTYPE_TUNNEL_VXLAN_GPE |
-						  RTE_PTYPE_L4_UDP,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_GRE)] = RTE_PTYPE_TUNNEL_GRE,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_GRE)] =
-		RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
-	[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_UDP)] =
-		RTE_PTYPE_TUNNEL_MPLS_IN_GRE | RTE_PTYPE_L4_UDP,
-};
-
-/** Structure to generate a simple graph of layers supported by the NIC. */
-struct mlx5_flow_items {
-	/** List of possible actions for these items. */
-	const enum rte_flow_action_type *const actions;
-	/** Bit-masks corresponding to the possibilities for the item. */
-	const void *mask;
-	/**
-	 * Default bit-masks to use when item->mask is not provided. When
-	 * \default_mask is also NULL, the full supported bit-mask (\mask) is
-	 * used instead.
-	 */
-	const void *default_mask;
-	/** Bit-masks size in bytes. */
-	const unsigned int mask_sz;
-	/**
-	 * Conversion function from rte_flow to NIC specific flow.
-	 *
-	 * @param item
-	 *   rte_flow item to convert.
-	 * @param default_mask
-	 *   Default bit-masks to use when item->mask is not provided.
-	 * @param data
-	 *   Internal structure to store the conversion.
-	 *
-	 * @return
-	 *   0 on success, a negative errno value otherwise and rte_errno is
-	 *   set.
-	 */
-	int (*convert)(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data);
-	/** Size in bytes of the destination structure. */
-	const unsigned int dst_sz;
-	/** List of possible following items.  */
-	const enum rte_flow_item_type *const items;
-};
-
-/** Valid action for this PMD. */
-static const enum rte_flow_action_type valid_actions[] = {
-	RTE_FLOW_ACTION_TYPE_DROP,
-	RTE_FLOW_ACTION_TYPE_QUEUE,
-	RTE_FLOW_ACTION_TYPE_MARK,
-	RTE_FLOW_ACTION_TYPE_FLAG,
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	RTE_FLOW_ACTION_TYPE_COUNT,
-#endif
-	RTE_FLOW_ACTION_TYPE_END,
-};
-
-/** Graph of supported items and associated actions. */
-static const struct mlx5_flow_items mlx5_flow_items[] = {
-	[RTE_FLOW_ITEM_TYPE_END] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_VXLAN,
-			       RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
-			       RTE_FLOW_ITEM_TYPE_GRE),
-	},
-	[RTE_FLOW_ITEM_TYPE_ETH] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_eth){
-			.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
-			.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
-			.type = -1,
-		},
-		.default_mask = &rte_flow_item_eth_mask,
-		.mask_sz = sizeof(struct rte_flow_item_eth),
-		.convert = mlx5_flow_create_eth,
-		.dst_sz = sizeof(struct ibv_flow_spec_eth),
-	},
-	[RTE_FLOW_ITEM_TYPE_VLAN] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vlan){
-			.tci = -1,
-			.inner_type = -1,
-		},
-		.default_mask = &rte_flow_item_vlan_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vlan),
-		.convert = mlx5_flow_create_vlan,
-		.dst_sz = 0,
-	},
-	[RTE_FLOW_ITEM_TYPE_IPV4] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
-			       RTE_FLOW_ITEM_TYPE_TCP,
-			       RTE_FLOW_ITEM_TYPE_GRE),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_ipv4){
-			.hdr = {
-				.src_addr = -1,
-				.dst_addr = -1,
-				.type_of_service = -1,
-				.next_proto_id = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_ipv4_mask,
-		.mask_sz = sizeof(struct rte_flow_item_ipv4),
-		.convert = mlx5_flow_create_ipv4,
-		.dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
-	},
-	[RTE_FLOW_ITEM_TYPE_IPV6] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
-			       RTE_FLOW_ITEM_TYPE_TCP,
-			       RTE_FLOW_ITEM_TYPE_GRE),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_ipv6){
-			.hdr = {
-				.src_addr = {
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-				},
-				.dst_addr = {
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-					0xff, 0xff, 0xff, 0xff,
-				},
-				.vtc_flow = -1,
-				.proto = -1,
-				.hop_limits = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_ipv6_mask,
-		.mask_sz = sizeof(struct rte_flow_item_ipv6),
-		.convert = mlx5_flow_create_ipv6,
-		.dst_sz = sizeof(struct ibv_flow_spec_ipv6),
-	},
-	[RTE_FLOW_ITEM_TYPE_UDP] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN,
-			       RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
-			       RTE_FLOW_ITEM_TYPE_MPLS),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_udp){
-			.hdr = {
-				.src_port = -1,
-				.dst_port = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_udp_mask,
-		.mask_sz = sizeof(struct rte_flow_item_udp),
-		.convert = mlx5_flow_create_udp,
-		.dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
-	},
-	[RTE_FLOW_ITEM_TYPE_TCP] = {
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_tcp){
-			.hdr = {
-				.src_port = -1,
-				.dst_port = -1,
-			},
-		},
-		.default_mask = &rte_flow_item_tcp_mask,
-		.mask_sz = sizeof(struct rte_flow_item_tcp),
-		.convert = mlx5_flow_create_tcp,
-		.dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
-	},
-	[RTE_FLOW_ITEM_TYPE_GRE] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6,
-			       RTE_FLOW_ITEM_TYPE_MPLS),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_gre){
-			.protocol = -1,
-		},
-		.default_mask = &rte_flow_item_gre_mask,
-		.mask_sz = sizeof(struct rte_flow_item_gre),
-		.convert = mlx5_flow_create_gre,
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-		.dst_sz = sizeof(struct ibv_flow_spec_gre),
-#else
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
-#endif
-	},
-	[RTE_FLOW_ITEM_TYPE_MPLS] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_mpls){
-			.label_tc_s = "\xff\xff\xf0",
-		},
-		.default_mask = &rte_flow_item_mpls_mask,
-		.mask_sz = sizeof(struct rte_flow_item_mpls),
-		.convert = mlx5_flow_create_mpls,
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-		.dst_sz = sizeof(struct ibv_flow_spec_mpls),
-#endif
-	},
-	[RTE_FLOW_ITEM_TYPE_VXLAN] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4, /* For L3 VXLAN. */
-			       RTE_FLOW_ITEM_TYPE_IPV6), /* For L3 VXLAN. */
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vxlan){
-			.vni = "\xff\xff\xff",
-		},
-		.default_mask = &rte_flow_item_vxlan_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vxlan),
-		.convert = mlx5_flow_create_vxlan,
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
-	},
-	[RTE_FLOW_ITEM_TYPE_VXLAN_GPE] = {
-		.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
-			       RTE_FLOW_ITEM_TYPE_IPV4,
-			       RTE_FLOW_ITEM_TYPE_IPV6),
-		.actions = valid_actions,
-		.mask = &(const struct rte_flow_item_vxlan_gpe){
-			.vni = "\xff\xff\xff",
-		},
-		.default_mask = &rte_flow_item_vxlan_gpe_mask,
-		.mask_sz = sizeof(struct rte_flow_item_vxlan_gpe),
-		.convert = mlx5_flow_create_vxlan_gpe,
-		.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
-	},
-};
-
-/** Structure to pass to the conversion function. */
-struct mlx5_flow_parse {
-	uint32_t inner; /**< Verbs value, set once tunnel is encountered. */
-	uint32_t create:1;
-	/**< Whether resources should remain after a validate. */
-	uint32_t drop:1; /**< Target is a drop queue. */
-	uint32_t mark:1; /**< Mark is present in the flow. */
-	uint32_t count:1; /**< Count is present in the flow. */
-	uint32_t mark_id; /**< Mark identifier. */
-	struct rte_flow_action_rss rss_conf; /**< RSS configuration */
-	uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
-	uint8_t rss_key[40]; /**< copy of the RSS key. */
-	enum hash_rxq_type layer; /**< Last pattern layer detected. */
-	enum hash_rxq_type out_layer; /**< Last outer pattern layer detected. */
-	uint32_t tunnel; /**< Tunnel type of RTE_PTYPE_TUNNEL_XXX. */
-	struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
-	struct {
-		struct ibv_flow_attr *ibv_attr;
-		/**< Pointer to Verbs attributes. */
-		unsigned int offset;
-		/**< Current position or total size of the attribute. */
-		uint64_t hash_fields; /**< Verbs hash fields. */
-	} queue[RTE_DIM(hash_rxq_init)];
-};
-
-static const struct rte_flow_ops mlx5_flow_ops = {
-	.validate = mlx5_flow_validate,
-	.create = mlx5_flow_create,
-	.destroy = mlx5_flow_destroy,
-	.flush = mlx5_flow_flush,
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	.query = mlx5_flow_query,
-#else
-	.query = NULL,
-#endif
-	.isolate = mlx5_flow_isolate,
-};
-
-/* Convert FDIR request to Generic flow. */
-struct mlx5_fdir {
-	struct rte_flow_attr attr;
-	struct rte_flow_action actions[2];
-	struct rte_flow_item items[4];
-	struct rte_flow_item_eth l2;
-	struct rte_flow_item_eth l2_mask;
-	union {
-		struct rte_flow_item_ipv4 ipv4;
-		struct rte_flow_item_ipv6 ipv6;
-	} l3;
-	union {
-		struct rte_flow_item_ipv4 ipv4;
-		struct rte_flow_item_ipv6 ipv6;
-	} l3_mask;
-	union {
-		struct rte_flow_item_udp udp;
-		struct rte_flow_item_tcp tcp;
-	} l4;
-	union {
-		struct rte_flow_item_udp udp;
-		struct rte_flow_item_tcp tcp;
-	} l4_mask;
-	struct rte_flow_action_queue queue;
-};
-
-/* Verbs specification header. */
-struct ibv_spec_header {
-	enum ibv_flow_spec_type type;
-	uint16_t size;
-};
-
-/**
- * Check item is fully supported by the NIC matching capability.
- *
- * @param item[in]
- *   Item specification.
- * @param mask[in]
- *   Bit-masks covering supported fields to compare with spec, last and mask in
- *   \item.
- * @param size
- *   Bit-Mask size in bytes.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_item_validate(const struct rte_flow_item *item,
-			const uint8_t *mask, unsigned int size)
-{
-	unsigned int i;
-	const uint8_t *spec = item->spec;
-	const uint8_t *last = item->last;
-	const uint8_t *m = item->mask ? item->mask : mask;
-
-	if (!spec && (item->mask || last))
-		goto error;
-	if (!spec)
-		return 0;
-	/*
-	 * Single-pass check to make sure that:
-	 * - item->mask is supported, no bits are set outside mask.
-	 * - Both masked item->spec and item->last are equal (no range
-	 *   supported).
-	 */
-	for (i = 0; i < size; i++) {
-		if (!m[i])
-			continue;
-		if ((m[i] | mask[i]) != mask[i])
-			goto error;
-		if (last && ((spec[i] & m[i]) != (last[i] & m[i])))
-			goto error;
-	}
-	return 0;
-error:
-	rte_errno = ENOTSUP;
-	return -rte_errno;
-}
-
-/**
- * Extract attribute to the parser.
- *
- * @param[in] attr
- *   Flow rule attributes.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_attributes(const struct rte_flow_attr *attr,
-			     struct rte_flow_error *error)
-{
-	if (attr->group) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
-				   NULL,
-				   "groups are not supported");
-		return -rte_errno;
-	}
-	if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
-				   NULL,
-				   "priorities are not supported");
-		return -rte_errno;
-	}
-	if (attr->egress) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
-				   NULL,
-				   "egress is not supported");
-		return -rte_errno;
-	}
-	if (attr->transfer) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
-				   NULL,
-				   "transfer is not supported");
-		return -rte_errno;
-	}
-	if (!attr->ingress) {
-		rte_flow_error_set(error, ENOTSUP,
-				   RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
-				   NULL,
-				   "only ingress is supported");
-		return -rte_errno;
-	}
-	return 0;
-}
-
-/**
- * Extract actions request to the parser.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[in] actions
- *   Associated actions (list terminated by the END action).
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_actions(struct rte_eth_dev *dev,
-			  const struct rte_flow_action actions[],
-			  struct rte_flow_error *error,
-			  struct mlx5_flow_parse *parser)
-{
-	enum { FATE = 1, MARK = 2, COUNT = 4, };
-	uint32_t overlap = 0;
-	struct priv *priv = dev->data->dev_private;
-
-	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
-		if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
-			continue;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			parser->drop = 1;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
-			const struct rte_flow_action_queue *queue =
-				(const struct rte_flow_action_queue *)
-				actions->conf;
-
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			if (!queue || (queue->index > (priv->rxqs_n - 1)))
-				goto exit_action_not_supported;
-			parser->queues[0] = queue->index;
-			parser->rss_conf = (struct rte_flow_action_rss){
-				.queue_num = 1,
-				.queue = parser->queues,
-			};
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
-			const struct rte_flow_action_rss *rss =
-				(const struct rte_flow_action_rss *)
-				actions->conf;
-			const uint8_t *rss_key;
-			uint32_t rss_key_len;
-			uint16_t n;
-
-			if (overlap & FATE)
-				goto exit_action_overlap;
-			overlap |= FATE;
-			if (rss->func &&
-			    rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "the only supported RSS hash"
-						   " function is Toeplitz");
-				return -rte_errno;
-			}
-#ifndef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-			if (parser->rss_conf.level > 1) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "a nonzero RSS encapsulation"
-						   " level is not supported");
-				return -rte_errno;
-			}
-#endif
-			if (parser->rss_conf.level > 2) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "RSS encapsulation level"
-						   " > 1 is not supported");
-				return -rte_errno;
-			}
-			if (rss->types & MLX5_RSS_HF_MASK) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "unsupported RSS type"
-						   " requested");
-				return -rte_errno;
-			}
-			if (rss->key_len) {
-				rss_key_len = rss->key_len;
-				rss_key = rss->key;
-			} else {
-				rss_key_len = rss_hash_default_key_len;
-				rss_key = rss_hash_default_key;
-			}
-			if (rss_key_len != RTE_DIM(parser->rss_key)) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "RSS hash key must be"
-						   " exactly 40 bytes long");
-				return -rte_errno;
-			}
-			if (!rss->queue_num) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "no valid queues");
-				return -rte_errno;
-			}
-			if (rss->queue_num > RTE_DIM(parser->queues)) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "too many queues for RSS"
-						   " context");
-				return -rte_errno;
-			}
-			for (n = 0; n < rss->queue_num; ++n) {
-				if (rss->queue[n] >= priv->rxqs_n) {
-					rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "queue id > number of"
-						   " queues");
-					return -rte_errno;
-				}
-			}
-			parser->rss_conf = (struct rte_flow_action_rss){
-				.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
-				.level = rss->level ? rss->level : 1,
-				.types = rss->types,
-				.key_len = rss_key_len,
-				.queue_num = rss->queue_num,
-				.key = memcpy(parser->rss_key, rss_key,
-					      sizeof(*rss_key) * rss_key_len),
-				.queue = memcpy(parser->queues, rss->queue,
-						sizeof(*rss->queue) *
-						rss->queue_num),
-			};
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
-			const struct rte_flow_action_mark *mark =
-				(const struct rte_flow_action_mark *)
-				actions->conf;
-
-			if (overlap & MARK)
-				goto exit_action_overlap;
-			overlap |= MARK;
-			if (!mark) {
-				rte_flow_error_set(error, EINVAL,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "mark must be defined");
-				return -rte_errno;
-			} else if (mark->id >= MLX5_FLOW_MARK_MAX) {
-				rte_flow_error_set(error, ENOTSUP,
-						   RTE_FLOW_ERROR_TYPE_ACTION,
-						   actions,
-						   "mark must be between 0"
-						   " and 16777199");
-				return -rte_errno;
-			}
-			parser->mark = 1;
-			parser->mark_id = mark->id;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
-			if (overlap & MARK)
-				goto exit_action_overlap;
-			overlap |= MARK;
-			parser->mark = 1;
-		} else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
-			   priv->config.flow_counter_en) {
-			if (overlap & COUNT)
-				goto exit_action_overlap;
-			overlap |= COUNT;
-			parser->count = 1;
-		} else {
-			goto exit_action_not_supported;
-		}
-	}
-	/* When fate is unknown, drop traffic. */
-	if (!(overlap & FATE))
-		parser->drop = 1;
-	if (parser->drop && parser->mark)
-		parser->mark = 0;
-	if (!parser->rss_conf.queue_num && !parser->drop) {
-		rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "no valid action");
-		return -rte_errno;
-	}
-	return 0;
-exit_action_not_supported:
-	rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
-			   actions, "action not supported");
-	return -rte_errno;
-exit_action_overlap:
-	rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
-			   actions, "overlapping actions are not supported");
-	return -rte_errno;
-}
-
-/**
- * Validate items.
- *
- * @param[in] items
- *   Pattern specification (list terminated by the END pattern item).
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_items_validate(struct rte_eth_dev *dev,
-				 const struct rte_flow_item items[],
-				 struct rte_flow_error *error,
-				 struct mlx5_flow_parse *parser)
-{
-	struct priv *priv = dev->data->dev_private;
-	const struct mlx5_flow_items *cur_item = mlx5_flow_items;
-	unsigned int i;
-	unsigned int last_voids = 0;
-	int ret = 0;
-
-	/* Initialise the offsets to start after verbs attribute. */
-	for (i = 0; i != hash_rxq_init_n; ++i)
-		parser->queue[i].offset = sizeof(struct ibv_flow_attr);
-	for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
-		const struct mlx5_flow_items *token = NULL;
-		unsigned int n;
-
-		if (items->type == RTE_FLOW_ITEM_TYPE_VOID) {
-			last_voids++;
-			continue;
-		}
-		for (i = 0;
-		     cur_item->items &&
-		     cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
-		     ++i) {
-			if (cur_item->items[i] == items->type) {
-				token = &mlx5_flow_items[items->type];
-				break;
-			}
-		}
-		if (!token) {
-			ret = -ENOTSUP;
-			goto exit_item_not_supported;
-		}
-		cur_item = token;
-		ret = mlx5_flow_item_validate(items,
-					      (const uint8_t *)cur_item->mask,
-					      cur_item->mask_sz);
-		if (ret)
-			goto exit_item_not_supported;
-		if (IS_TUNNEL(items->type)) {
-			if (parser->tunnel &&
-			    !((items - last_voids - 1)->type ==
-			      RTE_FLOW_ITEM_TYPE_GRE && items->type ==
-			      RTE_FLOW_ITEM_TYPE_MPLS)) {
-				rte_flow_error_set(error, ENOTSUP,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   items,
-						   "Cannot recognize multiple"
-						   " tunnel encapsulations.");
-				return -rte_errno;
-			}
-			if (items->type == RTE_FLOW_ITEM_TYPE_MPLS &&
-			    !priv->config.mpls_en) {
-				rte_flow_error_set(error, ENOTSUP,
-						   RTE_FLOW_ERROR_TYPE_ITEM,
-						   items,
-						   "MPLS not supported or"
-						   " disabled in firmware"
-						   " configuration.");
-				return -rte_errno;
-			}
-			if (!priv->config.tunnel_en &&
-			    parser->rss_conf.level > 1) {
-				rte_flow_error_set(error, ENOTSUP,
-					RTE_FLOW_ERROR_TYPE_ITEM,
-					items,
-					"RSS on tunnel is not supported");
-				return -rte_errno;
-			}
-			parser->inner = IBV_FLOW_SPEC_INNER;
-			parser->tunnel = flow_ptype[items->type];
-		}
-		if (parser->drop) {
-			parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
-		} else {
-			for (n = 0; n != hash_rxq_init_n; ++n)
-				parser->queue[n].offset += cur_item->dst_sz;
-		}
-		last_voids = 0;
-	}
-	if (parser->drop) {
-		parser->queue[HASH_RXQ_ETH].offset +=
-			sizeof(struct ibv_flow_spec_action_drop);
-	}
-	if (parser->mark) {
-		for (i = 0; i != hash_rxq_init_n; ++i)
-			parser->queue[i].offset +=
-				sizeof(struct ibv_flow_spec_action_tag);
-	}
-	if (parser->count) {
-		unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
-
-		for (i = 0; i != hash_rxq_init_n; ++i)
-			parser->queue[i].offset += size;
-	}
-	return 0;
-exit_item_not_supported:
-	return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
-				  items, "item not supported");
-}
-
-/**
- * Allocate memory space to store verbs flow attributes.
- *
- * @param[in] size
- *   Amount of byte to allocate.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   A verbs flow attribute on success, NULL otherwise and rte_errno is set.
- */
-static struct ibv_flow_attr *
-mlx5_flow_convert_allocate(unsigned int size, struct rte_flow_error *error)
-{
-	struct ibv_flow_attr *ibv_attr;
-
-	ibv_attr = rte_calloc(__func__, 1, size, 0);
-	if (!ibv_attr) {
-		rte_flow_error_set(error, ENOMEM,
-				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				   NULL,
-				   "cannot allocate verbs spec attributes");
-		return NULL;
-	}
-	return ibv_attr;
-}
-
-/**
- * Make inner packet matching with an higher priority from the non Inner
- * matching.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[in, out] parser
- *   Internal parser structure.
- * @param attr
- *   User flow attribute.
- */
-static void
-mlx5_flow_update_priority(struct rte_eth_dev *dev,
-			  struct mlx5_flow_parse *parser,
-			  const struct rte_flow_attr *attr)
-{
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-	uint16_t priority;
-
-	/*			8 priorities	>= 16 priorities
-	 * Control flow:	4-7		8-15
-	 * User normal flow:	1-3		4-7
-	 * User tunnel flow:	0-2		0-3
-	 */
-	priority = attr->priority * MLX5_VERBS_FLOW_PRIO_8;
-	if (priv->config.max_verbs_prio == MLX5_VERBS_FLOW_PRIO_8)
-		priority /= 2;
-	/*
-	 * Lower non-tunnel flow Verbs priority 1 if only support 8 Verbs
-	 * priorities, lower 4 otherwise.
-	 */
-	if (!parser->inner) {
-		if (priv->config.max_verbs_prio == MLX5_VERBS_FLOW_PRIO_8)
-			priority += 1;
-		else
-			priority += MLX5_VERBS_FLOW_PRIO_8 / 2;
-	}
-	if (parser->drop) {
-		parser->queue[HASH_RXQ_ETH].ibv_attr->priority = priority +
-				hash_rxq_init[HASH_RXQ_ETH].flow_priority;
-		return;
-	}
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		parser->queue[i].ibv_attr->priority = priority +
-				hash_rxq_init[i].flow_priority;
-	}
-}
-
-/**
- * Finalise verbs flow attributes.
- *
- * @param[in, out] parser
- *   Internal parser structure.
- */
-static void
-mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser)
-{
-	unsigned int i;
-	uint32_t inner = parser->inner;
-
-	/* Don't create extra flows for outer RSS. */
-	if (parser->tunnel && parser->rss_conf.level < 2)
-		return;
-	/*
-	 * Fill missing layers in verbs specifications, or compute the correct
-	 * offset to allocate the memory space for the attributes and
-	 * specifications.
-	 */
-	for (i = 0; i != hash_rxq_init_n - 1; ++i) {
-		union {
-			struct ibv_flow_spec_ipv4_ext ipv4;
-			struct ibv_flow_spec_ipv6 ipv6;
-			struct ibv_flow_spec_tcp_udp udp_tcp;
-			struct ibv_flow_spec_eth eth;
-		} specs;
-		void *dst;
-		uint16_t size;
-
-		if (i == parser->layer)
-			continue;
-		if (parser->layer == HASH_RXQ_ETH ||
-		    parser->layer == HASH_RXQ_TUNNEL) {
-			if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
-				size = sizeof(struct ibv_flow_spec_ipv4_ext);
-				specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
-					.type = inner | IBV_FLOW_SPEC_IPV4_EXT,
-					.size = size,
-				};
-			} else {
-				size = sizeof(struct ibv_flow_spec_ipv6);
-				specs.ipv6 = (struct ibv_flow_spec_ipv6){
-					.type = inner | IBV_FLOW_SPEC_IPV6,
-					.size = size,
-				};
-			}
-			if (parser->queue[i].ibv_attr) {
-				dst = (void *)((uintptr_t)
-					       parser->queue[i].ibv_attr +
-					       parser->queue[i].offset);
-				memcpy(dst, &specs, size);
-				++parser->queue[i].ibv_attr->num_of_specs;
-			}
-			parser->queue[i].offset += size;
-		}
-		if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
-		    (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
-			size = sizeof(struct ibv_flow_spec_tcp_udp);
-			specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
-				.type = inner | ((i == HASH_RXQ_UDPV4 ||
-					  i == HASH_RXQ_UDPV6) ?
-					 IBV_FLOW_SPEC_UDP :
-					 IBV_FLOW_SPEC_TCP),
-				.size = size,
-			};
-			if (parser->queue[i].ibv_attr) {
-				dst = (void *)((uintptr_t)
-					       parser->queue[i].ibv_attr +
-					       parser->queue[i].offset);
-				memcpy(dst, &specs, size);
-				++parser->queue[i].ibv_attr->num_of_specs;
-			}
-			parser->queue[i].offset += size;
-		}
-	}
-}
-
-/**
- * Update flows according to pattern and RSS hash fields.
- *
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_rss(struct mlx5_flow_parse *parser)
-{
-	unsigned int i;
-	enum hash_rxq_type start;
-	enum hash_rxq_type layer;
-	int outer = parser->tunnel && parser->rss_conf.level < 2;
-	uint64_t rss = parser->rss_conf.types;
-
-	layer = outer ? parser->out_layer : parser->layer;
-	if (layer == HASH_RXQ_TUNNEL)
-		layer = HASH_RXQ_ETH;
-	if (outer) {
-		/* Only one hash type for outer RSS. */
-		if (rss && layer == HASH_RXQ_ETH) {
-			start = HASH_RXQ_TCPV4;
-		} else if (rss && layer != HASH_RXQ_ETH &&
-			   !(rss & hash_rxq_init[layer].dpdk_rss_hf)) {
-			/* If RSS not match L4 pattern, try L3 RSS. */
-			if (layer < HASH_RXQ_IPV4)
-				layer = HASH_RXQ_IPV4;
-			else if (layer > HASH_RXQ_IPV4 && layer < HASH_RXQ_IPV6)
-				layer = HASH_RXQ_IPV6;
-			start = layer;
-		} else {
-			start = layer;
-		}
-		/* Scan first valid hash type. */
-		for (i = start; rss && i <= layer; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-			if (hash_rxq_init[i].dpdk_rss_hf & rss)
-				break;
-		}
-		if (rss && i <= layer)
-			parser->queue[layer].hash_fields =
-					hash_rxq_init[i].hash_fields;
-		/* Trim unused hash types. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (parser->queue[i].ibv_attr && i != layer) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			}
-		}
-	} else {
-		/* Expand for inner or normal RSS. */
-		if (rss && (layer == HASH_RXQ_ETH || layer == HASH_RXQ_IPV4))
-			start = HASH_RXQ_TCPV4;
-		else if (rss && layer == HASH_RXQ_IPV6)
-			start = HASH_RXQ_TCPV6;
-		else
-			start = layer;
-		/* For L4 pattern, try L3 RSS if no L4 RSS. */
-		/* Trim unused hash types. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-			if (i < start || i > layer) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-				continue;
-			}
-			if (!rss)
-				continue;
-			if (hash_rxq_init[i].dpdk_rss_hf & rss) {
-				parser->queue[i].hash_fields =
-						hash_rxq_init[i].hash_fields;
-			} else if (i != layer) {
-				/* Remove unused RSS expansion. */
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			} else if (layer < HASH_RXQ_IPV4 &&
-				   (hash_rxq_init[HASH_RXQ_IPV4].dpdk_rss_hf &
-				    rss)) {
-				/* Allow IPv4 RSS on L4 pattern. */
-				parser->queue[i].hash_fields =
-					hash_rxq_init[HASH_RXQ_IPV4]
-						.hash_fields;
-			} else if (i > HASH_RXQ_IPV4 && i < HASH_RXQ_IPV6 &&
-				   (hash_rxq_init[HASH_RXQ_IPV6].dpdk_rss_hf &
-				    rss)) {
-				/* Allow IPv4 RSS on L4 pattern. */
-				parser->queue[i].hash_fields =
-					hash_rxq_init[HASH_RXQ_IPV6]
-						.hash_fields;
-			}
-		}
-	}
-	return 0;
-}
-
-/**
- * Validate and convert a flow supported by the NIC.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param[in] attr
- *   Flow rule attributes.
- * @param[in] pattern
- *   Pattern specification (list terminated by the END pattern item).
- * @param[in] actions
- *   Associated actions (list terminated by the END action).
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- * @param[in, out] parser
- *   Internal parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert(struct rte_eth_dev *dev,
-		  const struct rte_flow_attr *attr,
-		  const struct rte_flow_item items[],
-		  const struct rte_flow_action actions[],
-		  struct rte_flow_error *error,
-		  struct mlx5_flow_parse *parser)
-{
-	const struct mlx5_flow_items *cur_item = mlx5_flow_items;
-	unsigned int i;
-	int ret;
-
-	/* First step. Validate the attributes, items and actions. */
-	*parser = (struct mlx5_flow_parse){
-		.create = parser->create,
-		.layer = HASH_RXQ_ETH,
-		.mark_id = MLX5_FLOW_MARK_DEFAULT,
-	};
-	ret = mlx5_flow_convert_attributes(attr, error);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert_actions(dev, actions, error, parser);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert_items_validate(dev, items, error, parser);
-	if (ret)
-		return ret;
-	mlx5_flow_convert_finalise(parser);
-	/*
-	 * Second step.
-	 * Allocate the memory space to store verbs specifications.
-	 */
-	if (parser->drop) {
-		unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
-
-		parser->queue[HASH_RXQ_ETH].ibv_attr =
-			mlx5_flow_convert_allocate(offset, error);
-		if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
-			goto exit_enomem;
-		parser->queue[HASH_RXQ_ETH].offset =
-			sizeof(struct ibv_flow_attr);
-	} else {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			unsigned int offset;
-
-			offset = parser->queue[i].offset;
-			parser->queue[i].ibv_attr =
-				mlx5_flow_convert_allocate(offset, error);
-			if (!parser->queue[i].ibv_attr)
-				goto exit_enomem;
-			parser->queue[i].offset = sizeof(struct ibv_flow_attr);
-		}
-	}
-	/* Third step. Conversion parse, fill the specifications. */
-	parser->inner = 0;
-	parser->tunnel = 0;
-	parser->layer = HASH_RXQ_ETH;
-	for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
-		struct mlx5_flow_data data = {
-			.dev = dev,
-			.parser = parser,
-			.error = error,
-		};
-
-		if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
-			continue;
-		cur_item = &mlx5_flow_items[items->type];
-		ret = cur_item->convert(items,
-					(cur_item->default_mask ?
-					 cur_item->default_mask :
-					 cur_item->mask),
-					 &data);
-		if (ret)
-			goto exit_free;
-	}
-	if (!parser->drop) {
-		/* RSS check, remove unused hash types. */
-		ret = mlx5_flow_convert_rss(parser);
-		if (ret)
-			goto exit_free;
-		/* Complete missing specification. */
-		mlx5_flow_convert_finalise(parser);
-	}
-	mlx5_flow_update_priority(dev, parser, attr);
-	if (parser->mark)
-		mlx5_flow_create_flag_mark(parser, parser->mark_id);
-	if (parser->count && parser->create) {
-		mlx5_flow_create_count(dev, parser);
-		if (!parser->cs)
-			goto exit_count_error;
-	}
-exit_free:
-	/* Only verification is expected, all resources should be released. */
-	if (!parser->create) {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (parser->queue[i].ibv_attr) {
-				rte_free(parser->queue[i].ibv_attr);
-				parser->queue[i].ibv_attr = NULL;
-			}
-		}
-	}
-	return ret;
-exit_enomem:
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser->queue[i].ibv_attr) {
-			rte_free(parser->queue[i].ibv_attr);
-			parser->queue[i].ibv_attr = NULL;
-		}
-	}
-	rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL, "cannot allocate verbs spec attributes");
-	return -rte_errno;
-exit_count_error:
-	rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL, "cannot create counter");
-	return -rte_errno;
-}
-
-/**
- * Copy the specification created into the flow.
- *
- * @param parser
- *   Internal parser structure.
- * @param src
- *   Create specification.
- * @param size
- *   Size in bytes of the specification to copy.
- */
-static void
-mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
-		      unsigned int size)
-{
-	unsigned int i;
-	void *dst;
-
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset);
-		memcpy(dst, src, size);
-		++parser->queue[i].ibv_attr->num_of_specs;
-		parser->queue[i].offset += size;
-	}
-}
-
-/**
- * Convert Ethernet item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_eth(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_eth *spec = item->spec;
-	const struct rte_flow_item_eth *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
-	struct ibv_flow_spec_eth eth = {
-		.type = parser->inner | IBV_FLOW_SPEC_ETH,
-		.size = eth_size,
-	};
-
-	parser->layer = HASH_RXQ_ETH;
-	if (spec) {
-		unsigned int i;
-
-		if (!mask)
-			mask = default_mask;
-		memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
-		memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
-		eth.val.ether_type = spec->type;
-		memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
-		memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
-		eth.mask.ether_type = mask->type;
-		/* Remove unwanted bits from values. */
-		for (i = 0; i < ETHER_ADDR_LEN; ++i) {
-			eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
-			eth.val.src_mac[i] &= eth.mask.src_mac[i];
-		}
-		eth.val.ether_type &= eth.mask.ether_type;
-	}
-	mlx5_flow_create_copy(parser, &eth, eth_size);
-	return 0;
-}
-
-/**
- * Convert VLAN item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_vlan(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_vlan *spec = item->spec;
-	const struct rte_flow_item_vlan *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	struct ibv_flow_spec_eth *eth;
-	const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
-	const char *msg = "VLAN cannot be empty";
-
-	if (spec) {
-		unsigned int i;
-		if (!mask)
-			mask = default_mask;
-
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!parser->queue[i].ibv_attr)
-				continue;
-
-			eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				       parser->queue[i].offset - eth_size);
-			eth->val.vlan_tag = spec->tci;
-			eth->mask.vlan_tag = mask->tci;
-			eth->val.vlan_tag &= eth->mask.vlan_tag;
-			/*
-			 * From verbs perspective an empty VLAN is equivalent
-			 * to a packet without VLAN layer.
-			 */
-			if (!eth->mask.vlan_tag)
-				goto error;
-			/* Outer TPID cannot be matched. */
-			if (eth->mask.ether_type) {
-				msg = "VLAN TPID matching is not supported";
-				goto error;
-			}
-			eth->val.ether_type = spec->inner_type;
-			eth->mask.ether_type = mask->inner_type;
-			eth->val.ether_type &= eth->mask.ether_type;
-		}
-		return 0;
-	}
-error:
-	return rte_flow_error_set(data->error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
-				  item, msg);
-}
-
-/**
- * Convert IPv4 item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_ipv4(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-	struct priv *priv = data->dev->data->dev_private;
-	const struct rte_flow_item_ipv4 *spec = item->spec;
-	const struct rte_flow_item_ipv4 *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
-	struct ibv_flow_spec_ipv4_ext ipv4 = {
-		.type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
-		.size = ipv4_size,
-	};
-
-	if (parser->layer == HASH_RXQ_TUNNEL &&
-	    parser->tunnel == ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] &&
-	    !priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	parser->layer = HASH_RXQ_IPV4;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
-			.src_ip = spec->hdr.src_addr,
-			.dst_ip = spec->hdr.dst_addr,
-			.proto = spec->hdr.next_proto_id,
-			.tos = spec->hdr.type_of_service,
-		};
-		ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
-			.src_ip = mask->hdr.src_addr,
-			.dst_ip = mask->hdr.dst_addr,
-			.proto = mask->hdr.next_proto_id,
-			.tos = mask->hdr.type_of_service,
-		};
-		/* Remove unwanted bits from values. */
-		ipv4.val.src_ip &= ipv4.mask.src_ip;
-		ipv4.val.dst_ip &= ipv4.mask.dst_ip;
-		ipv4.val.proto &= ipv4.mask.proto;
-		ipv4.val.tos &= ipv4.mask.tos;
-	}
-	mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
-	return 0;
-}
-
-/**
- * Convert IPv6 item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_ipv6(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-	struct priv *priv = data->dev->data->dev_private;
-	const struct rte_flow_item_ipv6 *spec = item->spec;
-	const struct rte_flow_item_ipv6 *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
-	struct ibv_flow_spec_ipv6 ipv6 = {
-		.type = parser->inner | IBV_FLOW_SPEC_IPV6,
-		.size = ipv6_size,
-	};
-
-	if (parser->layer == HASH_RXQ_TUNNEL &&
-	    parser->tunnel == ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] &&
-	    !priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	parser->layer = HASH_RXQ_IPV6;
-	if (spec) {
-		unsigned int i;
-		uint32_t vtc_flow_val;
-		uint32_t vtc_flow_mask;
-
-		if (!mask)
-			mask = default_mask;
-		memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
-		       RTE_DIM(ipv6.val.src_ip));
-		memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
-		       RTE_DIM(ipv6.val.dst_ip));
-		memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
-		       RTE_DIM(ipv6.mask.src_ip));
-		memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
-		       RTE_DIM(ipv6.mask.dst_ip));
-		vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
-		vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
-		ipv6.val.flow_label =
-			rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
-					 IPV6_HDR_FL_SHIFT);
-		ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
-					 IPV6_HDR_TC_SHIFT;
-		ipv6.val.next_hdr = spec->hdr.proto;
-		ipv6.val.hop_limit = spec->hdr.hop_limits;
-		ipv6.mask.flow_label =
-			rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
-					 IPV6_HDR_FL_SHIFT);
-		ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
-					  IPV6_HDR_TC_SHIFT;
-		ipv6.mask.next_hdr = mask->hdr.proto;
-		ipv6.mask.hop_limit = mask->hdr.hop_limits;
-		/* Remove unwanted bits from values. */
-		for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
-			ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
-			ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
-		}
-		ipv6.val.flow_label &= ipv6.mask.flow_label;
-		ipv6.val.traffic_class &= ipv6.mask.traffic_class;
-		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
-		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
-	}
-	mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
-	return 0;
-}
-
-/**
- * Convert UDP item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_udp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_udp *spec = item->spec;
-	const struct rte_flow_item_udp *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
-	struct ibv_flow_spec_tcp_udp udp = {
-		.type = parser->inner | IBV_FLOW_SPEC_UDP,
-		.size = udp_size,
-	};
-
-	if (parser->layer == HASH_RXQ_IPV4)
-		parser->layer = HASH_RXQ_UDPV4;
-	else
-		parser->layer = HASH_RXQ_UDPV6;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		udp.val.dst_port = spec->hdr.dst_port;
-		udp.val.src_port = spec->hdr.src_port;
-		udp.mask.dst_port = mask->hdr.dst_port;
-		udp.mask.src_port = mask->hdr.src_port;
-		/* Remove unwanted bits from values. */
-		udp.val.src_port &= udp.mask.src_port;
-		udp.val.dst_port &= udp.mask.dst_port;
-	}
-	mlx5_flow_create_copy(parser, &udp, udp_size);
-	return 0;
-}
-
-/**
- * Convert TCP item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_tcp(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_tcp *spec = item->spec;
-	const struct rte_flow_item_tcp *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
-	struct ibv_flow_spec_tcp_udp tcp = {
-		.type = parser->inner | IBV_FLOW_SPEC_TCP,
-		.size = tcp_size,
-	};
-
-	if (parser->layer == HASH_RXQ_IPV4)
-		parser->layer = HASH_RXQ_TCPV4;
-	else
-		parser->layer = HASH_RXQ_TCPV6;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		tcp.val.dst_port = spec->hdr.dst_port;
-		tcp.val.src_port = spec->hdr.src_port;
-		tcp.mask.dst_port = mask->hdr.dst_port;
-		tcp.mask.src_port = mask->hdr.src_port;
-		/* Remove unwanted bits from values. */
-		tcp.val.src_port &= tcp.mask.src_port;
-		tcp.val.dst_port &= tcp.mask.dst_port;
-	}
-	mlx5_flow_create_copy(parser, &tcp, tcp_size);
-	return 0;
-}
-
-/**
- * Convert VXLAN item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_vxlan(const struct rte_flow_item *item,
-		       const void *default_mask,
-		       struct mlx5_flow_data *data)
-{
-	const struct rte_flow_item_vxlan *spec = item->spec;
-	const struct rte_flow_item_vxlan *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel vxlan = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
-	union vni {
-		uint32_t vlan_id;
-		uint8_t vni[4];
-	} id;
-
-	id.vni[0] = 0;
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		memcpy(&id.vni[1], spec->vni, 3);
-		vxlan.val.tunnel_id = id.vlan_id;
-		memcpy(&id.vni[1], mask->vni, 3);
-		vxlan.mask.tunnel_id = id.vlan_id;
-		/* Remove unwanted bits from values. */
-		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
-	}
-	/*
-	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
-	 * layer is defined in the Verbs specification it is interpreted as
-	 * wildcard and all packets will match this rule, if it follows a full
-	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
-	 * before will also match this rule.
-	 * To avoid such situation, VNI 0 is currently refused.
-	 */
-	/* Only allow tunnel w/o tunnel id pattern after proper outer spec. */
-	if (parser->out_layer == HASH_RXQ_ETH && !vxlan.val.tunnel_id)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "VxLAN vni cannot be 0");
-	mlx5_flow_create_copy(parser, &vxlan, size);
-	return 0;
-}
-
-/**
- * Convert VXLAN-GPE item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_vxlan_gpe(const struct rte_flow_item *item,
-			   const void *default_mask,
-			   struct mlx5_flow_data *data)
-{
-	struct priv *priv = data->dev->data->dev_private;
-	const struct rte_flow_item_vxlan_gpe *spec = item->spec;
-	const struct rte_flow_item_vxlan_gpe *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel vxlan = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
-	union vni {
-		uint32_t vlan_id;
-		uint8_t vni[4];
-	} id;
-
-	if (!priv->config.l3_vxlan_en)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "L3 VXLAN not enabled by device"
-					  " parameter and/or not configured"
-					  " in firmware");
-	id.vni[0] = 0;
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN_GPE)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		memcpy(&id.vni[1], spec->vni, 3);
-		vxlan.val.tunnel_id = id.vlan_id;
-		memcpy(&id.vni[1], mask->vni, 3);
-		vxlan.mask.tunnel_id = id.vlan_id;
-		if (spec->protocol)
-			return rte_flow_error_set(data->error, EINVAL,
-						  RTE_FLOW_ERROR_TYPE_ITEM,
-						  item,
-						  "VxLAN-GPE protocol not"
-						  " supported");
-		/* Remove unwanted bits from values. */
-		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
-	}
-	/*
-	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
-	 * layer is defined in the Verbs specification it is interpreted as
-	 * wildcard and all packets will match this rule, if it follows a full
-	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
-	 * before will also match this rule.
-	 * To avoid such situation, VNI 0 is currently refused.
-	 */
-	/* Only allow tunnel w/o tunnel id pattern after proper outer spec. */
-	if (parser->out_layer == HASH_RXQ_ETH && !vxlan.val.tunnel_id)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "VxLAN-GPE vni cannot be 0");
-	mlx5_flow_create_copy(parser, &vxlan, size);
-	return 0;
-}
-
-/**
- * Convert GRE item to Verbs specification.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_gre(const struct rte_flow_item *item,
-		     const void *default_mask,
-		     struct mlx5_flow_data *data)
-{
-	struct mlx5_flow_parse *parser = data->parser;
-#ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	(void)default_mask;
-	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
-	struct ibv_flow_spec_tunnel tunnel = {
-		.type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
-		.size = size,
-	};
-#else
-	const struct rte_flow_item_gre *spec = item->spec;
-	const struct rte_flow_item_gre *mask = item->mask;
-	unsigned int size = sizeof(struct ibv_flow_spec_gre);
-	struct ibv_flow_spec_gre tunnel = {
-		.type = parser->inner | IBV_FLOW_SPEC_GRE,
-		.size = size,
-	};
-#endif
-	struct ibv_flow_spec_ipv4_ext *ipv4;
-	struct ibv_flow_spec_ipv6 *ipv6;
-	unsigned int i;
-
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_GRE)];
-	parser->out_layer = parser->layer;
-	parser->layer = HASH_RXQ_TUNNEL;
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
-		tunnel.val.protocol = spec->protocol;
-		tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
-		tunnel.mask.protocol = mask->protocol;
-		/* Remove unwanted bits from values. */
-		tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
-		tunnel.val.protocol &= tunnel.mask.protocol;
-		tunnel.val.key &= tunnel.mask.key;
-	}
-#endif
-	/* Update encapsulation IP layer protocol. */
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		if (parser->out_layer == HASH_RXQ_IPV4) {
-			ipv4 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset -
-				sizeof(struct ibv_flow_spec_ipv4_ext));
-			if (ipv4->mask.proto && ipv4->val.proto != MLX5_GRE)
-				break;
-			ipv4->val.proto = MLX5_GRE;
-			ipv4->mask.proto = 0xff;
-		} else if (parser->out_layer == HASH_RXQ_IPV6) {
-			ipv6 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
-				parser->queue[i].offset -
-				sizeof(struct ibv_flow_spec_ipv6));
-			if (ipv6->mask.next_hdr &&
-			    ipv6->val.next_hdr != MLX5_GRE)
-				break;
-			ipv6->val.next_hdr = MLX5_GRE;
-			ipv6->mask.next_hdr = 0xff;
-		}
-	}
-	if (i != hash_rxq_init_n)
-		return rte_flow_error_set(data->error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_ITEM,
-					  item,
-					  "IP protocol of GRE must be 47");
-	mlx5_flow_create_copy(parser, &tunnel, size);
-	return 0;
-}
-
-/**
- * Convert MPLS item to Verbs specification.
- * MPLS tunnel types currently supported are MPLS-in-GRE and MPLS-in-UDP.
- *
- * @param item[in]
- *   Item specification.
- * @param default_mask[in]
- *   Default bit-masks to use when item->mask is not provided.
- * @param data[in, out]
- *   User structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_mpls(const struct rte_flow_item *item,
-		      const void *default_mask,
-		      struct mlx5_flow_data *data)
-{
-#ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
-	(void)default_mask;
-	return rte_flow_error_set(data->error, ENOTSUP,
-				  RTE_FLOW_ERROR_TYPE_ITEM,
-				  item,
-				  "MPLS is not supported by driver");
-#else
-	const struct rte_flow_item_mpls *spec = item->spec;
-	const struct rte_flow_item_mpls *mask = item->mask;
-	struct mlx5_flow_parse *parser = data->parser;
-	unsigned int size = sizeof(struct ibv_flow_spec_mpls);
-	struct ibv_flow_spec_mpls mpls = {
-		.type = IBV_FLOW_SPEC_MPLS,
-		.size = size,
-	};
-
-	parser->inner = IBV_FLOW_SPEC_INNER;
-	if (parser->layer == HASH_RXQ_UDPV4 ||
-	    parser->layer == HASH_RXQ_UDPV6) {
-		parser->tunnel =
-			ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_UDP)];
-		parser->out_layer = parser->layer;
-	} else {
-		parser->tunnel =
-			ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_GRE)];
-		/* parser->out_layer stays as in GRE out_layer. */
-	}
-	parser->layer = HASH_RXQ_TUNNEL;
-	if (spec) {
-		if (!mask)
-			mask = default_mask;
-		/*
-		 * The verbs label field includes the entire MPLS header:
-		 * bits 0:19 - label value field.
-		 * bits 20:22 - traffic class field.
-		 * bits 23 - bottom of stack bit.
-		 * bits 24:31 - ttl field.
-		 */
-		mpls.val.label = *(const uint32_t *)spec;
-		mpls.mask.label = *(const uint32_t *)mask;
-		/* Remove unwanted bits from values. */
-		mpls.val.label &= mpls.mask.label;
-	}
-	mlx5_flow_create_copy(parser, &mpls, size);
-	return 0;
-#endif
-}
-
-/**
- * Convert mark/flag action to Verbs specification.
- *
- * @param parser
- *   Internal parser structure.
- * @param mark_id
- *   Mark identifier.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
-{
-	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
-	struct ibv_flow_spec_action_tag tag = {
-		.type = IBV_FLOW_SPEC_ACTION_TAG,
-		.size = size,
-		.tag_id = mlx5_flow_mark_set(mark_id),
-	};
-
-	assert(parser->mark);
-	mlx5_flow_create_copy(parser, &tag, size);
-	return 0;
-}
-
-/**
- * Convert count action to Verbs specification.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Pointer to MLX5 flow parser structure.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused,
-		       struct mlx5_flow_parse *parser __rte_unused)
-{
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-	struct priv *priv = dev->data->dev_private;
-	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
-	struct ibv_counter_set_init_attr init_attr = {0};
-	struct ibv_flow_spec_counter_action counter = {
-		.type = IBV_FLOW_SPEC_ACTION_COUNT,
-		.size = size,
-		.counter_set_handle = 0,
-	};
-
-	init_attr.counter_set_id = 0;
-	parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr);
-	if (!parser->cs) {
-		rte_errno = EINVAL;
-		return -rte_errno;
-	}
-	counter.counter_set_handle = parser->cs->handle;
-	mlx5_flow_create_copy(parser, &counter, size);
-#endif
-	return 0;
-}
-
-/**
- * Complete flow rule creation with a drop queue.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev,
-				   struct mlx5_flow_parse *parser,
-				   struct rte_flow *flow,
-				   struct rte_flow_error *error)
-{
-	struct priv *priv = dev->data->dev_private;
-	struct ibv_flow_spec_action_drop *drop;
-	unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
-
-	assert(priv->pd);
-	assert(priv->ctx);
-	flow->drop = 1;
-	drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr +
-			parser->queue[HASH_RXQ_ETH].offset);
-	*drop = (struct ibv_flow_spec_action_drop){
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-			.size = size,
-	};
-	++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs;
-	parser->queue[HASH_RXQ_ETH].offset += size;
-	flow->frxq[HASH_RXQ_ETH].ibv_attr =
-		parser->queue[HASH_RXQ_ETH].ibv_attr;
-	if (parser->count)
-		flow->cs = parser->cs;
-	if (!dev->data->dev_started)
-		return 0;
-	parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
-	flow->frxq[HASH_RXQ_ETH].ibv_flow =
-		mlx5_glue->create_flow(priv->flow_drop_queue->qp,
-				       flow->frxq[HASH_RXQ_ETH].ibv_attr);
-	if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-		rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "flow rule creation failure");
-		goto error;
-	}
-	return 0;
-error:
-	assert(flow);
-	if (flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-		claim_zero(mlx5_glue->destroy_flow
-			   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-		flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
-	}
-	if (flow->frxq[HASH_RXQ_ETH].ibv_attr) {
-		rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
-		flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL;
-	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-		parser->cs = NULL;
-	}
-	return -rte_errno;
-}
-
-/**
- * Create hash Rx queues when RSS is enabled.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev,
-				  struct mlx5_flow_parse *parser,
-				  struct rte_flow *flow,
-				  struct rte_flow_error *error)
-{
-	unsigned int i;
-
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!parser->queue[i].ibv_attr)
-			continue;
-		flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
-		parser->queue[i].ibv_attr = NULL;
-		flow->frxq[i].hash_fields = parser->queue[i].hash_fields;
-		if (!dev->data->dev_started)
-			continue;
-		flow->frxq[i].hrxq =
-			mlx5_hrxq_get(dev,
-				      parser->rss_conf.key,
-				      parser->rss_conf.key_len,
-				      flow->frxq[i].hash_fields,
-				      parser->rss_conf.queue,
-				      parser->rss_conf.queue_num,
-				      parser->tunnel,
-				      parser->rss_conf.level);
-		if (flow->frxq[i].hrxq)
-			continue;
-		flow->frxq[i].hrxq =
-			mlx5_hrxq_new(dev,
-				      parser->rss_conf.key,
-				      parser->rss_conf.key_len,
-				      flow->frxq[i].hash_fields,
-				      parser->rss_conf.queue,
-				      parser->rss_conf.queue_num,
-				      parser->tunnel,
-				      parser->rss_conf.level);
-		if (!flow->frxq[i].hrxq) {
-			return rte_flow_error_set(error, ENOMEM,
-						  RTE_FLOW_ERROR_TYPE_HANDLE,
-						  NULL,
-						  "cannot create hash rxq");
-		}
-	}
-	return 0;
-}
-
-/**
- * RXQ update after flow rule creation.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Pointer to the flow rule.
- */
-static void
-mlx5_flow_create_update_rxqs(struct rte_eth_dev *dev, struct rte_flow *flow)
-{
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-	unsigned int j;
-
-	if (!dev->data->dev_started)
-		return;
-	for (i = 0; i != flow->rss_conf.queue_num; ++i) {
-		struct mlx5_rxq_data *rxq_data = (*priv->rxqs)
-						 [(*flow->queues)[i]];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
-		uint8_t tunnel = PTYPE_IDX(flow->tunnel);
-
-		rxq_data->mark |= flow->mark;
-		if (!tunnel)
-			continue;
-		rxq_ctrl->tunnel_types[tunnel] += 1;
-		/* Clear tunnel type if more than one tunnel types set. */
-		for (j = 0; j != RTE_DIM(rxq_ctrl->tunnel_types); ++j) {
-			if (j == tunnel)
-				continue;
-			if (rxq_ctrl->tunnel_types[j] > 0) {
-				rxq_data->tunnel = 0;
-				break;
-			}
-		}
-		if (j == RTE_DIM(rxq_ctrl->tunnel_types))
-			rxq_data->tunnel = flow->tunnel;
-	}
-}
-
-/**
- * Dump flow hash RX queue detail.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Pointer to the rte_flow.
- * @param hrxq_idx
- *   Hash RX queue index.
- */
-static void
-mlx5_flow_dump(struct rte_eth_dev *dev __rte_unused,
-	       struct rte_flow *flow __rte_unused,
-	       unsigned int hrxq_idx __rte_unused)
-{
-#ifndef NDEBUG
-	uintptr_t spec_ptr;
-	uint16_t j;
-	char buf[256];
-	uint8_t off;
-	uint64_t extra_hash_fields = 0;
+/* Dev ops structure defined in mlx5.c */
+extern const struct eth_dev_ops mlx5_dev_ops;
+extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
-#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	if (flow->tunnel && flow->rss_conf.level > 1)
-		extra_hash_fields = (uint32_t)IBV_RX_HASH_INNER;
-#endif
-	spec_ptr = (uintptr_t)(flow->frxq[hrxq_idx].ibv_attr + 1);
-	for (j = 0, off = 0; j < flow->frxq[hrxq_idx].ibv_attr->num_of_specs;
-	     j++) {
-		struct ibv_flow_spec *spec = (void *)spec_ptr;
-		off += sprintf(buf + off, " %x(%hu)", spec->hdr.type,
-			       spec->hdr.size);
-		spec_ptr += spec->hdr.size;
-	}
-	DRV_LOG(DEBUG,
-		"port %u Verbs flow %p type %u: hrxq:%p qp:%p ind:%p,"
-		" hash:%" PRIx64 "/%u specs:%hhu(%hu), priority:%hu, type:%d,"
-		" flags:%x, comp_mask:%x specs:%s",
-		dev->data->port_id, (void *)flow, hrxq_idx,
-		(void *)flow->frxq[hrxq_idx].hrxq,
-		(void *)flow->frxq[hrxq_idx].hrxq->qp,
-		(void *)flow->frxq[hrxq_idx].hrxq->ind_table,
-		(flow->frxq[hrxq_idx].hash_fields | extra_hash_fields),
-		flow->rss_conf.queue_num,
-		flow->frxq[hrxq_idx].ibv_attr->num_of_specs,
-		flow->frxq[hrxq_idx].ibv_attr->size,
-		flow->frxq[hrxq_idx].ibv_attr->priority,
-		flow->frxq[hrxq_idx].ibv_attr->type,
-		flow->frxq[hrxq_idx].ibv_attr->flags,
-		flow->frxq[hrxq_idx].ibv_attr->comp_mask,
-		buf);
-#endif
-}
+struct rte_flow {
+	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
+};
 
-/**
- * Complete flow rule creation.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param parser
- *   Internal parser structure.
- * @param flow
- *   Pointer to the rte_flow.
- * @param[out] error
- *   Perform verbose error reporting if not NULL.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_create_action_queue(struct rte_eth_dev *dev,
-			      struct mlx5_flow_parse *parser,
-			      struct rte_flow *flow,
-			      struct rte_flow_error *error)
-{
-	struct priv *priv __rte_unused = dev->data->dev_private;
-	int ret;
-	unsigned int i;
-	unsigned int flows_n = 0;
+static const struct rte_flow_ops mlx5_flow_ops = {
+	.isolate = mlx5_flow_isolate,
+};
 
-	assert(priv->pd);
-	assert(priv->ctx);
-	assert(!parser->drop);
-	ret = mlx5_flow_create_action_queue_rss(dev, parser, flow, error);
-	if (ret)
-		goto error;
-	if (parser->count)
-		flow->cs = parser->cs;
-	if (!dev->data->dev_started)
-		return 0;
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (!flow->frxq[i].hrxq)
-			continue;
-		flow->frxq[i].ibv_flow =
-			mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
-					       flow->frxq[i].ibv_attr);
-		mlx5_flow_dump(dev, flow, i);
-		if (!flow->frxq[i].ibv_flow) {
-			rte_flow_error_set(error, ENOMEM,
-					   RTE_FLOW_ERROR_TYPE_HANDLE,
-					   NULL, "flow rule creation failure");
-			goto error;
-		}
-		++flows_n;
-	}
-	if (!flows_n) {
-		rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
-				   NULL, "internal error in flow creation");
-		goto error;
-	}
-	mlx5_flow_create_update_rxqs(dev, flow);
-	return 0;
-error:
-	ret = rte_errno; /* Save rte_errno before cleanup. */
-	assert(flow);
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (flow->frxq[i].ibv_flow) {
-			struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
+/* Convert FDIR request to Generic flow. */
+struct mlx5_fdir {
+	struct rte_flow_attr attr;
+	struct rte_flow_action actions[2];
+	struct rte_flow_item items[4];
+	struct rte_flow_item_eth l2;
+	struct rte_flow_item_eth l2_mask;
+	union {
+		struct rte_flow_item_ipv4 ipv4;
+		struct rte_flow_item_ipv6 ipv6;
+	} l3;
+	union {
+		struct rte_flow_item_ipv4 ipv4;
+		struct rte_flow_item_ipv6 ipv6;
+	} l3_mask;
+	union {
+		struct rte_flow_item_udp udp;
+		struct rte_flow_item_tcp tcp;
+	} l4;
+	union {
+		struct rte_flow_item_udp udp;
+		struct rte_flow_item_tcp tcp;
+	} l4_mask;
+	struct rte_flow_action_queue queue;
+};
 
-			claim_zero(mlx5_glue->destroy_flow(ibv_flow));
-		}
-		if (flow->frxq[i].hrxq)
-			mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
-		if (flow->frxq[i].ibv_attr)
-			rte_free(flow->frxq[i].ibv_attr);
-	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-		parser->cs = NULL;
-	}
-	rte_errno = ret; /* Restore rte_errno. */
-	return -rte_errno;
-}
+/* Verbs specification header. */
+struct ibv_spec_header {
+	enum ibv_flow_spec_type type;
+	uint16_t size;
+};
 
 /**
  * Convert a flow.
@@ -2452,69 +95,17 @@ mlx5_flow_create_action_queue(struct rte_eth_dev *dev,
  *   A flow on success, NULL otherwise and rte_errno is set.
  */
 static struct rte_flow *
-mlx5_flow_list_create(struct rte_eth_dev *dev,
-		      struct mlx5_flows *list,
-		      const struct rte_flow_attr *attr,
-		      const struct rte_flow_item items[],
-		      const struct rte_flow_action actions[],
+mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
+		      struct mlx5_flows *list __rte_unused,
+		      const struct rte_flow_attr *attr __rte_unused,
+		      const struct rte_flow_item items[] __rte_unused,
+		      const struct rte_flow_action actions[] __rte_unused,
 		      struct rte_flow_error *error)
 {
-	struct mlx5_flow_parse parser = { .create = 1, };
-	struct rte_flow *flow = NULL;
-	unsigned int i;
-	int ret;
-
-	ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser);
-	if (ret)
-		goto exit;
-	flow = rte_calloc(__func__, 1,
-			  sizeof(*flow) +
-			  parser.rss_conf.queue_num * sizeof(uint16_t),
-			  0);
-	if (!flow) {
-		rte_flow_error_set(error, ENOMEM,
-				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				   NULL,
-				   "cannot allocate flow memory");
-		return NULL;
-	}
-	/* Copy configuration. */
-	flow->queues = (uint16_t (*)[])(flow + 1);
-	flow->tunnel = parser.tunnel;
-	flow->rss_conf = (struct rte_flow_action_rss){
-		.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
-		.level = parser.rss_conf.level,
-		.types = parser.rss_conf.types,
-		.key_len = parser.rss_conf.key_len,
-		.queue_num = parser.rss_conf.queue_num,
-		.key = memcpy(flow->rss_key, parser.rss_conf.key,
-			      sizeof(*parser.rss_conf.key) *
-			      parser.rss_conf.key_len),
-		.queue = memcpy(flow->queues, parser.rss_conf.queue,
-				sizeof(*parser.rss_conf.queue) *
-				parser.rss_conf.queue_num),
-	};
-	flow->mark = parser.mark;
-	/* finalise the flow. */
-	if (parser.drop)
-		ret = mlx5_flow_create_action_queue_drop(dev, &parser, flow,
-							 error);
-	else
-		ret = mlx5_flow_create_action_queue(dev, &parser, flow, error);
-	if (ret)
-		goto exit;
-	TAILQ_INSERT_TAIL(list, flow, next);
-	DRV_LOG(DEBUG, "port %u flow created %p", dev->data->port_id,
-		(void *)flow);
-	return flow;
-exit:
-	DRV_LOG(ERR, "port %u flow creation error: %s", dev->data->port_id,
-		error->message);
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser.queue[i].ibv_attr)
-			rte_free(parser.queue[i].ibv_attr);
-	}
-	rte_free(flow);
+	rte_flow_error_set(error, ENOTSUP,
+			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			   NULL,
+			   "action not supported");
 	return NULL;
 }
 
@@ -2525,15 +116,16 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
  * @see rte_flow_ops
  */
 int
-mlx5_flow_validate(struct rte_eth_dev *dev,
-		   const struct rte_flow_attr *attr,
-		   const struct rte_flow_item items[],
-		   const struct rte_flow_action actions[],
+mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
+		   const struct rte_flow_attr *attr __rte_unused,
+		   const struct rte_flow_item items[] __rte_unused,
+		   const struct rte_flow_action actions[] __rte_unused,
 		   struct rte_flow_error *error)
 {
-	struct mlx5_flow_parse parser = { .create = 0, };
-
-	return mlx5_flow_convert(dev, attr, items, actions, error, &parser);
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "action not supported");
 }
 
 /**
@@ -2543,16 +135,17 @@ mlx5_flow_validate(struct rte_eth_dev *dev,
  * @see rte_flow_ops
  */
 struct rte_flow *
-mlx5_flow_create(struct rte_eth_dev *dev,
-		 const struct rte_flow_attr *attr,
-		 const struct rte_flow_item items[],
-		 const struct rte_flow_action actions[],
+mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
+		 const struct rte_flow_attr *attr __rte_unused,
+		 const struct rte_flow_item items[] __rte_unused,
+		 const struct rte_flow_action actions[] __rte_unused,
 		 struct rte_flow_error *error)
 {
-	struct priv *priv = dev->data->dev_private;
-
-	return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions,
-				     error);
+	rte_flow_error_set(error, ENOTSUP,
+			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			   NULL,
+			   "action not supported");
+	return NULL;
 }
 
 /**
@@ -2566,99 +159,10 @@ mlx5_flow_create(struct rte_eth_dev *dev,
  *   Flow to destroy.
  */
 static void
-mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
-		       struct rte_flow *flow)
+mlx5_flow_list_destroy(struct rte_eth_dev *dev __rte_unused,
+		       struct mlx5_flows *list __rte_unused,
+		       struct rte_flow *flow __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	unsigned int i;
-
-	if (flow->drop || !dev->data->dev_started)
-		goto free;
-	for (i = 0; flow->tunnel && i != flow->rss_conf.queue_num; ++i) {
-		/* Update queue tunnel type. */
-		struct mlx5_rxq_data *rxq_data = (*priv->rxqs)
-						 [(*flow->queues)[i]];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
-		uint8_t tunnel = PTYPE_IDX(flow->tunnel);
-
-		assert(rxq_ctrl->tunnel_types[tunnel] > 0);
-		rxq_ctrl->tunnel_types[tunnel] -= 1;
-		if (!rxq_ctrl->tunnel_types[tunnel]) {
-			/* Update tunnel type. */
-			uint8_t j;
-			uint8_t types = 0;
-			uint8_t last;
-
-			for (j = 0; j < RTE_DIM(rxq_ctrl->tunnel_types); j++)
-				if (rxq_ctrl->tunnel_types[j]) {
-					types += 1;
-					last = j;
-				}
-			/* Keep same if more than one tunnel types left. */
-			if (types == 1)
-				rxq_data->tunnel = ptype_ext[last];
-			else if (types == 0)
-				/* No tunnel type left. */
-				rxq_data->tunnel = 0;
-		}
-	}
-	for (i = 0; flow->mark && i != flow->rss_conf.queue_num; ++i) {
-		struct rte_flow *tmp;
-		int mark = 0;
-
-		/*
-		 * To remove the mark from the queue, the queue must not be
-		 * present in any other marked flow (RSS or not).
-		 */
-		TAILQ_FOREACH(tmp, list, next) {
-			unsigned int j;
-			uint16_t *tqs = NULL;
-			uint16_t tq_n = 0;
-
-			if (!tmp->mark)
-				continue;
-			for (j = 0; j != hash_rxq_init_n; ++j) {
-				if (!tmp->frxq[j].hrxq)
-					continue;
-				tqs = tmp->frxq[j].hrxq->ind_table->queues;
-				tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
-			}
-			if (!tq_n)
-				continue;
-			for (j = 0; (j != tq_n) && !mark; j++)
-				if (tqs[j] == (*flow->queues)[i])
-					mark = 1;
-		}
-		(*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
-	}
-free:
-	if (flow->drop) {
-		if (flow->frxq[HASH_RXQ_ETH].ibv_flow)
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-		rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
-	} else {
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			struct mlx5_flow *frxq = &flow->frxq[i];
-
-			if (frxq->ibv_flow)
-				claim_zero(mlx5_glue->destroy_flow
-					   (frxq->ibv_flow));
-			if (frxq->hrxq)
-				mlx5_hrxq_release(dev, frxq->hrxq);
-			if (frxq->ibv_attr)
-				rte_free(frxq->ibv_attr);
-		}
-	}
-	if (flow->cs) {
-		claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
-		flow->cs = NULL;
-	}
-	TAILQ_REMOVE(list, flow, next);
-	DRV_LOG(DEBUG, "port %u flow destroyed %p", dev->data->port_id,
-		(void *)flow);
-	rte_free(flow);
 }
 
 /**
@@ -2690,97 +194,9 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_flow_create_drop_queue(struct rte_eth_dev *dev)
+mlx5_flow_create_drop_queue(struct rte_eth_dev *dev __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_hrxq_drop *fdq = NULL;
-
-	assert(priv->pd);
-	assert(priv->ctx);
-	fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
-	if (!fdq) {
-		DRV_LOG(WARNING,
-			"port %u cannot allocate memory for drop queue",
-			dev->data->port_id);
-		rte_errno = ENOMEM;
-		return -rte_errno;
-	}
-	fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
-	if (!fdq->cq) {
-		DRV_LOG(WARNING, "port %u cannot allocate CQ for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->wq = mlx5_glue->create_wq
-		(priv->ctx,
-		 &(struct ibv_wq_init_attr){
-			.wq_type = IBV_WQT_RQ,
-			.max_wr = 1,
-			.max_sge = 1,
-			.pd = priv->pd,
-			.cq = fdq->cq,
-		 });
-	if (!fdq->wq) {
-		DRV_LOG(WARNING, "port %u cannot allocate WQ for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->ind_table = mlx5_glue->create_rwq_ind_table
-		(priv->ctx,
-		 &(struct ibv_rwq_ind_table_init_attr){
-			.log_ind_tbl_size = 0,
-			.ind_tbl = &fdq->wq,
-			.comp_mask = 0,
-		 });
-	if (!fdq->ind_table) {
-		DRV_LOG(WARNING,
-			"port %u cannot allocate indirection table for drop"
-			" queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	fdq->qp = mlx5_glue->create_qp_ex
-		(priv->ctx,
-		 &(struct ibv_qp_init_attr_ex){
-			.qp_type = IBV_QPT_RAW_PACKET,
-			.comp_mask =
-				IBV_QP_INIT_ATTR_PD |
-				IBV_QP_INIT_ATTR_IND_TABLE |
-				IBV_QP_INIT_ATTR_RX_HASH,
-			.rx_hash_conf = (struct ibv_rx_hash_conf){
-				.rx_hash_function =
-					IBV_RX_HASH_FUNC_TOEPLITZ,
-				.rx_hash_key_len = rss_hash_default_key_len,
-				.rx_hash_key = rss_hash_default_key,
-				.rx_hash_fields_mask = 0,
-				},
-			.rwq_ind_tbl = fdq->ind_table,
-			.pd = priv->pd
-		 });
-	if (!fdq->qp) {
-		DRV_LOG(WARNING, "port %u cannot allocate QP for drop queue",
-			dev->data->port_id);
-		rte_errno = errno;
-		goto error;
-	}
-	priv->flow_drop_queue = fdq;
 	return 0;
-error:
-	if (fdq->qp)
-		claim_zero(mlx5_glue->destroy_qp(fdq->qp));
-	if (fdq->ind_table)
-		claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
-	if (fdq->wq)
-		claim_zero(mlx5_glue->destroy_wq(fdq->wq));
-	if (fdq->cq)
-		claim_zero(mlx5_glue->destroy_cq(fdq->cq));
-	if (fdq)
-		rte_free(fdq);
-	priv->flow_drop_queue = NULL;
-	return -rte_errno;
 }
 
 /**
@@ -2790,23 +206,8 @@ mlx5_flow_create_drop_queue(struct rte_eth_dev *dev)
  *   Pointer to Ethernet device.
  */
 void
-mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
+mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
-
-	if (!fdq)
-		return;
-	if (fdq->qp)
-		claim_zero(mlx5_glue->destroy_qp(fdq->qp));
-	if (fdq->ind_table)
-		claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
-	if (fdq->wq)
-		claim_zero(mlx5_glue->destroy_wq(fdq->wq));
-	if (fdq->cq)
-		claim_zero(mlx5_glue->destroy_cq(fdq->cq));
-	rte_free(fdq);
-	priv->flow_drop_queue = NULL;
 }
 
 /**
@@ -2818,70 +219,9 @@ mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
  *   Pointer to a TAILQ flow list.
  */
 void
-mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
+mlx5_flow_stop(struct rte_eth_dev *dev __rte_unused,
+	       struct mlx5_flows *list __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct rte_flow *flow;
-	unsigned int i;
-
-	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
-		struct mlx5_ind_table_ibv *ind_tbl = NULL;
-
-		if (flow->drop) {
-			if (!flow->frxq[HASH_RXQ_ETH].ibv_flow)
-				continue;
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[HASH_RXQ_ETH].ibv_flow));
-			flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
-			DRV_LOG(DEBUG, "port %u flow %p removed",
-				dev->data->port_id, (void *)flow);
-			/* Next flow. */
-			continue;
-		}
-		/* Verify the flow has not already been cleaned. */
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_flow)
-				continue;
-			/*
-			 * Indirection table may be necessary to remove the
-			 * flags in the Rx queues.
-			 * This helps to speed-up the process by avoiding
-			 * another loop.
-			 */
-			ind_tbl = flow->frxq[i].hrxq->ind_table;
-			break;
-		}
-		if (i == hash_rxq_init_n)
-			return;
-		if (flow->mark) {
-			assert(ind_tbl);
-			for (i = 0; i != ind_tbl->queues_n; ++i)
-				(*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
-		}
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_flow)
-				continue;
-			claim_zero(mlx5_glue->destroy_flow
-				   (flow->frxq[i].ibv_flow));
-			flow->frxq[i].ibv_flow = NULL;
-			mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
-			flow->frxq[i].hrxq = NULL;
-		}
-		DRV_LOG(DEBUG, "port %u flow %p removed", dev->data->port_id,
-			(void *)flow);
-	}
-	/* Cleanup Rx queue tunnel info. */
-	for (i = 0; i != priv->rxqs_n; ++i) {
-		struct mlx5_rxq_data *q = (*priv->rxqs)[i];
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(q, struct mlx5_rxq_ctrl, rxq);
-
-		if (!q)
-			continue;
-		memset((void *)rxq_ctrl->tunnel_types, 0,
-		       sizeof(rxq_ctrl->tunnel_types));
-		q->tunnel = 0;
-	}
 }
 
 /**
@@ -2896,76 +236,9 @@ mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
+mlx5_flow_start(struct rte_eth_dev *dev __rte_unused,
+		struct mlx5_flows *list __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct rte_flow *flow;
-
-	TAILQ_FOREACH(flow, list, next) {
-		unsigned int i;
-
-		if (flow->drop) {
-			flow->frxq[HASH_RXQ_ETH].ibv_flow =
-				mlx5_glue->create_flow
-				(priv->flow_drop_queue->qp,
-				 flow->frxq[HASH_RXQ_ETH].ibv_attr);
-			if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p cannot be applied",
-					dev->data->port_id, (void *)flow);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-			DRV_LOG(DEBUG, "port %u flow %p applied",
-				dev->data->port_id, (void *)flow);
-			/* Next flow. */
-			continue;
-		}
-		for (i = 0; i != hash_rxq_init_n; ++i) {
-			if (!flow->frxq[i].ibv_attr)
-				continue;
-			flow->frxq[i].hrxq =
-				mlx5_hrxq_get(dev, flow->rss_conf.key,
-					      flow->rss_conf.key_len,
-					      flow->frxq[i].hash_fields,
-					      flow->rss_conf.queue,
-					      flow->rss_conf.queue_num,
-					      flow->tunnel,
-					      flow->rss_conf.level);
-			if (flow->frxq[i].hrxq)
-				goto flow_create;
-			flow->frxq[i].hrxq =
-				mlx5_hrxq_new(dev, flow->rss_conf.key,
-					      flow->rss_conf.key_len,
-					      flow->frxq[i].hash_fields,
-					      flow->rss_conf.queue,
-					      flow->rss_conf.queue_num,
-					      flow->tunnel,
-					      flow->rss_conf.level);
-			if (!flow->frxq[i].hrxq) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p cannot create hash"
-					" rxq",
-					dev->data->port_id, (void *)flow);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-flow_create:
-			mlx5_flow_dump(dev, flow, i);
-			flow->frxq[i].ibv_flow =
-				mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
-						       flow->frxq[i].ibv_attr);
-			if (!flow->frxq[i].ibv_flow) {
-				DRV_LOG(DEBUG,
-					"port %u flow %p type %u cannot be"
-					" applied",
-					dev->data->port_id, (void *)flow, i);
-				rte_errno = EINVAL;
-				return -rte_errno;
-			}
-		}
-		mlx5_flow_create_update_rxqs(dev, flow);
-	}
 	return 0;
 }
 
@@ -3019,7 +292,6 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
-		.priority = MLX5_CTRL_FLOW_PRIORITY,
 	};
 	struct rte_flow_item items[] = {
 		{
@@ -3129,83 +401,6 @@ mlx5_flow_flush(struct rte_eth_dev *dev,
 	return 0;
 }
 
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
-/**
- * Query flow counter.
- *
- * @param cs
- *   the counter set.
- * @param counter_value
- *   returned data from the counter.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_query_count(struct ibv_counter_set *cs,
-		      struct mlx5_flow_counter_stats *counter_stats,
-		      struct rte_flow_query_count *query_count,
-		      struct rte_flow_error *error)
-{
-	uint64_t counters[2];
-	struct ibv_query_counter_set_attr query_cs_attr = {
-		.cs = cs,
-		.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
-	};
-	struct ibv_counter_set_data query_out = {
-		.out = counters,
-		.outlen = 2 * sizeof(uint64_t),
-	};
-	int err = mlx5_glue->query_counter_set(&query_cs_attr, &query_out);
-
-	if (err)
-		return rte_flow_error_set(error, err,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "cannot read counter");
-	query_count->hits_set = 1;
-	query_count->bytes_set = 1;
-	query_count->hits = counters[0] - counter_stats->hits;
-	query_count->bytes = counters[1] - counter_stats->bytes;
-	if (query_count->reset) {
-		counter_stats->hits = counters[0];
-		counter_stats->bytes = counters[1];
-	}
-	return 0;
-}
-
-/**
- * Query a flows.
- *
- * @see rte_flow_query()
- * @see rte_flow_ops
- */
-int
-mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
-		struct rte_flow *flow,
-		const struct rte_flow_action *action __rte_unused,
-		void *data,
-		struct rte_flow_error *error)
-{
-	if (flow->cs) {
-		int ret;
-
-		ret = mlx5_flow_query_count(flow->cs,
-					    &flow->counter_stats,
-					    (struct rte_flow_query_count *)data,
-					    error);
-		if (ret)
-			return ret;
-	} else {
-		return rte_flow_error_set(error, EINVAL,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "no counter found for flow");
-	}
-	return 0;
-}
-#endif
-
 /**
  * Isolated mode.
  *
@@ -3445,18 +640,11 @@ mlx5_fdir_filter_add(struct rte_eth_dev *dev,
 			.type = 0,
 		},
 	};
-	struct mlx5_flow_parse parser = {
-		.layer = HASH_RXQ_ETH,
-	};
 	struct rte_flow_error error;
 	struct rte_flow *flow;
 	int ret;
 
 	ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
-				attributes.actions, &error, &parser);
 	if (ret)
 		return ret;
 	flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr,
@@ -3482,96 +670,11 @@ mlx5_fdir_filter_add(struct rte_eth_dev *dev,
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx5_fdir_filter_delete(struct rte_eth_dev *dev,
-			const struct rte_eth_fdir_filter *fdir_filter)
+mlx5_fdir_filter_delete(struct rte_eth_dev *dev __rte_unused,
+			const struct rte_eth_fdir_filter *fdir_filter
+			__rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	struct mlx5_fdir attributes = {
-		.attr.group = 0,
-	};
-	struct mlx5_flow_parse parser = {
-		.create = 1,
-		.layer = HASH_RXQ_ETH,
-	};
-	struct rte_flow_error error;
-	struct rte_flow *flow;
-	unsigned int i;
-	int ret;
-
-	ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
-	if (ret)
-		return ret;
-	ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
-				attributes.actions, &error, &parser);
-	if (ret)
-		goto exit;
-	/*
-	 * Special case for drop action which is only set in the
-	 * specifications when the flow is created.  In this situation the
-	 * drop specification is missing.
-	 */
-	if (parser.drop) {
-		struct ibv_flow_spec_action_drop *drop;
-
-		drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr +
-				parser.queue[HASH_RXQ_ETH].offset);
-		*drop = (struct ibv_flow_spec_action_drop){
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-			.size = sizeof(struct ibv_flow_spec_action_drop),
-		};
-		parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++;
-	}
-	TAILQ_FOREACH(flow, &priv->flows, next) {
-		struct ibv_flow_attr *attr;
-		struct ibv_spec_header *attr_h;
-		void *spec;
-		struct ibv_flow_attr *flow_attr;
-		struct ibv_spec_header *flow_h;
-		void *flow_spec;
-		unsigned int specs_n;
-		unsigned int queue_id = parser.drop ? HASH_RXQ_ETH :
-						      parser.layer;
-
-		attr = parser.queue[queue_id].ibv_attr;
-		flow_attr = flow->frxq[queue_id].ibv_attr;
-		/* Compare first the attributes. */
-		if (!flow_attr ||
-		    memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
-			continue;
-		if (attr->num_of_specs == 0)
-			continue;
-		spec = (void *)((uintptr_t)attr +
-				sizeof(struct ibv_flow_attr));
-		flow_spec = (void *)((uintptr_t)flow_attr +
-				     sizeof(struct ibv_flow_attr));
-		specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
-		for (i = 0; i != specs_n; ++i) {
-			attr_h = spec;
-			flow_h = flow_spec;
-			if (memcmp(spec, flow_spec,
-				   RTE_MIN(attr_h->size, flow_h->size)))
-				goto wrong_flow;
-			spec = (void *)((uintptr_t)spec + attr_h->size);
-			flow_spec = (void *)((uintptr_t)flow_spec +
-					     flow_h->size);
-		}
-		/* At this point, the flow match. */
-		break;
-wrong_flow:
-		/* The flow does not match. */
-		continue;
-	}
-	if (flow)
-		mlx5_flow_list_destroy(dev, &priv->flows, flow);
-exit:
-	if (ret)
-		ret = rte_errno; /* Save rte_errno before cleanup. */
-	for (i = 0; i != hash_rxq_init_n; ++i) {
-		if (parser.queue[i].ibv_attr)
-			rte_free(parser.queue[i].ibv_attr);
-	}
-	if (ret)
-		rte_errno = ret; /* Restore rte_errno. */
+	rte_errno = ENOTSUP;
 	return -rte_errno;
 }
 
@@ -3738,45 +841,7 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
  *   number of supported Verbs flow priority.
  */
 unsigned int
-mlx5_get_max_verbs_prio(struct rte_eth_dev *dev)
+mlx5_get_max_verbs_prio(struct rte_eth_dev *dev __rte_unused)
 {
-	struct priv *priv = dev->data->dev_private;
-	unsigned int verb_priorities = MLX5_VERBS_FLOW_PRIO_8;
-	struct {
-		struct ibv_flow_attr attr;
-		struct ibv_flow_spec_eth eth;
-		struct ibv_flow_spec_action_drop drop;
-	} flow_attr = {
-		.attr = {
-			.num_of_specs = 2,
-		},
-		.eth = {
-			.type = IBV_FLOW_SPEC_ETH,
-			.size = sizeof(struct ibv_flow_spec_eth),
-		},
-		.drop = {
-			.size = sizeof(struct ibv_flow_spec_action_drop),
-			.type = IBV_FLOW_SPEC_ACTION_DROP,
-		},
-	};
-	struct ibv_flow *flow;
-
-	do {
-		flow_attr.attr.priority = verb_priorities - 1;
-		flow = mlx5_glue->create_flow(priv->flow_drop_queue->qp,
-					      &flow_attr.attr);
-		if (flow) {
-			claim_zero(mlx5_glue->destroy_flow(flow));
-			/* Try more priorities. */
-			verb_priorities *= 2;
-		} else {
-			/* Failed, restore last right number. */
-			verb_priorities /= 2;
-			break;
-		}
-	} while (1);
-	DRV_LOG(DEBUG, "port %u Verbs flow priorities: %d,"
-		" user flow priorities: %d",
-		dev->data->port_id, verb_priorities, MLX5_CTRL_FLOW_PRIORITY);
-	return verb_priorities;
+	return 8;
 }
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 0007be08b..97b4d9eb6 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -136,7 +136,6 @@ struct mlx5_rxq_ctrl {
 	struct priv *priv; /* Back pointer to private data. */
 	struct mlx5_rxq_data rxq; /* Data path structure. */
 	unsigned int socket; /* CPU socket ID for allocations. */
-	uint32_t tunnel_types[16]; /* Tunnel type counter. */
 	unsigned int irq:1; /* Whether IRQ is enabled. */
 	uint16_t idx; /* Queue index. */
 };
-- 
2.18.0

[dpdk-dev] [PATCH v4 02/21] net/mlx5: handle drop queues as regular queues

[Nelio Laranjeiro]

Drop queues are essentially used in flows due to Verbs API, the
information if the fate of the flow is a drop or not is already present
in the flow.  Due to this, drop queues can be fully mapped on regular
queues.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5.c      |  24 ++--
 drivers/net/mlx5/mlx5.h      |  14 ++-
 drivers/net/mlx5/mlx5_flow.c |  94 +++++++-------
 drivers/net/mlx5/mlx5_rxq.c  | 232 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_rxtx.h |   6 +
 5 files changed, 308 insertions(+), 62 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index df7f39844..e9780ac8f 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -261,7 +261,6 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 		priv->txqs_n = 0;
 		priv->txqs = NULL;
 	}
-	mlx5_flow_delete_drop_queue(dev);
 	mlx5_mprq_free_mp(dev);
 	mlx5_mr_release(dev);
 	if (priv->pd != NULL) {
@@ -1139,22 +1138,15 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	mlx5_link_update(eth_dev, 0);
 	/* Store device configuration on private structure. */
 	priv->config = config;
-	/* Create drop queue. */
-	err = mlx5_flow_create_drop_queue(eth_dev);
-	if (err) {
-		DRV_LOG(ERR, "port %u drop queue allocation failed: %s",
-			eth_dev->data->port_id, strerror(rte_errno));
-		err = rte_errno;
-		goto error;
-	}
 	/* Supported Verbs flow priority number detection. */
-	if (verb_priorities == 0)
-		verb_priorities = mlx5_get_max_verbs_prio(eth_dev);
-	if (verb_priorities < MLX5_VERBS_FLOW_PRIO_8) {
-		DRV_LOG(ERR, "port %u wrong Verbs flow priorities: %u",
-			eth_dev->data->port_id, verb_priorities);
-		err = ENOTSUP;
-		goto error;
+	if (verb_priorities == 0) {
+		err = mlx5_verbs_max_prio(eth_dev);
+		if (err < 0) {
+			DRV_LOG(ERR, "port %u wrong Verbs flow priorities",
+				eth_dev->data->port_id);
+			goto error;
+		}
+		verb_priorities = err;
 	}
 	priv->config.max_verbs_prio = verb_priorities;
 	/*
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index cc01310e0..227429848 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -139,9 +139,6 @@ enum mlx5_verbs_alloc_type {
 	MLX5_VERBS_ALLOC_TYPE_RX_QUEUE,
 };
 
-/* 8 Verbs priorities. */
-#define MLX5_VERBS_FLOW_PRIO_8 8
-
 /**
  * Verbs allocator needs a context to know in the callback which kind of
  * resources it is allocating.
@@ -153,6 +150,12 @@ struct mlx5_verbs_alloc_ctx {
 
 LIST_HEAD(mlx5_mr_list, mlx5_mr);
 
+/* Flow drop context necessary due to Verbs API. */
+struct mlx5_drop {
+	struct mlx5_hrxq *hrxq; /* Hash Rx queue queue. */
+	struct mlx5_rxq_ibv *rxq; /* Verbs Rx queue. */
+};
+
 struct priv {
 	LIST_ENTRY(priv) mem_event_cb; /* Called by memory event callback. */
 	struct rte_eth_dev_data *dev_data;  /* Pointer to device data. */
@@ -182,7 +185,7 @@ struct priv {
 	struct rte_intr_handle intr_handle; /* Interrupt handler. */
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
-	struct mlx5_hrxq_drop *flow_drop_queue; /* Flow drop queue. */
+	struct mlx5_drop drop_queue; /* Flow drop queues. */
 	struct mlx5_flows flows; /* RTE Flow rules. */
 	struct mlx5_flows ctrl_flows; /* Control flow rules. */
 	struct {
@@ -314,7 +317,8 @@ int mlx5_traffic_restart(struct rte_eth_dev *dev);
 
 /* mlx5_flow.c */
 
-unsigned int mlx5_get_max_verbs_prio(struct rte_eth_dev *dev);
+int mlx5_verbs_max_prio(struct rte_eth_dev *dev);
+void mlx5_flow_print(struct rte_flow *flow);
 int mlx5_flow_validate(struct rte_eth_dev *dev,
 		       const struct rte_flow_attr *attr,
 		       const struct rte_flow_item items[],
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index a45cb06e1..5e325be37 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -75,6 +75,58 @@ struct ibv_spec_header {
 	uint16_t size;
 };
 
+ /**
+  * Get the maximum number of priority available.
+  *
+  * @param[in] dev
+  *   Pointer to Ethernet device.
+  *
+  * @return
+  *   number of supported Verbs flow priority on success, a negative errno
+  *   value otherwise and rte_errno is set.
+  */
+int
+mlx5_verbs_max_prio(struct rte_eth_dev *dev)
+{
+	struct {
+		struct ibv_flow_attr attr;
+		struct ibv_flow_spec_eth eth;
+		struct ibv_flow_spec_action_drop drop;
+	} flow_attr = {
+		.attr = {
+			.num_of_specs = 2,
+		},
+		.eth = {
+			.type = IBV_FLOW_SPEC_ETH,
+			.size = sizeof(struct ibv_flow_spec_eth),
+		},
+		.drop = {
+			.size = sizeof(struct ibv_flow_spec_action_drop),
+			.type = IBV_FLOW_SPEC_ACTION_DROP,
+		},
+	};
+	struct ibv_flow *flow;
+	uint32_t verb_priorities;
+	struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
+
+	if (!drop) {
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	for (verb_priorities = 0; 1; verb_priorities++) {
+		flow_attr.attr.priority = verb_priorities;
+		flow = mlx5_glue->create_flow(drop->qp,
+					      &flow_attr.attr);
+		if (!flow)
+			break;
+		claim_zero(mlx5_glue->destroy_flow(flow));
+	}
+	mlx5_hrxq_drop_release(dev);
+	DRV_LOG(INFO, "port %u flow maximum priority: %d",
+		dev->data->port_id, verb_priorities);
+	return verb_priorities;
+}
+
 /**
  * Convert a flow.
  *
@@ -184,32 +236,6 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
 	}
 }
 
-/**
- * Create drop queue.
- *
- * @param dev
- *   Pointer to Ethernet device.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-int
-mlx5_flow_create_drop_queue(struct rte_eth_dev *dev __rte_unused)
-{
-	return 0;
-}
-
-/**
- * Delete drop queue.
- *
- * @param dev
- *   Pointer to Ethernet device.
- */
-void
-mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev __rte_unused)
-{
-}
-
 /**
  * Remove all flows.
  *
@@ -292,6 +318,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
+		.priority = priv->config.max_verbs_prio - 1,
 	};
 	struct rte_flow_item items[] = {
 		{
@@ -830,18 +857,3 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
 	}
 	return 0;
 }
-
-/**
- * Detect number of Verbs flow priorities supported.
- *
- * @param dev
- *   Pointer to Ethernet device.
- *
- * @return
- *   number of supported Verbs flow priority.
- */
-unsigned int
-mlx5_get_max_verbs_prio(struct rte_eth_dev *dev __rte_unused)
-{
-	return 8;
-}
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index fd0df177e..d960daa43 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1957,3 +1957,235 @@ mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev)
 	}
 	return ret;
 }
+
+/**
+ * Create a drop Rx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_rxq_ibv *
+mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct ibv_cq *cq;
+	struct ibv_wq *wq = NULL;
+	struct mlx5_rxq_ibv *rxq;
+
+	if (priv->drop_queue.rxq)
+		return priv->drop_queue.rxq;
+	cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
+	if (!cq) {
+		DEBUG("port %u cannot allocate CQ for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	wq = mlx5_glue->create_wq(priv->ctx,
+		 &(struct ibv_wq_init_attr){
+			.wq_type = IBV_WQT_RQ,
+			.max_wr = 1,
+			.max_sge = 1,
+			.pd = priv->pd,
+			.cq = cq,
+		 });
+	if (!wq) {
+		DEBUG("port %u cannot allocate WQ for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	rxq = rte_calloc(__func__, 1, sizeof(*rxq), 0);
+	if (!rxq) {
+		DEBUG("port %u cannot allocate drop Rx queue memory",
+		      dev->data->port_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	rxq->cq = cq;
+	rxq->wq = wq;
+	priv->drop_queue.rxq = rxq;
+	return rxq;
+error:
+	if (wq)
+		claim_zero(mlx5_glue->destroy_wq(wq));
+	if (cq)
+		claim_zero(mlx5_glue->destroy_cq(cq));
+	return NULL;
+}
+
+/**
+ * Release a drop Rx queue Verbs object.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+void
+mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_ibv *rxq = priv->drop_queue.rxq;
+
+	if (rxq->wq)
+		claim_zero(mlx5_glue->destroy_wq(rxq->wq));
+	if (rxq->cq)
+		claim_zero(mlx5_glue->destroy_cq(rxq->cq));
+	rte_free(rxq);
+	priv->drop_queue.rxq = NULL;
+}
+
+/**
+ * Create a drop indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_ind_table_ibv *
+mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	struct mlx5_rxq_ibv *rxq;
+	struct mlx5_ind_table_ibv tmpl;
+
+	rxq = mlx5_rxq_ibv_drop_new(dev);
+	if (!rxq)
+		return NULL;
+	tmpl.ind_table = mlx5_glue->create_rwq_ind_table
+		(priv->ctx,
+		 &(struct ibv_rwq_ind_table_init_attr){
+			.log_ind_tbl_size = 0,
+			.ind_tbl = &rxq->wq,
+			.comp_mask = 0,
+		 });
+	if (!tmpl.ind_table) {
+		DEBUG("port %u cannot allocate indirection table for drop"
+		      " queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	ind_tbl = rte_calloc(__func__, 1, sizeof(*ind_tbl), 0);
+	if (!ind_tbl) {
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	ind_tbl->ind_table = tmpl.ind_table;
+	return ind_tbl;
+error:
+	mlx5_rxq_ibv_drop_release(dev);
+	return NULL;
+}
+
+/**
+ * Release a drop indirection table.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl = priv->drop_queue.hrxq->ind_table;
+
+	claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl->ind_table));
+	mlx5_rxq_ibv_drop_release(dev);
+	rte_free(ind_tbl);
+	priv->drop_queue.hrxq->ind_table = NULL;
+}
+
+/**
+ * Create a drop Rx Hash queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   The Verbs object initialised, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_hrxq *
+mlx5_hrxq_drop_new(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_ind_table_ibv *ind_tbl;
+	struct ibv_qp *qp;
+	struct mlx5_hrxq *hrxq;
+
+	if (priv->drop_queue.hrxq) {
+		rte_atomic32_inc(&priv->drop_queue.hrxq->refcnt);
+		return priv->drop_queue.hrxq;
+	}
+	ind_tbl = mlx5_ind_table_ibv_drop_new(dev);
+	if (!ind_tbl)
+		return NULL;
+	qp = mlx5_glue->create_qp_ex(priv->ctx,
+		 &(struct ibv_qp_init_attr_ex){
+			.qp_type = IBV_QPT_RAW_PACKET,
+			.comp_mask =
+				IBV_QP_INIT_ATTR_PD |
+				IBV_QP_INIT_ATTR_IND_TABLE |
+				IBV_QP_INIT_ATTR_RX_HASH,
+			.rx_hash_conf = (struct ibv_rx_hash_conf){
+				.rx_hash_function =
+					IBV_RX_HASH_FUNC_TOEPLITZ,
+				.rx_hash_key_len = rss_hash_default_key_len,
+				.rx_hash_key = rss_hash_default_key,
+				.rx_hash_fields_mask = 0,
+				},
+			.rwq_ind_tbl = ind_tbl->ind_table,
+			.pd = priv->pd
+		 });
+	if (!qp) {
+		DEBUG("port %u cannot allocate QP for drop queue",
+		      dev->data->port_id);
+		rte_errno = errno;
+		goto error;
+	}
+	hrxq = rte_calloc(__func__, 1, sizeof(*hrxq), 0);
+	if (!hrxq) {
+		DRV_LOG(WARNING,
+			"port %u cannot allocate memory for drop queue",
+			dev->data->port_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	hrxq->ind_table = ind_tbl;
+	hrxq->qp = qp;
+	priv->drop_queue.hrxq = hrxq;
+	rte_atomic32_set(&hrxq->refcnt, 1);
+	return hrxq;
+error:
+	if (ind_tbl)
+		mlx5_ind_table_ibv_drop_release(dev);
+	return NULL;
+}
+
+/**
+ * Release a drop hash Rx queue.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+void
+mlx5_hrxq_drop_release(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq;
+
+	if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
+		claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
+		mlx5_ind_table_ibv_drop_release(dev);
+		rte_free(hrxq);
+		priv->drop_queue.hrxq = NULL;
+	}
+}
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 97b4d9eb6..708fdd4fa 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -244,6 +244,8 @@ struct mlx5_rxq_ibv *mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
 struct mlx5_rxq_ibv *mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
 int mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv);
 int mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv);
+struct mlx5_rxq_ibv *mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev);
+void mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev);
 int mlx5_rxq_ibv_verify(struct rte_eth_dev *dev);
 struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx,
 				   uint16_t desc, unsigned int socket,
@@ -264,6 +266,8 @@ struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_get(struct rte_eth_dev *dev,
 int mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
 			       struct mlx5_ind_table_ibv *ind_tbl);
 int mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev);
+void mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev);
 struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev,
 				const uint8_t *rss_key, uint32_t rss_key_len,
 				uint64_t hash_fields,
@@ -276,6 +280,8 @@ struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev,
 				uint32_t tunnel, uint32_t rss_level);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
 int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev);
+void mlx5_hrxq_drop_release(struct rte_eth_dev *dev);
 uint64_t mlx5_get_rx_port_offloads(void);
 uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
 
-- 
2.18.0

[dpdk-dev] [PATCH v4 03/21] net/mlx5: replace verbs priorities by flow

[Nelio Laranjeiro]

Previous work introduce verbs priorities, whereas the PMD is making
translation between Flow priority into Verbs.  Rename this to make more
sense on what the PMD has to translate.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5.c      | 15 ++++-----------
 drivers/net/mlx5/mlx5.h      |  4 ++--
 drivers/net/mlx5/mlx5_flow.c | 24 ++++++++++++------------
 3 files changed, 18 insertions(+), 25 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index e9780ac8f..74248f098 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -717,7 +717,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	unsigned int tunnel_en = 0;
 	unsigned int mpls_en = 0;
 	unsigned int swp = 0;
-	unsigned int verb_priorities = 0;
 	unsigned int mprq = 0;
 	unsigned int mprq_min_stride_size_n = 0;
 	unsigned int mprq_max_stride_size_n = 0;
@@ -1139,16 +1138,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	/* Store device configuration on private structure. */
 	priv->config = config;
 	/* Supported Verbs flow priority number detection. */
-	if (verb_priorities == 0) {
-		err = mlx5_verbs_max_prio(eth_dev);
-		if (err < 0) {
-			DRV_LOG(ERR, "port %u wrong Verbs flow priorities",
-				eth_dev->data->port_id);
-			goto error;
-		}
-		verb_priorities = err;
-	}
-	priv->config.max_verbs_prio = verb_priorities;
+	err = mlx5_flow_discover_priorities(eth_dev);
+	if (err < 0)
+		goto error;
+	priv->config.flow_prio = err;
 	/*
 	 * Once the device is added to the list of memory event
 	 * callback, its global MR cache table cannot be expanded
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 227429848..9949cd3fa 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -122,7 +122,7 @@ struct mlx5_dev_config {
 		unsigned int min_rxqs_num;
 		/* Rx queue count threshold to enable MPRQ. */
 	} mprq; /* Configurations for Multi-Packet RQ. */
-	unsigned int max_verbs_prio; /* Number of Verb flow priorities. */
+	unsigned int flow_prio; /* Number of flow priorities. */
 	unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
 	unsigned int ind_table_max_size; /* Maximum indirection table size. */
 	int txq_inline; /* Maximum packet size for inlining. */
@@ -317,7 +317,7 @@ int mlx5_traffic_restart(struct rte_eth_dev *dev);
 
 /* mlx5_flow.c */
 
-int mlx5_verbs_max_prio(struct rte_eth_dev *dev);
+int mlx5_flow_discover_priorities(struct rte_eth_dev *dev);
 void mlx5_flow_print(struct rte_flow *flow);
 int mlx5_flow_validate(struct rte_eth_dev *dev,
 		       const struct rte_flow_attr *attr,
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 5e325be37..8fdc6d7bb 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -76,17 +76,17 @@ struct ibv_spec_header {
 };
 
  /**
-  * Get the maximum number of priority available.
+  * Discover the maximum number of priority available.
   *
   * @param[in] dev
   *   Pointer to Ethernet device.
   *
   * @return
-  *   number of supported Verbs flow priority on success, a negative errno
-  *   value otherwise and rte_errno is set.
+  *   number of supported flow priority on success, a negative errno value
+  *   otherwise and rte_errno is set.
   */
 int
-mlx5_verbs_max_prio(struct rte_eth_dev *dev)
+mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 {
 	struct {
 		struct ibv_flow_attr attr;
@@ -106,25 +106,25 @@ mlx5_verbs_max_prio(struct rte_eth_dev *dev)
 		},
 	};
 	struct ibv_flow *flow;
-	uint32_t verb_priorities;
 	struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
+	uint16_t vprio[] = { 8, 16 };
+	int i;
 
 	if (!drop) {
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	for (verb_priorities = 0; 1; verb_priorities++) {
-		flow_attr.attr.priority = verb_priorities;
-		flow = mlx5_glue->create_flow(drop->qp,
-					      &flow_attr.attr);
+	for (i = 0; i != RTE_DIM(vprio); i++) {
+		flow_attr.attr.priority = vprio[i] - 1;
+		flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
 		if (!flow)
 			break;
 		claim_zero(mlx5_glue->destroy_flow(flow));
 	}
 	mlx5_hrxq_drop_release(dev);
 	DRV_LOG(INFO, "port %u flow maximum priority: %d",
-		dev->data->port_id, verb_priorities);
-	return verb_priorities;
+		dev->data->port_id, vprio[i - 1]);
+	return vprio[i - 1];
 }
 
 /**
@@ -318,7 +318,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
-		.priority = priv->config.max_verbs_prio - 1,
+		.priority = priv->config.flow_prio - 1,
 	};
 	struct rte_flow_item items[] = {
 		{
-- 
2.18.0

[dpdk-dev] [PATCH v4 04/21] net/mlx5: support flow Ethernet item along with drop action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5.c      |   1 +
 drivers/net/mlx5/mlx5_flow.c | 664 +++++++++++++++++++++++++++++++++--
 2 files changed, 627 insertions(+), 38 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 74248f098..6d3421fae 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -242,6 +242,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	/* In case mlx5_dev_stop() has not been called. */
 	mlx5_dev_interrupt_handler_uninstall(dev);
 	mlx5_traffic_disable(dev);
+	mlx5_flow_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
 	dev->rx_pkt_burst = removed_rx_burst;
 	dev->tx_pkt_burst = removed_tx_burst;
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 8fdc6d7bb..036a8d440 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -35,11 +35,50 @@
 extern const struct eth_dev_ops mlx5_dev_ops;
 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
+/* Pattern Layer bits. */
+#define MLX5_FLOW_LAYER_OUTER_L2 (1u << 0)
+#define MLX5_FLOW_LAYER_OUTER_L3_IPV4 (1u << 1)
+#define MLX5_FLOW_LAYER_OUTER_L3_IPV6 (1u << 2)
+#define MLX5_FLOW_LAYER_OUTER_L4_UDP (1u << 3)
+#define MLX5_FLOW_LAYER_OUTER_L4_TCP (1u << 4)
+#define MLX5_FLOW_LAYER_OUTER_VLAN (1u << 5)
+/* Masks. */
+#define MLX5_FLOW_LAYER_OUTER_L3 \
+	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
+#define MLX5_FLOW_LAYER_OUTER_L4 \
+	(MLX5_FLOW_LAYER_OUTER_L4_UDP | MLX5_FLOW_LAYER_OUTER_L4_TCP)
+
+/* Actions that modify the fate of matching traffic. */
+#define MLX5_FLOW_FATE_DROP (1u << 0)
+
+/** Handles information leading to a drop fate. */
+struct mlx5_flow_verbs {
+	unsigned int size; /**< Size of the attribute. */
+	struct {
+		struct ibv_flow_attr *attr;
+		/**< Pointer to the Specification buffer. */
+		uint8_t *specs; /**< Pointer to the specifications. */
+	};
+	struct ibv_flow *flow; /**< Verbs flow pointer. */
+	struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */
+};
+
+/* Flow structure. */
 struct rte_flow {
 	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
+	struct rte_flow_attr attributes; /**< User flow attribute. */
+	uint32_t layers;
+	/**< Bit-fields of present layers see MLX5_FLOW_LAYER_*. */
+	uint32_t fate;
+	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
+	struct mlx5_flow_verbs verbs; /* Verbs drop flow. */
 };
 
 static const struct rte_flow_ops mlx5_flow_ops = {
+	.validate = mlx5_flow_validate,
+	.create = mlx5_flow_create,
+	.destroy = mlx5_flow_destroy,
+	.flush = mlx5_flow_flush,
 	.isolate = mlx5_flow_isolate,
 };
 
@@ -128,13 +167,415 @@ mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 }
 
 /**
- * Convert a flow.
+ * Verify the @p attributes will be correctly understood by the NIC and store
+ * them in the @p flow if everything is correct.
  *
- * @param dev
+ * @param[in] dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to a TAILQ flow list.
- * @param[in] attr
+ * @param[in] attributes
+ *   Pointer to flow attributes
+ * @param[in, out] flow
+ *   Pointer to the rte_flow structure.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_attributes(struct rte_eth_dev *dev,
+		     const struct rte_flow_attr *attributes,
+		     struct rte_flow *flow,
+		     struct rte_flow_error *error)
+{
+	uint32_t priority_max =
+		((struct priv *)dev->data->dev_private)->config.flow_prio;
+
+	if (attributes->group)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+					  NULL,
+					  "groups is not supported");
+	if (attributes->priority >= priority_max)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
+					  NULL,
+					  "priority out of range");
+	if (attributes->egress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
+					  NULL,
+					  "egress is not supported");
+	if (attributes->transfer)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
+					  NULL,
+					  "transfer is not supported");
+	if (!attributes->ingress)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+					  NULL,
+					  "ingress attribute is mandatory");
+	flow->attributes = *attributes;
+	return 0;
+}
+
+/**
+ * Verify the @p item specifications (spec, last, mask) are compatible with the
+ * NIC capabilities.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in] mask
+ *   @p item->mask or flow default bit-masks.
+ * @param[in] nic_mask
+ *   Bit-masks covering supported fields by the NIC to compare with user mask.
+ * @param[in] size
+ *   Bit-masks size in bytes.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_item_acceptable(const struct rte_flow_item *item,
+			  const uint8_t *mask,
+			  const uint8_t *nic_mask,
+			  unsigned int size,
+			  struct rte_flow_error *error)
+{
+	unsigned int i;
+
+	assert(nic_mask);
+	for (i = 0; i < size; ++i)
+		if ((nic_mask[i] | mask[i]) != nic_mask[i])
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "mask enables non supported"
+						  " bits");
+	if (!item->spec && (item->mask || item->last))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "mask/last without a spec is not"
+					  " supported");
+	if (item->spec && item->last) {
+		uint8_t spec[size];
+		uint8_t last[size];
+		unsigned int i;
+		int ret;
+
+		for (i = 0; i < size; ++i) {
+			spec[i] = ((const uint8_t *)item->spec)[i] & mask[i];
+			last[i] = ((const uint8_t *)item->last)[i] & mask[i];
+		}
+		ret = memcmp(spec, last, size);
+		if (ret != 0)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  item,
+						  "range is not supported");
+	}
+	return 0;
+}
+
+/**
+ * Add a verbs specification into @p flow.
+ *
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] src
+ *   Create specification.
+ * @param[in] size
+ *   Size in bytes of the specification to copy.
+ */
+static void
+mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
+{
+	if (flow->verbs.specs) {
+		void *dst;
+
+		dst = (void *)(flow->verbs.specs + flow->verbs.size);
+		memcpy(dst, src, size);
+		++flow->verbs.attr->num_of_specs;
+	}
+	flow->verbs.size += size;
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_eth *spec = item->spec;
+	const struct rte_flow_item_eth *mask = item->mask;
+	const struct rte_flow_item_eth nic_mask = {
+		.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
+		.type = RTE_BE16(0xffff),
+	};
+	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	struct ibv_flow_spec_eth eth = {
+		.type = IBV_FLOW_SPEC_ETH,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L2)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L2 layers already configured");
+	if (!mask)
+		mask = &rte_flow_item_eth_mask;
+	ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
+					(const uint8_t *)&nic_mask,
+					sizeof(struct rte_flow_item_eth),
+					error);
+	if (ret)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		unsigned int i;
+
+		memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
+		memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
+		eth.val.ether_type = spec->type;
+		memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
+		memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
+		eth.mask.ether_type = mask->type;
+		/* Remove unwanted bits from values. */
+		for (i = 0; i < ETHER_ADDR_LEN; ++i) {
+			eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
+			eth.val.src_mac[i] &= eth.mask.src_mac[i];
+		}
+		eth.val.ether_type &= eth.mask.ether_type;
+	}
+	mlx5_flow_spec_verbs_add(flow, &eth, size);
+	return size;
+}
+
+/**
+ * Convert the @p pattern into a Verbs specifications after ensuring the NIC
+ * will understand and process it correctly.
+ * The conversion is performed item per item, each of them is written into
+ * the @p flow if its size is lesser or equal to @p flow_size.
+ * Validation and memory consumption computation are still performed until the
+ * end of @p pattern, unless an error is encountered.
+ *
+ * @param[in] pattern
+ *   Flow pattern.
+ * @param[in, out] flow
+ *   Pointer to the rte_flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small some
+ *   garbage may be present.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @pattern  has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_items(const struct rte_flow_item pattern[],
+		struct rte_flow *flow, const size_t flow_size,
+		struct rte_flow_error *error)
+{
+	int remain = flow_size;
+	size_t size = 0;
+
+	for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
+		int ret = 0;
+
+		switch (pattern->type) {
+		case RTE_FLOW_ITEM_TYPE_VOID:
+			break;
+		case RTE_FLOW_ITEM_TYPE_ETH:
+			ret = mlx5_flow_item_eth(pattern, flow, remain, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ITEM,
+						  pattern,
+						  "item not supported");
+		}
+		if (ret < 0)
+			return ret;
+		if (remain > ret)
+			remain -= ret;
+		else
+			remain = 0;
+		size += ret;
+	}
+	if (!flow->layers) {
+		const struct rte_flow_item item = {
+			.type = RTE_FLOW_ITEM_TYPE_ETH,
+		};
+
+		return mlx5_flow_item_eth(&item, flow, flow_size, error);
+	}
+	return size;
+}
+
+/**
+ * Convert the @p action into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_drop(const struct rte_flow_action *action,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
+{
+	unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
+	struct ibv_flow_spec_action_drop drop = {
+			.type = IBV_FLOW_SPEC_ACTION_DROP,
+			.size = size,
+	};
+
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "multiple fate actions are not"
+					  " supported");
+	if (size < flow_size)
+		mlx5_flow_spec_verbs_add(flow, &drop, size);
+	flow->fate |= MLX5_FLOW_FATE_DROP;
+	return size;
+}
+
+/**
+ * Convert the @p action into @p flow after ensuring the NIC will understand
+ * and process it correctly.
+ * The conversion is performed action per action, each of them is written into
+ * the @p flow if its size is lesser or equal to @p flow_size.
+ * Validation and memory consumption computation are still performed until the
+ * end of @p action, unless an error is encountered.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device structure.
+ * @param[in] actions
+ *   Pointer to flow actions array.
+ * @param[in, out] flow
+ *   Pointer to the rte_flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small some
+ *   garbage may be present.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p actions has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_actions(struct rte_eth_dev *dev __rte_unused,
+		  const struct rte_flow_action actions[],
+		  struct rte_flow *flow, const size_t flow_size,
+		  struct rte_flow_error *error)
+{
+	size_t size = 0;
+	int remain = flow_size;
+	int ret = 0;
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_DROP:
+			ret = mlx5_flow_action_drop(actions, flow, remain,
+						    error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+		if (ret < 0)
+			return ret;
+		if (remain > ret)
+			remain -= ret;
+		else
+			remain = 0;
+		size += ret;
+	}
+	if (!flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "no fate action found");
+	return size;
+}
+
+/**
+ * Convert the @p attributes, @p pattern, @p action, into an flow for the NIC
+ * after ensuring the NIC will understand and process it correctly.
+ * The conversion is only performed item/action per item/action, each of
+ * them is written into the @p flow if its size is lesser or equal to @p
+ * flow_size.
+ * Validation and memory consumption computation are still performed until the
+ * end, unless an error is encountered.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small some
+ *   garbage may be present.
+ * @param[in] attributes
  *   Flow rule attributes.
  * @param[in] pattern
  *   Pattern specification (list terminated by the END pattern item).
@@ -144,21 +585,42 @@ mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
  *   Perform verbose error reporting if not NULL.
  *
  * @return
- *   A flow on success, NULL otherwise and rte_errno is set.
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the flow has fully been converted and
+ *   can be applied, otherwise another call with this returned memory size
+ *   should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
  */
-static struct rte_flow *
-mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
-		      struct mlx5_flows *list __rte_unused,
-		      const struct rte_flow_attr *attr __rte_unused,
-		      const struct rte_flow_item items[] __rte_unused,
-		      const struct rte_flow_action actions[] __rte_unused,
-		      struct rte_flow_error *error)
+static int
+mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
+		const size_t flow_size,
+		const struct rte_flow_attr *attributes,
+		const struct rte_flow_item pattern[],
+		const struct rte_flow_action actions[],
+		struct rte_flow_error *error)
 {
-	rte_flow_error_set(error, ENOTSUP,
-			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL,
-			   "action not supported");
-	return NULL;
+	struct rte_flow local_flow = { .layers = 0, };
+	size_t size = sizeof(*flow) + sizeof(struct ibv_flow_attr);
+	int remain = (flow_size > size) ? flow_size - size : 0;
+	int ret;
+
+	if (!remain)
+		flow = &local_flow;
+	ret = mlx5_flow_attributes(dev, attributes, flow, error);
+	if (ret < 0)
+		return ret;
+	ret = mlx5_flow_items(pattern, flow, remain, error);
+	if (ret < 0)
+		return ret;
+	size += ret;
+	remain = (flow_size > size) ? flow_size - size : 0;
+	ret = mlx5_flow_actions(dev, actions, flow, remain, error);
+	if (ret < 0)
+		return ret;
+	size += ret;
+	if (size <= flow_size)
+		flow->verbs.attr->priority = flow->attributes.priority;
+	return size;
 }
 
 /**
@@ -168,16 +630,142 @@ mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
  * @see rte_flow_ops
  */
 int
-mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
-		   const struct rte_flow_attr *attr __rte_unused,
-		   const struct rte_flow_item items[] __rte_unused,
-		   const struct rte_flow_action actions[] __rte_unused,
+mlx5_flow_validate(struct rte_eth_dev *dev,
+		   const struct rte_flow_attr *attr,
+		   const struct rte_flow_item items[],
+		   const struct rte_flow_action actions[],
 		   struct rte_flow_error *error)
 {
-	return rte_flow_error_set(error, ENOTSUP,
-				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				  NULL,
-				  "action not supported");
+	int ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+
+	if (ret < 0)
+		return ret;
+	return 0;
+}
+
+/**
+ * Remove the flow.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ */
+static void
+mlx5_flow_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	if (flow->fate & MLX5_FLOW_FATE_DROP) {
+		if (flow->verbs.flow) {
+			claim_zero(mlx5_glue->destroy_flow(flow->verbs.flow));
+			flow->verbs.flow = NULL;
+		}
+	}
+	if (flow->verbs.hrxq) {
+		mlx5_hrxq_drop_release(dev);
+		flow->verbs.hrxq = NULL;
+	}
+}
+
+/**
+ * Apply the flow.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device structure.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
+		struct rte_flow_error *error)
+{
+	flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
+	if (!flow->verbs.hrxq)
+		return rte_flow_error_set
+			(error, errno,
+			 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+			 NULL,
+			 "cannot allocate Drop queue");
+	flow->verbs.flow =
+		mlx5_glue->create_flow(flow->verbs.hrxq->qp, flow->verbs.attr);
+	if (!flow->verbs.flow) {
+		mlx5_hrxq_drop_release(dev);
+		flow->verbs.hrxq = NULL;
+		return rte_flow_error_set(error, errno,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "kernel module refuses to create"
+					  " flow");
+	}
+	return 0;
+}
+
+/**
+ * Create a flow and add it to @p list.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param list
+ *   Pointer to a TAILQ flow list.
+ * @param[in] attr
+ *   Flow rule attributes.
+ * @param[in] items
+ *   Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ *   Associated actions (list terminated by the END action).
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ *
+ * @return
+ *   A flow on success, NULL otherwise and rte_errno is set.
+ */
+static struct rte_flow *
+mlx5_flow_list_create(struct rte_eth_dev *dev,
+		      struct mlx5_flows *list,
+		      const struct rte_flow_attr *attr,
+		      const struct rte_flow_item items[],
+		      const struct rte_flow_action actions[],
+		      struct rte_flow_error *error)
+{
+	struct rte_flow *flow;
+	size_t size;
+	int ret;
+
+	ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+	if (ret < 0)
+		return NULL;
+	size = ret;
+	flow = rte_zmalloc(__func__, size, 0);
+	if (!flow) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "cannot allocate memory");
+		return NULL;
+	}
+	flow->verbs.attr = (struct ibv_flow_attr *)(flow + 1);
+	flow->verbs.specs = (uint8_t *)(flow->verbs.attr + 1);
+	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
+	if (ret < 0)
+		goto error;
+	assert((size_t)ret == size);
+	if (dev->data->dev_started) {
+		ret = mlx5_flow_apply(dev, flow, error);
+		if (ret < 0)
+			goto error;
+	}
+	TAILQ_INSERT_TAIL(list, flow, next);
+	return flow;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	mlx5_flow_remove(dev, flow);
+	rte_free(flow);
+	rte_errno = ret; /* Restore rte_errno. */
+	return NULL;
 }
 
 /**
@@ -187,17 +775,15 @@ mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
  * @see rte_flow_ops
  */
 struct rte_flow *
-mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
-		 const struct rte_flow_attr *attr __rte_unused,
-		 const struct rte_flow_item items[] __rte_unused,
-		 const struct rte_flow_action actions[] __rte_unused,
+mlx5_flow_create(struct rte_eth_dev *dev,
+		 const struct rte_flow_attr *attr,
+		 const struct rte_flow_item items[],
+		 const struct rte_flow_action actions[],
 		 struct rte_flow_error *error)
 {
-	rte_flow_error_set(error, ENOTSUP,
-			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			   NULL,
-			   "action not supported");
-	return NULL;
+	return mlx5_flow_list_create
+		(dev, &((struct priv *)dev->data->dev_private)->flows,
+		 attr, items, actions, error);
 }
 
 /**
@@ -211,10 +797,12 @@ mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
  *   Flow to destroy.
  */
 static void
-mlx5_flow_list_destroy(struct rte_eth_dev *dev __rte_unused,
-		       struct mlx5_flows *list __rte_unused,
-		       struct rte_flow *flow __rte_unused)
+mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
+		       struct rte_flow *flow)
 {
+	mlx5_flow_remove(dev, flow);
+	TAILQ_REMOVE(list, flow, next);
+	rte_free(flow);
 }
 
 /**
-- 
2.18.0

[dpdk-dev] [PATCH v4 05/21] net/mlx5: add flow queue action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 97 ++++++++++++++++++++++++++++++++----
 1 file changed, 86 insertions(+), 11 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 036a8d440..6041a4573 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -50,6 +50,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
 /* Actions that modify the fate of matching traffic. */
 #define MLX5_FLOW_FATE_DROP (1u << 0)
+#define MLX5_FLOW_FATE_QUEUE (1u << 1)
 
 /** Handles information leading to a drop fate. */
 struct mlx5_flow_verbs {
@@ -71,7 +72,8 @@ struct rte_flow {
 	/**< Bit-fields of present layers see MLX5_FLOW_LAYER_*. */
 	uint32_t fate;
 	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
-	struct mlx5_flow_verbs verbs; /* Verbs drop flow. */
+	struct mlx5_flow_verbs verbs; /* Verbs flow. */
+	uint16_t queue; /**< Destination queue to redirect traffic to. */
 };
 
 static const struct rte_flow_ops mlx5_flow_ops = {
@@ -492,6 +494,52 @@ mlx5_flow_action_drop(const struct rte_flow_action *action,
 	return size;
 }
 
+/**
+ * Convert the @p action into @p flow after ensuring the NIC will understand
+ * and process it correctly.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device structure.
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_action_queue(struct rte_eth_dev *dev,
+		       const struct rte_flow_action *action,
+		       struct rte_flow *flow,
+		       struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_action_queue *queue = action->conf;
+
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "multiple fate actions are not"
+					  " supported");
+	if (queue->index >= priv->rxqs_n)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &queue->index,
+					  "queue index out of range");
+	if (!(*priv->rxqs)[queue->index])
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &queue->index,
+					  "queue is not configured");
+	flow->queue = queue->index;
+	flow->fate |= MLX5_FLOW_FATE_QUEUE;
+	return 0;
+}
+
 /**
  * Convert the @p action into @p flow after ensuring the NIC will understand
  * and process it correctly.
@@ -520,7 +568,7 @@ mlx5_flow_action_drop(const struct rte_flow_action *action,
  *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_actions(struct rte_eth_dev *dev __rte_unused,
+mlx5_flow_actions(struct rte_eth_dev *dev,
 		  const struct rte_flow_action actions[],
 		  struct rte_flow *flow, const size_t flow_size,
 		  struct rte_flow_error *error)
@@ -537,6 +585,9 @@ mlx5_flow_actions(struct rte_eth_dev *dev __rte_unused,
 			ret = mlx5_flow_action_drop(actions, flow, remain,
 						    error);
 			break;
+		case RTE_FLOW_ACTION_TYPE_QUEUE:
+			ret = mlx5_flow_action_queue(dev, actions, flow, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -661,7 +712,10 @@ mlx5_flow_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
 		}
 	}
 	if (flow->verbs.hrxq) {
-		mlx5_hrxq_drop_release(dev);
+		if (flow->fate & MLX5_FLOW_FATE_DROP)
+			mlx5_hrxq_drop_release(dev);
+		else if (flow->fate & MLX5_FLOW_FATE_QUEUE)
+			mlx5_hrxq_release(dev, flow->verbs.hrxq);
 		flow->verbs.hrxq = NULL;
 	}
 }
@@ -683,17 +737,38 @@ static int
 mlx5_flow_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 		struct rte_flow_error *error)
 {
-	flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
-	if (!flow->verbs.hrxq)
-		return rte_flow_error_set
-			(error, errno,
-			 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-			 NULL,
-			 "cannot allocate Drop queue");
+	if (flow->fate & MLX5_FLOW_FATE_DROP) {
+		flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
+		if (!flow->verbs.hrxq)
+			return rte_flow_error_set
+				(error, errno,
+				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				 NULL,
+				 "cannot allocate Drop queue");
+	} else if (flow->fate & MLX5_FLOW_FATE_QUEUE) {
+		struct mlx5_hrxq *hrxq;
+
+		hrxq = mlx5_hrxq_get(dev, rss_hash_default_key,
+				     rss_hash_default_key_len, 0,
+				     &flow->queue, 1, 0, 0);
+		if (!hrxq)
+			hrxq = mlx5_hrxq_new(dev, rss_hash_default_key,
+					     rss_hash_default_key_len, 0,
+					     &flow->queue, 1, 0, 0);
+		if (!hrxq)
+			return rte_flow_error_set(error, rte_errno,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					NULL,
+					"cannot create flow");
+		flow->verbs.hrxq = hrxq;
+	}
 	flow->verbs.flow =
 		mlx5_glue->create_flow(flow->verbs.hrxq->qp, flow->verbs.attr);
 	if (!flow->verbs.flow) {
-		mlx5_hrxq_drop_release(dev);
+		if (flow->fate & MLX5_FLOW_FATE_DROP)
+			mlx5_hrxq_drop_release(dev);
+		else
+			mlx5_hrxq_release(dev, flow->verbs.hrxq);
 		flow->verbs.hrxq = NULL;
 		return rte_flow_error_set(error, errno,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-- 
2.18.0

[dpdk-dev] [PATCH v4 06/21] net/mlx5: add flow stop/start

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 6041a4573..77f1bd5cc 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -908,9 +908,12 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
  *   Pointer to a TAILQ flow list.
  */
 void
-mlx5_flow_stop(struct rte_eth_dev *dev __rte_unused,
-	       struct mlx5_flows *list __rte_unused)
+mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
 {
+	struct rte_flow *flow;
+
+	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
+		mlx5_flow_remove(dev, flow);
 }
 
 /**
@@ -925,10 +928,23 @@ mlx5_flow_stop(struct rte_eth_dev *dev __rte_unused,
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_flow_start(struct rte_eth_dev *dev __rte_unused,
-		struct mlx5_flows *list __rte_unused)
+mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
 {
+	struct rte_flow *flow;
+	struct rte_flow_error error;
+	int ret = 0;
+
+	TAILQ_FOREACH(flow, list, next) {
+		ret = mlx5_flow_apply(dev, flow, &error);
+		if (ret < 0)
+			goto error;
+	}
 	return 0;
+error:
+	ret = rte_errno; /* Save rte_errno before cleanup. */
+	mlx5_flow_stop(dev, list);
+	rte_errno = ret; /* Restore rte_errno. */
+	return -rte_errno;
 }
 
 /**
-- 
2.18.0

[dpdk-dev] [PATCH v4 07/21] net/mlx5: add flow VLAN item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 127 +++++++++++++++++++++++++++++++++++
 1 file changed, 127 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 77f1bd5cc..659979283 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -382,6 +382,130 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Update the VLAN tag in the Verbs Ethernet specification.
+ *
+ * @param[in, out] attr
+ *   Pointer to Verbs attributes structure.
+ * @param[in] eth
+ *   Verbs structure containing the VLAN information to copy.
+ */
+static void
+mlx5_flow_item_vlan_update(struct ibv_flow_attr *attr,
+			   struct ibv_flow_spec_eth *eth)
+{
+	unsigned int i;
+	enum ibv_flow_spec_type search = IBV_FLOW_SPEC_ETH;
+	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
+		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
+
+	for (i = 0; i != attr->num_of_specs; ++i) {
+		if (hdr->type == search) {
+			struct ibv_flow_spec_eth *e =
+				(struct ibv_flow_spec_eth *)hdr;
+
+			e->val.vlan_tag = eth->val.vlan_tag;
+			e->mask.vlan_tag = eth->mask.vlan_tag;
+			e->val.ether_type = eth->val.ether_type;
+			e->mask.ether_type = eth->mask.ether_type;
+			break;
+		}
+		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
+	}
+}
+
+/**
+ * Convert the @p item into @p flow (or by updating the already present
+ * Ethernet Verbs) specification after ensuring the NIC will understand and
+ * process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_vlan *spec = item->spec;
+	const struct rte_flow_item_vlan *mask = item->mask;
+	const struct rte_flow_item_vlan nic_mask = {
+		.tci = RTE_BE16(0x0fff),
+		.inner_type = RTE_BE16(0xffff),
+	};
+	unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	struct ibv_flow_spec_eth eth = {
+		.type = IBV_FLOW_SPEC_ETH,
+		.size = size,
+	};
+	int ret;
+	const uint32_t l34m = MLX5_FLOW_LAYER_OUTER_L3 |
+			MLX5_FLOW_LAYER_OUTER_L4;
+	const uint32_t vlanm = MLX5_FLOW_LAYER_OUTER_VLAN;
+	const uint32_t l2m = MLX5_FLOW_LAYER_OUTER_L2;
+
+	if (flow->layers & vlanm)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VLAN layer already configured");
+	else if ((flow->layers & l34m) != 0)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L2 layer cannot follow L3/L4 layer");
+	if (!mask)
+		mask = &rte_flow_item_vlan_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&nic_mask,
+		 sizeof(struct rte_flow_item_vlan), error);
+	if (ret)
+		return ret;
+	if (spec) {
+		eth.val.vlan_tag = spec->tci;
+		eth.mask.vlan_tag = mask->tci;
+		eth.val.vlan_tag &= eth.mask.vlan_tag;
+		eth.val.ether_type = spec->inner_type;
+		eth.mask.ether_type = mask->inner_type;
+		eth.val.ether_type &= eth.mask.ether_type;
+	}
+	/*
+	 * From verbs perspective an empty VLAN is equivalent
+	 * to a packet without VLAN layer.
+	 */
+	if (!eth.mask.vlan_tag)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+					  item->spec,
+					  "VLAN cannot be empty");
+	if (!(flow->layers & l2m)) {
+		if (size <= flow_size)
+			mlx5_flow_spec_verbs_add(flow, &eth, size);
+	} else {
+		if (flow->verbs.attr)
+			mlx5_flow_item_vlan_update(flow->verbs.attr, &eth);
+		size = 0; /* Only an update is done in eth specification. */
+	}
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2 |
+		MLX5_FLOW_LAYER_OUTER_VLAN;
+	return size;
+}
+
 /**
  * Convert the @p pattern into a Verbs specifications after ensuring the NIC
  * will understand and process it correctly.
@@ -424,6 +548,9 @@ mlx5_flow_items(const struct rte_flow_item pattern[],
 		case RTE_FLOW_ITEM_TYPE_ETH:
 			ret = mlx5_flow_item_eth(pattern, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			ret = mlx5_flow_item_vlan(pattern, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

[dpdk-dev] [PATCH v4 08/21] net/mlx5: add flow IPv4 item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 90 ++++++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 659979283..c05b8498d 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -506,6 +506,93 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_ipv4 *spec = item->spec;
+	const struct rte_flow_item_ipv4 *mask = item->mask;
+	const struct rte_flow_item_ipv4 nic_mask = {
+		.hdr = {
+			.src_addr = RTE_BE32(0xffffffff),
+			.dst_addr = RTE_BE32(0xffffffff),
+			.type_of_service = 0xff,
+			.next_proto_id = 0xff,
+		},
+	};
+	unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
+	struct ibv_flow_spec_ipv4_ext ipv4 = {
+		.type = IBV_FLOW_SPEC_IPV4_EXT,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "multiple L3 layers not supported");
+	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 cannot follow an L4 layer.");
+	if (!mask)
+		mask = &rte_flow_item_ipv4_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&nic_mask,
+		 sizeof(struct rte_flow_item_ipv4), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
+			.src_ip = spec->hdr.src_addr,
+			.dst_ip = spec->hdr.dst_addr,
+			.proto = spec->hdr.next_proto_id,
+			.tos = spec->hdr.type_of_service,
+		};
+		ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
+			.src_ip = mask->hdr.src_addr,
+			.dst_ip = mask->hdr.dst_addr,
+			.proto = mask->hdr.next_proto_id,
+			.tos = mask->hdr.type_of_service,
+		};
+		/* Remove unwanted bits from values. */
+		ipv4.val.src_ip &= ipv4.mask.src_ip;
+		ipv4.val.dst_ip &= ipv4.mask.dst_ip;
+		ipv4.val.proto &= ipv4.mask.proto;
+		ipv4.val.tos &= ipv4.mask.tos;
+	}
+	mlx5_flow_spec_verbs_add(flow, &ipv4, size);
+	return size;
+}
+
 /**
  * Convert the @p pattern into a Verbs specifications after ensuring the NIC
  * will understand and process it correctly.
@@ -551,6 +638,9 @@ mlx5_flow_items(const struct rte_flow_item pattern[],
 		case RTE_FLOW_ITEM_TYPE_VLAN:
 			ret = mlx5_flow_item_vlan(pattern, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+			ret = mlx5_flow_item_ipv4(pattern, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

[dpdk-dev] [PATCH v4 09/21] net/mlx5: add flow IPv6 item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 115 +++++++++++++++++++++++++++++++++++
 1 file changed, 115 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index c05b8498d..513f70d40 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -593,6 +593,118 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
+		    const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_ipv6 *spec = item->spec;
+	const struct rte_flow_item_ipv6 *mask = item->mask;
+	const struct rte_flow_item_ipv6 nic_mask = {
+		.hdr = {
+			.src_addr =
+				"\xff\xff\xff\xff\xff\xff\xff\xff"
+				"\xff\xff\xff\xff\xff\xff\xff\xff",
+			.dst_addr =
+				"\xff\xff\xff\xff\xff\xff\xff\xff"
+				"\xff\xff\xff\xff\xff\xff\xff\xff",
+			.vtc_flow = RTE_BE32(0xffffffff),
+			.proto = 0xff,
+			.hop_limits = 0xff,
+		},
+	};
+	unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
+	struct ibv_flow_spec_ipv6 ipv6 = {
+		.type = IBV_FLOW_SPEC_IPV6,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "multiple L3 layers not supported");
+	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 cannot follow an L4 layer.");
+	if (!mask)
+		mask = &rte_flow_item_ipv6_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&nic_mask,
+		 sizeof(struct rte_flow_item_ipv6), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		unsigned int i;
+		uint32_t vtc_flow_val;
+		uint32_t vtc_flow_mask;
+
+		memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
+		       RTE_DIM(ipv6.val.src_ip));
+		memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
+		       RTE_DIM(ipv6.val.dst_ip));
+		memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
+		       RTE_DIM(ipv6.mask.src_ip));
+		memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
+		       RTE_DIM(ipv6.mask.dst_ip));
+		vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
+		vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
+		ipv6.val.flow_label =
+			rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
+					 IPV6_HDR_FL_SHIFT);
+		ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
+					 IPV6_HDR_TC_SHIFT;
+		ipv6.val.next_hdr = spec->hdr.proto;
+		ipv6.val.hop_limit = spec->hdr.hop_limits;
+		ipv6.mask.flow_label =
+			rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
+					 IPV6_HDR_FL_SHIFT);
+		ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
+					  IPV6_HDR_TC_SHIFT;
+		ipv6.mask.next_hdr = mask->hdr.proto;
+		ipv6.mask.hop_limit = mask->hdr.hop_limits;
+		/* Remove unwanted bits from values. */
+		for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
+			ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
+			ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
+		}
+		ipv6.val.flow_label &= ipv6.mask.flow_label;
+		ipv6.val.traffic_class &= ipv6.mask.traffic_class;
+		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
+		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
+	}
+	mlx5_flow_spec_verbs_add(flow, &ipv6, size);
+	return size;
+}
+
 /**
  * Convert the @p pattern into a Verbs specifications after ensuring the NIC
  * will understand and process it correctly.
@@ -641,6 +753,9 @@ mlx5_flow_items(const struct rte_flow_item pattern[],
 		case RTE_FLOW_ITEM_TYPE_IPV4:
 			ret = mlx5_flow_item_ipv4(pattern, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+			ret = mlx5_flow_item_ipv6(pattern, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

[dpdk-dev] [PATCH v4 10/21] net/mlx5: add flow UDP item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 97 +++++++++++++++++++++++++++++++++---
 1 file changed, 91 insertions(+), 6 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 513f70d40..0096ed8a2 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -52,6 +52,9 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_FLOW_FATE_DROP (1u << 0)
 #define MLX5_FLOW_FATE_QUEUE (1u << 1)
 
+/* possible L3 layers protocols filtering. */
+#define MLX5_IP_PROTOCOL_UDP 17
+
 /** Handles information leading to a drop fate. */
 struct mlx5_flow_verbs {
 	unsigned int size; /**< Size of the attribute. */
@@ -68,10 +71,12 @@ struct mlx5_flow_verbs {
 struct rte_flow {
 	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
 	struct rte_flow_attr attributes; /**< User flow attribute. */
+	uint32_t l3_protocol_en:1; /**< Protocol filtering requested. */
 	uint32_t layers;
 	/**< Bit-fields of present layers see MLX5_FLOW_LAYER_*. */
 	uint32_t fate;
 	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
+	uint8_t l3_protocol; /**< valid when l3_protocol_en is set. */
 	struct mlx5_flow_verbs verbs; /* Verbs flow. */
 	uint16_t queue; /**< Destination queue to redirect traffic to. */
 };
@@ -568,8 +573,6 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 	if (ret < 0)
 		return ret;
 	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
-	if (size > flow_size)
-		return size;
 	if (spec) {
 		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 			.src_ip = spec->hdr.src_addr,
@@ -589,7 +592,10 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 		ipv4.val.proto &= ipv4.mask.proto;
 		ipv4.val.tos &= ipv4.mask.tos;
 	}
-	mlx5_flow_spec_verbs_add(flow, &ipv4, size);
+	flow->l3_protocol_en = !!ipv4.mask.proto;
+	flow->l3_protocol = ipv4.val.proto;
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &ipv4, size);
 	return size;
 }
 
@@ -660,8 +666,6 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 	if (ret < 0)
 		return ret;
 	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
-	if (size > flow_size)
-		return size;
 	if (spec) {
 		unsigned int i;
 		uint32_t vtc_flow_val;
@@ -701,7 +705,85 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 		ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 		ipv6.val.hop_limit &= ipv6.mask.hop_limit;
 	}
-	mlx5_flow_spec_verbs_add(flow, &ipv6, size);
+	flow->l3_protocol_en = !!ipv6.mask.next_hdr;
+	flow->l3_protocol = ipv6.val.next_hdr;
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &ipv6, size);
+	return size;
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_udp *spec = item->spec;
+	const struct rte_flow_item_udp *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
+	struct ibv_flow_spec_tcp_udp udp = {
+		.type = IBV_FLOW_SPEC_UDP,
+		.size = size,
+	};
+	int ret;
+
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 is mandatory to filter on L4");
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L4 layer is already present");
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_UDP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with UDP layer");
+	if (!mask)
+		mask = &rte_flow_item_udp_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_udp_mask,
+		 sizeof(struct rte_flow_item_udp), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		udp.val.dst_port = spec->hdr.dst_port;
+		udp.val.src_port = spec->hdr.src_port;
+		udp.mask.dst_port = mask->hdr.dst_port;
+		udp.mask.src_port = mask->hdr.src_port;
+		/* Remove unwanted bits from values. */
+		udp.val.src_port &= udp.mask.src_port;
+		udp.val.dst_port &= udp.mask.dst_port;
+	}
+	mlx5_flow_spec_verbs_add(flow, &udp, size);
 	return size;
 }
 
@@ -756,6 +838,9 @@ mlx5_flow_items(const struct rte_flow_item pattern[],
 		case RTE_FLOW_ITEM_TYPE_IPV6:
 			ret = mlx5_flow_item_ipv6(pattern, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_UDP:
+			ret = mlx5_flow_item_udp(pattern, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

[dpdk-dev] [PATCH v4 11/21] net/mlx5: add flow TCP item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 79 ++++++++++++++++++++++++++++++++++++
 1 file changed, 79 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 0096ed8a2..f646eee01 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -53,6 +53,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_FLOW_FATE_QUEUE (1u << 1)
 
 /* possible L3 layers protocols filtering. */
+#define MLX5_IP_PROTOCOL_TCP 6
 #define MLX5_IP_PROTOCOL_UDP 17
 
 /** Handles information leading to a drop fate. */
@@ -787,6 +788,81 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
+		   const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_tcp *spec = item->spec;
+	const struct rte_flow_item_tcp *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
+	struct ibv_flow_spec_tcp_udp tcp = {
+		.type = IBV_FLOW_SPEC_TCP,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_TCP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with TCP layer");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 is mandatory to filter on L4");
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L4 layer is already present");
+	if (!mask)
+		mask = &rte_flow_item_tcp_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_tcp_mask,
+		 sizeof(struct rte_flow_item_tcp), error);
+	if (ret < 0)
+		return ret;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
+	if (size > flow_size)
+		return size;
+	if (spec) {
+		tcp.val.dst_port = spec->hdr.dst_port;
+		tcp.val.src_port = spec->hdr.src_port;
+		tcp.mask.dst_port = mask->hdr.dst_port;
+		tcp.mask.src_port = mask->hdr.src_port;
+		/* Remove unwanted bits from values. */
+		tcp.val.src_port &= tcp.mask.src_port;
+		tcp.val.dst_port &= tcp.mask.dst_port;
+	}
+	mlx5_flow_spec_verbs_add(flow, &tcp, size);
+	return size;
+}
+
 /**
  * Convert the @p pattern into a Verbs specifications after ensuring the NIC
  * will understand and process it correctly.
@@ -841,6 +917,9 @@ mlx5_flow_items(const struct rte_flow_item pattern[],
 		case RTE_FLOW_ITEM_TYPE_UDP:
 			ret = mlx5_flow_item_udp(pattern, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_TCP:
+			ret = mlx5_flow_item_tcp(pattern, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
-- 
2.18.0

[dpdk-dev] [PATCH v4 12/21] net/mlx5: add mark/flag flow action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 252 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_rxtx.h |   1 +
 2 files changed, 253 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index f646eee01..1280db486 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -52,6 +52,10 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_FLOW_FATE_DROP (1u << 0)
 #define MLX5_FLOW_FATE_QUEUE (1u << 1)
 
+/* Modify a packet. */
+#define MLX5_FLOW_MOD_FLAG (1u << 0)
+#define MLX5_FLOW_MOD_MARK (1u << 1)
+
 /* possible L3 layers protocols filtering. */
 #define MLX5_IP_PROTOCOL_TCP 6
 #define MLX5_IP_PROTOCOL_UDP 17
@@ -75,6 +79,8 @@ struct rte_flow {
 	uint32_t l3_protocol_en:1; /**< Protocol filtering requested. */
 	uint32_t layers;
 	/**< Bit-fields of present layers see MLX5_FLOW_LAYER_*. */
+	uint32_t modifier;
+	/**< Bit-fields of present modifier see MLX5_FLOW_MOD_*. */
 	uint32_t fate;
 	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
 	uint8_t l3_protocol; /**< valid when l3_protocol_en is set. */
@@ -984,6 +990,12 @@ mlx5_flow_action_drop(const struct rte_flow_action *action,
 					  action,
 					  "multiple fate actions are not"
 					  " supported");
+	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "drop is not compatible with"
+					  " flag/mark action");
 	if (size < flow_size)
 		mlx5_flow_spec_verbs_add(flow, &drop, size);
 	flow->fate |= MLX5_FLOW_FATE_DROP;
@@ -1036,6 +1048,161 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/**
+ * Convert the @p action into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_flag(const struct rte_flow_action *action,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
+{
+	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
+	struct ibv_flow_spec_action_tag tag = {
+		.type = IBV_FLOW_SPEC_ACTION_TAG,
+		.size = size,
+		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
+	};
+
+	if (flow->modifier & MLX5_FLOW_MOD_FLAG)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "flag action already present");
+	if (flow->fate & MLX5_FLOW_FATE_DROP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "flag is not compatible with drop"
+					  " action");
+	if (flow->modifier & MLX5_FLOW_MOD_MARK)
+		return 0;
+	flow->modifier |= MLX5_FLOW_MOD_FLAG;
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &tag, size);
+	return size;
+}
+
+/**
+ * Update verbs specification to modify the flag to mark.
+ *
+ * @param[in, out] flow
+ *   Pointer to the rte_flow structure.
+ * @param[in] mark_id
+ *   Mark identifier to replace the flag.
+ */
+static void
+mlx5_flow_verbs_mark_update(struct rte_flow *flow, uint32_t mark_id)
+{
+	struct ibv_spec_header *hdr;
+	int i;
+
+	/* Update Verbs specification. */
+	hdr = (struct ibv_spec_header *)flow->verbs.specs;
+	if (!hdr)
+		return;
+	for (i = 0; i != flow->verbs.attr->num_of_specs; ++i) {
+		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
+			struct ibv_flow_spec_action_tag *t =
+				(struct ibv_flow_spec_action_tag *)hdr;
+
+			t->tag_id = mlx5_flow_mark_set(mark_id);
+		}
+		hdr = (struct ibv_spec_header *)((uintptr_t)hdr + hdr->size);
+	}
+}
+
+/**
+ * Convert the @p action into @p flow (or by updating the already present
+ * Flag Verbs specification) after ensuring the NIC will understand and
+ * process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] action
+ *   Action configuration.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_mark(const struct rte_flow_action *action,
+		      struct rte_flow *flow, const size_t flow_size,
+		      struct rte_flow_error *error)
+{
+	const struct rte_flow_action_mark *mark = action->conf;
+	unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
+	struct ibv_flow_spec_action_tag tag = {
+		.type = IBV_FLOW_SPEC_ACTION_TAG,
+		.size = size,
+	};
+
+	if (!mark)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "configuration cannot be null");
+	if (mark->id >= MLX5_FLOW_MARK_MAX)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &mark->id,
+					  "mark id must in 0 <= id < "
+					  RTE_STR(MLX5_FLOW_MARK_MAX));
+	if (flow->modifier & MLX5_FLOW_MOD_MARK)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "mark action already present");
+	if (flow->fate & MLX5_FLOW_FATE_DROP)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "mark is not compatible with drop"
+					  " action");
+	if (flow->modifier & MLX5_FLOW_MOD_FLAG) {
+		mlx5_flow_verbs_mark_update(flow, mark->id);
+		size = 0; /**< Only an update is done in the specification. */
+	} else {
+		tag.tag_id = mlx5_flow_mark_set(mark->id);
+		if (size <= flow_size) {
+			tag.tag_id = mlx5_flow_mark_set(mark->id);
+			mlx5_flow_spec_verbs_add(flow, &tag, size);
+		}
+	}
+	flow->modifier |= MLX5_FLOW_MOD_MARK;
+	return size;
+}
+
 /**
  * Convert the @p action into @p flow after ensuring the NIC will understand
  * and process it correctly.
@@ -1077,6 +1244,14 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 		switch (actions->type) {
 		case RTE_FLOW_ACTION_TYPE_VOID:
 			break;
+		case RTE_FLOW_ACTION_TYPE_FLAG:
+			ret = mlx5_flow_action_flag(actions, flow, remain,
+						    error);
+			break;
+		case RTE_FLOW_ACTION_TYPE_MARK:
+			ret = mlx5_flow_action_mark(actions, flow, remain,
+						    error);
+			break;
 		case RTE_FLOW_ACTION_TYPE_DROP:
 			ret = mlx5_flow_action_drop(actions, flow, remain,
 						    error);
@@ -1170,6 +1345,79 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Mark the Rx queues mark flag if the flow has a mark or flag modifier.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in] flow
+ *   Pointer to flow structure.
+ */
+static void
+mlx5_flow_rxq_mark_set(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK)) {
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of((*priv->rxqs)[flow->queue],
+				     struct mlx5_rxq_ctrl, rxq);
+
+		rxq_ctrl->rxq.mark = 1;
+		rxq_ctrl->flow_mark_n++;
+	}
+}
+
+/**
+ * Clear the Rx queue mark associated with the @p flow if no other flow uses
+ * it with a mark request.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[in] flow
+ *   Pointer to the flow.
+ */
+static void
+mlx5_flow_rxq_mark_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct priv *priv = dev->data->dev_private;
+
+	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK)) {
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of((*priv->rxqs)[flow->queue],
+				     struct mlx5_rxq_ctrl, rxq);
+
+		rxq_ctrl->flow_mark_n--;
+		rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
+	}
+}
+
+/**
+ * Clear the mark bit in all Rx queues.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ */
+static void
+mlx5_flow_rxq_mark_clear(struct rte_eth_dev *dev)
+{
+	struct priv *priv = dev->data->dev_private;
+	unsigned int i;
+	unsigned int idx;
+
+	for (idx = 0, i = 0; idx != priv->rxqs_n; ++i) {
+		struct mlx5_rxq_ctrl *rxq_ctrl;
+
+		if (!(*priv->rxqs)[idx])
+			continue;
+		rxq_ctrl = container_of((*priv->rxqs)[idx],
+					struct mlx5_rxq_ctrl, rxq);
+		rxq_ctrl->flow_mark_n = 0;
+		rxq_ctrl->rxq.mark = 0;
+		++idx;
+	}
+}
+
 /**
  * Validate a flow supported by the NIC.
  *
@@ -1329,6 +1577,7 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
 		if (ret < 0)
 			goto error;
 	}
+	mlx5_flow_rxq_mark_set(dev, flow);
 	TAILQ_INSERT_TAIL(list, flow, next);
 	return flow;
 error:
@@ -1373,6 +1622,7 @@ mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
 {
 	mlx5_flow_remove(dev, flow);
 	TAILQ_REMOVE(list, flow, next);
+	mlx5_flow_rxq_mark_trim(dev, flow);
 	rte_free(flow);
 }
 
@@ -1410,6 +1660,7 @@ mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
 
 	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
 		mlx5_flow_remove(dev, flow);
+	mlx5_flow_rxq_mark_clear(dev);
 }
 
 /**
@@ -1434,6 +1685,7 @@ mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
 		ret = mlx5_flow_apply(dev, flow, &error);
 		if (ret < 0)
 			goto error;
+		mlx5_flow_rxq_mark_set(dev, flow);
 	}
 	return 0;
 error:
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 708fdd4fa..99b769d8a 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -138,6 +138,7 @@ struct mlx5_rxq_ctrl {
 	unsigned int socket; /* CPU socket ID for allocations. */
 	unsigned int irq:1; /* Whether IRQ is enabled. */
 	uint16_t idx; /* Queue index. */
+	uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */
 };
 
 /* Indirection table. */
-- 
2.18.0

[dpdk-dev] [PATCH v4 13/21] net/mlx5: use a macro for the RSS key size

[Nelio Laranjeiro]

ConnectX 4-5 support only 40 bytes of RSS key, using a compiled size
hash key is not necessary.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_ethdev.c | 14 +++++++-------
 drivers/net/mlx5/mlx5_flow.c   |  4 ++--
 drivers/net/mlx5/mlx5_prm.h    |  3 +++
 drivers/net/mlx5/mlx5_rss.c    |  7 ++++---
 drivers/net/mlx5/mlx5_rxq.c    | 12 +++++++-----
 drivers/net/mlx5/mlx5_rxtx.h   |  1 -
 6 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 05f66f7b6..6e44d5ff0 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -377,15 +377,15 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
 
 	if (use_app_rss_key &&
 	    (dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len !=
-	     rss_hash_default_key_len)) {
-		DRV_LOG(ERR, "port %u RSS key len must be %zu Bytes long",
-			dev->data->port_id, rss_hash_default_key_len);
+	     MLX5_RSS_HASH_KEY_LEN)) {
+		DRV_LOG(ERR, "port %u RSS key len must be %s Bytes long",
+			dev->data->port_id, RTE_STR(MLX5_RSS_HASH_KEY_LEN));
 		rte_errno = EINVAL;
 		return -rte_errno;
 	}
 	priv->rss_conf.rss_key =
 		rte_realloc(priv->rss_conf.rss_key,
-			    rss_hash_default_key_len, 0);
+			    MLX5_RSS_HASH_KEY_LEN, 0);
 	if (!priv->rss_conf.rss_key) {
 		DRV_LOG(ERR, "port %u cannot allocate RSS hash key memory (%u)",
 			dev->data->port_id, rxqs_n);
@@ -396,8 +396,8 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
 	       use_app_rss_key ?
 	       dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key :
 	       rss_hash_default_key,
-	       rss_hash_default_key_len);
-	priv->rss_conf.rss_key_len = rss_hash_default_key_len;
+	       MLX5_RSS_HASH_KEY_LEN);
+	priv->rss_conf.rss_key_len = MLX5_RSS_HASH_KEY_LEN;
 	priv->rss_conf.rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
 	priv->rxqs = (void *)dev->data->rx_queues;
 	priv->txqs = (void *)dev->data->tx_queues;
@@ -515,7 +515,7 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 		info->if_index = if_nametoindex(ifname);
 	info->reta_size = priv->reta_idx_n ?
 		priv->reta_idx_n : config->ind_table_max_size;
-	info->hash_key_size = rss_hash_default_key_len;
+	info->hash_key_size = MLX5_RSS_HASH_KEY_LEN;
 	info->speed_capa = priv->link_speed_capa;
 	info->flow_type_rss_offloads = ~MLX5_RSS_HF_MASK;
 	mlx5_set_default_params(dev, info);
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 1280db486..77483bd1f 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -1493,11 +1493,11 @@ mlx5_flow_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 		struct mlx5_hrxq *hrxq;
 
 		hrxq = mlx5_hrxq_get(dev, rss_hash_default_key,
-				     rss_hash_default_key_len, 0,
+				     MLX5_RSS_HASH_KEY_LEN, 0,
 				     &flow->queue, 1, 0, 0);
 		if (!hrxq)
 			hrxq = mlx5_hrxq_new(dev, rss_hash_default_key,
-					     rss_hash_default_key_len, 0,
+					     MLX5_RSS_HASH_KEY_LEN, 0,
 					     &flow->queue, 1, 0, 0);
 		if (!hrxq)
 			return rte_flow_error_set(error, rte_errno,
diff --git a/drivers/net/mlx5/mlx5_prm.h b/drivers/net/mlx5/mlx5_prm.h
index f9fae1e50..0870d32fd 100644
--- a/drivers/net/mlx5/mlx5_prm.h
+++ b/drivers/net/mlx5/mlx5_prm.h
@@ -21,6 +21,9 @@
 #include <rte_vect.h>
 #include "mlx5_autoconf.h"
 
+/* RSS hash key size. */
+#define MLX5_RSS_HASH_KEY_LEN 40
+
 /* Get CQE owner bit. */
 #define MLX5_CQE_OWNER(op_own) ((op_own) & MLX5_CQE_OWNER_MASK)
 
diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c
index d69b4c09e..b95778a8c 100644
--- a/drivers/net/mlx5/mlx5_rss.c
+++ b/drivers/net/mlx5/mlx5_rss.c
@@ -50,10 +50,11 @@ mlx5_rss_hash_update(struct rte_eth_dev *dev,
 		return -rte_errno;
 	}
 	if (rss_conf->rss_key && rss_conf->rss_key_len) {
-		if (rss_conf->rss_key_len != rss_hash_default_key_len) {
+		if (rss_conf->rss_key_len != MLX5_RSS_HASH_KEY_LEN) {
 			DRV_LOG(ERR,
-				"port %u RSS key len must be %zu Bytes long",
-				dev->data->port_id, rss_hash_default_key_len);
+				"port %u RSS key len must be %s Bytes long",
+				dev->data->port_id,
+				RTE_STR(MLX5_RSS_HASH_KEY_LEN));
 			rte_errno = EINVAL;
 			return -rte_errno;
 		}
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index d960daa43..d50b82c69 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -52,7 +52,9 @@ uint8_t rss_hash_default_key[] = {
 };
 
 /* Length of the default RSS hash key. */
-const size_t rss_hash_default_key_len = sizeof(rss_hash_default_key);
+static_assert(MLX5_RSS_HASH_KEY_LEN ==
+	      (unsigned int)sizeof(rss_hash_default_key),
+	      "wrong RSS default key size.");
 
 /**
  * Check whether Multi-Packet RQ can be enabled for the device.
@@ -1771,7 +1773,7 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 		return NULL;
 	}
 	if (!rss_key_len) {
-		rss_key_len = rss_hash_default_key_len;
+		rss_key_len = MLX5_RSS_HASH_KEY_LEN;
 		rss_key = rss_hash_default_key;
 	}
 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
@@ -1791,7 +1793,7 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 			.rx_hash_conf = (struct ibv_rx_hash_conf){
 				.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ,
 				.rx_hash_key_len = rss_key_len ? rss_key_len :
-						   rss_hash_default_key_len,
+						   MLX5_RSS_HASH_KEY_LEN,
 				.rx_hash_key = rss_key ?
 					       (void *)(uintptr_t)rss_key :
 					       rss_hash_default_key,
@@ -1815,7 +1817,7 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 			.rx_hash_conf = (struct ibv_rx_hash_conf){
 				.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ,
 				.rx_hash_key_len = rss_key_len ? rss_key_len :
-						   rss_hash_default_key_len,
+						   MLX5_RSS_HASH_KEY_LEN,
 				.rx_hash_key = rss_key ?
 					       (void *)(uintptr_t)rss_key :
 					       rss_hash_default_key,
@@ -2138,7 +2140,7 @@ mlx5_hrxq_drop_new(struct rte_eth_dev *dev)
 			.rx_hash_conf = (struct ibv_rx_hash_conf){
 				.rx_hash_function =
 					IBV_RX_HASH_FUNC_TOEPLITZ,
-				.rx_hash_key_len = rss_hash_default_key_len,
+				.rx_hash_key_len = MLX5_RSS_HASH_KEY_LEN,
 				.rx_hash_key = rss_hash_default_key,
 				.rx_hash_fields_mask = 0,
 				},
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 99b769d8a..59e374d8d 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -225,7 +225,6 @@ struct mlx5_txq_ctrl {
 /* mlx5_rxq.c */
 
 extern uint8_t rss_hash_default_key[];
-extern const size_t rss_hash_default_key_len;
 
 int mlx5_check_mprq_support(struct rte_eth_dev *dev);
 int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq);
-- 
2.18.0

[dpdk-dev] [PATCH v4 14/21] net/mlx5: add RSS flow action

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 686 +++++++++++++++++++++++++++--------
 1 file changed, 541 insertions(+), 145 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 77483bd1f..758c611a6 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -51,6 +51,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* Actions that modify the fate of matching traffic. */
 #define MLX5_FLOW_FATE_DROP (1u << 0)
 #define MLX5_FLOW_FATE_QUEUE (1u << 1)
+#define MLX5_FLOW_FATE_RSS (1u << 2)
 
 /* Modify a packet. */
 #define MLX5_FLOW_MOD_FLAG (1u << 0)
@@ -60,8 +61,68 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_IP_PROTOCOL_TCP 6
 #define MLX5_IP_PROTOCOL_UDP 17
 
+/* Priority reserved for default flows. */
+#define MLX5_FLOW_PRIO_RSVD ((uint32_t)-1)
+
+enum mlx5_expansion {
+	MLX5_EXPANSION_ROOT,
+	MLX5_EXPANSION_ETH,
+	MLX5_EXPANSION_IPV4,
+	MLX5_EXPANSION_IPV4_UDP,
+	MLX5_EXPANSION_IPV4_TCP,
+	MLX5_EXPANSION_IPV6,
+	MLX5_EXPANSION_IPV6_UDP,
+	MLX5_EXPANSION_IPV6_TCP,
+};
+
+/** Supported expansion of items. */
+static const struct rte_flow_expand_node mlx5_support_expansion[] = {
+	[MLX5_EXPANSION_ROOT] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
+						 MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_END,
+	},
+	[MLX5_EXPANSION_ETH] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+	},
+	[MLX5_EXPANSION_IPV4] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP,
+						 MLX5_EXPANSION_IPV4_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV4,
+		.rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+			ETH_RSS_NONFRAG_IPV4_OTHER,
+	},
+	[MLX5_EXPANSION_IPV4_UDP] = {
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
+	},
+	[MLX5_EXPANSION_IPV4_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
+	},
+	[MLX5_EXPANSION_IPV6] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP,
+						 MLX5_EXPANSION_IPV6_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV6,
+		.rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
+			ETH_RSS_NONFRAG_IPV6_OTHER,
+	},
+	[MLX5_EXPANSION_IPV6_UDP] = {
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
+	},
+	[MLX5_EXPANSION_IPV6_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
+	},
+};
+
 /** Handles information leading to a drop fate. */
 struct mlx5_flow_verbs {
+	LIST_ENTRY(mlx5_flow_verbs) next;
 	unsigned int size; /**< Size of the attribute. */
 	struct {
 		struct ibv_flow_attr *attr;
@@ -70,6 +131,7 @@ struct mlx5_flow_verbs {
 	};
 	struct ibv_flow *flow; /**< Verbs flow pointer. */
 	struct mlx5_hrxq *hrxq; /**< Hash Rx queue object. */
+	uint64_t hash_fields; /**< Verbs hash Rx queue hash fields. */
 };
 
 /* Flow structure. */
@@ -84,8 +146,12 @@ struct rte_flow {
 	uint32_t fate;
 	/**< Bit-fields of present fate see MLX5_FLOW_FATE_*. */
 	uint8_t l3_protocol; /**< valid when l3_protocol_en is set. */
-	struct mlx5_flow_verbs verbs; /* Verbs flow. */
-	uint16_t queue; /**< Destination queue to redirect traffic to. */
+	LIST_HEAD(verbs, mlx5_flow_verbs) verbs; /**< Verbs flows list. */
+	struct mlx5_flow_verbs *cur_verbs;
+	/**< Current Verbs flow structure being filled. */
+	struct rte_flow_action_rss rss;/**< RSS context. */
+	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
+	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
 };
 
 static const struct rte_flow_ops mlx5_flow_ops = {
@@ -128,16 +194,38 @@ struct ibv_spec_header {
 	uint16_t size;
 };
 
- /**
-  * Discover the maximum number of priority available.
-  *
-  * @param[in] dev
-  *   Pointer to Ethernet device.
-  *
-  * @return
-  *   number of supported flow priority on success, a negative errno value
-  *   otherwise and rte_errno is set.
-  */
+/*
+ * Number of sub priorities.
+ * For each kind of pattern matching i.e. L2, L3, L4 to have a correct
+ * matching on the NIC (firmware dependent) L4 most have the higher priority
+ * followed by L3 and ending with L2.
+ */
+#define MLX5_PRIORITY_MAP_L2 2
+#define MLX5_PRIORITY_MAP_L3 1
+#define MLX5_PRIORITY_MAP_L4 0
+#define MLX5_PRIORITY_MAP_MAX 3
+
+/* Map of Verbs to Flow priority with 8 Verbs priorities. */
+static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
+	{ 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
+};
+
+/* Map of Verbs to Flow priority with 16 Verbs priorities. */
+static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
+	{ 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
+	{ 9, 10, 11 }, { 12, 13, 14 },
+};
+
+/**
+ * Discover the maximum number of priority available.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   number of supported flow priority on success, a negative errno
+ *   value otherwise and rte_errno is set.
+ */
 int
 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 {
@@ -162,6 +250,7 @@ mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 	struct mlx5_hrxq *drop = mlx5_hrxq_drop_new(dev);
 	uint16_t vprio[] = { 8, 16 };
 	int i;
+	int priority = 0;
 
 	if (!drop) {
 		rte_errno = ENOTSUP;
@@ -173,11 +262,52 @@ mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
 		if (!flow)
 			break;
 		claim_zero(mlx5_glue->destroy_flow(flow));
+		priority = vprio[i];
+	}
+	switch (priority) {
+	case 8:
+		priority = RTE_DIM(priority_map_3);
+		break;
+	case 16:
+		priority = RTE_DIM(priority_map_5);
+		break;
+	default:
+		rte_errno = ENOTSUP;
+		DRV_LOG(ERR,
+			"port %u verbs maximum priority: %d expected 8/16",
+			dev->data->port_id, vprio[i]);
+		return -rte_errno;
 	}
 	mlx5_hrxq_drop_release(dev);
 	DRV_LOG(INFO, "port %u flow maximum priority: %d",
-		dev->data->port_id, vprio[i - 1]);
-	return vprio[i - 1];
+		dev->data->port_id, priority);
+	return priority;
+}
+
+/**
+ * Adjust flow priority.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param flow
+ *   Pointer to an rte flow.
+ */
+static void
+mlx5_flow_adjust_priority(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+	struct priv *priv = dev->data->dev_private;
+	uint32_t priority = flow->attributes.priority;
+	uint32_t subpriority = flow->cur_verbs->attr->priority;
+
+	switch (priv->config.flow_prio) {
+	case RTE_DIM(priority_map_3):
+		priority = priority_map_3[priority][subpriority];
+		break;
+	case RTE_DIM(priority_map_5):
+		priority = priority_map_5[priority][subpriority];
+		break;
+	}
+	flow->cur_verbs->attr->priority = priority;
 }
 
 /**
@@ -203,14 +333,15 @@ mlx5_flow_attributes(struct rte_eth_dev *dev,
 		     struct rte_flow_error *error)
 {
 	uint32_t priority_max =
-		((struct priv *)dev->data->dev_private)->config.flow_prio;
+		((struct priv *)dev->data->dev_private)->config.flow_prio - 1;
 
 	if (attributes->group)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
 					  NULL,
 					  "groups is not supported");
-	if (attributes->priority >= priority_max)
+	if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
+	    attributes->priority >= priority_max)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
 					  NULL,
@@ -231,6 +362,8 @@ mlx5_flow_attributes(struct rte_eth_dev *dev,
 					  NULL,
 					  "ingress attribute is mandatory");
 	flow->attributes = *attributes;
+	if (attributes->priority == MLX5_FLOW_PRIO_RSVD)
+		flow->attributes.priority = priority_max;
 	return 0;
 }
 
@@ -296,7 +429,7 @@ mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 }
 
 /**
- * Add a verbs specification into @p flow.
+ * Add a verbs item specification into @p flow.
  *
  * @param[in, out] flow
  *   Pointer to flow structure.
@@ -308,14 +441,16 @@ mlx5_flow_item_acceptable(const struct rte_flow_item *item,
 static void
 mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
 {
-	if (flow->verbs.specs) {
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+
+	if (verbs->specs) {
 		void *dst;
 
-		dst = (void *)(flow->verbs.specs + flow->verbs.size);
+		dst = (void *)(verbs->specs + verbs->size);
 		memcpy(dst, src, size);
-		++flow->verbs.attr->num_of_specs;
+		++verbs->attr->num_of_specs;
 	}
-	flow->verbs.size += size;
+	verbs->size += size;
 }
 
 /**
@@ -390,6 +525,7 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 		}
 		eth.val.ether_type &= eth.mask.ether_type;
 	}
+	flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
 	mlx5_flow_spec_verbs_add(flow, &eth, size);
 	return size;
 }
@@ -460,6 +596,7 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 		.inner_type = RTE_BE16(0xffff),
 	};
 	unsigned int size = sizeof(struct ibv_flow_spec_eth);
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 	struct ibv_flow_spec_eth eth = {
 		.type = IBV_FLOW_SPEC_ETH,
 		.size = size,
@@ -506,15 +643,16 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 					  item->spec,
 					  "VLAN cannot be empty");
 	if (!(flow->layers & l2m)) {
-		if (size <= flow_size)
+		if (size <= flow_size) {
+			flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
 			mlx5_flow_spec_verbs_add(flow, &eth, size);
+		}
 	} else {
-		if (flow->verbs.attr)
-			mlx5_flow_item_vlan_update(flow->verbs.attr, &eth);
+		if (verbs->attr)
+			mlx5_flow_item_vlan_update(verbs->attr, &eth);
 		size = 0; /* Only an update is done in eth specification. */
 	}
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2 |
-		MLX5_FLOW_LAYER_OUTER_VLAN;
+	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_VLAN;
 	return size;
 }
 
@@ -601,8 +739,18 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 	}
 	flow->l3_protocol_en = !!ipv4.mask.proto;
 	flow->l3_protocol = ipv4.val.proto;
-	if (size <= flow_size)
+	if (size <= flow_size) {
+		uint64_t hash_fields = IBV_RX_HASH_SRC_IPV4 |
+			IBV_RX_HASH_DST_IPV4;
+
+		if (!(flow->rss.types &
+		      (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+		       ETH_RSS_NONFRAG_IPV4_OTHER)))
+			hash_fields = 0;
+		flow->cur_verbs->hash_fields |= hash_fields;
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L3;
 		mlx5_flow_spec_verbs_add(flow, &ipv4, size);
+	}
 	return size;
 }
 
@@ -714,8 +862,17 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 	}
 	flow->l3_protocol_en = !!ipv6.mask.next_hdr;
 	flow->l3_protocol = ipv6.val.next_hdr;
-	if (size <= flow_size)
+	if (size <= flow_size) {
+		uint64_t hash_fields = IBV_RX_HASH_SRC_IPV6 |
+			IBV_RX_HASH_DST_IPV6;
+
+		if (!(flow->rss.types &
+		      (ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER)))
+			hash_fields = 0;
+		flow->cur_verbs->hash_fields |= hash_fields;
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L3;
 		mlx5_flow_spec_verbs_add(flow, &ipv6, size);
+	}
 	return size;
 }
 
@@ -754,22 +911,24 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 	};
 	int ret;
 
-	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_UDP)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
-					  "L3 is mandatory to filter on L4");
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+					  "protocol filtering not compatible"
+					  " with UDP layer");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
-					  "L4 layer is already present");
-	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_UDP)
+					  "L3 is mandatory to filter"
+					  " on L4");
+	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
-					  "protocol filtering not compatible"
-					  " with UDP layer");
+					  "L4 layer is already"
+					  " present");
 	if (!mask)
 		mask = &rte_flow_item_udp_mask;
 	ret = mlx5_flow_item_acceptable
@@ -779,8 +938,6 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 	if (ret < 0)
 		return ret;
 	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
-	if (size > flow_size)
-		return size;
 	if (spec) {
 		udp.val.dst_port = spec->hdr.dst_port;
 		udp.val.src_port = spec->hdr.src_port;
@@ -790,7 +947,16 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 		udp.val.src_port &= udp.mask.src_port;
 		udp.val.dst_port &= udp.mask.dst_port;
 	}
-	mlx5_flow_spec_verbs_add(flow, &udp, size);
+	if (size <= flow_size) {
+		uint64_t hash_fields = IBV_RX_HASH_SRC_PORT_UDP |
+			IBV_RX_HASH_DST_PORT_UDP;
+
+		if (!(flow->rss.types & ETH_RSS_UDP))
+			hash_fields = 0;
+		flow->cur_verbs->hash_fields |= hash_fields;
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L4;
+		mlx5_flow_spec_verbs_add(flow, &udp, size);
+	}
 	return size;
 }
 
@@ -854,8 +1020,6 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 	if (ret < 0)
 		return ret;
 	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
-	if (size > flow_size)
-		return size;
 	if (spec) {
 		tcp.val.dst_port = spec->hdr.dst_port;
 		tcp.val.src_port = spec->hdr.src_port;
@@ -865,7 +1029,16 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 		tcp.val.src_port &= tcp.mask.src_port;
 		tcp.val.dst_port &= tcp.mask.dst_port;
 	}
-	mlx5_flow_spec_verbs_add(flow, &tcp, size);
+	if (size <= flow_size) {
+		uint64_t hash_fields = IBV_RX_HASH_SRC_PORT_TCP |
+			IBV_RX_HASH_DST_PORT_TCP;
+
+		if (!(flow->rss.types & ETH_RSS_TCP))
+			hash_fields = 0;
+		flow->cur_verbs->hash_fields |= hash_fields;
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L4;
+		mlx5_flow_spec_verbs_add(flow, &tcp, size);
+	}
 	return size;
 }
 
@@ -1043,11 +1216,95 @@ mlx5_flow_action_queue(struct rte_eth_dev *dev,
 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 					  &queue->index,
 					  "queue is not configured");
-	flow->queue = queue->index;
+	if (flow->queue)
+		(*flow->queue)[0] = queue->index;
+	flow->rss.queue_num = 1;
 	flow->fate |= MLX5_FLOW_FATE_QUEUE;
 	return 0;
 }
 
+/**
+ * Ensure the @p action will be understood and used correctly by the  NIC.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param action[in]
+ *   Pointer to flow actions array.
+ * @param flow[in, out]
+ *   Pointer to the rte_flow structure.
+ * @param error[in, out]
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success @p flow->queue array and @p flow->rss are filled and valid.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_rss(struct rte_eth_dev *dev,
+		     const struct rte_flow_action *action,
+		     struct rte_flow *flow,
+		     struct rte_flow_error *error)
+{
+	struct priv *priv = dev->data->dev_private;
+	const struct rte_flow_action_rss *rss = action->conf;
+	unsigned int i;
+
+	if (flow->fate)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "multiple fate actions are not"
+					  " supported");
+	if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT &&
+	    rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->func,
+					  "RSS hash function not supported");
+	if (rss->level > 1)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->level,
+					  "tunnel RSS is not supported");
+	if (rss->key_len < MLX5_RSS_HASH_KEY_LEN)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->key_len,
+					  "RSS hash key too small");
+	if (rss->key_len > MLX5_RSS_HASH_KEY_LEN)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->key_len,
+					  "RSS hash key too large");
+	if (rss->queue_num > priv->config.ind_table_max_size)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->queue_num,
+					  "number of queues too large");
+	if (rss->types & MLX5_RSS_HF_MASK)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+					  &rss->types,
+					  "some RSS protocols are not"
+					  " supported");
+	for (i = 0; i != rss->queue_num; ++i) {
+		if (!(*priv->rxqs)[rss->queue[i]])
+			return rte_flow_error_set
+				(error, EINVAL,
+				 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
+				 &rss->queue[i],
+				 "queue is not configured");
+	}
+	if (flow->queue)
+		memcpy((*flow->queue), rss->queue,
+		       rss->queue_num * sizeof(uint16_t));
+	flow->rss.queue_num = rss->queue_num;
+	memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
+	flow->rss.types = rss->types;
+	flow->fate |= MLX5_FLOW_FATE_RSS;
+	return 0;
+}
+
 /**
  * Convert the @p action into a Verbs specification after ensuring the NIC
  * will understand and process it correctly.
@@ -1082,6 +1339,7 @@ mlx5_flow_action_flag(const struct rte_flow_action *action,
 		.size = size,
 		.tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
 	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 
 	if (flow->modifier & MLX5_FLOW_MOD_FLAG)
 		return rte_flow_error_set(error, ENOTSUP,
@@ -1095,32 +1353,34 @@ mlx5_flow_action_flag(const struct rte_flow_action *action,
 					  "flag is not compatible with drop"
 					  " action");
 	if (flow->modifier & MLX5_FLOW_MOD_MARK)
-		return 0;
-	flow->modifier |= MLX5_FLOW_MOD_FLAG;
-	if (size <= flow_size)
+		size = 0;
+	else if (size <= flow_size && verbs)
 		mlx5_flow_spec_verbs_add(flow, &tag, size);
+	flow->modifier |= MLX5_FLOW_MOD_FLAG;
 	return size;
 }
 
 /**
  * Update verbs specification to modify the flag to mark.
  *
- * @param[in, out] flow
- *   Pointer to the rte_flow structure.
+ * @param[in, out] verbs
+ *   Pointer to the mlx5_flow_verbs structure.
  * @param[in] mark_id
  *   Mark identifier to replace the flag.
  */
 static void
-mlx5_flow_verbs_mark_update(struct rte_flow *flow, uint32_t mark_id)
+mlx5_flow_verbs_mark_update(struct mlx5_flow_verbs *verbs, uint32_t mark_id)
 {
 	struct ibv_spec_header *hdr;
 	int i;
 
+	if (!verbs)
+		return;
 	/* Update Verbs specification. */
-	hdr = (struct ibv_spec_header *)flow->verbs.specs;
+	hdr = (struct ibv_spec_header *)verbs->specs;
 	if (!hdr)
 		return;
-	for (i = 0; i != flow->verbs.attr->num_of_specs; ++i) {
+	for (i = 0; i != verbs->attr->num_of_specs; ++i) {
 		if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
 			struct ibv_flow_spec_action_tag *t =
 				(struct ibv_flow_spec_action_tag *)hdr;
@@ -1166,6 +1426,7 @@ mlx5_flow_action_mark(const struct rte_flow_action *action,
 		.type = IBV_FLOW_SPEC_ACTION_TAG,
 		.size = size,
 	};
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
 
 	if (!mark)
 		return rte_flow_error_set(error, EINVAL,
@@ -1190,14 +1451,11 @@ mlx5_flow_action_mark(const struct rte_flow_action *action,
 					  "mark is not compatible with drop"
 					  " action");
 	if (flow->modifier & MLX5_FLOW_MOD_FLAG) {
-		mlx5_flow_verbs_mark_update(flow, mark->id);
-		size = 0; /**< Only an update is done in the specification. */
-	} else {
+		mlx5_flow_verbs_mark_update(verbs, mark->id);
+		size = 0;
+	} else if (size <= flow_size) {
 		tag.tag_id = mlx5_flow_mark_set(mark->id);
-		if (size <= flow_size) {
-			tag.tag_id = mlx5_flow_mark_set(mark->id);
-			mlx5_flow_spec_verbs_add(flow, &tag, size);
-		}
+		mlx5_flow_spec_verbs_add(flow, &tag, size);
 	}
 	flow->modifier |= MLX5_FLOW_MOD_MARK;
 	return size;
@@ -1259,6 +1517,9 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 		case RTE_FLOW_ACTION_TYPE_QUEUE:
 			ret = mlx5_flow_action_queue(dev, actions, flow, error);
 			break;
+		case RTE_FLOW_ACTION_TYPE_RSS:
+			ret = mlx5_flow_action_rss(dev, actions, flow, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -1322,26 +1583,126 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 		struct rte_flow_error *error)
 {
 	struct rte_flow local_flow = { .layers = 0, };
-	size_t size = sizeof(*flow) + sizeof(struct ibv_flow_attr);
-	int remain = (flow_size > size) ? flow_size - size : 0;
+	size_t size = sizeof(*flow);
+	union {
+		struct rte_flow_expand_rss buf;
+		uint8_t buffer[2048];
+	} expand_buffer;
+	struct rte_flow_expand_rss *buf = &expand_buffer.buf;
+	struct mlx5_flow_verbs *original_verbs = NULL;
+	size_t original_verbs_size = 0;
+	uint32_t original_layers = 0;
+	int expanded_pattern_idx = 0;
 	int ret;
+	uint32_t i;
 
-	if (!remain)
+	if (size > flow_size)
 		flow = &local_flow;
 	ret = mlx5_flow_attributes(dev, attributes, flow, error);
 	if (ret < 0)
 		return ret;
-	ret = mlx5_flow_items(pattern, flow, remain, error);
-	if (ret < 0)
-		return ret;
-	size += ret;
-	remain = (flow_size > size) ? flow_size - size : 0;
-	ret = mlx5_flow_actions(dev, actions, flow, remain, error);
+	ret = mlx5_flow_actions(dev, actions, &local_flow, 0, error);
 	if (ret < 0)
 		return ret;
-	size += ret;
+	if (local_flow.rss.types) {
+		ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
+					  pattern, local_flow.rss.types,
+					  mlx5_support_expansion,
+					  MLX5_EXPANSION_ROOT);
+		assert(ret > 0 &&
+		       (unsigned int)ret < sizeof(expand_buffer.buffer));
+	} else {
+		buf->entries = 1;
+		buf->entry[0].pattern = (void *)(uintptr_t)pattern;
+	}
+	size += RTE_ALIGN_CEIL(local_flow.rss.queue_num * sizeof(uint16_t),
+			       sizeof(void *));
 	if (size <= flow_size)
-		flow->verbs.attr->priority = flow->attributes.priority;
+		flow->queue = (void *)(flow + 1);
+	LIST_INIT(&flow->verbs);
+	flow->layers = 0;
+	flow->modifier = 0;
+	flow->fate = 0;
+	for (i = 0; i != buf->entries; ++i) {
+		size_t off = size;
+		size_t off2;
+
+		flow->layers = original_layers;
+		size += sizeof(struct ibv_flow_attr) +
+			sizeof(struct mlx5_flow_verbs);
+		off2 = size;
+		if (size < flow_size) {
+			flow->cur_verbs = (void *)((uintptr_t)flow + off);
+			flow->cur_verbs->attr = (void *)(flow->cur_verbs + 1);
+			flow->cur_verbs->specs =
+				(void *)(flow->cur_verbs->attr + 1);
+		}
+		/* First iteration convert the pattern into Verbs. */
+		if (i == 0) {
+			/* Actions don't need to be converted several time. */
+			ret = mlx5_flow_actions(dev, actions, flow,
+						(size < flow_size) ?
+						flow_size - size : 0,
+						error);
+			if (ret < 0)
+				return ret;
+			size += ret;
+		} else {
+			/*
+			 * Next iteration means the pattern has already been
+			 * converted and an expansion is necessary to match
+			 * the user RSS request.  For that only the expanded
+			 * items will be converted, the common part with the
+			 * user pattern are just copied into the next buffer
+			 * zone.
+			 */
+			size += original_verbs_size;
+			if (size < flow_size) {
+				rte_memcpy(flow->cur_verbs->attr,
+					   original_verbs->attr,
+					   original_verbs_size +
+					   sizeof(struct ibv_flow_attr));
+				flow->cur_verbs->size = original_verbs_size;
+			}
+		}
+		ret = mlx5_flow_items
+			((const struct rte_flow_item *)
+			 &buf->entry[i].pattern[expanded_pattern_idx],
+			 flow,
+			 (size < flow_size) ? flow_size - size : 0, error);
+		if (ret < 0)
+			return ret;
+		size += ret;
+		if (size <= flow_size) {
+			mlx5_flow_adjust_priority(dev, flow);
+			LIST_INSERT_HEAD(&flow->verbs, flow->cur_verbs, next);
+		}
+		/*
+		 * Keep a pointer of the first verbs conversion and the layers
+		 * it has encountered.
+		 */
+		if (i == 0) {
+			original_verbs = flow->cur_verbs;
+			original_verbs_size = size - off2;
+			original_layers = flow->layers;
+			/*
+			 * move the index of the expanded pattern to the
+			 * first item not addressed yet.
+			 */
+			if (pattern->type == RTE_FLOW_ITEM_TYPE_END) {
+				expanded_pattern_idx++;
+			} else {
+				const struct rte_flow_item *item = pattern;
+
+				for (item = pattern;
+				     item->type != RTE_FLOW_ITEM_TYPE_END;
+				     ++item)
+					expanded_pattern_idx++;
+			}
+		}
+	}
+	/* Restore the origin layers in the flow. */
+	flow->layers = original_layers;
 	return size;
 }
 
@@ -1359,12 +1720,17 @@ mlx5_flow_rxq_mark_set(struct rte_eth_dev *dev, struct rte_flow *flow)
 	struct priv *priv = dev->data->dev_private;
 
 	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK)) {
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of((*priv->rxqs)[flow->queue],
-				     struct mlx5_rxq_ctrl, rxq);
+		unsigned int i;
+
+		for (i = 0; i != flow->rss.queue_num; ++i) {
+			int idx = (*flow->queue)[i];
+			struct mlx5_rxq_ctrl *rxq_ctrl =
+				container_of((*priv->rxqs)[idx],
+					     struct mlx5_rxq_ctrl, rxq);
 
-		rxq_ctrl->rxq.mark = 1;
-		rxq_ctrl->flow_mark_n++;
+			rxq_ctrl->rxq.mark = 1;
+			rxq_ctrl->flow_mark_n++;
+		}
 	}
 }
 
@@ -1383,12 +1749,17 @@ mlx5_flow_rxq_mark_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
 	struct priv *priv = dev->data->dev_private;
 
 	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK)) {
-		struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of((*priv->rxqs)[flow->queue],
-				     struct mlx5_rxq_ctrl, rxq);
+		unsigned int i;
+
+		for (i = 0; i != flow->rss.queue_num; ++i) {
+			int idx = (*flow->queue)[i];
+			struct mlx5_rxq_ctrl *rxq_ctrl =
+				container_of((*priv->rxqs)[idx],
+					     struct mlx5_rxq_ctrl, rxq);
 
-		rxq_ctrl->flow_mark_n--;
-		rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
+			rxq_ctrl->flow_mark_n--;
+			rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
+		}
 	}
 }
 
@@ -1449,18 +1820,20 @@ mlx5_flow_validate(struct rte_eth_dev *dev,
 static void
 mlx5_flow_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
-	if (flow->fate & MLX5_FLOW_FATE_DROP) {
-		if (flow->verbs.flow) {
-			claim_zero(mlx5_glue->destroy_flow(flow->verbs.flow));
-			flow->verbs.flow = NULL;
+	struct mlx5_flow_verbs *verbs;
+
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (verbs->flow) {
+			claim_zero(mlx5_glue->destroy_flow(verbs->flow));
+			verbs->flow = NULL;
+		}
+		if (verbs->hrxq) {
+			if (flow->fate & MLX5_FLOW_FATE_DROP)
+				mlx5_hrxq_drop_release(dev);
+			else
+				mlx5_hrxq_release(dev, verbs->hrxq);
+			verbs->hrxq = NULL;
 		}
-	}
-	if (flow->verbs.hrxq) {
-		if (flow->fate & MLX5_FLOW_FATE_DROP)
-			mlx5_hrxq_drop_release(dev);
-		else if (flow->fate & MLX5_FLOW_FATE_QUEUE)
-			mlx5_hrxq_release(dev, flow->verbs.hrxq);
-		flow->verbs.hrxq = NULL;
 	}
 }
 
@@ -1481,46 +1854,68 @@ static int
 mlx5_flow_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 		struct rte_flow_error *error)
 {
-	if (flow->fate & MLX5_FLOW_FATE_DROP) {
-		flow->verbs.hrxq = mlx5_hrxq_drop_new(dev);
-		if (!flow->verbs.hrxq)
-			return rte_flow_error_set
-				(error, errno,
-				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				 NULL,
-				 "cannot allocate Drop queue");
-	} else if (flow->fate & MLX5_FLOW_FATE_QUEUE) {
-		struct mlx5_hrxq *hrxq;
-
-		hrxq = mlx5_hrxq_get(dev, rss_hash_default_key,
-				     MLX5_RSS_HASH_KEY_LEN, 0,
-				     &flow->queue, 1, 0, 0);
-		if (!hrxq)
-			hrxq = mlx5_hrxq_new(dev, rss_hash_default_key,
-					     MLX5_RSS_HASH_KEY_LEN, 0,
-					     &flow->queue, 1, 0, 0);
-		if (!hrxq)
-			return rte_flow_error_set(error, rte_errno,
-					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					NULL,
-					"cannot create flow");
-		flow->verbs.hrxq = hrxq;
-	}
-	flow->verbs.flow =
-		mlx5_glue->create_flow(flow->verbs.hrxq->qp, flow->verbs.attr);
-	if (!flow->verbs.flow) {
-		if (flow->fate & MLX5_FLOW_FATE_DROP)
-			mlx5_hrxq_drop_release(dev);
-		else
-			mlx5_hrxq_release(dev, flow->verbs.hrxq);
-		flow->verbs.hrxq = NULL;
-		return rte_flow_error_set(error, errno,
-					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-					  NULL,
-					  "kernel module refuses to create"
-					  " flow");
+	struct mlx5_flow_verbs *verbs;
+	int err;
+
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (flow->fate & MLX5_FLOW_FATE_DROP) {
+			verbs->hrxq = mlx5_hrxq_drop_new(dev);
+			if (!verbs->hrxq) {
+				rte_flow_error_set
+					(error, errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					 NULL,
+					 "cannot get drop hash queue");
+				goto error;
+			}
+		} else {
+			struct mlx5_hrxq *hrxq;
+
+			hrxq = mlx5_hrxq_get(dev, flow->key,
+					     MLX5_RSS_HASH_KEY_LEN,
+					     verbs->hash_fields,
+					     (*flow->queue),
+					     flow->rss.queue_num, 0, 0);
+			if (!hrxq)
+				hrxq = mlx5_hrxq_new(dev, flow->key,
+						     MLX5_RSS_HASH_KEY_LEN,
+						     verbs->hash_fields,
+						     (*flow->queue),
+						     flow->rss.queue_num, 0, 0);
+			if (!hrxq) {
+				rte_flow_error_set
+					(error, rte_errno,
+					 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					 NULL,
+					 "cannot get hash queue");
+				goto error;
+			}
+			verbs->hrxq = hrxq;
+		}
+		verbs->flow =
+			mlx5_glue->create_flow(verbs->hrxq->qp, verbs->attr);
+		if (!verbs->flow) {
+			rte_flow_error_set(error, errno,
+					   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					   NULL,
+					   "hardware refuses to create flow");
+			goto error;
+		}
 	}
 	return 0;
+error:
+	err = rte_errno; /* Save rte_errno before cleanup. */
+	LIST_FOREACH(verbs, &flow->verbs, next) {
+		if (verbs->hrxq) {
+			if (flow->fate & MLX5_FLOW_FATE_DROP)
+				mlx5_hrxq_drop_release(dev);
+			else
+				mlx5_hrxq_release(dev, verbs->hrxq);
+			verbs->hrxq = NULL;
+		}
+	}
+	rte_errno = err; /* Restore rte_errno. */
+	return -rte_errno;
 }
 
 /**
@@ -1550,42 +1945,43 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
 		      const struct rte_flow_action actions[],
 		      struct rte_flow_error *error)
 {
-	struct rte_flow *flow;
-	size_t size;
+	struct rte_flow *flow = NULL;
+	size_t size = 0;
 	int ret;
 
-	ret = mlx5_flow_merge(dev, NULL, 0, attr, items, actions, error);
+	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
 	if (ret < 0)
 		return NULL;
 	size = ret;
-	flow = rte_zmalloc(__func__, size, 0);
+	flow = rte_calloc(__func__, 1, size, 0);
 	if (!flow) {
 		rte_flow_error_set(error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL,
-				   "cannot allocate memory");
+				   "not enough memory to create flow");
 		return NULL;
 	}
-	flow->verbs.attr = (struct ibv_flow_attr *)(flow + 1);
-	flow->verbs.specs = (uint8_t *)(flow->verbs.attr + 1);
 	ret = mlx5_flow_merge(dev, flow, size, attr, items, actions, error);
-	if (ret < 0)
-		goto error;
+	if (ret < 0) {
+		rte_free(flow);
+		return NULL;
+	}
 	assert((size_t)ret == size);
 	if (dev->data->dev_started) {
 		ret = mlx5_flow_apply(dev, flow, error);
-		if (ret < 0)
-			goto error;
+		if (ret < 0) {
+			ret = rte_errno; /* Save rte_errno before cleanup. */
+			if (flow) {
+				mlx5_flow_remove(dev, flow);
+				rte_free(flow);
+			}
+			rte_errno = ret; /* Restore rte_errno. */
+			return NULL;
+		}
 	}
 	mlx5_flow_rxq_mark_set(dev, flow);
 	TAILQ_INSERT_TAIL(list, flow, next);
 	return flow;
-error:
-	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_remove(dev, flow);
-	rte_free(flow);
-	rte_errno = ret; /* Restore rte_errno. */
-	return NULL;
 }
 
 /**
@@ -1745,7 +2141,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 	struct priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.ingress = 1,
-		.priority = priv->config.flow_prio - 1,
+		.priority = MLX5_FLOW_PRIO_RSVD,
 	};
 	struct rte_flow_item items[] = {
 		{
-- 
2.18.0

[dpdk-dev] [PATCH v4 15/21] net/mlx5: remove useless arguments in hrxq API

[Nelio Laranjeiro]

RSS level is necessary to had a bit in the hash_fields which is already
provided in this API, for the tunnel, it is necessary to request such
queue to compute the checksum on the inner most, this last one should
always be activated.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c |  4 ++--
 drivers/net/mlx5/mlx5_rxq.c  | 39 +++++++++---------------------------
 drivers/net/mlx5/mlx5_rxtx.h |  8 ++------
 3 files changed, 13 insertions(+), 38 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 758c611a6..730360b22 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -1875,13 +1875,13 @@ mlx5_flow_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 					     MLX5_RSS_HASH_KEY_LEN,
 					     verbs->hash_fields,
 					     (*flow->queue),
-					     flow->rss.queue_num, 0, 0);
+					     flow->rss.queue_num);
 			if (!hrxq)
 				hrxq = mlx5_hrxq_new(dev, flow->key,
 						     MLX5_RSS_HASH_KEY_LEN,
 						     verbs->hash_fields,
 						     (*flow->queue),
-						     flow->rss.queue_num, 0, 0);
+						     flow->rss.queue_num);
 			if (!hrxq) {
 				rte_flow_error_set
 					(error, rte_errno,
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index d50b82c69..071740b6d 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1740,10 +1740,6 @@ mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev)
  *   first queue index will be taken for the indirection table.
  * @param queues_n
  *   Number of queues.
- * @param tunnel
- *   Tunnel type, implies tunnel offloading like inner checksum if available.
- * @param rss_level
- *   RSS hash on tunnel level.
  *
  * @return
  *   The Verbs object initialised, NULL otherwise and rte_errno is set.
@@ -1752,17 +1748,13 @@ struct mlx5_hrxq *
 mlx5_hrxq_new(struct rte_eth_dev *dev,
 	      const uint8_t *rss_key, uint32_t rss_key_len,
 	      uint64_t hash_fields,
-	      const uint16_t *queues, uint32_t queues_n,
-	      uint32_t tunnel, uint32_t rss_level)
+	      const uint16_t *queues, uint32_t queues_n)
 {
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
 	struct mlx5_ind_table_ibv *ind_tbl;
 	struct ibv_qp *qp;
 	int err;
-#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	struct mlx5dv_qp_init_attr qp_init_attr = {0};
-#endif
 
 	queues_n = hash_fields ? queues_n : 1;
 	ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n);
@@ -1777,11 +1769,6 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 		rss_key = rss_hash_default_key;
 	}
 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
-	if (tunnel) {
-		qp_init_attr.comp_mask =
-				MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
-		qp_init_attr.create_flags = MLX5DV_QP_CREATE_TUNNEL_OFFLOADS;
-	}
 	qp = mlx5_glue->dv_create_qp
 		(priv->ctx,
 		 &(struct ibv_qp_init_attr_ex){
@@ -1797,14 +1784,17 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 				.rx_hash_key = rss_key ?
 					       (void *)(uintptr_t)rss_key :
 					       rss_hash_default_key,
-				.rx_hash_fields_mask = hash_fields |
-					(tunnel && rss_level > 1 ?
-					(uint32_t)IBV_RX_HASH_INNER : 0),
+				.rx_hash_fields_mask = hash_fields,
 			},
 			.rwq_ind_tbl = ind_tbl->ind_table,
 			.pd = priv->pd,
 		 },
-		 &qp_init_attr);
+		 &(struct mlx5dv_qp_init_attr){
+			.comp_mask = (hash_fields & IBV_RX_HASH_INNER) ?
+				 MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS :
+				 0,
+			.create_flags = MLX5DV_QP_CREATE_TUNNEL_OFFLOADS,
+		 });
 #else
 	qp = mlx5_glue->create_qp_ex
 		(priv->ctx,
@@ -1838,8 +1828,6 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
 	hrxq->qp = qp;
 	hrxq->rss_key_len = rss_key_len;
 	hrxq->hash_fields = hash_fields;
-	hrxq->tunnel = tunnel;
-	hrxq->rss_level = rss_level;
 	memcpy(hrxq->rss_key, rss_key, rss_key_len);
 	rte_atomic32_inc(&hrxq->refcnt);
 	LIST_INSERT_HEAD(&priv->hrxqs, hrxq, next);
@@ -1865,10 +1853,6 @@ mlx5_hrxq_new(struct rte_eth_dev *dev,
  *   first queue index will be taken for the indirection table.
  * @param queues_n
  *   Number of queues.
- * @param tunnel
- *   Tunnel type, implies tunnel offloading like inner checksum if available.
- * @param rss_level
- *   RSS hash on tunnel level
  *
  * @return
  *   An hash Rx queue on success.
@@ -1877,8 +1861,7 @@ struct mlx5_hrxq *
 mlx5_hrxq_get(struct rte_eth_dev *dev,
 	      const uint8_t *rss_key, uint32_t rss_key_len,
 	      uint64_t hash_fields,
-	      const uint16_t *queues, uint32_t queues_n,
-	      uint32_t tunnel, uint32_t rss_level)
+	      const uint16_t *queues, uint32_t queues_n)
 {
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
@@ -1893,10 +1876,6 @@ mlx5_hrxq_get(struct rte_eth_dev *dev,
 			continue;
 		if (hrxq->hash_fields != hash_fields)
 			continue;
-		if (hrxq->tunnel != tunnel)
-			continue;
-		if (hrxq->rss_level != rss_level)
-			continue;
 		ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n);
 		if (!ind_tbl)
 			continue;
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 59e374d8d..808118e50 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -157,8 +157,6 @@ struct mlx5_hrxq {
 	struct mlx5_ind_table_ibv *ind_table; /* Indirection table. */
 	struct ibv_qp *qp; /* Verbs queue pair. */
 	uint64_t hash_fields; /* Verbs Hash fields. */
-	uint32_t tunnel; /* Tunnel type. */
-	uint32_t rss_level; /* RSS on tunnel level. */
 	uint32_t rss_key_len; /* Hash key length in bytes. */
 	uint8_t rss_key[]; /* Hash key. */
 };
@@ -271,13 +269,11 @@ void mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev);
 struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev,
 				const uint8_t *rss_key, uint32_t rss_key_len,
 				uint64_t hash_fields,
-				const uint16_t *queues, uint32_t queues_n,
-				uint32_t tunnel, uint32_t rss_level);
+				const uint16_t *queues, uint32_t queues_n);
 struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev,
 				const uint8_t *rss_key, uint32_t rss_key_len,
 				uint64_t hash_fields,
-				const uint16_t *queues, uint32_t queues_n,
-				uint32_t tunnel, uint32_t rss_level);
+				const uint16_t *queues, uint32_t queues_n);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
 int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev);
 struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev);
-- 
2.18.0

[dpdk-dev] [PATCH v4 16/21] net/mlx5: support inner RSS computation

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 245 ++++++++++++++++++++++++++---------
 1 file changed, 185 insertions(+), 60 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 730360b22..84bd99b3e 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -35,18 +35,42 @@
 extern const struct eth_dev_ops mlx5_dev_ops;
 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
-/* Pattern Layer bits. */
+/* Pattern outer Layer bits. */
 #define MLX5_FLOW_LAYER_OUTER_L2 (1u << 0)
 #define MLX5_FLOW_LAYER_OUTER_L3_IPV4 (1u << 1)
 #define MLX5_FLOW_LAYER_OUTER_L3_IPV6 (1u << 2)
 #define MLX5_FLOW_LAYER_OUTER_L4_UDP (1u << 3)
 #define MLX5_FLOW_LAYER_OUTER_L4_TCP (1u << 4)
 #define MLX5_FLOW_LAYER_OUTER_VLAN (1u << 5)
-/* Masks. */
+
+/* Pattern inner Layer bits. */
+#define MLX5_FLOW_LAYER_INNER_L2 (1u << 6)
+#define MLX5_FLOW_LAYER_INNER_L3_IPV4 (1u << 7)
+#define MLX5_FLOW_LAYER_INNER_L3_IPV6 (1u << 8)
+#define MLX5_FLOW_LAYER_INNER_L4_UDP (1u << 9)
+#define MLX5_FLOW_LAYER_INNER_L4_TCP (1u << 10)
+#define MLX5_FLOW_LAYER_INNER_VLAN (1u << 11)
+
+/* Outer Masks. */
 #define MLX5_FLOW_LAYER_OUTER_L3 \
 	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 #define MLX5_FLOW_LAYER_OUTER_L4 \
 	(MLX5_FLOW_LAYER_OUTER_L4_UDP | MLX5_FLOW_LAYER_OUTER_L4_TCP)
+#define MLX5_FLOW_LAYER_OUTER \
+	(MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_L3 | \
+	 MLX5_FLOW_LAYER_OUTER_L4)
+
+/* Tunnel Masks. */
+#define MLX5_FLOW_LAYER_TUNNEL 0
+
+/* Inner Masks. */
+#define MLX5_FLOW_LAYER_INNER_L3 \
+	(MLX5_FLOW_LAYER_INNER_L3_IPV4 | MLX5_FLOW_LAYER_INNER_L3_IPV6)
+#define MLX5_FLOW_LAYER_INNER_L4 \
+	(MLX5_FLOW_LAYER_INNER_L4_UDP | MLX5_FLOW_LAYER_INNER_L4_TCP)
+#define MLX5_FLOW_LAYER_INNER \
+	(MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3 | \
+	 MLX5_FLOW_LAYER_INNER_L4)
 
 /* Actions that modify the fate of matching traffic. */
 #define MLX5_FLOW_FATE_DROP (1u << 0)
@@ -66,6 +90,14 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
 enum mlx5_expansion {
 	MLX5_EXPANSION_ROOT,
+	MLX5_EXPANSION_ROOT_OUTER,
+	MLX5_EXPANSION_OUTER_ETH,
+	MLX5_EXPANSION_OUTER_IPV4,
+	MLX5_EXPANSION_OUTER_IPV4_UDP,
+	MLX5_EXPANSION_OUTER_IPV4_TCP,
+	MLX5_EXPANSION_OUTER_IPV6,
+	MLX5_EXPANSION_OUTER_IPV6_UDP,
+	MLX5_EXPANSION_OUTER_IPV6_TCP,
 	MLX5_EXPANSION_ETH,
 	MLX5_EXPANSION_IPV4,
 	MLX5_EXPANSION_IPV4_UDP,
@@ -83,6 +115,50 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 						 MLX5_EXPANSION_IPV6),
 		.type = RTE_FLOW_ITEM_TYPE_END,
 	},
+	[MLX5_EXPANSION_ROOT_OUTER] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH,
+						 MLX5_EXPANSION_OUTER_IPV4,
+						 MLX5_EXPANSION_OUTER_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_END,
+	},
+	[MLX5_EXPANSION_OUTER_ETH] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
+						 MLX5_EXPANSION_OUTER_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_ETH,
+		.rss_types = 0,
+	},
+	[MLX5_EXPANSION_OUTER_IPV4] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT
+			(MLX5_EXPANSION_OUTER_IPV4_UDP,
+			 MLX5_EXPANSION_OUTER_IPV4_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV4,
+		.rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+			ETH_RSS_NONFRAG_IPV4_OTHER,
+	},
+	[MLX5_EXPANSION_OUTER_IPV4_UDP] = {
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
+	},
+	[MLX5_EXPANSION_OUTER_IPV4_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV4_TCP,
+	},
+	[MLX5_EXPANSION_OUTER_IPV6] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT
+			(MLX5_EXPANSION_OUTER_IPV6_UDP,
+			 MLX5_EXPANSION_OUTER_IPV6_TCP),
+		.type = RTE_FLOW_ITEM_TYPE_IPV6,
+		.rss_types = ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
+			ETH_RSS_NONFRAG_IPV6_OTHER,
+	},
+	[MLX5_EXPANSION_OUTER_IPV6_UDP] = {
+		.type = RTE_FLOW_ITEM_TYPE_UDP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
+	},
+	[MLX5_EXPANSION_OUTER_IPV6_TCP] = {
+		.type = RTE_FLOW_ITEM_TYPE_TCP,
+		.rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
+	},
 	[MLX5_EXPANSION_ETH] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 						 MLX5_EXPANSION_IPV6),
@@ -453,6 +529,34 @@ mlx5_flow_spec_verbs_add(struct rte_flow *flow, void *src, unsigned int size)
 	verbs->size += size;
 }
 
+/**
+ * Adjust verbs hash fields according to the @p flow information.
+ *
+ * @param[in, out] flow.
+ *   Pointer to flow structure.
+ * @param[in] tunnel
+ *   1 when the hash field is for a tunnel item.
+ * @param[in] layer_types
+ *   ETH_RSS_* types.
+ * @param[in] hash_fields
+ *   Item hash fields.
+ */
+static void
+mlx5_flow_verbs_hashfields_adjust(struct rte_flow *flow, int tunnel __rte_unused,
+				  uint32_t layer_types, uint64_t hash_fields)
+{
+#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+	hash_fields |= (tunnel ? IBV_RX_HASH_INNER : 0);
+	if (flow->rss.level == 2 && !tunnel)
+		hash_fields = 0;
+	else if (flow->rss.level < 2 && tunnel)
+		hash_fields = 0;
+#endif
+	if (!(flow->rss.types & layer_types))
+		hash_fields = 0;
+	flow->cur_verbs->hash_fields |= hash_fields;
+}
+
 /**
  * Convert the @p item into a Verbs specification after ensuring the NIC
  * will understand and process it correctly.
@@ -486,14 +590,16 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 		.src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
 		.type = RTE_BE16(0xffff),
 	};
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 	const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 	struct ibv_flow_spec_eth eth = {
-		.type = IBV_FLOW_SPEC_ETH,
+		.type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
 
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L2)
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+			    MLX5_FLOW_LAYER_OUTER_L2))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -506,7 +612,8 @@ mlx5_flow_item_eth(const struct rte_flow_item *item, struct rte_flow *flow,
 					error);
 	if (ret)
 		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+		MLX5_FLOW_LAYER_OUTER_L2;
 	if (size > flow_size)
 		return size;
 	if (spec) {
@@ -543,7 +650,7 @@ mlx5_flow_item_vlan_update(struct ibv_flow_attr *attr,
 			   struct ibv_flow_spec_eth *eth)
 {
 	unsigned int i;
-	enum ibv_flow_spec_type search = IBV_FLOW_SPEC_ETH;
+	const enum ibv_flow_spec_type search = eth->type;
 	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 
@@ -596,16 +703,19 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 		.inner_type = RTE_BE16(0xffff),
 	};
 	unsigned int size = sizeof(struct ibv_flow_spec_eth);
-	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 	struct ibv_flow_spec_eth eth = {
-		.type = IBV_FLOW_SPEC_ETH,
+		.type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
-	const uint32_t l34m = MLX5_FLOW_LAYER_OUTER_L3 |
-			MLX5_FLOW_LAYER_OUTER_L4;
-	const uint32_t vlanm = MLX5_FLOW_LAYER_OUTER_VLAN;
-	const uint32_t l2m = MLX5_FLOW_LAYER_OUTER_L2;
+	const uint32_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
+					MLX5_FLOW_LAYER_INNER_L4) :
+		(MLX5_FLOW_LAYER_OUTER_L3 | MLX5_FLOW_LAYER_OUTER_L4);
+	const uint32_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
+		MLX5_FLOW_LAYER_OUTER_VLAN;
+	const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
+		MLX5_FLOW_LAYER_OUTER_L2;
 
 	if (flow->layers & vlanm)
 		return rte_flow_error_set(error, ENOTSUP,
@@ -648,11 +758,14 @@ mlx5_flow_item_vlan(const struct rte_flow_item *item, struct rte_flow *flow,
 			mlx5_flow_spec_verbs_add(flow, &eth, size);
 		}
 	} else {
-		if (verbs->attr)
-			mlx5_flow_item_vlan_update(verbs->attr, &eth);
+		if (flow->cur_verbs)
+			mlx5_flow_item_vlan_update(flow->cur_verbs->attr,
+						   &eth);
 		size = 0; /* Only an update is done in eth specification. */
 	}
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_VLAN;
+	flow->layers |= tunnel ?
+		(MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_VLAN) :
+		(MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_VLAN);
 	return size;
 }
 
@@ -692,19 +805,23 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 			.next_proto_id = 0xff,
 		},
 	};
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 	unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 	struct ibv_flow_spec_ipv4_ext ipv4 = {
-		.type = IBV_FLOW_SPEC_IPV4_EXT,
+		.type = IBV_FLOW_SPEC_IPV4_EXT |
+			(tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
 
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			    MLX5_FLOW_LAYER_OUTER_L3))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
 					  "multiple L3 layers not supported");
-	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+	else if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+				 MLX5_FLOW_LAYER_OUTER_L4))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -717,7 +834,8 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 		 sizeof(struct rte_flow_item_ipv4), error);
 	if (ret < 0)
 		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
+		MLX5_FLOW_LAYER_OUTER_L3_IPV4;
 	if (spec) {
 		ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 			.src_ip = spec->hdr.src_addr,
@@ -740,14 +858,11 @@ mlx5_flow_item_ipv4(const struct rte_flow_item *item, struct rte_flow *flow,
 	flow->l3_protocol_en = !!ipv4.mask.proto;
 	flow->l3_protocol = ipv4.val.proto;
 	if (size <= flow_size) {
-		uint64_t hash_fields = IBV_RX_HASH_SRC_IPV4 |
-			IBV_RX_HASH_DST_IPV4;
-
-		if (!(flow->rss.types &
-		      (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
-		       ETH_RSS_NONFRAG_IPV4_OTHER)))
-			hash_fields = 0;
-		flow->cur_verbs->hash_fields |= hash_fields;
+		mlx5_flow_verbs_hashfields_adjust
+			(flow, tunnel,
+			 (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
+			  ETH_RSS_NONFRAG_IPV4_OTHER),
+			 (IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4));
 		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L3;
 		mlx5_flow_spec_verbs_add(flow, &ipv4, size);
 	}
@@ -795,19 +910,22 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 			.hop_limits = 0xff,
 		},
 	};
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 	unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 	struct ibv_flow_spec_ipv6 ipv6 = {
-		.type = IBV_FLOW_SPEC_IPV6,
+		.type = IBV_FLOW_SPEC_IPV6 | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
 
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3)
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			    MLX5_FLOW_LAYER_OUTER_L3))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
 					  "multiple L3 layers not supported");
-	else if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+	else if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+				 MLX5_FLOW_LAYER_OUTER_L4))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -820,7 +938,8 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 		 sizeof(struct rte_flow_item_ipv6), error);
 	if (ret < 0)
 		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
+		MLX5_FLOW_LAYER_OUTER_L3_IPV6;
 	if (spec) {
 		unsigned int i;
 		uint32_t vtc_flow_val;
@@ -863,13 +982,10 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 	flow->l3_protocol_en = !!ipv6.mask.next_hdr;
 	flow->l3_protocol = ipv6.val.next_hdr;
 	if (size <= flow_size) {
-		uint64_t hash_fields = IBV_RX_HASH_SRC_IPV6 |
-			IBV_RX_HASH_DST_IPV6;
-
-		if (!(flow->rss.types &
-		      (ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER)))
-			hash_fields = 0;
-		flow->cur_verbs->hash_fields |= hash_fields;
+		mlx5_flow_verbs_hashfields_adjust
+			(flow, tunnel,
+			 (ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER),
+			 (IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6));
 		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L3;
 		mlx5_flow_spec_verbs_add(flow, &ipv6, size);
 	}
@@ -904,9 +1020,10 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 {
 	const struct rte_flow_item_udp *spec = item->spec;
 	const struct rte_flow_item_udp *mask = item->mask;
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 	struct ibv_flow_spec_tcp_udp udp = {
-		.type = IBV_FLOW_SPEC_UDP,
+		.type = IBV_FLOW_SPEC_UDP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
@@ -917,13 +1034,15 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 					  item,
 					  "protocol filtering not compatible"
 					  " with UDP layer");
-	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+	if (!(flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			      MLX5_FLOW_LAYER_OUTER_L3)))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
 					  "L3 is mandatory to filter"
 					  " on L4");
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+			    MLX5_FLOW_LAYER_OUTER_L4))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -937,7 +1056,8 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 		 sizeof(struct rte_flow_item_udp), error);
 	if (ret < 0)
 		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
+	flow->layers |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
+		MLX5_FLOW_LAYER_OUTER_L4_UDP;
 	if (spec) {
 		udp.val.dst_port = spec->hdr.dst_port;
 		udp.val.src_port = spec->hdr.src_port;
@@ -948,12 +1068,9 @@ mlx5_flow_item_udp(const struct rte_flow_item *item, struct rte_flow *flow,
 		udp.val.dst_port &= udp.mask.dst_port;
 	}
 	if (size <= flow_size) {
-		uint64_t hash_fields = IBV_RX_HASH_SRC_PORT_UDP |
-			IBV_RX_HASH_DST_PORT_UDP;
-
-		if (!(flow->rss.types & ETH_RSS_UDP))
-			hash_fields = 0;
-		flow->cur_verbs->hash_fields |= hash_fields;
+		mlx5_flow_verbs_hashfields_adjust(flow, tunnel, ETH_RSS_UDP,
+						  (IBV_RX_HASH_SRC_PORT_UDP |
+						   IBV_RX_HASH_DST_PORT_UDP));
 		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L4;
 		mlx5_flow_spec_verbs_add(flow, &udp, size);
 	}
@@ -988,9 +1105,10 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 {
 	const struct rte_flow_item_tcp *spec = item->spec;
 	const struct rte_flow_item_tcp *mask = item->mask;
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 	unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 	struct ibv_flow_spec_tcp_udp tcp = {
-		.type = IBV_FLOW_SPEC_TCP,
+		.type = IBV_FLOW_SPEC_TCP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 		.size = size,
 	};
 	int ret;
@@ -1001,12 +1119,14 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 					  item,
 					  "protocol filtering not compatible"
 					  " with TCP layer");
-	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+	if (!(flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L3 :
+			      MLX5_FLOW_LAYER_OUTER_L3)))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
 					  "L3 is mandatory to filter on L4");
-	if (flow->layers & MLX5_FLOW_LAYER_OUTER_L4)
+	if (flow->layers & (tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
+			    MLX5_FLOW_LAYER_OUTER_L4))
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
@@ -1019,7 +1139,8 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 		 sizeof(struct rte_flow_item_tcp), error);
 	if (ret < 0)
 		return ret;
-	flow->layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
+	flow->layers |=  tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
+		MLX5_FLOW_LAYER_OUTER_L4_TCP;
 	if (spec) {
 		tcp.val.dst_port = spec->hdr.dst_port;
 		tcp.val.src_port = spec->hdr.src_port;
@@ -1030,12 +1151,9 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 		tcp.val.dst_port &= tcp.mask.dst_port;
 	}
 	if (size <= flow_size) {
-		uint64_t hash_fields = IBV_RX_HASH_SRC_PORT_TCP |
-			IBV_RX_HASH_DST_PORT_TCP;
-
-		if (!(flow->rss.types & ETH_RSS_TCP))
-			hash_fields = 0;
-		flow->cur_verbs->hash_fields |= hash_fields;
+		mlx5_flow_verbs_hashfields_adjust(flow, tunnel, ETH_RSS_TCP,
+						  (IBV_RX_HASH_SRC_PORT_TCP |
+						   IBV_RX_HASH_DST_PORT_TCP));
 		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L4;
 		mlx5_flow_spec_verbs_add(flow, &tcp, size);
 	}
@@ -1261,7 +1379,11 @@ mlx5_flow_action_rss(struct rte_eth_dev *dev,
 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 					  &rss->func,
 					  "RSS hash function not supported");
+#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+	if (rss->level > 2)
+#else
 	if (rss->level > 1)
+#endif
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 					  &rss->level,
@@ -1301,6 +1423,7 @@ mlx5_flow_action_rss(struct rte_eth_dev *dev,
 	flow->rss.queue_num = rss->queue_num;
 	memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
 	flow->rss.types = rss->types;
+	flow->rss.level = rss->level;
 	flow->fate |= MLX5_FLOW_FATE_RSS;
 	return 0;
 }
@@ -1608,7 +1731,9 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 		ret = rte_flow_expand_rss(buf, sizeof(expand_buffer.buffer),
 					  pattern, local_flow.rss.types,
 					  mlx5_support_expansion,
-					  MLX5_EXPANSION_ROOT);
+					  local_flow.rss.level < 2 ?
+					  MLX5_EXPANSION_ROOT :
+					  MLX5_EXPANSION_ROOT_OUTER);
 		assert(ret > 0 &&
 		       (unsigned int)ret < sizeof(expand_buffer.buffer));
 	} else {
@@ -1979,8 +2104,8 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
 			return NULL;
 		}
 	}
-	mlx5_flow_rxq_mark_set(dev, flow);
 	TAILQ_INSERT_TAIL(list, flow, next);
+	mlx5_flow_rxq_mark_set(dev, flow);
 	return flow;
 }
 
-- 
2.18.0

[dpdk-dev] [PATCH v4 17/21] net/mlx5: add flow VXLAN item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 190 +++++++++++++++++++++++++++++------
 drivers/net/mlx5/mlx5_rxtx.h |   1 +
 2 files changed, 163 insertions(+), 28 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 84bd99b3e..7eb5d7da3 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -51,6 +51,9 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_FLOW_LAYER_INNER_L4_TCP (1u << 10)
 #define MLX5_FLOW_LAYER_INNER_VLAN (1u << 11)
 
+/* Pattern tunnel Layer bits. */
+#define MLX5_FLOW_LAYER_VXLAN (1u << 12)
+
 /* Outer Masks. */
 #define MLX5_FLOW_LAYER_OUTER_L3 \
 	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
@@ -61,7 +64,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 	 MLX5_FLOW_LAYER_OUTER_L4)
 
 /* Tunnel Masks. */
-#define MLX5_FLOW_LAYER_TUNNEL 0
+#define MLX5_FLOW_LAYER_TUNNEL MLX5_FLOW_LAYER_VXLAN
 
 /* Inner Masks. */
 #define MLX5_FLOW_LAYER_INNER_L3 \
@@ -98,6 +101,7 @@ enum mlx5_expansion {
 	MLX5_EXPANSION_OUTER_IPV6,
 	MLX5_EXPANSION_OUTER_IPV6_UDP,
 	MLX5_EXPANSION_OUTER_IPV6_TCP,
+	MLX5_EXPANSION_VXLAN,
 	MLX5_EXPANSION_ETH,
 	MLX5_EXPANSION_IPV4,
 	MLX5_EXPANSION_IPV4_UDP,
@@ -136,6 +140,7 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 			ETH_RSS_NONFRAG_IPV4_OTHER,
 	},
 	[MLX5_EXPANSION_OUTER_IPV4_UDP] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN),
 		.type = RTE_FLOW_ITEM_TYPE_UDP,
 		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 	},
@@ -152,6 +157,7 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 			ETH_RSS_NONFRAG_IPV6_OTHER,
 	},
 	[MLX5_EXPANSION_OUTER_IPV6_UDP] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN),
 		.type = RTE_FLOW_ITEM_TYPE_UDP,
 		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 	},
@@ -159,6 +165,10 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 		.type = RTE_FLOW_ITEM_TYPE_TCP,
 		.rss_types = ETH_RSS_NONFRAG_IPV6_TCP,
 	},
+	[MLX5_EXPANSION_VXLAN] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
+		.type = RTE_FLOW_ITEM_TYPE_VXLAN,
+	},
 	[MLX5_EXPANSION_ETH] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 						 MLX5_EXPANSION_IPV6),
@@ -226,6 +236,8 @@ struct rte_flow {
 	struct mlx5_flow_verbs *cur_verbs;
 	/**< Current Verbs flow structure being filled. */
 	struct rte_flow_action_rss rss;/**< RSS context. */
+	uint32_t tunnel_ptype;
+	/**< Store tunnel packet type data to store in Rx queue. */
 	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
 	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
 };
@@ -1160,6 +1172,103 @@ mlx5_flow_item_tcp(const struct rte_flow_item *item, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
+		     const size_t flow_size, struct rte_flow_error *error)
+{
+	const struct rte_flow_item_vxlan *spec = item->spec;
+	const struct rte_flow_item_vxlan *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel vxlan = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+	int ret;
+	union vni {
+		uint32_t vlan_id;
+		uint8_t vni[4];
+	} id = { .vlan_id = 0, };
+
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already present");
+	/*
+	 * Verify only UDPv4 is present as defined in
+	 * https://tools.ietf.org/html/rfc7348
+	 */
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L4_UDP))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "no outer UDP layer found");
+	if (!mask)
+		mask = &rte_flow_item_vxlan_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_vxlan_mask,
+		 sizeof(struct rte_flow_item_vxlan), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		memcpy(&id.vni[1], spec->vni, 3);
+		vxlan.val.tunnel_id = id.vlan_id;
+		memcpy(&id.vni[1], mask->vni, 3);
+		vxlan.mask.tunnel_id = id.vlan_id;
+		/* Remove unwanted bits from values. */
+		vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
+	}
+	/*
+	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if
+	 * only this layer is defined in the Verbs specification it is
+	 * interpreted as wildcard and all packets will match this
+	 * rule, if it follows a full stack layer (ex: eth / ipv4 /
+	 * udp), all packets matching the layers before will also
+	 * match this rule.  To avoid such situation, VNI 0 is
+	 * currently refused.
+	 */
+	if (!vxlan.val.tunnel_id)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN vni cannot be 0");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN tunnel must be fully defined");
+	if (size <= flow_size) {
+		mlx5_flow_spec_verbs_add(flow, &vxlan, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_VXLAN;
+	flow->tunnel_ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP;
+	return size;
+}
+
 /**
  * Convert the @p pattern into a Verbs specifications after ensuring the NIC
  * will understand and process it correctly.
@@ -1217,6 +1326,10 @@ mlx5_flow_items(const struct rte_flow_item pattern[],
 		case RTE_FLOW_ITEM_TYPE_TCP:
 			ret = mlx5_flow_item_tcp(pattern, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN:
+			ret = mlx5_flow_item_vxlan(pattern, flow, remain,
+						   error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
@@ -1832,7 +1945,7 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 }
 
 /**
- * Mark the Rx queues mark flag if the flow has a mark or flag modifier.
+ * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the flow.
  *
  * @param[in] dev
  *   Pointer to Ethernet device.
@@ -1840,28 +1953,34 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
  *   Pointer to flow structure.
  */
 static void
-mlx5_flow_rxq_mark_set(struct rte_eth_dev *dev, struct rte_flow *flow)
+mlx5_flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
 	struct priv *priv = dev->data->dev_private;
+	const int mark = !!(flow->modifier &
+			    (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK));
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int i;
 
-	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK)) {
-		unsigned int i;
-
-		for (i = 0; i != flow->rss.queue_num; ++i) {
-			int idx = (*flow->queue)[i];
-			struct mlx5_rxq_ctrl *rxq_ctrl =
-				container_of((*priv->rxqs)[idx],
-					     struct mlx5_rxq_ctrl, rxq);
+	for (i = 0; i != flow->rss.queue_num; ++i) {
+		int idx = (*flow->queue)[i];
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of((*priv->rxqs)[idx],
+				     struct mlx5_rxq_ctrl, rxq);
 
+		if (mark) {
 			rxq_ctrl->rxq.mark = 1;
 			rxq_ctrl->flow_mark_n++;
 		}
+		if (tunnel) {
+			rxq_ctrl->rxq.tunnel = flow->tunnel_ptype;
+			rxq_ctrl->flow_vxlan_n++;
+		}
 	}
 }
 
 /**
- * Clear the Rx queue mark associated with the @p flow if no other flow uses
- * it with a mark request.
+ * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the
+ * @p flow if no other flow uses it with the same kind of request.
  *
  * @param dev
  *   Pointer to Ethernet device.
@@ -1869,33 +1988,41 @@ mlx5_flow_rxq_mark_set(struct rte_eth_dev *dev, struct rte_flow *flow)
  *   Pointer to the flow.
  */
 static void
-mlx5_flow_rxq_mark_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
+mlx5_flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
 	struct priv *priv = dev->data->dev_private;
+	const int mark = !!(flow->modifier &
+			    (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK));
+	const int tunnel = !!(flow->layers & MLX5_FLOW_LAYER_TUNNEL);
+	unsigned int i;
 
-	if (flow->modifier & (MLX5_FLOW_MOD_FLAG | MLX5_FLOW_MOD_MARK)) {
-		unsigned int i;
-
-		for (i = 0; i != flow->rss.queue_num; ++i) {
-			int idx = (*flow->queue)[i];
-			struct mlx5_rxq_ctrl *rxq_ctrl =
-				container_of((*priv->rxqs)[idx],
-					     struct mlx5_rxq_ctrl, rxq);
+	assert(dev->data->dev_started);
+	for (i = 0; i != flow->rss.queue_num; ++i) {
+		int idx = (*flow->queue)[i];
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of((*priv->rxqs)[idx],
+				     struct mlx5_rxq_ctrl, rxq);
 
+		if (mark) {
 			rxq_ctrl->flow_mark_n--;
 			rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
 		}
+		if (tunnel) {
+			rxq_ctrl->flow_vxlan_n++;
+			if (!rxq_ctrl->flow_vxlan_n)
+				rxq_ctrl->rxq.tunnel = 0;
+		}
 	}
 }
 
 /**
- * Clear the mark bit in all Rx queues.
+ * Clear the Mark/Flag and Tunnel ptype information in all Rx queues.
  *
  * @param dev
  *   Pointer to Ethernet device.
  */
 static void
-mlx5_flow_rxq_mark_clear(struct rte_eth_dev *dev)
+mlx5_flow_rxq_flags_clear(struct rte_eth_dev *dev)
 {
 	struct priv *priv = dev->data->dev_private;
 	unsigned int i;
@@ -1910,6 +2037,8 @@ mlx5_flow_rxq_mark_clear(struct rte_eth_dev *dev)
 					struct mlx5_rxq_ctrl, rxq);
 		rxq_ctrl->flow_mark_n = 0;
 		rxq_ctrl->rxq.mark = 0;
+		rxq_ctrl->flow_vxlan_n = 0;
+		rxq_ctrl->rxq.tunnel = 0;
 		++idx;
 	}
 }
@@ -2105,7 +2234,7 @@ mlx5_flow_list_create(struct rte_eth_dev *dev,
 		}
 	}
 	TAILQ_INSERT_TAIL(list, flow, next);
-	mlx5_flow_rxq_mark_set(dev, flow);
+	mlx5_flow_rxq_flags_set(dev, flow);
 	return flow;
 }
 
@@ -2143,7 +2272,12 @@ mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
 {
 	mlx5_flow_remove(dev, flow);
 	TAILQ_REMOVE(list, flow, next);
-	mlx5_flow_rxq_mark_trim(dev, flow);
+	/*
+	 * Update RX queue flags only if port is started, otherwise it is
+	 * already clean.
+	 */
+	if (dev->data->dev_started)
+		mlx5_flow_rxq_flags_trim(dev, flow);
 	rte_free(flow);
 }
 
@@ -2181,7 +2315,7 @@ mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
 
 	TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next)
 		mlx5_flow_remove(dev, flow);
-	mlx5_flow_rxq_mark_clear(dev);
+	mlx5_flow_rxq_flags_clear(dev);
 }
 
 /**
@@ -2206,7 +2340,7 @@ mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
 		ret = mlx5_flow_apply(dev, flow, &error);
 		if (ret < 0)
 			goto error;
-		mlx5_flow_rxq_mark_set(dev, flow);
+		mlx5_flow_rxq_flags_set(dev, flow);
 	}
 	return 0;
 error:
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 808118e50..ae9b564dd 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -139,6 +139,7 @@ struct mlx5_rxq_ctrl {
 	unsigned int irq:1; /* Whether IRQ is enabled. */
 	uint16_t idx; /* Queue index. */
 	uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */
+	uint32_t flow_vxlan_n; /* Number of VXLAN flows using this queue. */
 };
 
 /* Indirection table. */
-- 
2.18.0

[dpdk-dev] [PATCH v4 18/21] net/mlx5: add flow VXLAN-GPE item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 219 ++++++++++++++++++++++++++++++++---
 drivers/net/mlx5/mlx5_rxtx.h |   5 +-
 2 files changed, 209 insertions(+), 15 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 7eb5d7da3..5d0ad4a04 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -53,6 +53,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
 /* Pattern tunnel Layer bits. */
 #define MLX5_FLOW_LAYER_VXLAN (1u << 12)
+#define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
 
 /* Outer Masks. */
 #define MLX5_FLOW_LAYER_OUTER_L3 \
@@ -64,7 +65,8 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 	 MLX5_FLOW_LAYER_OUTER_L4)
 
 /* Tunnel Masks. */
-#define MLX5_FLOW_LAYER_TUNNEL MLX5_FLOW_LAYER_VXLAN
+#define MLX5_FLOW_LAYER_TUNNEL \
+	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE)
 
 /* Inner Masks. */
 #define MLX5_FLOW_LAYER_INNER_L3 \
@@ -102,6 +104,7 @@ enum mlx5_expansion {
 	MLX5_EXPANSION_OUTER_IPV6_UDP,
 	MLX5_EXPANSION_OUTER_IPV6_TCP,
 	MLX5_EXPANSION_VXLAN,
+	MLX5_EXPANSION_VXLAN_GPE,
 	MLX5_EXPANSION_ETH,
 	MLX5_EXPANSION_IPV4,
 	MLX5_EXPANSION_IPV4_UDP,
@@ -140,7 +143,8 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 			ETH_RSS_NONFRAG_IPV4_OTHER,
 	},
 	[MLX5_EXPANSION_OUTER_IPV4_UDP] = {
-		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN),
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
+						 MLX5_EXPANSION_VXLAN_GPE),
 		.type = RTE_FLOW_ITEM_TYPE_UDP,
 		.rss_types = ETH_RSS_NONFRAG_IPV4_UDP,
 	},
@@ -157,7 +161,8 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 			ETH_RSS_NONFRAG_IPV6_OTHER,
 	},
 	[MLX5_EXPANSION_OUTER_IPV6_UDP] = {
-		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN),
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN,
+						 MLX5_EXPANSION_VXLAN_GPE),
 		.type = RTE_FLOW_ITEM_TYPE_UDP,
 		.rss_types = ETH_RSS_NONFRAG_IPV6_UDP,
 	},
@@ -169,6 +174,12 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH),
 		.type = RTE_FLOW_ITEM_TYPE_VXLAN,
 	},
+	[MLX5_EXPANSION_VXLAN_GPE] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH,
+						 MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
+	},
 	[MLX5_EXPANSION_ETH] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 						 MLX5_EXPANSION_IPV6),
@@ -236,8 +247,6 @@ struct rte_flow {
 	struct mlx5_flow_verbs *cur_verbs;
 	/**< Current Verbs flow structure being filled. */
 	struct rte_flow_action_rss rss;/**< RSS context. */
-	uint32_t tunnel_ptype;
-	/**< Store tunnel packet type data to store in Rx queue. */
 	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
 	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
 };
@@ -304,6 +313,23 @@ static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
 	{ 9, 10, 11 }, { 12, 13, 14 },
 };
 
+/* Tunnel information. */
+struct mlx5_flow_tunnel_info {
+	uint32_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */
+	uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */
+};
+
+static struct mlx5_flow_tunnel_info tunnels_info[] = {
+	{
+		.tunnel = MLX5_FLOW_LAYER_VXLAN,
+		.ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP,
+	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
+		.ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
+	},
+};
+
 /**
  * Discover the maximum number of priority available.
  *
@@ -1265,7 +1291,119 @@ mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
 		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
 	}
 	flow->layers |= MLX5_FLOW_LAYER_VXLAN;
-	flow->tunnel_ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP;
+	return size;
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_vxlan_gpe(struct rte_eth_dev *dev,
+			 const struct rte_flow_item *item,
+			 struct rte_flow *flow, const size_t flow_size,
+			 struct rte_flow_error *error)
+{
+	const struct rte_flow_item_vxlan_gpe *spec = item->spec;
+	const struct rte_flow_item_vxlan_gpe *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel vxlan_gpe = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+	int ret;
+	union vni {
+		uint32_t vlan_id;
+		uint8_t vni[4];
+	} id = { .vlan_id = 0, };
+
+	if (!((struct priv *)dev->data->dev_private)->config.l3_vxlan_en)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 VXLAN is not enabled by device"
+					  " parameter and/or not configured in"
+					  " firmware");
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already present");
+	/*
+	 * Verify only UDPv4 is present as defined in
+	 * https://tools.ietf.org/html/rfc7348
+	 */
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L4_UDP))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "no outer UDP layer found");
+	if (!mask)
+		mask = &rte_flow_item_vxlan_gpe_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask,
+		 sizeof(struct rte_flow_item_vxlan_gpe), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		memcpy(&id.vni[1], spec->vni, 3);
+		vxlan_gpe.val.tunnel_id = id.vlan_id;
+		memcpy(&id.vni[1], mask->vni, 3);
+		vxlan_gpe.mask.tunnel_id = id.vlan_id;
+		if (spec->protocol)
+			return rte_flow_error_set
+				(error, EINVAL,
+				 RTE_FLOW_ERROR_TYPE_ITEM,
+				 item,
+				 "VxLAN-GPE protocol not supported");
+		/* Remove unwanted bits from values. */
+		vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
+	}
+	/*
+	 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
+	 * layer is defined in the Verbs specification it is interpreted as
+	 * wildcard and all packets will match this rule, if it follows a full
+	 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
+	 * before will also match this rule.  To avoid such situation, VNI 0
+	 * is currently refused.
+	 */
+	if (!vxlan_gpe.val.tunnel_id)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN-GPE vni cannot be 0");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "VXLAN-GPE tunnel must be fully"
+					  " defined");
+	if (size <= flow_size) {
+		mlx5_flow_spec_verbs_add(flow, &vxlan_gpe, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_VXLAN_GPE;
 	return size;
 }
 
@@ -1295,7 +1433,8 @@ mlx5_flow_item_vxlan(const struct rte_flow_item *item, struct rte_flow *flow,
  *   On error, a negative errno value is returned and rte_errno is set.
  */
 static int
-mlx5_flow_items(const struct rte_flow_item pattern[],
+mlx5_flow_items(struct rte_eth_dev *dev,
+		const struct rte_flow_item pattern[],
 		struct rte_flow *flow, const size_t flow_size,
 		struct rte_flow_error *error)
 {
@@ -1330,6 +1469,10 @@ mlx5_flow_items(const struct rte_flow_item pattern[],
 			ret = mlx5_flow_item_vxlan(pattern, flow, remain,
 						   error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
+			ret = mlx5_flow_item_vxlan_gpe(dev, pattern, flow,
+						       remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
@@ -1904,7 +2047,8 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 			}
 		}
 		ret = mlx5_flow_items
-			((const struct rte_flow_item *)
+			(dev,
+			 (const struct rte_flow_item *)
 			 &buf->entry[i].pattern[expanded_pattern_idx],
 			 flow,
 			 (size < flow_size) ? flow_size - size : 0, error);
@@ -1944,6 +2088,34 @@ mlx5_flow_merge(struct rte_eth_dev *dev, struct rte_flow *flow,
 	return size;
 }
 
+/**
+ * Lookup and set the ptype in the data Rx part.  A single Ptype can be used,
+ * if several tunnel rules are used on this queue, the tunnel ptype will be
+ * cleared.
+ *
+ * @param rxq_ctrl
+ *   Rx queue to update.
+ */
+static void
+mlx5_flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl)
+{
+	unsigned int i;
+	uint32_t tunnel_ptype = 0;
+
+	/* Look up for the ptype to use. */
+	for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) {
+		if (!rxq_ctrl->flow_tunnels_n[i])
+			continue;
+		if (!tunnel_ptype) {
+			tunnel_ptype = tunnels_info[i].ptype;
+		} else {
+			tunnel_ptype = 0;
+			break;
+		}
+	}
+	rxq_ctrl->rxq.tunnel = tunnel_ptype;
+}
+
 /**
  * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the flow.
  *
@@ -1972,8 +2144,17 @@ mlx5_flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
 			rxq_ctrl->flow_mark_n++;
 		}
 		if (tunnel) {
-			rxq_ctrl->rxq.tunnel = flow->tunnel_ptype;
-			rxq_ctrl->flow_vxlan_n++;
+			unsigned int j;
+
+			/* Increase the counter matching the flow. */
+			for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
+				if ((tunnels_info[j].tunnel & flow->layers) ==
+				    tunnels_info[j].tunnel) {
+					rxq_ctrl->flow_tunnels_n[j]++;
+					break;
+				}
+			}
+			mlx5_flow_rxq_tunnel_ptype_update(rxq_ctrl);
 		}
 	}
 }
@@ -2008,9 +2189,17 @@ mlx5_flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
 			rxq_ctrl->rxq.mark = !!rxq_ctrl->flow_mark_n;
 		}
 		if (tunnel) {
-			rxq_ctrl->flow_vxlan_n++;
-			if (!rxq_ctrl->flow_vxlan_n)
-				rxq_ctrl->rxq.tunnel = 0;
+			unsigned int j;
+
+			/* Decrease the counter matching the flow. */
+			for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
+				if ((tunnels_info[j].tunnel & flow->layers) ==
+				    tunnels_info[j].tunnel) {
+					rxq_ctrl->flow_tunnels_n[j]--;
+					break;
+				}
+			}
+			mlx5_flow_rxq_tunnel_ptype_update(rxq_ctrl);
 		}
 	}
 }
@@ -2030,6 +2219,7 @@ mlx5_flow_rxq_flags_clear(struct rte_eth_dev *dev)
 
 	for (idx = 0, i = 0; idx != priv->rxqs_n; ++i) {
 		struct mlx5_rxq_ctrl *rxq_ctrl;
+		unsigned int j;
 
 		if (!(*priv->rxqs)[idx])
 			continue;
@@ -2037,7 +2227,8 @@ mlx5_flow_rxq_flags_clear(struct rte_eth_dev *dev)
 					struct mlx5_rxq_ctrl, rxq);
 		rxq_ctrl->flow_mark_n = 0;
 		rxq_ctrl->rxq.mark = 0;
-		rxq_ctrl->flow_vxlan_n = 0;
+		for (j = 0; j != MLX5_FLOW_TUNNEL; ++j)
+			rxq_ctrl->flow_tunnels_n[j] = 0;
 		rxq_ctrl->rxq.tunnel = 0;
 		++idx;
 	}
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index ae9b564dd..e97f5766c 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -34,6 +34,9 @@
 #include "mlx5_defs.h"
 #include "mlx5_prm.h"
 
+/* Support tunnel matching. */
+#define MLX5_FLOW_TUNNEL 2
+
 struct mlx5_rxq_stats {
 	unsigned int idx; /**< Mapping index. */
 #ifdef MLX5_PMD_SOFT_COUNTERS
@@ -139,7 +142,7 @@ struct mlx5_rxq_ctrl {
 	unsigned int irq:1; /* Whether IRQ is enabled. */
 	uint16_t idx; /* Queue index. */
 	uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */
-	uint32_t flow_vxlan_n; /* Number of VXLAN flows using this queue. */
+	uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels counters. */
 };
 
 /* Indirection table. */
-- 
2.18.0

[dpdk-dev] [PATCH v4 19/21] net/mlx5: add flow GRE item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 193 ++++++++++++++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rxtx.h |   2 +-
 2 files changed, 192 insertions(+), 3 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 5d0ad4a04..b05e30204 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -54,6 +54,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* Pattern tunnel Layer bits. */
 #define MLX5_FLOW_LAYER_VXLAN (1u << 12)
 #define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
+#define MLX5_FLOW_LAYER_GRE (1u << 14)
 
 /* Outer Masks. */
 #define MLX5_FLOW_LAYER_OUTER_L3 \
@@ -66,7 +67,8 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 
 /* Tunnel Masks. */
 #define MLX5_FLOW_LAYER_TUNNEL \
-	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE)
+	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE | \
+	 MLX5_FLOW_LAYER_GRE)
 
 /* Inner Masks. */
 #define MLX5_FLOW_LAYER_INNER_L3 \
@@ -89,6 +91,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* possible L3 layers protocols filtering. */
 #define MLX5_IP_PROTOCOL_TCP 6
 #define MLX5_IP_PROTOCOL_UDP 17
+#define MLX5_IP_PROTOCOL_GRE 47
 
 /* Priority reserved for default flows. */
 #define MLX5_FLOW_PRIO_RSVD ((uint32_t)-1)
@@ -105,6 +108,7 @@ enum mlx5_expansion {
 	MLX5_EXPANSION_OUTER_IPV6_TCP,
 	MLX5_EXPANSION_VXLAN,
 	MLX5_EXPANSION_VXLAN_GPE,
+	MLX5_EXPANSION_GRE,
 	MLX5_EXPANSION_ETH,
 	MLX5_EXPANSION_IPV4,
 	MLX5_EXPANSION_IPV4_UDP,
@@ -137,7 +141,8 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 	[MLX5_EXPANSION_OUTER_IPV4] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT
 			(MLX5_EXPANSION_OUTER_IPV4_UDP,
-			 MLX5_EXPANSION_OUTER_IPV4_TCP),
+			 MLX5_EXPANSION_OUTER_IPV4_TCP,
+			 MLX5_EXPANSION_GRE),
 		.type = RTE_FLOW_ITEM_TYPE_IPV4,
 		.rss_types = ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 			ETH_RSS_NONFRAG_IPV4_OTHER,
@@ -180,6 +185,10 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 						 MLX5_EXPANSION_IPV6),
 		.type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
 	},
+	[MLX5_EXPANSION_GRE] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
+		.type = RTE_FLOW_ITEM_TYPE_GRE,
+	},
 	[MLX5_EXPANSION_ETH] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 						 MLX5_EXPANSION_IPV6),
@@ -328,6 +337,10 @@ static struct mlx5_flow_tunnel_info tunnels_info[] = {
 		.tunnel = MLX5_FLOW_LAYER_VXLAN_GPE,
 		.ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP,
 	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_GRE,
+		.ptype = RTE_PTYPE_TUNNEL_GRE,
+	},
 };
 
 /**
@@ -968,6 +981,18 @@ mlx5_flow_item_ipv6(const struct rte_flow_item *item, struct rte_flow *flow,
 					  RTE_FLOW_ERROR_TYPE_ITEM,
 					  item,
 					  "L3 cannot follow an L4 layer.");
+	/*
+	 * IPv6 is not recognised by the NIC inside a GRE tunnel.
+	 * Such support has to be disabled as the rule will be
+	 * accepted.  Issue reproduced with Mellanox OFED 4.3-3.0.2.1 and
+	 * Mellanox OFED 4.4-1.0.0.0.
+	 */
+	if (tunnel && flow->layers & MLX5_FLOW_LAYER_GRE)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "IPv6 inside a GRE tunnel is"
+					  " not recognised.");
 	if (!mask)
 		mask = &rte_flow_item_ipv6_mask;
 	ret = mlx5_flow_item_acceptable
@@ -1407,6 +1432,167 @@ mlx5_flow_item_vxlan_gpe(struct rte_eth_dev *dev,
 	return size;
 }
 
+/**
+ * Update the protocol in Verbs IPv4/IPv6 spec.
+ *
+ * @param[in, out] attr
+ *   Pointer to Verbs attributes structure.
+ * @param[in] search
+ *   Specification type to search in order to update the IP protocol.
+ * @param[in] protocol
+ *   Protocol value to set if none is present in the specification.
+ */
+static void
+mlx5_flow_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
+				      enum ibv_flow_spec_type search,
+				      uint8_t protocol)
+{
+	unsigned int i;
+	struct ibv_spec_header *hdr = (struct ibv_spec_header *)
+		((uint8_t *)attr + sizeof(struct ibv_flow_attr));
+
+	if (!attr)
+		return;
+	for (i = 0; i != attr->num_of_specs; ++i) {
+		if (hdr->type == search) {
+			union {
+				struct ibv_flow_spec_ipv4_ext *ipv4;
+				struct ibv_flow_spec_ipv6 *ipv6;
+			} ip;
+
+			switch (search) {
+			case IBV_FLOW_SPEC_IPV4_EXT:
+				ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
+				if (!ip.ipv4->val.proto) {
+					ip.ipv4->val.proto = protocol;
+					ip.ipv4->mask.proto = 0xff;
+				}
+				break;
+			case IBV_FLOW_SPEC_IPV6:
+				ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
+				if (!ip.ipv6->val.next_hdr) {
+					ip.ipv6->val.next_hdr = protocol;
+					ip.ipv6->mask.next_hdr = 0xff;
+				}
+				break;
+			default:
+				break;
+			}
+			break;
+		}
+		hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
+	}
+}
+
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * It will also update the previous L3 layer with the protocol value matching
+ * the GRE.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_gre(const struct rte_flow_item *item,
+		   struct rte_flow *flow, const size_t flow_size,
+		   struct rte_flow_error *error)
+{
+	struct mlx5_flow_verbs *verbs = flow->cur_verbs;
+	const struct rte_flow_item_gre *spec = item->spec;
+	const struct rte_flow_item_gre *mask = item->mask;
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	unsigned int size = sizeof(struct ibv_flow_spec_gre);
+	struct ibv_flow_spec_gre tunnel = {
+		.type = IBV_FLOW_SPEC_GRE,
+		.size = size,
+	};
+#else
+	unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+	struct ibv_flow_spec_tunnel tunnel = {
+		.type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
+		.size = size,
+	};
+#endif
+	int ret;
+
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_GRE)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with this GRE layer");
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already present");
+	if (!(flow->layers & MLX5_FLOW_LAYER_OUTER_L3))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "L3 Layer is missing");
+	if (!mask)
+		mask = &rte_flow_item_gre_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_gre_mask,
+		 sizeof(struct rte_flow_item_gre), error);
+	if (ret < 0)
+		return ret;
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	if (spec) {
+		tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
+		tunnel.val.protocol = spec->protocol;
+		tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
+		tunnel.mask.protocol = mask->protocol;
+		/* Remove unwanted bits from values. */
+		tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
+		tunnel.val.protocol &= tunnel.mask.protocol;
+		tunnel.val.key &= tunnel.mask.key;
+	}
+#else
+	if (spec && (spec->protocol & mask->protocol))
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "without MPLS support the"
+					  " specification cannot be used for"
+					  " filtering");
+#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
+	if (size <= flow_size) {
+		if (flow->layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
+			mlx5_flow_item_gre_ip_protocol_update
+				(verbs->attr, IBV_FLOW_SPEC_IPV4_EXT,
+				 MLX5_IP_PROTOCOL_GRE);
+		else
+			mlx5_flow_item_gre_ip_protocol_update
+				(verbs->attr, IBV_FLOW_SPEC_IPV6,
+				 MLX5_IP_PROTOCOL_GRE);
+		mlx5_flow_spec_verbs_add(flow, &tunnel, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_GRE;
+	return size;
+}
+
 /**
  * Convert the @p pattern into a Verbs specifications after ensuring the NIC
  * will understand and process it correctly.
@@ -1473,6 +1659,9 @@ mlx5_flow_items(struct rte_eth_dev *dev,
 			ret = mlx5_flow_item_vxlan_gpe(dev, pattern, flow,
 						       remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_GRE:
+			ret = mlx5_flow_item_gre(pattern, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index e97f5766c..53992e8db 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -35,7 +35,7 @@
 #include "mlx5_prm.h"
 
 /* Support tunnel matching. */
-#define MLX5_FLOW_TUNNEL 2
+#define MLX5_FLOW_TUNNEL 3
 
 struct mlx5_rxq_stats {
 	unsigned int idx; /**< Mapping index. */
-- 
2.18.0

[dpdk-dev] [PATCH v4 20/21] net/mlx5: add flow MPLS item

[Nelio Laranjeiro]

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c | 102 ++++++++++++++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rxtx.h |   2 +-
 2 files changed, 101 insertions(+), 3 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index b05e30204..1d7b72ac1 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -55,6 +55,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_FLOW_LAYER_VXLAN (1u << 12)
 #define MLX5_FLOW_LAYER_VXLAN_GPE (1u << 13)
 #define MLX5_FLOW_LAYER_GRE (1u << 14)
+#define MLX5_FLOW_LAYER_MPLS (1u << 15)
 
 /* Outer Masks. */
 #define MLX5_FLOW_LAYER_OUTER_L3 \
@@ -68,7 +69,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* Tunnel Masks. */
 #define MLX5_FLOW_LAYER_TUNNEL \
 	(MLX5_FLOW_LAYER_VXLAN | MLX5_FLOW_LAYER_VXLAN_GPE | \
-	 MLX5_FLOW_LAYER_GRE)
+	 MLX5_FLOW_LAYER_GRE | MLX5_FLOW_LAYER_MPLS)
 
 /* Inner Masks. */
 #define MLX5_FLOW_LAYER_INNER_L3 \
@@ -92,6 +93,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 #define MLX5_IP_PROTOCOL_TCP 6
 #define MLX5_IP_PROTOCOL_UDP 17
 #define MLX5_IP_PROTOCOL_GRE 47
+#define MLX5_IP_PROTOCOL_MPLS 147
 
 /* Priority reserved for default flows. */
 #define MLX5_FLOW_PRIO_RSVD ((uint32_t)-1)
@@ -109,6 +111,7 @@ enum mlx5_expansion {
 	MLX5_EXPANSION_VXLAN,
 	MLX5_EXPANSION_VXLAN_GPE,
 	MLX5_EXPANSION_GRE,
+	MLX5_EXPANSION_MPLS,
 	MLX5_EXPANSION_ETH,
 	MLX5_EXPANSION_IPV4,
 	MLX5_EXPANSION_IPV4_UDP,
@@ -134,7 +137,8 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 	},
 	[MLX5_EXPANSION_OUTER_ETH] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4,
-						 MLX5_EXPANSION_OUTER_IPV6),
+						 MLX5_EXPANSION_OUTER_IPV6,
+						 MLX5_EXPANSION_MPLS),
 		.type = RTE_FLOW_ITEM_TYPE_ETH,
 		.rss_types = 0,
 	},
@@ -189,6 +193,11 @@ static const struct rte_flow_expand_node mlx5_support_expansion[] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4),
 		.type = RTE_FLOW_ITEM_TYPE_GRE,
 	},
+	[MLX5_EXPANSION_MPLS] = {
+		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
+						 MLX5_EXPANSION_IPV6),
+		.type = RTE_FLOW_ITEM_TYPE_MPLS,
+	},
 	[MLX5_EXPANSION_ETH] = {
 		.next = RTE_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4,
 						 MLX5_EXPANSION_IPV6),
@@ -341,6 +350,14 @@ static struct mlx5_flow_tunnel_info tunnels_info[] = {
 		.tunnel = MLX5_FLOW_LAYER_GRE,
 		.ptype = RTE_PTYPE_TUNNEL_GRE,
 	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP,
+		.ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE | RTE_PTYPE_L4_UDP,
+	},
+	{
+		.tunnel = MLX5_FLOW_LAYER_MPLS,
+		.ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
+	},
 };
 
 /**
@@ -1593,6 +1610,84 @@ mlx5_flow_item_gre(const struct rte_flow_item *item,
 	return size;
 }
 
+/**
+ * Convert the @p item into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param[in] item
+ *   Item specification.
+ * @param[in, out] flow
+ *   Pointer to flow structure.
+ * @param[in] flow_size
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p item has fully been converted,
+ *   otherwise another call with this returned memory size should be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_item_mpls(const struct rte_flow_item *item __rte_unused,
+		    struct rte_flow *flow __rte_unused,
+		    const size_t flow_size __rte_unused,
+		    struct rte_flow_error *error)
+{
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+	const struct rte_flow_item_mpls *spec = item->spec;
+	const struct rte_flow_item_mpls *mask = item->mask;
+	unsigned int size = sizeof(struct ibv_flow_spec_mpls);
+	struct ibv_flow_spec_mpls mpls = {
+		.type = IBV_FLOW_SPEC_MPLS,
+		.size = size,
+	};
+	int ret;
+
+	if (flow->l3_protocol_en && flow->l3_protocol != MLX5_IP_PROTOCOL_MPLS)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "protocol filtering not compatible"
+					  " with MPLS layer");
+	if (flow->layers & MLX5_FLOW_LAYER_TUNNEL)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  item,
+					  "a tunnel is already"
+					  " present");
+	if (!mask)
+		mask = &rte_flow_item_mpls_mask;
+	ret = mlx5_flow_item_acceptable
+		(item, (const uint8_t *)mask,
+		 (const uint8_t *)&rte_flow_item_mpls_mask,
+		 sizeof(struct rte_flow_item_mpls), error);
+	if (ret < 0)
+		return ret;
+	if (spec) {
+		memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
+		memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
+		/* Remove unwanted bits from values.  */
+		mpls.val.label &= mpls.mask.label;
+	}
+	if (size <= flow_size) {
+		mlx5_flow_spec_verbs_add(flow, &mpls, size);
+		flow->cur_verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
+	}
+	flow->layers |= MLX5_FLOW_LAYER_MPLS;
+	return size;
+#endif /* !HAVE_IBV_DEVICE_MPLS_SUPPORT */
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_ITEM,
+				  item,
+				  "MPLS is not supported by Verbs, please"
+				  " update.");
+}
+
 /**
  * Convert the @p pattern into a Verbs specifications after ensuring the NIC
  * will understand and process it correctly.
@@ -1662,6 +1757,9 @@ mlx5_flow_items(struct rte_eth_dev *dev,
 		case RTE_FLOW_ITEM_TYPE_GRE:
 			ret = mlx5_flow_item_gre(pattern, flow, remain, error);
 			break;
+		case RTE_FLOW_ITEM_TYPE_MPLS:
+			ret = mlx5_flow_item_mpls(pattern, flow, remain, error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 53992e8db..a04a84fec 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -35,7 +35,7 @@
 #include "mlx5_prm.h"
 
 /* Support tunnel matching. */
-#define MLX5_FLOW_TUNNEL 3
+#define MLX5_FLOW_TUNNEL 5
 
 struct mlx5_rxq_stats {
 	unsigned int idx; /**< Mapping index. */
-- 
2.18.0

[dpdk-dev] [PATCH v4 21/21] net/mlx5: add count flow action

[Nelio Laranjeiro]

This is only supported by Mellanox OFED.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5.h      |   2 +
 drivers/net/mlx5/mlx5_flow.c | 242 +++++++++++++++++++++++++++++++++++
 2 files changed, 244 insertions(+)

diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 9949cd3fa..131be334c 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -188,6 +188,8 @@ struct priv {
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
 	struct mlx5_flows flows; /* RTE Flow rules. */
 	struct mlx5_flows ctrl_flows; /* Control flow rules. */
+	LIST_HEAD(counters, mlx5_flow_counter) flow_counters;
+	/* Flow counters. */
 	struct {
 		uint32_t dev_gen; /* Generation number to flush local caches. */
 		rte_rwlock_t rwlock; /* MR Lock. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 1d7b72ac1..89bfc670f 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -88,6 +88,7 @@ extern const struct eth_dev_ops mlx5_dev_ops_isolate;
 /* Modify a packet. */
 #define MLX5_FLOW_MOD_FLAG (1u << 0)
 #define MLX5_FLOW_MOD_MARK (1u << 1)
+#define MLX5_FLOW_MOD_COUNT (1u << 2)
 
 /* possible L3 layers protocols filtering. */
 #define MLX5_IP_PROTOCOL_TCP 6
@@ -249,6 +250,17 @@ struct mlx5_flow_verbs {
 	uint64_t hash_fields; /**< Verbs hash Rx queue hash fields. */
 };
 
+/* Counters information. */
+struct mlx5_flow_counter {
+	LIST_ENTRY(mlx5_flow_counter) next; /**< Pointer to the next counter. */
+	uint32_t shared:1; /**< Share counter ID with other flow rules. */
+	uint32_t ref_cnt:31; /**< Reference counter. */
+	uint32_t id; /**< Counter ID. */
+	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
+	uint64_t hits; /**< Number of packets matched by the rule. */
+	uint64_t bytes; /**< Number of bytes matched by the rule. */
+};
+
 /* Flow structure. */
 struct rte_flow {
 	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
@@ -264,6 +276,7 @@ struct rte_flow {
 	LIST_HEAD(verbs, mlx5_flow_verbs) verbs; /**< Verbs flows list. */
 	struct mlx5_flow_verbs *cur_verbs;
 	/**< Current Verbs flow structure being filled. */
+	struct mlx5_flow_counter *counter; /**< Holds Verbs flow counter. */
 	struct rte_flow_action_rss rss;/**< RSS context. */
 	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
 	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
@@ -275,6 +288,7 @@ static const struct rte_flow_ops mlx5_flow_ops = {
 	.destroy = mlx5_flow_destroy,
 	.flush = mlx5_flow_flush,
 	.isolate = mlx5_flow_isolate,
+	.query = mlx5_flow_query,
 };
 
 /* Convert FDIR request to Generic flow. */
@@ -454,6 +468,80 @@ mlx5_flow_adjust_priority(struct rte_eth_dev *dev, struct rte_flow *flow)
 	flow->cur_verbs->attr->priority = priority;
 }
 
+/**
+ * Get a flow counter.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in] shared
+ *   Indicate if this counter is shared with other flows.
+ * @param[in] id
+ *   Counter identifier.
+ *
+ * @return
+ *   A pointer to the counter, NULL otherwise and rte_errno is set.
+ */
+static struct mlx5_flow_counter *
+mlx5_flow_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
+{
+	struct priv *priv = dev->data->dev_private;
+	struct mlx5_flow_counter *cnt;
+
+	LIST_FOREACH(cnt, &priv->flow_counters, next) {
+		if (cnt->shared != shared)
+			continue;
+		if (cnt->id != id)
+			continue;
+		cnt->ref_cnt++;
+		return cnt;
+	}
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+
+	struct mlx5_flow_counter tmpl = {
+		.shared = shared,
+		.id = id,
+		.cs = mlx5_glue->create_counter_set
+			(priv->ctx,
+			 &(struct ibv_counter_set_init_attr){
+				 .counter_set_id = id,
+			 }),
+		.hits = 0,
+		.bytes = 0,
+	};
+
+	if (!tmpl.cs) {
+		rte_errno = errno;
+		return NULL;
+	}
+	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
+	if (!cnt) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	*cnt = tmpl;
+	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
+	return cnt;
+#endif
+	rte_errno = ENOTSUP;
+	return NULL;
+}
+
+/**
+ * Release a flow counter.
+ *
+ * @param[in] counter
+ *   Pointer to the counter handler.
+ */
+static void
+mlx5_flow_counter_release(struct mlx5_flow_counter *counter)
+{
+	if (--counter->ref_cnt == 0) {
+		claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
+		LIST_REMOVE(counter, next);
+		rte_free(counter);
+	}
+}
+
 /**
  * Verify the @p attributes will be correctly understood by the NIC and store
  * them in the @p flow if everything is correct.
@@ -2127,6 +2215,70 @@ mlx5_flow_action_mark(const struct rte_flow_action *action,
 	return size;
 }
 
+/**
+ * Convert the @p action into a Verbs specification after ensuring the NIC
+ * will understand and process it correctly.
+ * If the necessary size for the conversion is greater than the @p flow_size,
+ * nothing is written in @p flow, the validation is still performed.
+ *
+ * @param action[in]
+ *   Action configuration.
+ * @param flow[in, out]
+ *   Pointer to flow structure.
+ * @param flow_size[in]
+ *   Size in bytes of the available space in @p flow, if too small, nothing is
+ *   written.
+ * @param error[int, out]
+ *   Pointer to error structure.
+ *
+ * @return
+ *   On success the number of bytes consumed/necessary, if the returned value
+ *   is lesser or equal to @p flow_size, the @p action has fully been
+ *   converted, otherwise another call with this returned memory size should
+ *   be done.
+ *   On error, a negative errno value is returned and rte_errno is set.
+ */
+static int
+mlx5_flow_action_count(struct rte_eth_dev *dev,
+		       const struct rte_flow_action *action,
+		       struct rte_flow *flow,
+		       const size_t flow_size __rte_unused,
+		       struct rte_flow_error *error)
+{
+	const struct rte_flow_action_count *count = action->conf;
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
+	struct ibv_flow_spec_counter_action counter = {
+		.type = IBV_FLOW_SPEC_ACTION_COUNT,
+		.size = size,
+	};
+#endif
+
+	if (!flow->counter) {
+		flow->counter = mlx5_flow_counter_new(dev, count->shared,
+						      count->id);
+		if (!flow->counter)
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  action,
+						  "cannot get counter"
+						  " context.");
+	}
+	if (!((struct priv *)dev->data->dev_private)->config.flow_counter_en)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_ACTION,
+					  action,
+					  "flow counters are not supported.");
+	flow->modifier |= MLX5_FLOW_MOD_COUNT;
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	counter.counter_set_handle = flow->counter->cs->handle;
+	if (size <= flow_size)
+		mlx5_flow_spec_verbs_add(flow, &counter, size);
+	return size;
+#endif
+	return 0;
+}
+
 /**
  * Convert the @p action into @p flow after ensuring the NIC will understand
  * and process it correctly.
@@ -2186,6 +2338,10 @@ mlx5_flow_actions(struct rte_eth_dev *dev,
 		case RTE_FLOW_ACTION_TYPE_RSS:
 			ret = mlx5_flow_action_rss(dev, actions, flow, error);
 			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = mlx5_flow_action_count(dev, actions, flow, remain,
+						     error);
+			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -2567,6 +2723,10 @@ mlx5_flow_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
 			verbs->hrxq = NULL;
 		}
 	}
+	if (flow->counter) {
+		mlx5_flow_counter_release(flow->counter);
+		flow->counter = NULL;
+	}
 }
 
 /**
@@ -3016,6 +3176,88 @@ mlx5_flow_isolate(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/**
+ * Query flow counter.
+ *
+ * @param flow
+ *   Pointer to the flow.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_query_count(struct rte_flow *flow __rte_unused,
+		      void *data __rte_unused,
+		      struct rte_flow_error *error)
+{
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+	struct rte_flow_query_count *qc = data;
+	uint64_t counters[2] = {0, 0};
+	struct ibv_query_counter_set_attr query_cs_attr = {
+		.cs = flow->counter->cs,
+		.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
+	};
+	struct ibv_counter_set_data query_out = {
+		.out = counters,
+		.outlen = 2 * sizeof(uint64_t),
+	};
+	int err = mlx5_glue->query_counter_set(&query_cs_attr, &query_out);
+
+	if (err)
+		return rte_flow_error_set(error, err,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "cannot read counter");
+	qc->hits_set = 1;
+	qc->bytes_set = 1;
+	qc->hits = counters[0] - flow->counter->hits;
+	qc->bytes = counters[1] - flow->counter->bytes;
+	if (qc->reset) {
+		flow->counter->hits = counters[0];
+		flow->counter->bytes = counters[1];
+	}
+	return 0;
+#endif
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "counters are not available");
+}
+
+/**
+ * Query a flows.
+ *
+ * @see rte_flow_query()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
+		struct rte_flow *flow,
+		const struct rte_flow_action *actions,
+		void *data,
+		struct rte_flow_error *error)
+{
+	int ret = 0;
+
+	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+		switch (actions->type) {
+		case RTE_FLOW_ACTION_TYPE_VOID:
+			break;
+		case RTE_FLOW_ACTION_TYPE_COUNT:
+			ret = mlx5_flow_query_count(flow, data, error);
+			break;
+		default:
+			return rte_flow_error_set(error, ENOTSUP,
+						  RTE_FLOW_ERROR_TYPE_ACTION,
+						  actions,
+						  "action not supported");
+		}
+		if (ret < 0)
+			return ret;
+	}
+	return 0;
+}
+
 /**
  * Convert a flow director filter to a generic flow.
  *
-- 
2.18.0

Re: [dpdk-dev] [PATCH v4 00/21] net/mlx5: flow rework

[Shahaf Shuler]

Thursday, July 12, 2018 12:31 PM, Nelio Laranjeiro:
> Subject: [dpdk-dev] [PATCH v4 00/21] net/mlx5: flow rework
> 
> Re-work flow engine to support port redirection actions through TC.
> 
> This first series depends on [1] which is implemented in commit
> "net/mlx5: support inner RSS computation" and on [2].
> Next series will bring the port redirection as announced[3].
> 
> [1]
> https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fma
> ils.dpdk.org%2Farchives%2Fdev%2F2018-
> July%2F107378.html&data=02%7C01%7Cshahafs%40mellanox.com%7Cd
> 4468642a7bc46f895a408d5e7da3f23%7Ca652971c7d2e4d9ba6a4d149256f461b
> %7C0%7C0%7C636669846916246569&sdata=INTMuEpP%2BGQP2xj05JM
> ti4jCyHL8rqJiUgrrHNMpAC4%3D&reserved=0
> [2]
> https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fma
> ils.dpdk.org%2Farchives%2Fdev%2F2018-
> June%2F104192.html&data=02%7C01%7Cshahafs%40mellanox.com%7C
> d4468642a7bc46f895a408d5e7da3f23%7Ca652971c7d2e4d9ba6a4d149256f461
> b%7C0%7C0%7C636669846916246569&sdata=aq%2Bl5GWgAgnIdWYxG2
> VB4JlhLvAOVdB7JuLMScgYaN0%3D&reserved=0
> [3]
> https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fma
> ils.dpdk.org%2Farchives%2Fdev%2F2018-
> May%2F103043.html&data=02%7C01%7Cshahafs%40mellanox.com%7C
> d4468642a7bc46f895a408d5e7da3f23%7Ca652971c7d2e4d9ba6a4d149256f461
> b%7C0%7C0%7C636669846916246569&sdata=G4uhC1VCDA4z6UgC9Vfo
> CdUsndngUgMNbTHYZ4UjAqM%3D&reserved=0
> 
> Changes in v4:
> 
> - fix compilation on redhat 7.5 without Mellanox OFED.
> - avoid multiple pattern parsing for the expansion.
> 
> Changes in v3:
> 
> - remove redundant parameters in drop queues internal API.
> - simplify the RSS expansion by only adding missing items in the pattern.
> - document all functions.
> 
> 
> Nelio Laranjeiro (21):
>   net/mlx5: remove flow support
>   net/mlx5: handle drop queues as regular queues
>   net/mlx5: replace verbs priorities by flow
>   net/mlx5: support flow Ethernet item along with drop action
>   net/mlx5: add flow queue action
>   net/mlx5: add flow stop/start
>   net/mlx5: add flow VLAN item
>   net/mlx5: add flow IPv4 item
>   net/mlx5: add flow IPv6 item
>   net/mlx5: add flow UDP item
>   net/mlx5: add flow TCP item
>   net/mlx5: add mark/flag flow action
>   net/mlx5: use a macro for the RSS key size
>   net/mlx5: add RSS flow action
>   net/mlx5: remove useless arguments in hrxq API
>   net/mlx5: support inner RSS computation
>   net/mlx5: add flow VXLAN item
>   net/mlx5: add flow VXLAN-GPE item
>   net/mlx5: add flow GRE item
>   net/mlx5: add flow MPLS item
>   net/mlx5: add count flow action
> 
>  drivers/net/mlx5/mlx5.c        |   22 +-
>  drivers/net/mlx5/mlx5.h        |   18 +-
>  drivers/net/mlx5/mlx5_ethdev.c |   14 +-
>  drivers/net/mlx5/mlx5_flow.c   | 4821 ++++++++++++++++----------------
>  drivers/net/mlx5/mlx5_prm.h    |    3 +
>  drivers/net/mlx5/mlx5_rss.c    |    7 +-
>  drivers/net/mlx5/mlx5_rxq.c    |  281 +-
>  drivers/net/mlx5/mlx5_rxtx.h   |   21 +-
>  8 files changed, 2640 insertions(+), 2547 deletions(-)

Series applied to next-net-mlx, thanks. 

> 
> --
> 2.18.0