[dpdk-dev] [PATCH v2 5/5] mlx5: add VLAN insertion offload

[Adrien Mazarguil <adrien.mazarguil@6wind.com>]

From: Yaacov Hazan <yaacovh@mellanox.com>

VLAN insertion is done in software by the PMD by default unless
CONFIG_RTE_LIBRTE_MLX5_VERBS_VLAN_INSERTION is enabled and Verbs provides
support for hardware insertion.

When enabled, this option improves performance when VLAN insertion is
requested, however ConnectX-4 Lx boards cannot take advantage of
multi-packet send optimizations anymore.

Signed-off-by: Yaacov Hazan <yaacovh@mellanox.com>
Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
---
 config/common_linuxapp                 |   1 +
 doc/guides/nics/mlx5.rst               |   9 +++
 doc/guides/rel_notes/release_16_04.rst |   6 ++
 drivers/net/mlx5/Makefile              |   9 +++
 drivers/net/mlx5/mlx5_defs.h           |   9 +++
 drivers/net/mlx5/mlx5_ethdev.c         |  12 ++--
 drivers/net/mlx5/mlx5_rxtx.c           | 109 +++++++++++++++++++++++++++------
 drivers/net/mlx5/mlx5_rxtx.h           |  13 ++++
 drivers/net/mlx5/mlx5_txq.c            |  15 ++++-
 9 files changed, 158 insertions(+), 25 deletions(-)

diff --git a/config/common_linuxapp b/config/common_linuxapp
index 7b5e49f..793d262 100644
--- a/config/common_linuxapp
+++ b/config/common_linuxapp
@@ -220,6 +220,7 @@ CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
 CONFIG_RTE_LIBRTE_MLX5_SGE_WR_N=4
 CONFIG_RTE_LIBRTE_MLX5_MAX_INLINE=0
 CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
+CONFIG_RTE_LIBRTE_MLX5_VERBS_VLAN_INSERTION=n
 
 #
 # Compile burst-oriented Broadcom PMD driver
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 9df30be..e391518 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -84,6 +84,7 @@ Features
 - Support for multiple MAC addresses.
 - VLAN filtering.
 - RX VLAN stripping.
+- TX VLAN insertion.
 - RX CRC stripping configuration.
 - Promiscuous mode.
 - Multicast promiscuous mode.
@@ -143,6 +144,13 @@ These options can be modified in the ``.config`` file.
 
   This value is always 1 for RX queues since they use a single MP.
 
+- ``CONFIG_RTE_LIBRTE_MLX5_VERBS_VLAN_INSERTION`` (default **n**)
+
+  Use Verbs instead of PMD implementation for VLAN insertion. Disabled by
+  default since it prevents ConnectX-4 Lx adapters from taking advantage of
+  multi-packet send optimizations, otherwise provides better performance
+  when VLAN insertion is requested.
+
 Environment variables
 ~~~~~~~~~~~~~~~~~~~~~
 
@@ -247,6 +255,7 @@ Currently supported by DPDK:
 
     - Flow director.
     - RX VLAN stripping.
+    - TX VLAN insertion.
     - RX CRC stripping configuration.
 
 - Minimum firmware version:
diff --git a/doc/guides/rel_notes/release_16_04.rst b/doc/guides/rel_notes/release_16_04.rst
index 6bcfad1..238ef84 100644
--- a/doc/guides/rel_notes/release_16_04.rst
+++ b/doc/guides/rel_notes/release_16_04.rst
@@ -108,6 +108,12 @@ This section should contain new features added in this release. Sample format:
 
   Only available with Mellanox OFED >= 3.2.
 
+* **mlx5: TX VLAN insertion support.**
+
+  Added support for TX VLAN insertion.
+
+  Only available with Mellanox OFED >= 3.2.
+
 
 Resolved Issues
 ---------------
diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index a6a3cab..7d24fd2 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -101,6 +101,10 @@ ifdef CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE
 CFLAGS += -DMLX5_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE)
 endif
 
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_VERBS_VLAN_INSERTION),y)
+CFLAGS += -DMLX5_VERBS_VLAN_INSERTION
+endif
+
 include $(RTE_SDK)/mk/rte.lib.mk
 
 # Generate and clean-up mlx5_autoconf.h.
@@ -147,6 +151,11 @@ mlx5_autoconf.h: $(RTE_SDK)/scripts/auto-config-h.sh
 		infiniband/verbs.h \
 		enum IBV_EXP_CREATE_WQ_FLAG_RX_END_PADDING \
 		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_VERBS_VLAN_INSERTION \
+		infiniband/verbs.h \
+		enum IBV_EXP_RECEIVE_WQ_CVLAN_INSERTION \
+		$(AUTOCONF_OUTPUT)
 
 $(SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD):.c=.o): mlx5_autoconf.h
 
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 5b00d8e..fb8db2e 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -95,4 +95,13 @@
 #define MLX5_FDIR_SUPPORT 1
 #endif
 
+/*
+ * Prevent compilation when HW VLAN insertion is requested by configuration
+ * but not supported by Verbs.
+ */
+#if defined(MLX5_VERBS_VLAN_INSERTION) && !defined(HAVE_VERBS_VLAN_INSERTION)
+#error CONFIG_RTE_LIBRTE_MLX5_VERBS_VLAN_INSERTION \
+	enabled in configuration but not supported by libibverbs.
+#endif
+
 #endif /* RTE_PMD_MLX5_DEFS_H_ */
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 6b674a2..66115d2 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -544,12 +544,12 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 		  DEV_RX_OFFLOAD_UDP_CKSUM |
 		  DEV_RX_OFFLOAD_TCP_CKSUM) :
 		 0);
-	info->tx_offload_capa =
-		(priv->hw_csum ?
-		 (DEV_TX_OFFLOAD_IPV4_CKSUM |
-		  DEV_TX_OFFLOAD_UDP_CKSUM |
-		  DEV_TX_OFFLOAD_TCP_CKSUM) :
-		 0);
+	info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT;
+	if (priv->hw_csum)
+		info->tx_offload_capa |=
+			(DEV_TX_OFFLOAD_IPV4_CKSUM |
+			 DEV_TX_OFFLOAD_UDP_CKSUM |
+			 DEV_TX_OFFLOAD_TCP_CKSUM);
 	if (priv_get_ifname(priv, &ifname) == 0)
 		info->if_index = if_nametoindex(ifname);
 	/* FIXME: RETA update/query API expects the callee to know the size of
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 34340d2..819060e 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -333,6 +333,40 @@ txq_mp2mr_iter(const struct rte_mempool *mp, void *arg)
 	txq_mp2mr(txq, mp);
 }
 
+#ifndef MLX5_VERBS_VLAN_INSERTION
+
+/**
+ * Insert VLAN to specific packet, using the mbuf's headroom space.
+ *
+ * @param buf
+ *   Buffer to insert the vlan.
+ *
+ * @return
+ *   0 on success, errno value on failure.
+ */
+static inline int
+insert_vlan_sw(struct rte_mbuf *buf)
+{
+	uintptr_t addr;
+	uint32_t vlan;
+	uint16_t head_room_len = rte_pktmbuf_headroom(buf);
+
+	if (head_room_len < 4)
+		return EINVAL;
+
+	addr = rte_pktmbuf_mtod(buf, uintptr_t);
+	vlan = htonl(0x81000000 | buf->vlan_tci);
+	memmove((void *)(addr - 4), (void *)addr, 12);
+	memcpy((void *)(addr + 8), &vlan, sizeof(vlan));
+
+	SET_DATA_OFF(buf, head_room_len - 4);
+	DATA_LEN(buf) += 4;
+
+	return 0;
+}
+
+#endif /* !MLX5_VERBS_VLAN_INSERTION */
+
 #if MLX5_PMD_SGE_WR_N > 1
 
 /**
@@ -554,6 +588,14 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			if (RTE_ETH_IS_TUNNEL_PKT(buf->packet_type))
 				send_flags |= IBV_EXP_QP_BURST_TUNNEL;
 		}
+#ifndef MLX5_VERBS_VLAN_INSERTION
+		if (buf->ol_flags & PKT_TX_VLAN_PKT) {
+			err = insert_vlan_sw(buf);
+
+			if (unlikely(err))
+				goto stop;
+		}
+#endif /* !MLX5_VERBS_VLAN_INSERTION */
 		if (likely(segs == 1)) {
 			uintptr_t addr;
 			uint32_t length;
@@ -577,13 +619,23 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			}
 			/* Put packet into send queue. */
 #if MLX5_PMD_MAX_INLINE > 0
-			if (length <= txq->max_inline)
-				err = txq->send_pending_inline
-					(txq->qp,
-					 (void *)addr,
-					 length,
-					 send_flags);
-			else
+			if (length <= txq->max_inline) {
+#ifdef MLX5_VERBS_VLAN_INSERTION
+				if (buf->ol_flags & PKT_TX_VLAN_PKT)
+					err = txq->send_pending_inline_vlan
+						(txq->qp,
+						 (void *)addr,
+						 length,
+						 send_flags,
+						 &buf->vlan_tci);
+				else
+#endif /* MLX5_VERBS_VLAN_INSERTION */
+					err = txq->send_pending_inline
+						(txq->qp,
+						 (void *)addr,
+						 length,
+						 send_flags);
+			} else
 #endif
 			{
 				/* Retrieve Memory Region key for this
@@ -597,12 +649,23 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 					elt->buf = NULL;
 					goto stop;
 				}
-				err = txq->send_pending
-					(txq->qp,
-					 addr,
-					 length,
-					 lkey,
-					 send_flags);
+#ifdef MLX5_VERBS_VLAN_INSERTION
+				if (buf->ol_flags & PKT_TX_VLAN_PKT)
+					err = txq->send_pending_vlan
+						(txq->qp,
+						 addr,
+						 length,
+						 lkey,
+						 send_flags,
+						 &buf->vlan_tci);
+				else
+#endif /* MLX5_VERBS_VLAN_INSERTION */
+					err = txq->send_pending
+						(txq->qp,
+						 addr,
+						 length,
+						 lkey,
+						 send_flags);
 			}
 			if (unlikely(err))
 				goto stop;
@@ -619,11 +682,21 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			if (ret.length == (unsigned int)-1)
 				goto stop;
 			/* Put SG list into send queue. */
-			err = txq->send_pending_sg_list
-				(txq->qp,
-				 sges,
-				 ret.num,
-				 send_flags);
+#ifdef MLX5_VERBS_VLAN_INSERTION
+			if (buf->ol_flags & PKT_TX_VLAN_PKT)
+				err = txq->send_pending_sg_list_vlan
+					(txq->qp,
+					 sges,
+					 ret.num,
+					 send_flags,
+					 &buf->vlan_tci);
+			else
+#endif /* MLX5_VERBS_VLAN_INSERTION */
+				err = txq->send_pending_sg_list
+					(txq->qp,
+					 sges,
+					 ret.num,
+					 send_flags);
 			if (unlikely(err))
 				goto stop;
 #ifdef MLX5_PMD_SOFT_COUNTERS
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 61be3e4..dd9aa25 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -255,11 +255,20 @@ struct txq {
 	struct priv *priv; /* Back pointer to private data. */
 	int32_t (*poll_cnt)(struct ibv_cq *cq, uint32_t max);
 	int (*send_pending)();
+#ifdef MLX5_VERBS_VLAN_INSERTION
+	int (*send_pending_vlan)();
+#endif
 #if MLX5_PMD_MAX_INLINE > 0
 	int (*send_pending_inline)();
+#ifdef MLX5_VERBS_VLAN_INSERTION
+	int (*send_pending_inline_vlan)();
+#endif
 #endif
 #if MLX5_PMD_SGE_WR_N > 1
 	int (*send_pending_sg_list)();
+#ifdef MLX5_VERBS_VLAN_INSERTION
+	int (*send_pending_sg_list_vlan)();
+#endif
 #endif
 	int (*send_flush)(struct ibv_qp *qp);
 	struct ibv_cq *cq; /* Completion Queue. */
@@ -283,7 +292,11 @@ struct txq {
 	/* Elements used only for init part are here. */
 	linear_t (*elts_linear)[]; /* Linearized buffers. */
 	struct ibv_mr *mr_linear; /* Memory Region for linearized buffers. */
+#ifdef HAVE_VERBS_VLAN_INSERTION
+	struct ibv_exp_qp_burst_family_v1 *if_qp; /* QP burst interface. */
+#else
 	struct ibv_exp_qp_burst_family *if_qp; /* QP burst interface. */
+#endif
 	struct ibv_exp_cq_family *if_cq; /* CQ interface. */
 	struct ibv_exp_res_domain *rd; /* Resource Domain. */
 	unsigned int socket; /* CPU socket ID for allocations. */
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 6700af4..c643cf4 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -400,7 +400,11 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
 		.intf_scope = IBV_EXP_INTF_GLOBAL,
 		.intf = IBV_EXP_INTF_QP_BURST,
 		.obj = tmpl.qp,
-#ifdef HAVE_EXP_QP_BURST_CREATE_ENABLE_MULTI_PACKET_SEND_WR
+#ifdef HAVE_VERBS_VLAN_INSERTION
+		.intf_version = 1,
+#endif
+#if defined(HAVE_EXP_QP_BURST_CREATE_ENABLE_MULTI_PACKET_SEND_WR) && \
+	!defined(MLX5_VERBS_VLAN_INSERTION)
 		/* Multi packet send WR can only be used outside of VF. */
 		.family_flags =
 			(!priv->vf ?
@@ -422,11 +426,20 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
 	txq->poll_cnt = txq->if_cq->poll_cnt;
 #if MLX5_PMD_MAX_INLINE > 0
 	txq->send_pending_inline = txq->if_qp->send_pending_inline;
+#ifdef MLX5_VERBS_VLAN_INSERTION
+	txq->send_pending_inline_vlan = txq->if_qp->send_pending_inline_vlan;
+#endif
 #endif
 #if MLX5_PMD_SGE_WR_N > 1
 	txq->send_pending_sg_list = txq->if_qp->send_pending_sg_list;
+#ifdef MLX5_VERBS_VLAN_INSERTION
+	txq->send_pending_sg_list_vlan = txq->if_qp->send_pending_sg_list_vlan;
+#endif
 #endif
 	txq->send_pending = txq->if_qp->send_pending;
+#ifdef MLX5_VERBS_VLAN_INSERTION
+	txq->send_pending_vlan = txq->if_qp->send_pending_vlan;
+#endif
 	txq->send_flush = txq->if_qp->send_flush;
 	DEBUG("%p: txq updated with %p", (void *)txq, (void *)&tmpl);
 	/* Pre-register known mempools. */
-- 
2.1.4