[dpdk-dev] [PATCH v3 6/7] net/mlx4: get back Rx checksum offloads

[Ophir Munk <ophirmu@mellanox.com>]

From: Moti Haimovsky <motih@mellanox.com>

This patch adds hardware offloading support for IPV4, UDP and TCP
checksum verification.
This commit also includes support for offloading IPV4, UDP and TCP tunnel
checksum verification to the hardware.

Signed-off-by: Vasily Philipov <vasilyf@mellanox.com>
---
 drivers/net/mlx4/mlx4.c        |   2 +
 drivers/net/mlx4/mlx4_ethdev.c |   8 ++-
 drivers/net/mlx4/mlx4_prm.h    |  19 +++++++
 drivers/net/mlx4/mlx4_rxq.c    |   5 ++
 drivers/net/mlx4/mlx4_rxtx.c   | 120 ++++++++++++++++++++++++++++++++++++++++-
 drivers/net/mlx4/mlx4_rxtx.h   |   2 +
 6 files changed, 152 insertions(+), 4 deletions(-)

diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index a0e76ee..865ffdd 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -535,6 +535,8 @@ struct mlx4_conf {
 		priv->vf = vf;
 		priv->hw_csum =
 		     !!(device_attr.device_cap_flags & IBV_DEVICE_RAW_IP_CSUM);
+		DEBUG("checksum offloading is %ssupported",
+		      (priv->hw_csum ? "" : "not "));
 		priv->hw_csum_l2tun = tunnel_en;
 		DEBUG("L2 tunnel checksum offloads are %ssupported",
 		      (priv->hw_csum_l2tun ? "" : "not "));
diff --git a/drivers/net/mlx4/mlx4_ethdev.c b/drivers/net/mlx4/mlx4_ethdev.c
index 95cc6e4..6dbf273 100644
--- a/drivers/net/mlx4/mlx4_ethdev.c
+++ b/drivers/net/mlx4/mlx4_ethdev.c
@@ -553,10 +553,14 @@
 	info->max_mac_addrs = 1;
 	info->rx_offload_capa = 0;
 	info->tx_offload_capa = 0;
-	if (priv->hw_csum)
+	if (priv->hw_csum) {
 		info->tx_offload_capa |= (DEV_TX_OFFLOAD_IPV4_CKSUM |
-					  DEV_TX_OFFLOAD_UDP_CKSUM  |
+					  DEV_TX_OFFLOAD_UDP_CKSUM |
 					  DEV_TX_OFFLOAD_TCP_CKSUM);
+		info->rx_offload_capa |= (DEV_RX_OFFLOAD_IPV4_CKSUM |
+					  DEV_RX_OFFLOAD_UDP_CKSUM |
+					  DEV_RX_OFFLOAD_TCP_CKSUM);
+	}
 	if (priv->hw_csum_l2tun)
 		info->tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
 	if (mlx4_get_ifname(priv, &ifname) == 0)
diff --git a/drivers/net/mlx4/mlx4_prm.h b/drivers/net/mlx4/mlx4_prm.h
index 57f5a46..73c3d55 100644
--- a/drivers/net/mlx4/mlx4_prm.h
+++ b/drivers/net/mlx4/mlx4_prm.h
@@ -70,6 +70,25 @@
 #define MLX4_SIZE_TO_TXBBS(size) \
 		(RTE_ALIGN((size), (MLX4_TXBB_SIZE)) >> (MLX4_TXBB_SHIFT))
 
+/* Generic macro to convert MLX4 to IBV flags. */
+#define MLX4_TRANSPOSE(val, from, to) \
+		(__extension__({ \
+			typeof(val) _val = (val); \
+			typeof(from) _from = (from); \
+			typeof(to) _to = (to); \
+			(((_from) >= (_to)) ? \
+			(((_val) & (_from)) / ((_from) / (_to))) : \
+			(((_val) & (_from)) * ((_to) / (_from)))); \
+		}))
+
+/* CQE checksum flags */
+enum {
+	MLX4_CQE_L2_TUNNEL_IPV4 = (int)(1U << 25),
+	MLX4_CQE_L2_TUNNEL_L4_CSUM = (int)(1U << 26),
+	MLX4_CQE_L2_TUNNEL = (int)(1U << 27),
+	MLX4_CQE_L2_TUNNEL_IPOK = (int)(1U << 31),
+};
+
 /* Send queue information. */
 struct mlx4_sq {
 	char *buf; /**< SQ buffer. */
diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c
index 7d13121..889f05c 100644
--- a/drivers/net/mlx4/mlx4_rxq.c
+++ b/drivers/net/mlx4/mlx4_rxq.c
@@ -267,6 +267,11 @@
 	int ret;
 
 	(void)conf; /* Thresholds configuration (ignored). */
+	/* Toggle Rx checksum offload if hardware supports it. */
+	if (priv->hw_csum)
+		tmpl.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
+	if (priv->hw_csum_l2tun)
+		tmpl.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
 	mb_len = rte_pktmbuf_data_room_size(mp);
 	/* Enable scattered packets support for this queue if necessary. */
 	assert(mb_len >= RTE_PKTMBUF_HEADROOM);
diff --git a/drivers/net/mlx4/mlx4_rxtx.c b/drivers/net/mlx4/mlx4_rxtx.c
index ea92ebb..ca66b1d 100644
--- a/drivers/net/mlx4/mlx4_rxtx.c
+++ b/drivers/net/mlx4/mlx4_rxtx.c
@@ -563,6 +563,110 @@ struct pv {
 }
 
 /**
+ * Translate Rx completion flags to packet type.
+ *
+ * @param flags
+ *   Rx completion flags returned by poll_length_flags().
+ *
+ * @return
+ *   Packet type for struct rte_mbuf.
+ */
+static inline uint32_t
+rxq_cq_to_pkt_type(uint32_t flags)
+{
+	uint32_t pkt_type;
+
+	if (flags & MLX4_CQE_L2_TUNNEL)
+		pkt_type =
+			MLX4_TRANSPOSE(flags,
+			       (uint32_t)MLX4_CQE_L2_TUNNEL_IPV4,
+			       (uint32_t)RTE_PTYPE_L3_IPV4_EXT_UNKNOWN) |
+			MLX4_TRANSPOSE(flags,
+			       (uint32_t)MLX4_CQE_STATUS_IPV4_PKT,
+			       (uint32_t)RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN);
+	else
+		pkt_type =
+			MLX4_TRANSPOSE(flags,
+			       (uint32_t)MLX4_CQE_STATUS_IPV4_PKT,
+			       (uint32_t)RTE_PTYPE_L3_IPV4_EXT_UNKNOWN);
+	ERROR("pkt_type 0x%x", pkt_type); //
+	return pkt_type;
+}
+
+/**
+ * Translate Rx completion flags to offload flags.
+ *
+ * @param  flags
+ *   Rx completion flags returned by poll_length_flags().
+ * @param csum
+ *   Rx checksum enable flag
+ * @param csum_l2tun
+ *   Rx L2 tunnel checksum enable flag
+ *
+ * @return
+ *   Offload flags (ol_flags) for struct rte_mbuf.
+ */
+static inline uint32_t
+rxq_cq_to_ol_flags(uint32_t flags, unsigned int csum, unsigned int csum_l2tun)
+{
+	uint32_t ol_flags = 0;
+
+	if (csum)
+		ol_flags |=
+			MLX4_TRANSPOSE(flags,
+				(uint64_t)MLX4_CQE_STATUS_IP_HDR_CSUM_OK,
+				PKT_RX_IP_CKSUM_GOOD) |
+			MLX4_TRANSPOSE(flags,
+				(uint64_t)MLX4_CQE_STATUS_TCP_UDP_CSUM_OK,
+				PKT_RX_L4_CKSUM_GOOD);
+	if ((flags & MLX4_CQE_L2_TUNNEL) && csum_l2tun)
+		ol_flags |=
+			MLX4_TRANSPOSE(flags,
+				       (uint64_t)MLX4_CQE_L2_TUNNEL_IPOK,
+				       PKT_RX_IP_CKSUM_GOOD) |
+			MLX4_TRANSPOSE(flags,
+				       (uint64_t)MLX4_CQE_L2_TUNNEL_L4_CSUM,
+				       PKT_RX_L4_CKSUM_GOOD);
+	return ol_flags;
+}
+
+/**
+ * Get Rx checksum CQE flags.
+ *
+ * @param cqe
+ *   Pointer to cqe structure.
+ * @param csum
+ *   Rx checksum enable flag
+ * @param csum_l2tun
+ *   RX L2 tunnel checksum enable flag
+ *
+ * @return
+ *   CQE flags in CPU order
+ */
+static inline uint32_t
+mlx4_cqe_flags(struct mlx4_cqe *cqe,
+	       int csum, unsigned int csum_l2tun)
+{
+	uint32_t flags = 0;
+
+	/*
+	 * The relevant bits are in different locations on their
+	 * CQE fields therefore we can join them in one 32-bit
+	 * variable.
+	 */
+	if (csum)
+		flags = rte_be_to_cpu_32(cqe->status) &
+			MLX4_CQE_STATUS_IPV4_CSUM_OK;
+	if (csum_l2tun)
+		flags |= rte_be_to_cpu_32(cqe->vlan_my_qpn) &
+			 (MLX4_CQE_L2_TUNNEL |
+			  MLX4_CQE_L2_TUNNEL_IPOK |
+			  MLX4_CQE_L2_TUNNEL_L4_CSUM |
+			  MLX4_CQE_L2_TUNNEL_IPV4);
+		return flags;
+}
+
+/**
  * Poll one CQE from CQ.
  *
  * @param rxq
@@ -601,7 +705,7 @@ struct pv {
 }
 
 /**
- * DPDK callback for RX with scattered packets support.
+ * DPDK callback for Rx with scattered packets support.
  *
  * @param dpdk_rxq
  *   Generic pointer to Rx queue structure.
@@ -666,7 +770,7 @@ struct pv {
 				break;
 			}
 			if (unlikely(len < 0)) {
-				/* RX error, packet is likely too large. */
+				/* Rx error, packet is likely too large. */
 				rte_mbuf_raw_free(rep);
 				++rxq->stats.idropped;
 				goto skip;
@@ -674,6 +778,18 @@ struct pv {
 			pkt = seg;
 			pkt->packet_type = 0;
 			pkt->ol_flags = 0;
+			if (rxq->csum | rxq->csum_l2tun) {
+				uint32_t flags =
+					mlx4_cqe_flags(cqe, rxq->csum,
+						       rxq->csum_l2tun);
+				pkt->ol_flags =
+					rxq_cq_to_ol_flags(flags, rxq->csum,
+							   rxq->csum_l2tun);
+				pkt->packet_type = rxq_cq_to_pkt_type(flags);
+			} else {
+				pkt->packet_type = 0;
+				pkt->ol_flags = 0;
+			}
 			pkt->pkt_len = len;
 		}
 		rep->nb_segs = 1;
diff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h
index dc283e1..75c98c1 100644
--- a/drivers/net/mlx4/mlx4_rxtx.h
+++ b/drivers/net/mlx4/mlx4_rxtx.h
@@ -80,6 +80,8 @@ struct rxq {
 	} hw;
 	struct mlx4_cq mcq;  /**< Info for directly manipulating the CQ. */
 	unsigned int sge_n; /**< Log 2 of SGEs number. */
+	unsigned int csum:1; /**< Enable checksum offloading. */
+	unsigned int csum_l2tun:1; /**< Enable checksum for L2 tunnels. */
 	struct mlx4_rxq_stats stats; /**< Rx queue counters. */
 	unsigned int socket; /**< CPU socket ID for allocations. */
 };
-- 
1.8.3.1