[PATCH 1/4] net/ena: add fast mbuf free support

[Michal Krawczyk <mk@semihalf.com>]

From: Dawid Gorecki <dgr@semihalf.com>

Add support for RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE offload. It can be
enabled if all the mbufs for a given queue belong to the same mempool
and their reference count is equal to 1.

Signed-off-by: Dawid Gorecki <dgr@semihalf.com>
Reviewed-by: Michal Krawczyk <mk@semihalf.com>
Reviewed-by: Shai Brandes <shaibran@amazon.com>
Reviewed-by: Amit Bernstein <amitbern@amazon.com>
---
 doc/guides/nics/features/ena.ini       |  1 +
 doc/guides/rel_notes/release_22_07.rst |  7 ++++
 drivers/net/ena/ena_ethdev.c           | 49 ++++++++++++++++++++++++--
 3 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/doc/guides/nics/features/ena.ini b/doc/guides/nics/features/ena.ini
index 59c1ae85fa..1fe7a71e3d 100644
--- a/doc/guides/nics/features/ena.ini
+++ b/doc/guides/nics/features/ena.ini
@@ -7,6 +7,7 @@
 Link status          = Y
 Link status event    = Y
 Rx interrupt         = Y
+Fast mbuf free       = Y
 Free Tx mbuf on demand = Y
 MTU update           = Y
 Scattered Rx         = Y
diff --git a/doc/guides/rel_notes/release_22_07.rst b/doc/guides/rel_notes/release_22_07.rst
index d46f773df0..73f566e5fc 100644
--- a/doc/guides/rel_notes/release_22_07.rst
+++ b/doc/guides/rel_notes/release_22_07.rst
@@ -87,6 +87,13 @@ New Features
 
   Added an API which can get the device type of vDPA device.
 
+
+* **Updated Amazon ena driver.**
+
+  The new driver version (v2.7.0) includes:
+
+  * Added fast mbuf free feature support.
+
 * **Updated Intel iavf driver.**
 
   * Added Tx QoS queue rate limitation support.
diff --git a/drivers/net/ena/ena_ethdev.c b/drivers/net/ena/ena_ethdev.c
index 68768cab70..68a4478410 100644
--- a/drivers/net/ena/ena_ethdev.c
+++ b/drivers/net/ena/ena_ethdev.c
@@ -36,6 +36,12 @@
 
 #define ENA_MIN_RING_DESC	128
 
+/*
+ * We should try to keep ENA_CLEANUP_BUF_SIZE lower than
+ * RTE_MEMPOOL_CACHE_MAX_SIZE, so we can fit this in mempool local cache.
+ */
+#define ENA_CLEANUP_BUF_SIZE	256
+
 #define ENA_PTYPE_HAS_HASH	(RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP)
 
 struct ena_stats {
@@ -2402,6 +2408,8 @@ static uint64_t ena_get_tx_port_offloads(struct ena_adapter *adapter)
 
 	port_offloads |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS;
 
+	port_offloads |= RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
+
 	return port_offloads;
 }
 
@@ -2414,9 +2422,12 @@ static uint64_t ena_get_rx_queue_offloads(struct ena_adapter *adapter)
 
 static uint64_t ena_get_tx_queue_offloads(struct ena_adapter *adapter)
 {
+	uint64_t queue_offloads = 0;
 	RTE_SET_USED(adapter);
 
-	return 0;
+	queue_offloads |= RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
+
+	return queue_offloads;
 }
 
 static int ena_infos_get(struct rte_eth_dev *dev,
@@ -3001,13 +3012,38 @@ static int ena_xmit_mbuf(struct ena_ring *tx_ring, struct rte_mbuf *mbuf)
 	return 0;
 }
 
+static __rte_always_inline size_t
+ena_tx_cleanup_mbuf_fast(struct rte_mbuf **mbufs_to_clean,
+			 struct rte_mbuf *mbuf,
+			 size_t mbuf_cnt,
+			 size_t buf_size)
+{
+	struct rte_mbuf *m_next;
+
+	while (mbuf != NULL) {
+		m_next = mbuf->next;
+		mbufs_to_clean[mbuf_cnt++] = mbuf;
+		if (mbuf_cnt == buf_size) {
+			rte_mempool_put_bulk(mbufs_to_clean[0]->pool, (void **)mbufs_to_clean,
+				(unsigned int)mbuf_cnt);
+			mbuf_cnt = 0;
+		}
+		mbuf = m_next;
+	}
+
+	return mbuf_cnt;
+}
+
 static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt)
 {
+	struct rte_mbuf *mbufs_to_clean[ENA_CLEANUP_BUF_SIZE];
 	struct ena_ring *tx_ring = (struct ena_ring *)txp;
+	size_t mbuf_cnt = 0;
 	unsigned int total_tx_descs = 0;
 	unsigned int total_tx_pkts = 0;
 	uint16_t cleanup_budget;
 	uint16_t next_to_clean = tx_ring->next_to_clean;
+	bool fast_free = tx_ring->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
 
 	/*
 	 * If free_pkt_cnt is equal to 0, it means that the user requested
@@ -3032,7 +3068,12 @@ static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt)
 		tx_info->timestamp = 0;
 
 		mbuf = tx_info->mbuf;
-		rte_pktmbuf_free(mbuf);
+		if (fast_free) {
+			mbuf_cnt = ena_tx_cleanup_mbuf_fast(mbufs_to_clean, mbuf, mbuf_cnt,
+				ENA_CLEANUP_BUF_SIZE);
+		} else {
+			rte_pktmbuf_free(mbuf);
+		}
 
 		tx_info->mbuf = NULL;
 		tx_ring->empty_tx_reqs[next_to_clean] = req_id;
@@ -3052,6 +3093,10 @@ static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt)
 		ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);
 	}
 
+	if (mbuf_cnt != 0)
+		rte_mempool_put_bulk(mbufs_to_clean[0]->pool,
+			(void **)mbufs_to_clean, mbuf_cnt);
+
 	/* Notify completion handler that full cleanup was performed */
 	if (free_pkt_cnt == 0 || total_tx_pkts < cleanup_budget)
 		tx_ring->last_cleanup_ticks = rte_get_timer_cycles();
-- 
2.25.1