[dpdk-dev] [PATCH v2 10/15] net/enic: reduce Tx completion updates

[John Daley <johndale@cisco.com>]

From: Hyong Youb Kim <hyonkim@cisco.com>

Request one completion update per roughly 32 buffers. It saves DMA
resources on the NIC, PCIe utilization, and cache miss rates.

Signed-off-by: Hyong Youb Kim <hyonkim@cisco.com>
Reviewed-by: John Daley <johndale@cisco.com>
---
 drivers/net/enic/base/vnic_wq.c |  1 +
 drivers/net/enic/base/vnic_wq.h |  1 +
 drivers/net/enic/enic_res.h     |  3 +++
 drivers/net/enic/enic_rxtx.c    | 23 +++++++++++++++++------
 4 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/drivers/net/enic/base/vnic_wq.c b/drivers/net/enic/base/vnic_wq.c
index a4c08a769..c9bf3572c 100644
--- a/drivers/net/enic/base/vnic_wq.c
+++ b/drivers/net/enic/base/vnic_wq.c
@@ -113,6 +113,7 @@ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index,
 	vnic_wq_init_start(wq, cq_index, 0, 0,
 		error_interrupt_enable,
 		error_interrupt_offset);
+	wq->cq_pend = 0;
 	wq->last_completed_index = 0;
 }
 
diff --git a/drivers/net/enic/base/vnic_wq.h b/drivers/net/enic/base/vnic_wq.h
index 6622a8a2d..236cf6962 100644
--- a/drivers/net/enic/base/vnic_wq.h
+++ b/drivers/net/enic/base/vnic_wq.h
@@ -44,6 +44,7 @@ struct vnic_wq {
 	struct vnic_dev_ring ring;
 	struct rte_mbuf **bufs;
 	unsigned int head_idx;
+	unsigned int cq_pend;
 	unsigned int tail_idx;
 	unsigned int socket_id;
 	const struct rte_memzone *cqmsg_rz;
diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h
index 6a3a0c5cc..6b1f6acad 100644
--- a/drivers/net/enic/enic_res.h
+++ b/drivers/net/enic/enic_res.h
@@ -20,6 +20,9 @@
 #define ENIC_ALIGN_DESCS		32
 #define ENIC_ALIGN_DESCS_MASK		~(ENIC_ALIGN_DESCS - 1)
 
+/* Request a completion index every 32 buffers (roughly packets) */
+#define ENIC_WQ_CQ_THRESH		32
+
 #define ENIC_MIN_MTU			68
 
 /* Does not include (possible) inserted VLAN tag and FCS */
diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c
index 89a1e66fe..7cddb53d9 100644
--- a/drivers/net/enic/enic_rxtx.c
+++ b/drivers/net/enic/enic_rxtx.c
@@ -603,7 +603,7 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	struct wq_enet_desc *descs, *desc_p, desc_tmp;
 	uint16_t mss;
 	uint8_t vlan_tag_insert;
-	uint8_t eop;
+	uint8_t eop, cq;
 	uint64_t bus_addr;
 	uint8_t offload_mode;
 	uint16_t header_len;
@@ -686,10 +686,14 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 				break;
 			}
 		}
-
-
+		wq->cq_pend++;
+		cq = 0;
+		if (eop && wq->cq_pend >= ENIC_WQ_CQ_THRESH) {
+			cq = 1;
+			wq->cq_pend = 0;
+		}
 		wq_enet_desc_enc(&desc_tmp, bus_addr, data_len, mss, header_len,
-				 offload_mode, eop, eop, 0, vlan_tag_insert,
+				 offload_mode, eop, cq, 0, vlan_tag_insert,
 				 vlan_id, 0);
 
 		*desc_p = desc_tmp;
@@ -702,14 +706,21 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 			    tx_pkt->next) {
 				data_len = tx_pkt->data_len;
 
-				if (tx_pkt->next == NULL)
+				wq->cq_pend++;
+				cq = 0;
+				if (tx_pkt->next == NULL) {
 					eop = 1;
+					if (wq->cq_pend >= ENIC_WQ_CQ_THRESH) {
+						cq = 1;
+						wq->cq_pend = 0;
+					}
+				}
 				desc_p = descs + head_idx;
 				bus_addr = (dma_addr_t)(tx_pkt->buf_iova
 					   + tx_pkt->data_off);
 				wq_enet_desc_enc((struct wq_enet_desc *)
 						 &desc_tmp, bus_addr, data_len,
-						 mss, 0, offload_mode, eop, eop,
+						 mss, 0, offload_mode, eop, cq,
 						 0, vlan_tag_insert, vlan_id,
 						 0);
 
-- 
2.16.2