From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <johndale@cisco.com>
Received: from alln-iport-4.cisco.com (alln-iport-4.cisco.com [173.37.142.91])
 by dpdk.org (Postfix) with ESMTP id 5F8105A24
 for <dev@dpdk.org>; Tue, 24 May 2016 08:33:04 +0200 (CEST)
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple;
 d=cisco.com; i=@cisco.com; l=5584; q=dns/txt; s=iport;
 t=1464071584; x=1465281184;
 h=from:to:cc:subject:date:message-id:in-reply-to: references;
 bh=0FduklNvVhuBybLCGHL7tqiNyIys0001/aVrRG7phFw=;
 b=C3ibizBwNKZqX+c7R8xdPbhJjtZ6m3dduZooye0UKQMTjAkoBkkk+g5r
 Uyi7rTv9Yus6PKel7+LPchAp3ykmdVbNU9QyB56VUn+53ovu229OW/n4f
 hVCMVRKElbXwO1MWPFIBI029SvefZ3oTEcxFuTcod1Y1/EvIPxLwryLqD s=;
X-IronPort-AV: E=Sophos;i="5.26,359,1459814400"; d="scan'208";a="276290013"
Received: from rcdn-core-12.cisco.com ([173.37.93.148])
 by alln-iport-4.cisco.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;
 24 May 2016 06:33:02 +0000
Received: from cisco.com (savbu-usnic-a.cisco.com [10.193.184.48])
 by rcdn-core-12.cisco.com (8.14.5/8.14.5) with ESMTP id u4O6X2Mu010476;
 Tue, 24 May 2016 06:33:02 GMT
Received: by cisco.com (Postfix, from userid 392789)
 id 7728B3FAAE20; Mon, 23 May 2016 23:33:02 -0700 (PDT)
From: John Daley <johndale@cisco.com>
To: dev@dpdk.org
Cc: John Daley <johndale@cisco.com>
Date: Mon, 23 May 2016 23:32:55 -0700
Message-Id: <1464071579-30072-8-git-send-email-johndale@cisco.com>
X-Mailer: git-send-email 2.7.0
In-Reply-To: <1464071579-30072-1-git-send-email-johndale@cisco.com>
References: <1464071579-30072-1-git-send-email-johndale@cisco.com>
Subject: [dpdk-dev] [PATCH v2 07/11] enic: use Tx completion messages
	instead of descriptors
X-BeenThere: dev@dpdk.org
X-Mailman-Version: 2.1.15
Precedence: list
List-Id: patches and discussions about DPDK <dev.dpdk.org>
List-Unsubscribe: <http://dpdk.org/ml/options/dev>,
 <mailto:dev-request@dpdk.org?subject=unsubscribe>
List-Archive: <http://dpdk.org/ml/archives/dev/>
List-Post: <mailto:dev@dpdk.org>
List-Help: <mailto:dev-request@dpdk.org?subject=help>
List-Subscribe: <http://dpdk.org/ml/listinfo/dev>,
 <mailto:dev-request@dpdk.org?subject=subscribe>
X-List-Received-Date: Tue, 24 May 2016 06:33:04 -0000

The NIC can either DMA a separate completion message for each
completed send or periodically just DMA an index of the last
completed send. Switch to the second method which improves
cache locality and performance.

Signed-off-by: John Daley <johndale@cisco.com>
---
 drivers/net/enic/base/vnic_wq.c |  1 +
 drivers/net/enic/base/vnic_wq.h |  3 +++
 drivers/net/enic/enic_main.c    | 43 ++++++++++++++++++++++++++++++++---------
 drivers/net/enic/enic_rxtx.c    | 11 +++++++----
 4 files changed, 45 insertions(+), 13 deletions(-)

diff --git a/drivers/net/enic/base/vnic_wq.c b/drivers/net/enic/base/vnic_wq.c
index ab81c7e..cfef1af 100644
--- a/drivers/net/enic/base/vnic_wq.c
+++ b/drivers/net/enic/base/vnic_wq.c
@@ -142,6 +142,7 @@ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index,
 	vnic_wq_init_start(wq, cq_index, 0, 0,
 		error_interrupt_enable,
 		error_interrupt_offset);
+	wq->last_completed_index = 0;
 }
 
 void vnic_wq_error_out(struct vnic_wq *wq, unsigned int error)
diff --git a/drivers/net/enic/base/vnic_wq.h b/drivers/net/enic/base/vnic_wq.h
index a6759f5..fe46bb4 100644
--- a/drivers/net/enic/base/vnic_wq.h
+++ b/drivers/net/enic/base/vnic_wq.h
@@ -38,6 +38,7 @@
 
 #include "vnic_dev.h"
 #include "vnic_cq.h"
+#include <rte_memzone.h>
 
 /* Work queue control */
 struct vnic_wq_ctrl {
@@ -79,6 +80,8 @@ struct vnic_wq {
 	unsigned int tail_idx;
 	unsigned int pkts_outstanding;
 	unsigned int socket_id;
+	const struct rte_memzone *cqmsg_rz;
+	uint16_t last_completed_index;
 };
 
 static inline unsigned int vnic_wq_desc_avail(struct vnic_wq *wq)
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index 5bf5fcf..eaa206e 100644
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -97,7 +97,6 @@ enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq)
 	}
 }
 
-
 void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size)
 {
 	vnic_set_hdr_split_size(enic->vdev, split_hdr_size);
@@ -235,12 +234,26 @@ void enic_init_vnic_resources(struct enic *enic)
 	unsigned int error_interrupt_enable = 1;
 	unsigned int error_interrupt_offset = 0;
 	unsigned int index = 0;
+	unsigned int cq_idx;
 
 	for (index = 0; index < enic->rq_count; index++) {
 		vnic_rq_init(&enic->rq[index],
 			enic_cq_rq(enic, index),
 			error_interrupt_enable,
 			error_interrupt_offset);
+
+		cq_idx = enic_cq_rq(enic, index);
+		vnic_cq_init(&enic->cq[cq_idx],
+			0 /* flow_control_enable */,
+			1 /* color_enable */,
+			0 /* cq_head */,
+			0 /* cq_tail */,
+			1 /* cq_tail_color */,
+			0 /* interrupt_enable */,
+			1 /* cq_entry_enable */,
+			0 /* cq_message_enable */,
+			0 /* interrupt offset */,
+			0 /* cq_message_addr */);
 	}
 
 	for (index = 0; index < enic->wq_count; index++) {
@@ -248,22 +261,19 @@ void enic_init_vnic_resources(struct enic *enic)
 			enic_cq_wq(enic, index),
 			error_interrupt_enable,
 			error_interrupt_offset);
-	}
 
-	vnic_dev_stats_clear(enic->vdev);
-
-	for (index = 0; index < enic->cq_count; index++) {
-		vnic_cq_init(&enic->cq[index],
+		cq_idx = enic_cq_wq(enic, index);
+		vnic_cq_init(&enic->cq[cq_idx],
 			0 /* flow_control_enable */,
 			1 /* color_enable */,
 			0 /* cq_head */,
 			0 /* cq_tail */,
 			1 /* cq_tail_color */,
 			0 /* interrupt_enable */,
-			1 /* cq_entry_enable */,
-			0 /* cq_message_enable */,
+			0 /* cq_entry_enable */,
+			1 /* cq_message_enable */,
 			0 /* interrupt offset */,
-			0 /* cq_message_addr */);
+			(u64)enic->wq[index].cqmsg_rz->phys_addr);
 	}
 
 	vnic_intr_init(&enic->intr,
@@ -507,6 +517,7 @@ void enic_free_wq(void *txq)
 	struct vnic_wq *wq = (struct vnic_wq *)txq;
 	struct enic *enic = vnic_dev_priv(wq->vdev);
 
+	rte_memzone_free(wq->cqmsg_rz);
 	vnic_wq_free(wq);
 	vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
 }
@@ -517,6 +528,8 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
 	int err;
 	struct vnic_wq *wq = &enic->wq[queue_idx];
 	unsigned int cq_index = enic_cq_wq(enic, queue_idx);
+	char name[NAME_MAX];
+	static int instance;
 
 	wq->socket_id = socket_id;
 	if (nb_desc) {
@@ -552,6 +565,18 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
 		dev_err(enic, "error in allocation of cq for wq\n");
 	}
 
+	/* setup up CQ message */
+	snprintf((char *)name, sizeof(name),
+		 "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
+		instance++);
+
+	wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
+						   sizeof(uint32_t),
+						   SOCKET_ID_ANY, 0,
+						   ENIC_ALIGN);
+	if (!wq->cqmsg_rz)
+		return -ENOMEM;
+
 	return err;
 }
 
diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c
index ea31dfa..2a54333 100644
--- a/drivers/net/enic/enic_rxtx.c
+++ b/drivers/net/enic/enic_rxtx.c
@@ -348,11 +348,14 @@ static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc,
 
 unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq)
 {
-	unsigned int cq = enic_cq_wq(enic, wq->index);
+	u16 completed_index = *((uint32_t *)wq->cqmsg_rz->addr) & 0xffff;
 
-	/* Return the work done */
-	return vnic_cq_service(&enic->cq[cq],
-		-1 /*wq_work_to_do*/, enic_wq_service, NULL);
+	if (wq->last_completed_index != completed_index) {
+		enic_wq_service(enic->vdev, NULL, 0, wq->index,
+				completed_index, NULL);
+		wq->last_completed_index = completed_index;
+	}
+	return 0;
 }
 
 void enic_post_wq_index(struct vnic_wq *wq)
-- 
2.7.0