From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from alln-iport-4.cisco.com (alln-iport-4.cisco.com [173.37.142.91]) by dpdk.org (Postfix) with ESMTP id 5F8105A24 for ; Tue, 24 May 2016 08:33:04 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=cisco.com; i=@cisco.com; l=5584; q=dns/txt; s=iport; t=1464071584; x=1465281184; h=from:to:cc:subject:date:message-id:in-reply-to: references; bh=0FduklNvVhuBybLCGHL7tqiNyIys0001/aVrRG7phFw=; b=C3ibizBwNKZqX+c7R8xdPbhJjtZ6m3dduZooye0UKQMTjAkoBkkk+g5r Uyi7rTv9Yus6PKel7+LPchAp3ykmdVbNU9QyB56VUn+53ovu229OW/n4f hVCMVRKElbXwO1MWPFIBI029SvefZ3oTEcxFuTcod1Y1/EvIPxLwryLqD s=; X-IronPort-AV: E=Sophos;i="5.26,359,1459814400"; d="scan'208";a="276290013" Received: from rcdn-core-12.cisco.com ([173.37.93.148]) by alln-iport-4.cisco.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 24 May 2016 06:33:02 +0000 Received: from cisco.com (savbu-usnic-a.cisco.com [10.193.184.48]) by rcdn-core-12.cisco.com (8.14.5/8.14.5) with ESMTP id u4O6X2Mu010476; Tue, 24 May 2016 06:33:02 GMT Received: by cisco.com (Postfix, from userid 392789) id 7728B3FAAE20; Mon, 23 May 2016 23:33:02 -0700 (PDT) From: John Daley To: dev@dpdk.org Cc: John Daley Date: Mon, 23 May 2016 23:32:55 -0700 Message-Id: <1464071579-30072-8-git-send-email-johndale@cisco.com> X-Mailer: git-send-email 2.7.0 In-Reply-To: <1464071579-30072-1-git-send-email-johndale@cisco.com> References: <1464071579-30072-1-git-send-email-johndale@cisco.com> Subject: [dpdk-dev] [PATCH v2 07/11] enic: use Tx completion messages instead of descriptors X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 24 May 2016 06:33:04 -0000 The NIC can either DMA a separate completion message for each completed send or periodically just DMA an index of the last completed send. Switch to the second method which improves cache locality and performance. Signed-off-by: John Daley --- drivers/net/enic/base/vnic_wq.c | 1 + drivers/net/enic/base/vnic_wq.h | 3 +++ drivers/net/enic/enic_main.c | 43 ++++++++++++++++++++++++++++++++--------- drivers/net/enic/enic_rxtx.c | 11 +++++++---- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/drivers/net/enic/base/vnic_wq.c b/drivers/net/enic/base/vnic_wq.c index ab81c7e..cfef1af 100644 --- a/drivers/net/enic/base/vnic_wq.c +++ b/drivers/net/enic/base/vnic_wq.c @@ -142,6 +142,7 @@ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, vnic_wq_init_start(wq, cq_index, 0, 0, error_interrupt_enable, error_interrupt_offset); + wq->last_completed_index = 0; } void vnic_wq_error_out(struct vnic_wq *wq, unsigned int error) diff --git a/drivers/net/enic/base/vnic_wq.h b/drivers/net/enic/base/vnic_wq.h index a6759f5..fe46bb4 100644 --- a/drivers/net/enic/base/vnic_wq.h +++ b/drivers/net/enic/base/vnic_wq.h @@ -38,6 +38,7 @@ #include "vnic_dev.h" #include "vnic_cq.h" +#include /* Work queue control */ struct vnic_wq_ctrl { @@ -79,6 +80,8 @@ struct vnic_wq { unsigned int tail_idx; unsigned int pkts_outstanding; unsigned int socket_id; + const struct rte_memzone *cqmsg_rz; + uint16_t last_completed_index; }; static inline unsigned int vnic_wq_desc_avail(struct vnic_wq *wq) diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c index 5bf5fcf..eaa206e 100644 --- a/drivers/net/enic/enic_main.c +++ b/drivers/net/enic/enic_main.c @@ -97,7 +97,6 @@ enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq) } } - void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size) { vnic_set_hdr_split_size(enic->vdev, split_hdr_size); @@ -235,12 +234,26 @@ void enic_init_vnic_resources(struct enic *enic) unsigned int error_interrupt_enable = 1; unsigned int error_interrupt_offset = 0; unsigned int index = 0; + unsigned int cq_idx; for (index = 0; index < enic->rq_count; index++) { vnic_rq_init(&enic->rq[index], enic_cq_rq(enic, index), error_interrupt_enable, error_interrupt_offset); + + cq_idx = enic_cq_rq(enic, index); + vnic_cq_init(&enic->cq[cq_idx], + 0 /* flow_control_enable */, + 1 /* color_enable */, + 0 /* cq_head */, + 0 /* cq_tail */, + 1 /* cq_tail_color */, + 0 /* interrupt_enable */, + 1 /* cq_entry_enable */, + 0 /* cq_message_enable */, + 0 /* interrupt offset */, + 0 /* cq_message_addr */); } for (index = 0; index < enic->wq_count; index++) { @@ -248,22 +261,19 @@ void enic_init_vnic_resources(struct enic *enic) enic_cq_wq(enic, index), error_interrupt_enable, error_interrupt_offset); - } - vnic_dev_stats_clear(enic->vdev); - - for (index = 0; index < enic->cq_count; index++) { - vnic_cq_init(&enic->cq[index], + cq_idx = enic_cq_wq(enic, index); + vnic_cq_init(&enic->cq[cq_idx], 0 /* flow_control_enable */, 1 /* color_enable */, 0 /* cq_head */, 0 /* cq_tail */, 1 /* cq_tail_color */, 0 /* interrupt_enable */, - 1 /* cq_entry_enable */, - 0 /* cq_message_enable */, + 0 /* cq_entry_enable */, + 1 /* cq_message_enable */, 0 /* interrupt offset */, - 0 /* cq_message_addr */); + (u64)enic->wq[index].cqmsg_rz->phys_addr); } vnic_intr_init(&enic->intr, @@ -507,6 +517,7 @@ void enic_free_wq(void *txq) struct vnic_wq *wq = (struct vnic_wq *)txq; struct enic *enic = vnic_dev_priv(wq->vdev); + rte_memzone_free(wq->cqmsg_rz); vnic_wq_free(wq); vnic_cq_free(&enic->cq[enic->rq_count + wq->index]); } @@ -517,6 +528,8 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, int err; struct vnic_wq *wq = &enic->wq[queue_idx]; unsigned int cq_index = enic_cq_wq(enic, queue_idx); + char name[NAME_MAX]; + static int instance; wq->socket_id = socket_id; if (nb_desc) { @@ -552,6 +565,18 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, dev_err(enic, "error in allocation of cq for wq\n"); } + /* setup up CQ message */ + snprintf((char *)name, sizeof(name), + "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx, + instance++); + + wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name, + sizeof(uint32_t), + SOCKET_ID_ANY, 0, + ENIC_ALIGN); + if (!wq->cqmsg_rz) + return -ENOMEM; + return err; } diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c index ea31dfa..2a54333 100644 --- a/drivers/net/enic/enic_rxtx.c +++ b/drivers/net/enic/enic_rxtx.c @@ -348,11 +348,14 @@ static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq) { - unsigned int cq = enic_cq_wq(enic, wq->index); + u16 completed_index = *((uint32_t *)wq->cqmsg_rz->addr) & 0xffff; - /* Return the work done */ - return vnic_cq_service(&enic->cq[cq], - -1 /*wq_work_to_do*/, enic_wq_service, NULL); + if (wq->last_completed_index != completed_index) { + enic_wq_service(enic->vdev, NULL, 0, wq->index, + completed_index, NULL); + wq->last_completed_index = completed_index; + } + return 0; } void enic_post_wq_index(struct vnic_wq *wq) -- 2.7.0