From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 2CB67A04B6; Tue, 13 Oct 2020 03:56:46 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 026B61D9F4; Tue, 13 Oct 2020 03:56:22 +0200 (CEST) Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by dpdk.org (Postfix) with ESMTP id 42AE61D9F3 for ; Tue, 13 Oct 2020 03:56:18 +0200 (CEST) IronPort-SDR: qrJvZ91W6gsRDbgMaIfNyKK8ObCCl/KxLezK/662pHFHkcgilbI1WMzYijCrrflJjjIKJPpWs9 KwHNCh4YGztg== X-IronPort-AV: E=McAfee;i="6000,8403,9772"; a="183300518" X-IronPort-AV: E=Sophos;i="5.77,369,1596524400"; d="scan'208";a="183300518" X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga006.fm.intel.com ([10.253.24.20]) by fmsmga101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 12 Oct 2020 18:56:09 -0700 IronPort-SDR: ffllLOMCXS3+tAsXzBwMgYTBz2HSpl1xyjA0p7Pbw2FExX2TLJnoXm+AOmjjbqVQD5JJmxiXOq oYgSuHBo99Pw== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.77,369,1596524400"; d="scan'208";a="519805843" Received: from npg-dpdk-patrickfu-casc2.sh.intel.com ([10.67.119.92]) by fmsmga006.fm.intel.com with ESMTP; 12 Oct 2020 18:56:07 -0700 From: Patrick Fu To: dev@dpdk.org, maxime.coquelin@redhat.com, chenbo.xia@intel.com Cc: zhihong.wang@intel.com, cheng1.jiang@intel.com, patrick.fu@intel.com Date: Tue, 13 Oct 2020 09:45:44 +0800 Message-Id: <20201013014546.2896162-3-patrick.fu@intel.com> X-Mailer: git-send-email 2.18.4 In-Reply-To: <20201013014546.2896162-1-patrick.fu@intel.com> References: <20200911015316.1903181-1-patrick.fu@intel.com> <20201013014546.2896162-1-patrick.fu@intel.com> Subject: [dpdk-dev] [PATCH v4 2/4] vhost: dynamically allocate async memory X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Allocate async internal memory buffer by rte_malloc(), replacing array declaration inside vq structure. Dynamic allocation can help to save memory footprint when async path is not registered. Signed-off-by: Patrick Fu --- lib/librte_vhost/vhost.c | 69 ++++++++++++++++++++++++++-------------- lib/librte_vhost/vhost.h | 4 +-- 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c index 80adc8e59..323565898 100644 --- a/lib/librte_vhost/vhost.c +++ b/lib/librte_vhost/vhost.c @@ -324,6 +324,24 @@ cleanup_device(struct virtio_net *dev, int destroy) } } +static void +vhost_free_async_mem(struct vhost_virtqueue *vq) +{ + if (vq->async_pkts_pending) + rte_free(vq->async_pkts_pending); + if (vq->async_pkts_info) + rte_free(vq->async_pkts_info); + if (vq->it_pool) + rte_free(vq->it_pool); + if (vq->vec_pool) + rte_free(vq->vec_pool); + + vq->async_pkts_pending = NULL; + vq->async_pkts_info = NULL; + vq->it_pool = NULL; + vq->vec_pool = NULL; +} + void free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq) { @@ -331,10 +349,7 @@ free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq) rte_free(vq->shadow_used_packed); else { rte_free(vq->shadow_used_split); - if (vq->async_pkts_pending) - rte_free(vq->async_pkts_pending); - if (vq->async_pkts_info) - rte_free(vq->async_pkts_info); + vhost_free_async_mem(vq); } rte_free(vq->batch_copy_elems); rte_mempool_free(vq->iotlb_pool); @@ -1524,6 +1539,7 @@ int rte_vhost_async_channel_register(int vid, uint16_t queue_id, struct vhost_virtqueue *vq; struct virtio_net *dev = get_device(vid); struct rte_vhost_async_features f; + int node; if (dev == NULL || ops == NULL) return -1; @@ -1556,19 +1572,32 @@ int rte_vhost_async_channel_register(int vid, uint16_t queue_id, goto reg_out; } - vq->async_pkts_pending = rte_malloc(NULL, +#ifdef RTE_LIBRTE_VHOST_NUMA + if (get_mempolicy(&node, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR)) { + VHOST_LOG_CONFIG(ERR, + "unable to get numa information in async register. " + "allocating async buffer memory on the caller thread node\n"); + node = SOCKET_ID_ANY; + } +#else + node = SOCKET_ID_ANY; +#endif + + vq->async_pkts_pending = rte_malloc_socket(NULL, vq->size * sizeof(uintptr_t), - RTE_CACHE_LINE_SIZE); - vq->async_pkts_info = rte_malloc(NULL, + RTE_CACHE_LINE_SIZE, node); + vq->async_pkts_info = rte_malloc_socket(NULL, vq->size * sizeof(struct async_inflight_info), - RTE_CACHE_LINE_SIZE); - if (!vq->async_pkts_pending || !vq->async_pkts_info) { - if (vq->async_pkts_pending) - rte_free(vq->async_pkts_pending); - - if (vq->async_pkts_info) - rte_free(vq->async_pkts_info); - + RTE_CACHE_LINE_SIZE, node); + vq->it_pool = rte_malloc_socket(NULL, + VHOST_MAX_ASYNC_IT * sizeof(struct rte_vhost_iov_iter), + RTE_CACHE_LINE_SIZE, node); + vq->vec_pool = rte_malloc_socket(NULL, + VHOST_MAX_ASYNC_VEC * sizeof(struct iovec), + RTE_CACHE_LINE_SIZE, node); + if (!vq->async_pkts_pending || !vq->async_pkts_info || + !vq->it_pool || !vq->vec_pool) { + vhost_free_async_mem(vq); VHOST_LOG_CONFIG(ERR, "async register failed: cannot allocate memory for vq data " "(vid %d, qid: %d)\n", vid, queue_id); @@ -1616,15 +1645,7 @@ int rte_vhost_async_channel_unregister(int vid, uint16_t queue_id) goto out; } - if (vq->async_pkts_pending) { - rte_free(vq->async_pkts_pending); - vq->async_pkts_pending = NULL; - } - - if (vq->async_pkts_info) { - rte_free(vq->async_pkts_info); - vq->async_pkts_info = NULL; - } + vhost_free_async_mem(vq); vq->async_ops.transfer_data = NULL; vq->async_ops.check_completed_copies = NULL; diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index 08bfc41b3..7d1a8d2b7 100644 --- a/lib/librte_vhost/vhost.h +++ b/lib/librte_vhost/vhost.h @@ -198,8 +198,8 @@ struct vhost_virtqueue { /* operation callbacks for async dma */ struct rte_vhost_async_channel_ops async_ops; - struct rte_vhost_iov_iter it_pool[VHOST_MAX_ASYNC_IT]; - struct iovec vec_pool[VHOST_MAX_ASYNC_VEC]; + struct rte_vhost_iov_iter *it_pool; + struct iovec *vec_pool; /* async data transfer status */ uintptr_t **async_pkts_pending; -- 2.18.4