From: "Xia, Chenbo" <chenbo.xia@intel.com>
To: Maxime Coquelin <maxime.coquelin@redhat.com>,
"dev@dpdk.org" <dev@dpdk.org>,
"david.marchand@redhat.com" <david.marchand@redhat.com>
Subject: Re: [dpdk-dev] [PATCH v7 7/7] vhost: convert inflight data to DPDK allocation API
Date: Wed, 30 Jun 2021 05:20:45 +0000 [thread overview]
Message-ID: <MN2PR11MB406338BF79519F5F890DE4279C019@MN2PR11MB4063.namprd11.prod.outlook.com> (raw)
In-Reply-To: <20210629161133.79472-8-maxime.coquelin@redhat.com>
> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Wednesday, June 30, 2021 12:12 AM
> To: dev@dpdk.org; Xia, Chenbo <chenbo.xia@intel.com>;
> david.marchand@redhat.com
> Cc: Maxime Coquelin <maxime.coquelin@redhat.com>
> Subject: [PATCH v7 7/7] vhost: convert inflight data to DPDK allocation API
>
> Inflight metadata are allocated using glibc's calloc.
> This patch converts them to rte_zmalloc_socket to take
> care of the NUMA affinity.
>
> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> ---
> lib/vhost/vhost.c | 4 +--
> lib/vhost/vhost_user.c | 71 ++++++++++++++++++++++++++++++++++--------
> 2 files changed, 60 insertions(+), 15 deletions(-)
>
> diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c
> index 0000cd3297..53a470f547 100644
> --- a/lib/vhost/vhost.c
> +++ b/lib/vhost/vhost.c
> @@ -312,10 +312,10 @@ cleanup_vq_inflight(struct virtio_net *dev, struct
> vhost_virtqueue *vq)
>
> if (vq->resubmit_inflight) {
> if (vq->resubmit_inflight->resubmit_list) {
> - free(vq->resubmit_inflight->resubmit_list);
> + rte_free(vq->resubmit_inflight->resubmit_list);
> vq->resubmit_inflight->resubmit_list = NULL;
> }
> - free(vq->resubmit_inflight);
> + rte_free(vq->resubmit_inflight);
> vq->resubmit_inflight = NULL;
> }
> }
> diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
> index d8ec087dfc..67935c4ccc 100644
> --- a/lib/vhost/vhost_user.c
> +++ b/lib/vhost/vhost_user.c
> @@ -188,7 +188,7 @@ vhost_backend_cleanup(struct virtio_net *dev)
> dev->inflight_info->fd = -1;
> }
>
> - free(dev->inflight_info);
> + rte_free(dev->inflight_info);
> dev->inflight_info = NULL;
> }
>
> @@ -562,6 +562,31 @@ numa_realloc(struct virtio_net *dev, int index)
> vq->log_cache = lc;
> }
>
> + if (vq->resubmit_inflight) {
> + struct rte_vhost_resubmit_info *ri;
> +
> + ri = rte_realloc_socket(vq->resubmit_inflight, sizeof(*ri), 0,
> node);
> + if (!ri) {
> + VHOST_LOG_CONFIG(ERR, "Failed to realloc resubmit inflight
> on node %d\n",
> + node);
> + return dev;
> + }
> + vq->resubmit_inflight = ri;
> +
> + if (ri->resubmit_list) {
> + struct rte_vhost_resubmit_desc *rd;
> +
> + rd = rte_realloc_socket(ri->resubmit_list, sizeof(*rd) * ri-
> >resubmit_num,
> + 0, node);
> + if (!rd) {
> + VHOST_LOG_CONFIG(ERR, "Failed to realloc resubmit list
> on node %d\n",
> + node);
> + return dev;
> + }
> + ri->resubmit_list = rd;
> + }
> + }
> +
> vq->numa_node = node;
>
> out_dev_realloc:
> @@ -1491,6 +1516,7 @@ vhost_user_get_inflight_fd(struct virtio_net **pdev,
> uint16_t num_queues, queue_size;
> struct virtio_net *dev = *pdev;
> int fd, i, j;
> + int numa_node = SOCKET_ID_ANY;
> void *addr;
>
> if (msg->size != sizeof(msg->payload.inflight)) {
> @@ -1500,9 +1526,16 @@ vhost_user_get_inflight_fd(struct virtio_net **pdev,
> return RTE_VHOST_MSG_RESULT_ERR;
> }
>
> + /*
> + * If VQ 0 has already been allocated, try to allocate on the same
> + * NUMA node. It can be reallocated later in numa_realloc().
> + */
> + if (dev->nr_vring > 0)
> + numa_node = dev->virtqueue[0]->numa_node;
> +
> if (dev->inflight_info == NULL) {
> - dev->inflight_info = calloc(1,
> - sizeof(struct inflight_mem_info));
> + dev->inflight_info = rte_zmalloc_socket("inflight_info",
> + sizeof(struct inflight_mem_info), 0, numa_node);
> if (!dev->inflight_info) {
> VHOST_LOG_CONFIG(ERR,
> "failed to alloc dev inflight area\n");
> @@ -1585,6 +1618,7 @@ vhost_user_set_inflight_fd(struct virtio_net **pdev,
> VhostUserMsg *msg,
> struct vhost_virtqueue *vq;
> void *addr;
> int fd, i;
> + int numa_node = SOCKET_ID_ANY;
>
> fd = msg->fds[0];
> if (msg->size != sizeof(msg->payload.inflight) || fd < 0) {
> @@ -1618,9 +1652,16 @@ vhost_user_set_inflight_fd(struct virtio_net **pdev,
> VhostUserMsg *msg,
> "set_inflight_fd pervq_inflight_size: %d\n",
> pervq_inflight_size);
>
> + /*
> + * If VQ 0 has already been allocated, try to allocate on the same
> + * NUMA node. It can be reallocated later in numa_realloc().
> + */
> + if (dev->nr_vring > 0)
> + numa_node = dev->virtqueue[0]->numa_node;
> +
> if (!dev->inflight_info) {
> - dev->inflight_info = calloc(1,
> - sizeof(struct inflight_mem_info));
> + dev->inflight_info = rte_zmalloc_socket("inflight_info",
> + sizeof(struct inflight_mem_info), 0, numa_node);
> if (dev->inflight_info == NULL) {
> VHOST_LOG_CONFIG(ERR,
> "failed to alloc dev inflight area\n");
> @@ -1779,19 +1820,21 @@ vhost_check_queue_inflights_split(struct virtio_net
> *dev,
> vq->last_avail_idx += resubmit_num;
>
> if (resubmit_num) {
> - resubmit = calloc(1, sizeof(struct rte_vhost_resubmit_info));
> + resubmit = rte_zmalloc_socket("resubmit", sizeof(struct
> rte_vhost_resubmit_info),
> + 0, vq->numa_node);
> if (!resubmit) {
> VHOST_LOG_CONFIG(ERR,
> "failed to allocate memory for resubmit info.\n");
> return RTE_VHOST_MSG_RESULT_ERR;
> }
>
> - resubmit->resubmit_list = calloc(resubmit_num,
> - sizeof(struct rte_vhost_resubmit_desc));
> + resubmit->resubmit_list = rte_zmalloc_socket("resubmit_list",
> + resubmit_num * sizeof(struct rte_vhost_resubmit_desc),
> + 0, vq->numa_node);
> if (!resubmit->resubmit_list) {
> VHOST_LOG_CONFIG(ERR,
> "failed to allocate memory for inflight desc.\n");
> - free(resubmit);
> + rte_free(resubmit);
> return RTE_VHOST_MSG_RESULT_ERR;
> }
>
> @@ -1873,19 +1916,21 @@ vhost_check_queue_inflights_packed(struct virtio_net
> *dev,
> }
>
> if (resubmit_num) {
> - resubmit = calloc(1, sizeof(struct rte_vhost_resubmit_info));
> + resubmit = rte_zmalloc_socket("resubmit", sizeof(struct
> rte_vhost_resubmit_info),
> + 0, vq->numa_node);
> if (resubmit == NULL) {
> VHOST_LOG_CONFIG(ERR,
> "failed to allocate memory for resubmit info.\n");
> return RTE_VHOST_MSG_RESULT_ERR;
> }
>
> - resubmit->resubmit_list = calloc(resubmit_num,
> - sizeof(struct rte_vhost_resubmit_desc));
> + resubmit->resubmit_list = rte_zmalloc_socket("resubmit_list",
> + resubmit_num * sizeof(struct rte_vhost_resubmit_desc),
> + 0, vq->numa_node);
> if (resubmit->resubmit_list == NULL) {
> VHOST_LOG_CONFIG(ERR,
> "failed to allocate memory for resubmit desc.\n");
> - free(resubmit);
> + rte_free(resubmit);
> return RTE_VHOST_MSG_RESULT_ERR;
> }
>
> --
> 2.31.1
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
next prev parent reply other threads:[~2021-06-30 5:20 UTC|newest]
Thread overview: 17+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-06-29 16:11 [dpdk-dev] [PATCH v7 0/7] vhost: Fix and improve NUMA reallocation Maxime Coquelin
2021-06-29 16:11 ` [dpdk-dev] [PATCH v7 1/7] vhost: fix missing memory table NUMA realloc Maxime Coquelin
2021-06-29 16:11 ` [dpdk-dev] [PATCH v7 2/7] vhost: fix missing guest pages " Maxime Coquelin
2021-06-29 16:11 ` [dpdk-dev] [PATCH v7 3/7] vhost: fix missing cache logging " Maxime Coquelin
2021-06-30 5:20 ` Xia, Chenbo
2021-06-29 16:11 ` [dpdk-dev] [PATCH v7 4/7] vhost: fix NUMA reallocation with multiqueue Maxime Coquelin
2021-06-30 5:20 ` Xia, Chenbo
2021-06-30 7:24 ` David Marchand
2021-06-30 7:47 ` Maxime Coquelin
2021-06-29 16:11 ` [dpdk-dev] [PATCH v7 5/7] vhost: improve NUMA reallocation Maxime Coquelin
2021-06-29 16:11 ` [dpdk-dev] [PATCH v7 6/7] vhost: allocate all data on same node as virtqueue Maxime Coquelin
2021-06-29 16:11 ` [dpdk-dev] [PATCH v7 7/7] vhost: convert inflight data to DPDK allocation API Maxime Coquelin
2021-06-30 5:20 ` Xia, Chenbo [this message]
2021-06-30 7:55 ` David Marchand
2021-06-30 11:16 ` Maxime Coquelin
2021-06-30 7:57 ` [dpdk-dev] [PATCH v7 0/7] vhost: Fix and improve NUMA reallocation David Marchand
2021-06-30 11:53 ` Xia, Chenbo
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=MN2PR11MB406338BF79519F5F890DE4279C019@MN2PR11MB4063.namprd11.prod.outlook.com \
--to=chenbo.xia@intel.com \
--cc=david.marchand@redhat.com \
--cc=dev@dpdk.org \
--cc=maxime.coquelin@redhat.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).