DPDK patches and discussions
 help / color / mirror / Atom feed
From: Tetsuya Mukawa <mukawa@igel.co.jp>
To: Huawei Xie <huawei.xie@intel.com>, dev@dpdk.org
Cc: haifeng.lin@intel.com
Subject: Re: [dpdk-dev] [PATCH RFC v2 10/12] lib/librte_vhost: vhost-user memory region map
Date: Tue, 16 Dec 2014 11:38:15 +0900
Message-ID: <548F9B17.40807@igel.co.jp> (raw)
In-Reply-To: <1418247477-13920-11-git-send-email-huawei.xie@intel.com>

(2014/12/11 6:37), Huawei Xie wrote:
> deals with vhost user memory map/unmap alignment
>
> Signed-off-by: Huawei Xie <huawei.xie@intel.com>
> ---
>  lib/librte_vhost/rte_virtio_net.h             |   2 +
>  lib/librte_vhost/vhost-net.h                  |   2 -
>  lib/librte_vhost/vhost_user/vhost-net-user.h  |   3 +-
>  lib/librte_vhost/vhost_user/virtio-net-user.c | 105 ++++++++++++++++++++++++--
>  4 files changed, 100 insertions(+), 12 deletions(-)
>
> diff --git a/lib/librte_vhost/rte_virtio_net.h b/lib/librte_vhost/rte_virtio_net.h
> index 00b1328..77db80b 100644
> --- a/lib/librte_vhost/rte_virtio_net.h
> +++ b/lib/librte_vhost/rte_virtio_net.h
> @@ -48,6 +48,8 @@
>  #include <rte_mempool.h>
>  #include <rte_mbuf.h>
>  
> +#define VHOST_MEMORY_MAX_NREGIONS 8
> +
>  /* Used to indicate that the device is running on a data core */
>  #define VIRTIO_DEV_RUNNING 1
>  
> diff --git a/lib/librte_vhost/vhost-net.h b/lib/librte_vhost/vhost-net.h
> index f9ec40b..ec2584f 100644
> --- a/lib/librte_vhost/vhost-net.h
> +++ b/lib/librte_vhost/vhost-net.h
> @@ -43,8 +43,6 @@
>  
>  #include "rte_virtio_net.h"
>  
> -#define VHOST_MEMORY_MAX_NREGIONS 8
> -
>  extern struct vhost_net_device_ops const *ops;
>  
>  /* Macros for printing using RTE_LOG */
> diff --git a/lib/librte_vhost/vhost_user/vhost-net-user.h b/lib/librte_vhost/vhost_user/vhost-net-user.h
> index c138844..f4c9d01 100644
> --- a/lib/librte_vhost/vhost_user/vhost-net-user.h
> +++ b/lib/librte_vhost/vhost_user/vhost-net-user.h
> @@ -37,6 +37,7 @@
>  #include <stdint.h>
>  #include <linux/vhost.h>
>  
> +#include "rte_virtio_net.h"
>  #include "fd_man.h"
>  
>  struct vhost_server {
> @@ -47,8 +48,6 @@ struct vhost_server {
>  
>  /* refer to hw/virtio/vhost-user.c */
>  
> -#define VHOST_MEMORY_MAX_NREGIONS    8
> -
>  typedef enum VhostUserRequest {
>  	VHOST_USER_NONE = 0,
>  	VHOST_USER_GET_FEATURES = 1,
> diff --git a/lib/librte_vhost/vhost_user/virtio-net-user.c b/lib/librte_vhost/vhost_user/virtio-net-user.c
> index ad59fcc..3aecb17 100644
> --- a/lib/librte_vhost/vhost_user/virtio-net-user.c
> +++ b/lib/librte_vhost/vhost_user/virtio-net-user.c
> @@ -36,7 +36,11 @@
>  #include <stdlib.h>
>  #include <unistd.h>
>  #include <sys/mman.h>
> +#include <sys/types.h>
> +#include <sys/stat.h>
> +#include <unistd.h>
>  
> +#include <rte_common.h>
>  #include <rte_log.h>
>  
>  #include "virtio-net.h"
> @@ -44,13 +48,56 @@
>  #include "vhost-net-user.h"
>  #include "vhost-net.h"
>  
> +struct orig_region_map {
> +	int fd;
> +	uint64_t mapped_address;
> +	uint64_t mapped_size;
> +	uint64_t blksz;
> +};
> +
> +#define orig_region(ptr, nregions) (struct orig_region_map *)RTE_PTR_ADD(ptr, sizeof(struct virtio_memory) + sizeof(struct virtio_memory_regions) * (nregions))
> +
> +static uint64_t
> +get_blk_size(int fd)
> +{
> +	struct stat stat;
> +	fstat(fd, &stat);
> +	return (uint64_t)stat.st_blksize;
> +}

I've also confirmed we can get hugepage size of the fd using st_blksize.
If someone wants to run QEMU on 2MB hugepage, but DPDK backend is on
1GB, even in such a case, we will also be able to mmap and munmap QEMU
backend memory correctly.
So I guess using st_blksize is smart workaround not to hit munmap issue.

> +
>  int
>  user_set_mem_table(struct vhost_device_ctx ctx, struct VhostUserMsg *pmsg)
>  {
> -	unsigned int idx;
>  	struct VhostUserMemory memory = pmsg->payload.memory;
>  	struct virtio_memory_regions regions[VHOST_MEMORY_MAX_NREGIONS];
> -	uint64_t mapped_address, base_address = 0;
> +	uint64_t mapped_address, mapped_size, base_address = 0;
> +	struct virtio_net *dev;
> +	unsigned int idx = 0;
> +	struct orig_region_map tmp[VHOST_MEMORY_MAX_NREGIONS] =
> +		{ [0 ... VHOST_MEMORY_MAX_NREGIONS - 1] = { 0 } };
> +	struct orig_region_map *region;
> +	uint64_t alignment;
> +	int ret;
> +
> +	/* unmap old memory regions one by one*/
> +	dev = get_device(ctx);
> +	if (dev->mem) {
> +		region = orig_region(dev->mem, dev->mem->nregions);
> +		for (idx = 0; idx < dev->mem->nregions; idx++) {
> +			if (region[idx].mapped_address) {
> +				alignment = region[idx].blksz;
> +				printf("Freeing %p\n",
> +					(void *)(uintptr_t)region[idx].mapped_address);
> +				ret = munmap((void *)RTE_ALIGN_FLOOR(region[idx].mapped_address, alignment),
> +					RTE_ALIGN_CEIL(region[idx].mapped_size, alignment));
> +				printf("munmap ret= %d\n", ret);
> +				printf("close file %d\n", region[idx].fd);
> +				close(region[idx].fd);
> +			}
> +		}
> +		free(dev->mem);
> +		dev->mem = NULL;
> +	}
>  
>  	for (idx = 0; idx < memory.nregions; idx++) {
>  		if (memory.regions[idx].guest_phys_addr == 0)
> @@ -73,22 +120,30 @@ user_set_mem_table(struct vhost_device_ctx ctx, struct VhostUserMsg *pmsg)
>  			memory.regions[idx].userspace_addr;
>  
>  		/* This is ugly */
> +		mapped_size = regions[idx].memory_size +
> +			memory.regions[idx].mmap_offset;
>  		mapped_address = (uint64_t)(uintptr_t)mmap(NULL,
> -			regions[idx].memory_size +
> -				memory.regions[idx].mmap_offset,
> +			mapped_size,
>  			PROT_READ | PROT_WRITE, MAP_SHARED,
>  			pmsg->fds[idx],
>  			0);
> +
>  		RTE_LOG(INFO, VHOST_CONFIG,
> -			"mapped region %d to %p\n",
> -			idx, (void *)mapped_address);
> +			"mapped region %d fd:%d to %p sz:0x%"PRIx64" off:0x%"PRIx64"\n",
> +			idx, pmsg->fds[idx], (void *)mapped_address,
> +			mapped_size, memory.regions[idx].mmap_offset);
>  
>  		if (mapped_address == (uint64_t)(uintptr_t)MAP_FAILED) {
>  			RTE_LOG(ERR, VHOST_CONFIG,
>  				"mmap qemu guest failed.\n");
> -			return -1;
> +			goto err;
>  		}
>  
> +		tmp[idx].mapped_address = mapped_address;
> +		tmp[idx].mapped_size = mapped_size;
> +		tmp[idx].blksz = get_blk_size(pmsg->fds[idx]);
> +		tmp[idx].fd = pmsg->fds[idx];
> +
>  		mapped_address +=  memory.regions[idx].mmap_offset;
>  
>  		regions[idx].address_offset = mapped_address -
> @@ -100,10 +155,44 @@ user_set_mem_table(struct vhost_device_ctx ctx, struct VhostUserMsg *pmsg)
>  			(void *)(uintptr_t)regions[idx].userspace_address,
>  			 regions[idx].memory_size);
>  	}
> +
>  	ops->set_mem_table(ctx, regions, memory.nregions);
> +
> +	if (dev->mem) {
> +		void *tmp_mem;
> +		tmp_mem = realloc(dev->mem,
> +			sizeof(struct virtio_memory) +
> +			sizeof(struct virtio_memory_regions) * memory.nregions +
> +			sizeof(struct orig_region_map) * memory.nregions);
> +		if (tmp_mem == NULL) {
> +			goto err_realloc;
> +		}
> +		dev->mem = tmp_mem;
> +		region = orig_region(dev->mem, memory.nregions);
> +		for (idx = 0; idx < memory.nregions; idx++) {
> +			region[idx].mapped_address = tmp[idx].mapped_address;
> +			region[idx].mapped_size = tmp[idx].mapped_size;
> +			region[idx].blksz = tmp[idx].blksz;
> +			region[idx].fd = tmp[idx].fd;
> +		}
> +	} else
> +		goto err_set_mem_table;
> +
>  	return 0;
> -}
>  
> +err_realloc:
> +	free(dev->mem);
> +err_set_mem_table:
> +err:
> +	while (idx--) {
> +		alignment = tmp[idx].blksz;
> +		munmap((void *)RTE_ALIGN_FLOOR(tmp[idx].mapped_address, alignment),
> +			RTE_ALIGN_CEIL(tmp[idx].mapped_size, alignment));
> +		close(tmp[idx].fd);
> +	}
> +	dev->mem = NULL;
> +	return -1;
> +}
>  
>  static int
>  virtio_is_ready(struct virtio_net *dev)

  reply	other threads:[~2014-12-16  2:38 UTC|newest]

Thread overview: 43+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2014-12-10 21:37 [dpdk-dev] [PATCH RFC v2 00/12] lib/librte_vhost: vhost-user support Huawei Xie
2014-12-10 21:37 ` [dpdk-dev] [PATCH RFC v2 01/12] lib/librte_vhost: mov vhost-cuse implementation to vhost_cuse directory Huawei Xie
2014-12-10 21:37 ` [dpdk-dev] [PATCH RFC v2 02/12] lib/librte_vhost: rename vhost-net-cdev.h as vhost-net.h Huawei Xie
2014-12-10 21:37 ` [dpdk-dev] [PATCH RFC v2 03/12] lib/librte_vhost: move event_copy logic from virtio-net.c to vhost-net-cdev.c Huawei Xie
2015-01-07  9:10   ` Xie, Huawei
2014-12-10 21:37 ` [dpdk-dev] [PATCH RFC v2 04/12] lib/librte_vhost: copy of host_memory_map from virtio-net.c to new file virtio-net-cdev.c Huawei Xie
2014-12-10 21:37 ` [dpdk-dev] [PATCH RFC v2 05/12] lib/librte_vhost: host_memory_map refine Huawei Xie
2014-12-10 21:37 ` [dpdk-dev] [PATCH RFC v2 06/12] lib/librte_vhost: cuse_set_memory_table Huawei Xie
2014-12-15  5:20   ` Tetsuya Mukawa
2014-12-10 21:37 ` [dpdk-dev] [PATCH RFC v2 07/12] lib/librte_vhost: async event and callback Huawei Xie
2014-12-15  5:20   ` Tetsuya Mukawa
2014-12-17 17:51     ` Xie, Huawei
2014-12-10 21:37 ` [dpdk-dev] [PATCH RFC v2 08/12] lib/librte_vhost: vhost-user support Huawei Xie
2014-12-11  5:36   ` Linhaifeng
2015-01-05 10:21     ` Xie, Huawei
2015-01-23  3:40     ` Xie, Huawei
2015-01-23  3:53       ` Linhaifeng
2014-12-11  6:04   ` Linhaifeng
2014-12-11 17:13     ` Xie, Huawei
2014-12-12  2:25       ` Linhaifeng
2014-12-11 20:16     ` Xie, Huawei
2015-01-23  3:36     ` Xie, Huawei
2015-01-23  8:36       ` Linhaifeng
2014-12-16  3:05   ` Tetsuya Mukawa
2014-12-17  1:06     ` Xie, Huawei
2014-12-17  3:31       ` Tetsuya Mukawa
2014-12-17  4:22         ` Tetsuya Mukawa
2014-12-17 17:31           ` Xie, Huawei
2014-12-19  3:36             ` Tetsuya Mukawa
2014-12-24  7:21   ` Tetsuya Mukawa
2015-01-04  9:53     ` Xie, Huawei
2014-12-10 21:37 ` [dpdk-dev] [PATCH RFC v2 09/12] lib/librte_vhost: minor fix Huawei Xie
2014-12-10 21:37 ` [dpdk-dev] [PATCH RFC v2 10/12] lib/librte_vhost: vhost-user memory region map Huawei Xie
2014-12-16  2:38   ` Tetsuya Mukawa [this message]
2014-12-10 21:37 ` [dpdk-dev] [PATCH RFC v2 11/12] lib/librte_vhost: kick/callfd fix Huawei Xie
2014-12-10 21:37 ` [dpdk-dev] [PATCH RFC v2 12/12] lib/librte_vhost: cleanup when vhost user socket connection is closed Huawei Xie
2014-12-10 22:04 ` [dpdk-dev] [PATCH RFC v2 00/12] lib/librte_vhost: vhost-user support Xie, Huawei
2014-12-11  2:21   ` Tetsuya Mukawa
2014-12-15  5:26 ` Tetsuya Mukawa
2014-12-17 17:43   ` Xie, Huawei
2015-01-07 12:43     ` Qiu, Michael
2015-01-23  8:16 ` Linhaifeng
2015-01-26  7:24   ` Xie, Huawei

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=548F9B17.40807@igel.co.jp \
    --to=mukawa@igel.co.jp \
    --cc=dev@dpdk.org \
    --cc=haifeng.lin@intel.com \
    --cc=huawei.xie@intel.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

DPDK patches and discussions

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://inbox.dpdk.org/dev/0 dev/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 dev dev/ https://inbox.dpdk.org/dev \
		dev@dpdk.org
	public-inbox-index dev

Example config snippet for mirrors.
Newsgroup available over NNTP:
	nntp://inbox.dpdk.org/inbox.dpdk.dev


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git