From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga02.intel.com (mga02.intel.com [134.134.136.20]) by dpdk.org (Postfix) with ESMTP id 680B43792 for ; Wed, 24 Aug 2016 09:30:34 +0200 (CEST) Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by orsmga101.jf.intel.com with ESMTP; 24 Aug 2016 00:30:33 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.28,569,1464678000"; d="scan'208";a="752811329" Received: from yliu-dev.sh.intel.com (HELO yliu-dev) ([10.239.67.162]) by FMSMGA003.fm.intel.com with ESMTP; 24 Aug 2016 00:30:32 -0700 Date: Wed, 24 Aug 2016 15:40:26 +0800 From: Yuanhan Liu To: "Xu, Qian Q" Cc: "dev@dpdk.org" , Maxime Coquelin Message-ID: <20160824074026.GT30752@yliu-dev.sh.intel.com> References: <1471939839-29778-1-git-send-email-yuanhan.liu@linux.intel.com> <1471939839-29778-2-git-send-email-yuanhan.liu@linux.intel.com> <82F45D86ADE5454A95A89742C8D1410E03322F77@shsmsx102.ccr.corp.intel.com> MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline In-Reply-To: <82F45D86ADE5454A95A89742C8D1410E03322F77@shsmsx102.ccr.corp.intel.com> User-Agent: Mutt/1.5.23 (2014-03-12) Subject: Re: [dpdk-dev] [PATCH 1/6] vhost: simplify memory regions handling X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 24 Aug 2016 07:30:35 -0000 Yes, it depends on the vhost-cuse removal patchset I sent last week. --yliu On Wed, Aug 24, 2016 at 07:26:07AM +0000, Xu, Qian Q wrote: > I want to apply the patch on the latest DPDK, see below commit ID but failed since no vhost.h and vhost-user.h files. So do you have any dependency on other patches? > > commit 28d8abaf250c3fb4dcb6416518f4c54b4ae67205 > Author: Deirdre O'Connor > Date: Mon Aug 22 17:20:08 2016 +0100 > > doc: fix patchwork link > > Fixes: 58abf6e77c6b ("doc: add contributors guide") > > Reported-by: Jon Loeliger > Signed-off-by: Deirdre O'Connor > Acked-by: John McNamara > > > -----Original Message----- > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Yuanhan Liu > Sent: Tuesday, August 23, 2016 4:11 PM > To: dev@dpdk.org > Cc: Maxime Coquelin ; Yuanhan Liu > Subject: [dpdk-dev] [PATCH 1/6] vhost: simplify memory regions handling > > Due to history reason (that vhost-cuse comes before vhost-user), some fields for maintaining the vhost-user memory mappings (such as mmapped address and size, with those we then can unmap on destroy) are kept in "orig_region_map" struct, a structure that is defined only in vhost-user source file. > > The right way to go is to remove the structure and move all those fields into virtio_memory_region struct. But we simply can't do that before, because it breaks the ABI. > > Now, thanks to the ABI refactoring, it's never been a blocking issue any more. And here it goes: this patch removes orig_region_map and redefines virtio_memory_region, to include all necessary info. > > With that, we can simplify the guest/host address convert a bit. > > Signed-off-by: Yuanhan Liu > --- > lib/librte_vhost/vhost.h | 49 ++++++------ > lib/librte_vhost/vhost_user.c | 172 +++++++++++++++++------------------------- > 2 files changed, 90 insertions(+), 131 deletions(-) > > diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index c2dfc3c..df2107b 100644 > --- a/lib/librte_vhost/vhost.h > +++ b/lib/librte_vhost/vhost.h > @@ -143,12 +143,14 @@ struct virtio_net { > * Information relating to memory regions including offsets to > * addresses in QEMUs memory file. > */ > -struct virtio_memory_regions { > - uint64_t guest_phys_address; > - uint64_t guest_phys_address_end; > - uint64_t memory_size; > - uint64_t userspace_address; > - uint64_t address_offset; > +struct virtio_memory_region { > + uint64_t guest_phys_addr; > + uint64_t guest_user_addr; > + uint64_t host_user_addr; > + uint64_t size; > + void *mmap_addr; > + uint64_t mmap_size; > + int fd; > }; > > > @@ -156,12 +158,8 @@ struct virtio_memory_regions { > * Memory structure includes region and mapping information. > */ > struct virtio_memory { > - /* Base QEMU userspace address of the memory file. */ > - uint64_t base_address; > - uint64_t mapped_address; > - uint64_t mapped_size; > uint32_t nregions; > - struct virtio_memory_regions regions[0]; > + struct virtio_memory_region regions[0]; > }; > > > @@ -200,26 +198,23 @@ extern uint64_t VHOST_FEATURES; > #define MAX_VHOST_DEVICE 1024 > extern struct virtio_net *vhost_devices[MAX_VHOST_DEVICE]; > > -/** > - * Function to convert guest physical addresses to vhost virtual addresses. > - * This is used to convert guest virtio buffer addresses. > - */ > +/* Convert guest physical Address to host virtual address */ > static inline uint64_t __attribute__((always_inline)) -gpa_to_vva(struct virtio_net *dev, uint64_t guest_pa) > +gpa_to_vva(struct virtio_net *dev, uint64_t gpa) > { > - struct virtio_memory_regions *region; > - uint32_t regionidx; > - uint64_t vhost_va = 0; > - > - for (regionidx = 0; regionidx < dev->mem->nregions; regionidx++) { > - region = &dev->mem->regions[regionidx]; > - if ((guest_pa >= region->guest_phys_address) && > - (guest_pa <= region->guest_phys_address_end)) { > - vhost_va = region->address_offset + guest_pa; > - break; > + struct virtio_memory_region *reg; > + uint32_t i; > + > + for (i = 0; i < dev->mem->nregions; i++) { > + reg = &dev->mem->regions[i]; > + if (gpa >= reg->guest_phys_addr && > + gpa < reg->guest_phys_addr + reg->size) { > + return gpa - reg->guest_phys_addr + > + reg->host_user_addr; > } > } > - return vhost_va; > + > + return 0; > } > > struct virtio_net_device_ops const *notify_ops; diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c index eee99e9..d2071fd 100644 > --- a/lib/librte_vhost/vhost_user.c > +++ b/lib/librte_vhost/vhost_user.c > @@ -74,18 +74,6 @@ static const char *vhost_message_str[VHOST_USER_MAX] = { > [VHOST_USER_SEND_RARP] = "VHOST_USER_SEND_RARP", }; > > -struct orig_region_map { > - int fd; > - uint64_t mapped_address; > - uint64_t mapped_size; > - uint64_t blksz; > -}; > - > -#define orig_region(ptr, nregions) \ > - ((struct orig_region_map *)RTE_PTR_ADD((ptr), \ > - sizeof(struct virtio_memory) + \ > - sizeof(struct virtio_memory_regions) * (nregions))) > - > static uint64_t > get_blk_size(int fd) > { > @@ -99,18 +87,17 @@ get_blk_size(int fd) static void free_mem_region(struct virtio_net *dev) { > - struct orig_region_map *region; > - unsigned int idx; > + uint32_t i; > + struct virtio_memory_region *reg; > > if (!dev || !dev->mem) > return; > > - region = orig_region(dev->mem, dev->mem->nregions); > - for (idx = 0; idx < dev->mem->nregions; idx++) { > - if (region[idx].mapped_address) { > - munmap((void *)(uintptr_t)region[idx].mapped_address, > - region[idx].mapped_size); > - close(region[idx].fd); > + for (i = 0; i < dev->mem->nregions; i++) { > + reg = &dev->mem->regions[i]; > + if (reg->host_user_addr) { > + munmap(reg->mmap_addr, reg->mmap_size); > + close(reg->fd); > } > } > } > @@ -120,7 +107,7 @@ vhost_backend_cleanup(struct virtio_net *dev) { > if (dev->mem) { > free_mem_region(dev); > - free(dev->mem); > + rte_free(dev->mem); > dev->mem = NULL; > } > if (dev->log_addr) { > @@ -286,25 +273,23 @@ numa_realloc(struct virtio_net *dev, int index __rte_unused) > * used to convert the ring addresses to our address space. > */ > static uint64_t > -qva_to_vva(struct virtio_net *dev, uint64_t qemu_va) > +qva_to_vva(struct virtio_net *dev, uint64_t qva) > { > - struct virtio_memory_regions *region; > - uint64_t vhost_va = 0; > - uint32_t regionidx = 0; > + struct virtio_memory_region *reg; > + uint32_t i; > > /* Find the region where the address lives. */ > - for (regionidx = 0; regionidx < dev->mem->nregions; regionidx++) { > - region = &dev->mem->regions[regionidx]; > - if ((qemu_va >= region->userspace_address) && > - (qemu_va <= region->userspace_address + > - region->memory_size)) { > - vhost_va = qemu_va + region->guest_phys_address + > - region->address_offset - > - region->userspace_address; > - break; > + for (i = 0; i < dev->mem->nregions; i++) { > + reg = &dev->mem->regions[i]; > + > + if (qva >= reg->guest_user_addr && > + qva < reg->guest_user_addr + reg->size) { > + return qva - reg->guest_user_addr + > + reg->host_user_addr; > } > } > - return vhost_va; > + > + return 0; > } > > /* > @@ -391,11 +376,13 @@ static int > vhost_user_set_mem_table(struct virtio_net *dev, struct VhostUserMsg *pmsg) { > struct VhostUserMemory memory = pmsg->payload.memory; > - struct virtio_memory_regions *pregion; > - uint64_t mapped_address, mapped_size; > - unsigned int idx = 0; > - struct orig_region_map *pregion_orig; > + struct virtio_memory_region *reg; > + void *mmap_addr; > + uint64_t mmap_size; > + uint64_t mmap_offset; > uint64_t alignment; > + uint32_t i; > + int fd; > > /* Remove from the data plane. */ > if (dev->flags & VIRTIO_DEV_RUNNING) { @@ -405,14 +392,12 @@ vhost_user_set_mem_table(struct virtio_net *dev, struct VhostUserMsg *pmsg) > > if (dev->mem) { > free_mem_region(dev); > - free(dev->mem); > + rte_free(dev->mem); > dev->mem = NULL; > } > > - dev->mem = calloc(1, > - sizeof(struct virtio_memory) + > - sizeof(struct virtio_memory_regions) * memory.nregions + > - sizeof(struct orig_region_map) * memory.nregions); > + dev->mem = rte_zmalloc("vhost-mem-table", sizeof(struct virtio_memory) + > + sizeof(struct virtio_memory_region) * memory.nregions, 0); > if (dev->mem == NULL) { > RTE_LOG(ERR, VHOST_CONFIG, > "(%d) failed to allocate memory for dev->mem\n", @@ -421,22 +406,17 @@ vhost_user_set_mem_table(struct virtio_net *dev, struct VhostUserMsg *pmsg) > } > dev->mem->nregions = memory.nregions; > > - pregion_orig = orig_region(dev->mem, memory.nregions); > - for (idx = 0; idx < memory.nregions; idx++) { > - pregion = &dev->mem->regions[idx]; > - pregion->guest_phys_address = > - memory.regions[idx].guest_phys_addr; > - pregion->guest_phys_address_end = > - memory.regions[idx].guest_phys_addr + > - memory.regions[idx].memory_size; > - pregion->memory_size = > - memory.regions[idx].memory_size; > - pregion->userspace_address = > - memory.regions[idx].userspace_addr; > - > - /* This is ugly */ > - mapped_size = memory.regions[idx].memory_size + > - memory.regions[idx].mmap_offset; > + for (i = 0; i < memory.nregions; i++) { > + fd = pmsg->fds[i]; > + reg = &dev->mem->regions[i]; > + > + reg->guest_phys_addr = memory.regions[i].guest_phys_addr; > + reg->guest_user_addr = memory.regions[i].userspace_addr; > + reg->size = memory.regions[i].memory_size; > + reg->fd = fd; > + > + mmap_offset = memory.regions[i].mmap_offset; > + mmap_size = reg->size + mmap_offset; > > /* mmap() without flag of MAP_ANONYMOUS, should be called > * with length argument aligned with hugepagesz at older @@ -446,67 +426,51 @@ vhost_user_set_mem_table(struct virtio_net *dev, struct VhostUserMsg *pmsg) > * to avoid failure, make sure in caller to keep length > * aligned. > */ > - alignment = get_blk_size(pmsg->fds[idx]); > + alignment = get_blk_size(fd); > if (alignment == (uint64_t)-1) { > RTE_LOG(ERR, VHOST_CONFIG, > "couldn't get hugepage size through fstat\n"); > goto err_mmap; > } > - mapped_size = RTE_ALIGN_CEIL(mapped_size, alignment); > + mmap_size = RTE_ALIGN_CEIL(mmap_size, alignment); > > - mapped_address = (uint64_t)(uintptr_t)mmap(NULL, > - mapped_size, > - PROT_READ | PROT_WRITE, MAP_SHARED, > - pmsg->fds[idx], > - 0); > + mmap_addr = mmap(NULL, mmap_size, > + PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); > > - RTE_LOG(INFO, VHOST_CONFIG, > - "mapped region %d fd:%d to:%p sz:0x%"PRIx64" " > - "off:0x%"PRIx64" align:0x%"PRIx64"\n", > - idx, pmsg->fds[idx], (void *)(uintptr_t)mapped_address, > - mapped_size, memory.regions[idx].mmap_offset, > - alignment); > - > - if (mapped_address == (uint64_t)(uintptr_t)MAP_FAILED) { > + if (mmap_addr == MAP_FAILED) { > RTE_LOG(ERR, VHOST_CONFIG, > - "mmap qemu guest failed.\n"); > + "mmap region %u failed.\n", i); > goto err_mmap; > } > > - pregion_orig[idx].mapped_address = mapped_address; > - pregion_orig[idx].mapped_size = mapped_size; > - pregion_orig[idx].blksz = alignment; > - pregion_orig[idx].fd = pmsg->fds[idx]; > - > - mapped_address += memory.regions[idx].mmap_offset; > + reg->mmap_addr = mmap_addr; > + reg->mmap_size = mmap_size; > + reg->host_user_addr = (uint64_t)(uintptr_t)mmap_addr + mmap_offset; > > - pregion->address_offset = mapped_address - > - pregion->guest_phys_address; > - > - if (memory.regions[idx].guest_phys_addr == 0) { > - dev->mem->base_address = > - memory.regions[idx].userspace_addr; > - dev->mem->mapped_address = > - pregion->address_offset; > - } > - > - LOG_DEBUG(VHOST_CONFIG, > - "REGION: %u GPA: %p QEMU VA: %p SIZE (%"PRIu64")\n", > - idx, > - (void *)(uintptr_t)pregion->guest_phys_address, > - (void *)(uintptr_t)pregion->userspace_address, > - pregion->memory_size); > + RTE_LOG(INFO, VHOST_CONFIG, > + "guest memory region %u, size: 0x%" PRIx64 "\n" > + "\t guest physical addr: 0x%" PRIx64 "\n" > + "\t guest virtual addr: 0x%" PRIx64 "\n" > + "\t host virtual addr: 0x%" PRIx64 "\n" > + "\t mmap addr : 0x%" PRIx64 "\n" > + "\t mmap size : 0x%" PRIx64 "\n" > + "\t mmap align: 0x%" PRIx64 "\n" > + "\t mmap off : 0x%" PRIx64 "\n", > + i, reg->size, > + reg->guest_phys_addr, > + reg->guest_user_addr, > + reg->host_user_addr, > + (uint64_t)(uintptr_t)mmap_addr, > + mmap_size, > + alignment, > + mmap_offset); > } > > return 0; > > err_mmap: > - while (idx--) { > - munmap((void *)(uintptr_t)pregion_orig[idx].mapped_address, > - pregion_orig[idx].mapped_size); > - close(pregion_orig[idx].fd); > - } > - free(dev->mem); > + free_mem_region(dev); > + rte_free(dev->mem); > dev->mem = NULL; > return -1; > } > -- > 1.9.0