Re: [dpdk-dev] [RFC 1/2] vhost: populate guest memory for DMA-accelerated vhost-user

[Maxime Coquelin <maxime.coquelin@redhat.com>]

On 9/26/19 4:26 PM, Jiayu Hu wrote:
> DMA engines, like I/OAT, are efficient in moving large data
> within memory. Offloading large copies in vhost side to DMA
> engines can save precious CPU cycles and improve vhost
> performance.
> 
> However, using the DMA engine requires to populate guest's
> memory. This patch is to enable DMA-accelerated vhost-user
> to populate guest's memory.
> 
> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
> ---
>  lib/librte_vhost/rte_vhost.h  |  1 +
>  lib/librte_vhost/socket.c     | 10 ++++++++++
>  lib/librte_vhost/vhost.h      |  2 ++
>  lib/librte_vhost/vhost_user.c |  3 ++-
>  4 files changed, 15 insertions(+), 1 deletion(-)
> 
> diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
> index 7fb1729..61f8bef 100644
> --- a/lib/librte_vhost/rte_vhost.h
> +++ b/lib/librte_vhost/rte_vhost.h
> @@ -30,6 +30,7 @@ extern "C" {
>  #define RTE_VHOST_USER_DEQUEUE_ZERO_COPY	(1ULL << 2)
>  #define RTE_VHOST_USER_IOMMU_SUPPORT	(1ULL << 3)
>  #define RTE_VHOST_USER_POSTCOPY_SUPPORT		(1ULL << 4)
> +#define RTE_VHOST_USER_DMA_COPY		(1ULL << 5)
>  
>  /** Protocol features. */
>  #ifndef VHOST_USER_PROTOCOL_F_MQ
> diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
> index 274988c..55498c4 100644
> --- a/lib/librte_vhost/socket.c
> +++ b/lib/librte_vhost/socket.c
> @@ -60,6 +60,8 @@ struct vhost_user_socket {
>  	 */
>  	int vdpa_dev_id;
>  
> +	bool dma_enabled;
> +
>  	struct vhost_device_ops const *notify_ops;
>  };
>  
> @@ -232,6 +234,13 @@ vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket)
>  	if (vsocket->dequeue_zero_copy)
>  		vhost_enable_dequeue_zero_copy(vid);
>  
> +	if (vsocket->dma_enabled) {
> +		struct virtio_net *dev;
> +
> +		dev = get_device(vid);
> +		dev->dma_enabled = true;
> +	}
> +
>  	RTE_LOG(INFO, VHOST_CONFIG, "new device, handle is %d\n", vid);
>  
>  	if (vsocket->notify_ops->new_connection) {
> @@ -870,6 +879,7 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
>  		goto out_free;
>  	}
>  	vsocket->dequeue_zero_copy = flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
> +	vsocket->dma_enabled = flags & RTE_VHOST_USER_DMA_COPY;
>  
>  	/*
>  	 * Set the supported features correctly for the builtin vhost-user

I think you also need to disable VHOST_USER_PROTOCOL_F_PAGEFAULT from
the supported protocol features if DMA is enabled, since it requires
MAP_POPULATE which is not compatible with postcopy.

> diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
> index 884befa..8f564f1 100644
> --- a/lib/librte_vhost/vhost.h
> +++ b/lib/librte_vhost/vhost.h
> @@ -329,6 +329,8 @@ struct virtio_net {
>  	 */
>  	int			vdpa_dev_id;
>  
> +	bool			dma_enabled;
> +
>  	/* context data for the external message handlers */
>  	void			*extern_data;
>  	/* pre and post vhost user message handlers for the device */
> diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
> index 0b72648..6a4969b 100644
> --- a/lib/librte_vhost/vhost_user.c
> +++ b/lib/librte_vhost/vhost_user.c
> @@ -990,7 +990,8 @@ vhost_user_set_mem_table(struct virtio_net **pdev, struct VhostUserMsg *msg,
>  		}
>  		mmap_size = RTE_ALIGN_CEIL(mmap_size, alignment);
>  
> -		populate = (dev->dequeue_zero_copy) ? MAP_POPULATE : 0;
> +		populate = (dev->dequeue_zero_copy || dev->dma_enabled) ?
> +			MAP_POPULATE : 0;
>  		mmap_addr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
>  				 MAP_SHARED | populate, fd, 0);
>  
>