[dpdk-dev] [PATCH v3 0/3] vhost: add support for IOVA_VA mode

[dpdk-dev] [PATCH v3 0/3] vhost: add support for IOVA_VA mode

[Adrian Moreno]

Currently, IOVAs are assumed to be GPA when the dirty page logging
is performed.

Add support for guest IOVA_VA mode by translating addresses
to GPA before doing the dirty page logging.

Also, the guest_pages tracking mechanism used for zero_copy will not work
either if IOVA_VA is enabled. In fact, enabling dequeue_zero_copy and
IOMMU is dangerous altogether because IOTLB invalidation of guest
pages can happen while the buffers are being used. So, ensure this
cannot happen by preventing both features to be activated
simultaneously.

Further information: https://bugs.dpdk.org/show_bug.cgi?id=337

Fixes: 69c90e98f483 ("vhost: enable IOMMU support")
Cc: maxime.coquelin@redhat.com
Cc: stable@dpdk.org

----
v3 changes: Address style warnings
----
----
v2 changes: Also translate buffer addresses
----

Adrian Moreno (3):
  vhost: translate incoming log address to gpa
  vhost: convert buffer addresses to GPA for logging
  vhost: prevent zero copy mode if iommu is on

 lib/librte_vhost/rte_vhost.h  |  2 +-
 lib/librte_vhost/socket.c     |  8 +++++++
 lib/librte_vhost/vdpa.c       |  4 +++-
 lib/librte_vhost/vhost.c      | 22 ++++++++++++++++++
 lib/librte_vhost/vhost.h      | 36 ++++++++++++++++++++++++++++
 lib/librte_vhost/vhost_user.c | 44 ++++++++++++++++++++++++++++++++++-
 lib/librte_vhost/virtio_net.c | 12 ++++++----
 7 files changed, 120 insertions(+), 8 deletions(-)

-- 
2.21.0

[dpdk-dev] [PATCH v3 1/3] vhost: translate incoming log address to gpa

[Adrian Moreno]

When IOMMU is enabled the incoming log address is in IOVA space. In that
case, look in IOTLB table and translate the resulting HVA to GPA.

If IOMMU is not enabled, the incoming log address is already a GPA so no
transformation is needed.

Fixes: 69c90e98f483 ("vhost: enable IOMMU support")
Cc: maxime.coquelin@redhat.com
Cc: stable@dpdk.org

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 lib/librte_vhost/vhost.c      |  1 +
 lib/librte_vhost/vhost.h      | 20 ++++++++++++++++
 lib/librte_vhost/vhost_user.c | 44 ++++++++++++++++++++++++++++++++++-
 3 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index 981837b5d..e57dda22f 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -383,6 +383,7 @@ vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq)
 	vq->desc = NULL;
 	vq->avail = NULL;
 	vq->used = NULL;
+	vq->log_guest_addr = 0;
 
 	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 		vhost_user_iotlb_wr_unlock(vq);
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 884befa85..9d3883385 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -447,6 +447,26 @@ gpa_to_hpa(struct virtio_net *dev, uint64_t gpa, uint64_t size)
 	return 0;
 }
 
+static __rte_always_inline uint64_t
+hva_to_gpa(struct virtio_net *dev, uint64_t vva, uint64_t len)
+{
+	struct rte_vhost_mem_region *r;
+	uint32_t i;
+
+	if (unlikely(!dev || !dev->mem))
+		return 0;
+
+	for (i = 0; i < dev->mem->nregions; i++) {
+		r = &dev->mem->regions[i];
+
+		if (vva >= r->host_user_addr &&
+		    vva + len <  r->host_user_addr + r->size) {
+			return r->guest_phys_addr + vva - r->host_user_addr;
+		}
+	}
+	return 0;
+}
+
 static __rte_always_inline struct virtio_net *
 get_device(int vid)
 {
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index 0b72648a5..fec448ea2 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -570,6 +570,40 @@ ring_addr_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	return qva_to_vva(dev, ra, size);
 }
 
+/*
+ * Converts vring log address to GPA
+ * If IOMMU is enabled, the log address is IOVA
+ * If IOMMU not enabled, the log address is already GPA
+ */
+static uint64_t
+translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
+		uint64_t log_addr)
+{
+	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
+		const uint64_t exp_size = sizeof(struct vring_used) +
+			sizeof(struct vring_used_elem) * vq->size;
+		uint64_t hva, gpa;
+		uint64_t size = exp_size;
+
+		hva = vhost_iova_to_vva(dev, vq, log_addr,
+					&size, VHOST_ACCESS_RW);
+		if (size != exp_size)
+			return 0;
+
+		gpa = hva_to_gpa(dev, hva, exp_size);
+		if (!gpa) {
+			RTE_LOG(ERR, VHOST_CONFIG,
+				"VQ: Failed to find GPA mapping for log_addr."
+				"log_addr: 0x%0lx hva: 0x%0lx\n",
+				log_addr, hva);
+			return 0;
+		}
+		return gpa;
+
+	} else
+		return log_addr;
+}
+
 static struct virtio_net *
 translate_ring_addresses(struct virtio_net *dev, int vq_index)
 {
@@ -676,7 +710,15 @@ translate_ring_addresses(struct virtio_net *dev, int vq_index)
 		vq->last_avail_idx = vq->used->idx;
 	}
 
-	vq->log_guest_addr = addr->log_guest_addr;
+	vq->log_guest_addr =
+		translate_log_addr(dev, vq, addr->log_guest_addr);
+	if (vq->log_guest_addr == 0) {
+		RTE_LOG(DEBUG, VHOST_CONFIG,
+			"(%d) failed to map log_guest_addr .\n",
+			dev->vid);
+		return dev;
+	}
+
 
 	VHOST_LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address desc: %p\n",
 			dev->vid, vq->desc);
-- 
2.21.0

[dpdk-dev] [PATCH v3 2/3] vhost: convert buffer addresses to GPA for logging

[Adrian Moreno]

Note that the API facing rte_vhost_log_write is not modified.
So, make explicit that it expects the address in GPA space.

Fixes: 69c90e98f483 ("vhost: enable IOMMU support")
Cc: maxime.coquelin@redhat.com
Cc: stable@dpdk.org

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 lib/librte_vhost/rte_vhost.h  |  2 +-
 lib/librte_vhost/vdpa.c       |  4 +++-
 lib/librte_vhost/vhost.c      | 21 +++++++++++++++++++++
 lib/librte_vhost/vhost.h      | 16 ++++++++++++++++
 lib/librte_vhost/virtio_net.c | 12 +++++++-----
 5 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
index 7fb172912..16c7e4243 100644
--- a/lib/librte_vhost/rte_vhost.h
+++ b/lib/librte_vhost/rte_vhost.h
@@ -265,7 +265,7 @@ rte_vhost_va_from_guest_pa(struct rte_vhost_memory *mem,
  * @param vid
  *  vhost device ID
  * @param addr
- *  the starting address for write
+ *  the starting address for write (in guest physical address space)
  * @param len
  *  the length to write
  */
diff --git a/lib/librte_vhost/vdpa.c b/lib/librte_vhost/vdpa.c
index 8e45ce9f8..11fd5aede 100644
--- a/lib/librte_vhost/vdpa.c
+++ b/lib/librte_vhost/vdpa.c
@@ -201,7 +201,9 @@ rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
 				goto fail;
 			desc = desc_ring[desc_id];
 			if (desc.flags & VRING_DESC_F_WRITE)
-				vhost_log_write(dev, desc.addr, desc.len);
+				vhost_log_cache_write_iova(dev, vq,
+							   desc.addr,
+							   desc.len);
 			desc_id = desc.next;
 		} while (desc.flags & VRING_DESC_F_NEXT);
 
diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index e57dda22f..78457a77c 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -200,6 +200,27 @@ __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	}
 }
 
+void
+__vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
+			     uint64_t iova, uint64_t len)
+{
+	uint64_t hva, gpa, map_len;
+	map_len = len;
+
+	hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW);
+	if (map_len != len) {
+		RTE_LOG(ERR, VHOST_CONFIG,
+			"Failed to write log for IOVA 0x%0lx."
+			"No IOTLB entry found\n",
+			iova);
+		return;
+	}
+
+	gpa = hva_to_gpa(dev, hva, len);
+	if (gpa)
+		__vhost_log_cache_write(dev, vq, gpa, len);
+}
+
 void *
 vhost_alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		uint64_t desc_addr, uint64_t desc_len)
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 9d3883385..9fa70bc3c 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -353,6 +353,9 @@ desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 void __vhost_log_cache_write(struct virtio_net *dev,
 		struct vhost_virtqueue *vq,
 		uint64_t addr, uint64_t len);
+void __vhost_log_cache_write_iova(struct virtio_net *dev,
+		struct vhost_virtqueue *vq,
+		uint64_t iova, uint64_t len);
 void __vhost_log_cache_sync(struct virtio_net *dev,
 		struct vhost_virtqueue *vq);
 void __vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len);
@@ -393,6 +396,19 @@ vhost_log_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	vhost_log_write(dev, vq->log_guest_addr + offset, len);
 }
 
+static __rte_always_inline void
+vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
+			   uint64_t iova, uint64_t len)
+{
+	if (likely(!(dev->features & (1ULL << VHOST_F_LOG_ALL))))
+		return;
+
+	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
+		__vhost_log_cache_write_iova(dev, vq, iova, len);
+	else
+		__vhost_log_cache_write(dev, vq, iova, len);
+}
+
 /* Macros for printing using RTE_LOG */
 #define RTE_LOGTYPE_VHOST_CONFIG RTE_LOGTYPE_USER1
 #define RTE_LOGTYPE_VHOST_DATA   RTE_LOGTYPE_USER1
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 5b85b832d..22961cafe 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -178,7 +178,8 @@ do_data_copy_enqueue(struct virtio_net *dev, struct vhost_virtqueue *vq)
 
 	for (i = 0; i < count; i++) {
 		rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
-		vhost_log_cache_write(dev, vq, elem[i].log_addr, elem[i].len);
+		vhost_log_cache_write_iova(dev, vq, elem[i].log_addr,
+					   elem[i].len);
 		PRINT_PACKET(dev, (uintptr_t)elem[i].dst, elem[i].len, 0);
 	}
 
@@ -633,7 +634,7 @@ copy_vnet_hdr_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 
 		PRINT_PACKET(dev, (uintptr_t)dst,
 				(uint32_t)len, 0);
-		vhost_log_cache_write(dev, vq,
+		vhost_log_cache_write_iova(dev, vq,
 				iova, len);
 
 		remain -= len;
@@ -733,7 +734,7 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			} else {
 				PRINT_PACKET(dev, (uintptr_t)hdr_addr,
 						dev->vhost_hlen, 0);
-				vhost_log_cache_write(dev, vq,
+				vhost_log_cache_write_iova(dev, vq,
 						buf_vec[0].buf_iova,
 						dev->vhost_hlen);
 			}
@@ -748,8 +749,9 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			rte_memcpy((void *)((uintptr_t)(buf_addr + buf_offset)),
 				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
 				cpy_len);
-			vhost_log_cache_write(dev, vq, buf_iova + buf_offset,
-					cpy_len);
+			vhost_log_cache_write_iova(dev, vq,
+						   buf_iova + buf_offset,
+						   cpy_len);
 			PRINT_PACKET(dev, (uintptr_t)(buf_addr + buf_offset),
 				cpy_len, 0);
 		} else {
-- 
2.21.0

[dpdk-dev] [PATCH v3 3/3] vhost: prevent zero copy mode if iommu is on

[Adrian Moreno]

The simltaneous use of dequeue_zero_copy and IOMMU is problematic.
Not only because IOVA_VA mode is not supported but also because the
potential invalidation of guest pages while the buffers are in use,
is not handled.

Prevent these two features to be enabled simultaneosly.

Fixes: 69c90e98f483 ("vhost: enable IOMMU support")
Cc: maxime.coquelin@redhat.com
Cc: stable@dpdk.org

Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
---
 lib/librte_vhost/socket.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
index 274988c4d..534a7ffe3 100644
--- a/lib/librte_vhost/socket.c
+++ b/lib/librte_vhost/socket.c
@@ -871,6 +871,14 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
 	}
 	vsocket->dequeue_zero_copy = flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
 
+	if (vsocket->dequeue_zero_copy &&
+	   (flags & RTE_VHOST_USER_IOMMU_SUPPORT)) {
+		RTE_LOG(ERR, VHOST_CONFIG,
+			"error: enabling dequeue zero copy and IOMMU features "
+			"simultaneously is not supported\n");
+		goto out_mutex;
+	}
+
 	/*
 	 * Set the supported features correctly for the builtin vhost-user
 	 * net driver.
-- 
2.21.0

Re: [dpdk-dev] [PATCH v3 3/3] vhost: prevent zero copy mode if iommu is on

[Tiwei Bie]

On Fri, Sep 27, 2019 at 08:37:59PM +0200, Adrian Moreno wrote:
> The simltaneous use of dequeue_zero_copy and IOMMU is problematic.
> Not only because IOVA_VA mode is not supported but also because the
> potential invalidation of guest pages while the buffers are in use,
> is not handled.
> 
> Prevent these two features to be enabled simultaneosly.
> 
> Fixes: 69c90e98f483 ("vhost: enable IOMMU support")
> Cc: maxime.coquelin@redhat.com
> Cc: stable@dpdk.org
> 
> Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
> ---
>  lib/librte_vhost/socket.c | 8 ++++++++
>  1 file changed, 8 insertions(+)

Reviewed-by: Tiwei Bie <tiwei.bie@intel.com>

> 
> diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
> index 274988c4d..534a7ffe3 100644
> --- a/lib/librte_vhost/socket.c
> +++ b/lib/librte_vhost/socket.c
> @@ -871,6 +871,14 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
>  	}
>  	vsocket->dequeue_zero_copy = flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
>  
> +	if (vsocket->dequeue_zero_copy &&
> +	   (flags & RTE_VHOST_USER_IOMMU_SUPPORT)) {

It's better to add one more space before '(flags ...'

> +		RTE_LOG(ERR, VHOST_CONFIG,
> +			"error: enabling dequeue zero copy and IOMMU features "
> +			"simultaneously is not supported\n");
> +		goto out_mutex;
> +	}
> +
>  	/*
>  	 * Set the supported features correctly for the builtin vhost-user
>  	 * net driver.
> -- 
> 2.21.0
> 

Re: [dpdk-dev] [PATCH v3 2/3] vhost: convert buffer addresses to GPA for logging

[Tiwei Bie]

On Fri, Sep 27, 2019 at 08:37:58PM +0200, Adrian Moreno wrote:
> Note that the API facing rte_vhost_log_write is not modified.
> So, make explicit that it expects the address in GPA space.
> 
> Fixes: 69c90e98f483 ("vhost: enable IOMMU support")
> Cc: maxime.coquelin@redhat.com
> Cc: stable@dpdk.org
> 
> Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
> ---
>  lib/librte_vhost/rte_vhost.h  |  2 +-
>  lib/librte_vhost/vdpa.c       |  4 +++-
>  lib/librte_vhost/vhost.c      | 21 +++++++++++++++++++++
>  lib/librte_vhost/vhost.h      | 16 ++++++++++++++++
>  lib/librte_vhost/virtio_net.c | 12 +++++++-----
>  5 files changed, 48 insertions(+), 7 deletions(-)
> 
> diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
> index 7fb172912..16c7e4243 100644
> --- a/lib/librte_vhost/rte_vhost.h
> +++ b/lib/librte_vhost/rte_vhost.h
> @@ -265,7 +265,7 @@ rte_vhost_va_from_guest_pa(struct rte_vhost_memory *mem,
>   * @param vid
>   *  vhost device ID
>   * @param addr
> - *  the starting address for write
> + *  the starting address for write (in guest physical address space)
>   * @param len
>   *  the length to write
>   */
> diff --git a/lib/librte_vhost/vdpa.c b/lib/librte_vhost/vdpa.c
> index 8e45ce9f8..11fd5aede 100644
> --- a/lib/librte_vhost/vdpa.c
> +++ b/lib/librte_vhost/vdpa.c
> @@ -201,7 +201,9 @@ rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
>  				goto fail;
>  			desc = desc_ring[desc_id];
>  			if (desc.flags & VRING_DESC_F_WRITE)
> -				vhost_log_write(dev, desc.addr, desc.len);
> +				vhost_log_cache_write_iova(dev, vq,
> +							   desc.addr,
> +							   desc.len);

This will only log the dirty pages in cache.


>  			desc_id = desc.next;
>  		} while (desc.flags & VRING_DESC_F_NEXT);
>  
> diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
> index e57dda22f..78457a77c 100644
> --- a/lib/librte_vhost/vhost.c
> +++ b/lib/librte_vhost/vhost.c
> @@ -200,6 +200,27 @@ __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq,
>  	}
>  }
>  
> +void
> +__vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
> +			     uint64_t iova, uint64_t len)
> +{
> +	uint64_t hva, gpa, map_len;
> +	map_len = len;
> +
> +	hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW);
> +	if (map_len != len) {
> +		RTE_LOG(ERR, VHOST_CONFIG,
> +			"Failed to write log for IOVA 0x%0lx."

PRIx64 should be used here, otherwise the build will be
broken on i686.

It's better to add a space after '.'


> +			"No IOTLB entry found\n",
> +			iova);
> +		return;
> +	}
> +
> +	gpa = hva_to_gpa(dev, hva, len);
> +	if (gpa)
> +		__vhost_log_cache_write(dev, vq, gpa, len);
> +}
> +

Re: [dpdk-dev] [PATCH v3 2/3] vhost: convert buffer addresses to GPA for logging

[Adrian Moreno]

On 10/8/19 8:30 AM, Tiwei Bie wrote:
> On Fri, Sep 27, 2019 at 08:37:58PM +0200, Adrian Moreno wrote:
>> Note that the API facing rte_vhost_log_write is not modified.
>> So, make explicit that it expects the address in GPA space.
>>
>> Fixes: 69c90e98f483 ("vhost: enable IOMMU support")
>> Cc: maxime.coquelin@redhat.com
>> Cc: stable@dpdk.org
>>
>> Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
>> ---
>>  lib/librte_vhost/rte_vhost.h  |  2 +-
>>  lib/librte_vhost/vdpa.c       |  4 +++-
>>  lib/librte_vhost/vhost.c      | 21 +++++++++++++++++++++
>>  lib/librte_vhost/vhost.h      | 16 ++++++++++++++++
>>  lib/librte_vhost/virtio_net.c | 12 +++++++-----
>>  5 files changed, 48 insertions(+), 7 deletions(-)
>>
>> diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
>> index 7fb172912..16c7e4243 100644
>> --- a/lib/librte_vhost/rte_vhost.h
>> +++ b/lib/librte_vhost/rte_vhost.h
>> @@ -265,7 +265,7 @@ rte_vhost_va_from_guest_pa(struct rte_vhost_memory *mem,
>>   * @param vid
>>   *  vhost device ID
>>   * @param addr
>> - *  the starting address for write
>> + *  the starting address for write (in guest physical address space)
>>   * @param len
>>   *  the length to write
>>   */
>> diff --git a/lib/librte_vhost/vdpa.c b/lib/librte_vhost/vdpa.c
>> index 8e45ce9f8..11fd5aede 100644
>> --- a/lib/librte_vhost/vdpa.c
>> +++ b/lib/librte_vhost/vdpa.c
>> @@ -201,7 +201,9 @@ rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
>>  				goto fail;
>>  			desc = desc_ring[desc_id];
>>  			if (desc.flags & VRING_DESC_F_WRITE)
>> -				vhost_log_write(dev, desc.addr, desc.len);
>> +				vhost_log_cache_write_iova(dev, vq,
>> +							   desc.addr,
>> +							   desc.len);
> 
> This will only log the dirty pages in cache.
You're right. Thanks for spotting it. I will change it in the next version.
> 
> 
>>  			desc_id = desc.next;
>>  		} while (desc.flags & VRING_DESC_F_NEXT);
>>  
>> diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
>> index e57dda22f..78457a77c 100644
>> --- a/lib/librte_vhost/vhost.c
>> +++ b/lib/librte_vhost/vhost.c
>> @@ -200,6 +200,27 @@ __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>  	}
>>  }
>>  
>> +void
>> +__vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
>> +			     uint64_t iova, uint64_t len)
>> +{
>> +	uint64_t hva, gpa, map_len;
>> +	map_len = len;
>> +
>> +	hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW);
>> +	if (map_len != len) {
>> +		RTE_LOG(ERR, VHOST_CONFIG,
>> +			"Failed to write log for IOVA 0x%0lx."
> 
> PRIx64 should be used here, otherwise the build will be
> broken on i686.
> 
> It's better to add a space after '.'
Thank you. Will do.
> 
> 
>> +			"No IOTLB entry found\n",
>> +			iova);
>> +		return;
>> +	}
>> +
>> +	gpa = hva_to_gpa(dev, hva, len);
>> +	if (gpa)
>> +		__vhost_log_cache_write(dev, vq, gpa, len);
>> +}
>> +

Re: [dpdk-dev] [PATCH v3 3/3] vhost: prevent zero copy mode if iommu is on

[Adrian Moreno]

On 10/8/19 8:23 AM, Tiwei Bie wrote:
> On Fri, Sep 27, 2019 at 08:37:59PM +0200, Adrian Moreno wrote:
>> The simltaneous use of dequeue_zero_copy and IOMMU is problematic.
>> Not only because IOVA_VA mode is not supported but also because the
>> potential invalidation of guest pages while the buffers are in use,
>> is not handled.
>>
>> Prevent these two features to be enabled simultaneosly.
>>
>> Fixes: 69c90e98f483 ("vhost: enable IOMMU support")
>> Cc: maxime.coquelin@redhat.com
>> Cc: stable@dpdk.org
>>
>> Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
>> ---
>>  lib/librte_vhost/socket.c | 8 ++++++++
>>  1 file changed, 8 insertions(+)
> 
> Reviewed-by: Tiwei Bie <tiwei.bie@intel.com>
> 
>>
>> diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
>> index 274988c4d..534a7ffe3 100644
>> --- a/lib/librte_vhost/socket.c
>> +++ b/lib/librte_vhost/socket.c
>> @@ -871,6 +871,14 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
>>  	}
>>  	vsocket->dequeue_zero_copy = flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
>>  
>> +	if (vsocket->dequeue_zero_copy &&
>> +	   (flags & RTE_VHOST_USER_IOMMU_SUPPORT)) {
> 
> It's better to add one more space before '(flags ...'
OK. I'll add it in the next version.
> 
>> +		RTE_LOG(ERR, VHOST_CONFIG,
>> +			"error: enabling dequeue zero copy and IOMMU features "
>> +			"simultaneously is not supported\n");
>> +		goto out_mutex;
>> +	}
>> +
>>  	/*
>>  	 * Set the supported features correctly for the builtin vhost-user
>>  	 * net driver.
>> -- 
>> 2.21.0
>>

Thanks.
-Adrián

Re: [dpdk-dev] [PATCH v3 2/3] vhost: convert buffer addresses to GPA for logging

[Maxime Coquelin]

On 10/8/19 9:38 AM, Adrian Moreno wrote:
> On 10/8/19 8:30 AM, Tiwei Bie wrote:
>> On Fri, Sep 27, 2019 at 08:37:58PM +0200, Adrian Moreno wrote:
>>> Note that the API facing rte_vhost_log_write is not modified.
>>> So, make explicit that it expects the address in GPA space.
>>>
>>> Fixes: 69c90e98f483 ("vhost: enable IOMMU support")
>>> Cc: maxime.coquelin@redhat.com
>>> Cc: stable@dpdk.org
>>>
>>> Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
>>> ---
>>>  lib/librte_vhost/rte_vhost.h  |  2 +-
>>>  lib/librte_vhost/vdpa.c       |  4 +++-
>>>  lib/librte_vhost/vhost.c      | 21 +++++++++++++++++++++
>>>  lib/librte_vhost/vhost.h      | 16 ++++++++++++++++
>>>  lib/librte_vhost/virtio_net.c | 12 +++++++-----
>>>  5 files changed, 48 insertions(+), 7 deletions(-)
>>>
>>> diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
>>> index 7fb172912..16c7e4243 100644
>>> --- a/lib/librte_vhost/rte_vhost.h
>>> +++ b/lib/librte_vhost/rte_vhost.h
>>> @@ -265,7 +265,7 @@ rte_vhost_va_from_guest_pa(struct rte_vhost_memory *mem,
>>>   * @param vid
>>>   *  vhost device ID
>>>   * @param addr
>>> - *  the starting address for write
>>> + *  the starting address for write (in guest physical address space)
>>>   * @param len
>>>   *  the length to write
>>>   */
>>> diff --git a/lib/librte_vhost/vdpa.c b/lib/librte_vhost/vdpa.c
>>> index 8e45ce9f8..11fd5aede 100644
>>> --- a/lib/librte_vhost/vdpa.c
>>> +++ b/lib/librte_vhost/vdpa.c
>>> @@ -201,7 +201,9 @@ rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
>>>  				goto fail;
>>>  			desc = desc_ring[desc_id];
>>>  			if (desc.flags & VRING_DESC_F_WRITE)
>>> -				vhost_log_write(dev, desc.addr, desc.len);
>>> +				vhost_log_cache_write_iova(dev, vq,
>>> +							   desc.addr,
>>> +							   desc.len);
>>
>> This will only log the dirty pages in cache.
> You're right. Thanks for spotting it. I will change it in the next version.
>>
>>
>>>  			desc_id = desc.next;
>>>  		} while (desc.flags & VRING_DESC_F_NEXT);
>>>  
>>> diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
>>> index e57dda22f..78457a77c 100644
>>> --- a/lib/librte_vhost/vhost.c
>>> +++ b/lib/librte_vhost/vhost.c
>>> @@ -200,6 +200,27 @@ __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>>  	}
>>>  }
>>>  
>>> +void
>>> +__vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>> +			     uint64_t iova, uint64_t len)
>>> +{
>>> +	uint64_t hva, gpa, map_len;
>>> +	map_len = len;
>>> +
>>> +	hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW);
>>> +	if (map_len != len) {
>>> +		RTE_LOG(ERR, VHOST_CONFIG,
>>> +			"Failed to write log for IOVA 0x%0lx."
>>
>> PRIx64 should be used here, otherwise the build will be
>> broken on i686.
>>
>> It's better to add a space after '.'
> Thank you. Will do.
>>
>>
>>> +			"No IOTLB entry found\n",

I personally prefer not to split error messages even it is is slightly
above 80 chars. It makes grepping for error messages easier.

>>> +			iova);
>>> +		return;
>>> +	}
>>> +
>>> +	gpa = hva_to_gpa(dev, hva, len);
>>> +	if (gpa)
>>> +		__vhost_log_cache_write(dev, vq, gpa, len);
>>> +}
>>> +
> 

Re: [dpdk-dev] [PATCH v3 2/3] vhost: convert buffer addresses to GPA for logging

[Tiwei Bie]

On Tue, Oct 08, 2019 at 09:43:48AM +0200, Maxime Coquelin wrote:
> On 10/8/19 9:38 AM, Adrian Moreno wrote:
> > On 10/8/19 8:30 AM, Tiwei Bie wrote:
> >> On Fri, Sep 27, 2019 at 08:37:58PM +0200, Adrian Moreno wrote:
> >>> Note that the API facing rte_vhost_log_write is not modified.
> >>> So, make explicit that it expects the address in GPA space.
> >>>
> >>> Fixes: 69c90e98f483 ("vhost: enable IOMMU support")
> >>> Cc: maxime.coquelin@redhat.com
> >>> Cc: stable@dpdk.org
> >>>
> >>> Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
> >>> ---
> >>>  lib/librte_vhost/rte_vhost.h  |  2 +-
> >>>  lib/librte_vhost/vdpa.c       |  4 +++-
> >>>  lib/librte_vhost/vhost.c      | 21 +++++++++++++++++++++
> >>>  lib/librte_vhost/vhost.h      | 16 ++++++++++++++++
> >>>  lib/librte_vhost/virtio_net.c | 12 +++++++-----
> >>>  5 files changed, 48 insertions(+), 7 deletions(-)
> >>>
> >>> diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
> >>> index 7fb172912..16c7e4243 100644
> >>> --- a/lib/librte_vhost/rte_vhost.h
> >>> +++ b/lib/librte_vhost/rte_vhost.h
> >>> @@ -265,7 +265,7 @@ rte_vhost_va_from_guest_pa(struct rte_vhost_memory *mem,
> >>>   * @param vid
> >>>   *  vhost device ID
> >>>   * @param addr
> >>> - *  the starting address for write
> >>> + *  the starting address for write (in guest physical address space)
> >>>   * @param len
> >>>   *  the length to write
> >>>   */
> >>> diff --git a/lib/librte_vhost/vdpa.c b/lib/librte_vhost/vdpa.c
> >>> index 8e45ce9f8..11fd5aede 100644
> >>> --- a/lib/librte_vhost/vdpa.c
> >>> +++ b/lib/librte_vhost/vdpa.c
> >>> @@ -201,7 +201,9 @@ rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
> >>>  				goto fail;
> >>>  			desc = desc_ring[desc_id];
> >>>  			if (desc.flags & VRING_DESC_F_WRITE)
> >>> -				vhost_log_write(dev, desc.addr, desc.len);
> >>> +				vhost_log_cache_write_iova(dev, vq,
> >>> +							   desc.addr,
> >>> +							   desc.len);
> >>
> >> This will only log the dirty pages in cache.
> > You're right. Thanks for spotting it. I will change it in the next version.
> >>
> >>
> >>>  			desc_id = desc.next;
> >>>  		} while (desc.flags & VRING_DESC_F_NEXT);
> >>>  
> >>> diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
> >>> index e57dda22f..78457a77c 100644
> >>> --- a/lib/librte_vhost/vhost.c
> >>> +++ b/lib/librte_vhost/vhost.c
> >>> @@ -200,6 +200,27 @@ __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq,
> >>>  	}
> >>>  }
> >>>  
> >>> +void
> >>> +__vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
> >>> +			     uint64_t iova, uint64_t len)
> >>> +{
> >>> +	uint64_t hva, gpa, map_len;
> >>> +	map_len = len;
> >>> +
> >>> +	hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW);
> >>> +	if (map_len != len) {
> >>> +		RTE_LOG(ERR, VHOST_CONFIG,
> >>> +			"Failed to write log for IOVA 0x%0lx."
> >>
> >> PRIx64 should be used here, otherwise the build will be
> >> broken on i686.
> >>
> >> It's better to add a space after '.'
> > Thank you. Will do.
> >>
> >>
> >>> +			"No IOTLB entry found\n",
> 
> I personally prefer not to split error messages even it is is slightly
> above 80 chars. It makes grepping for error messages easier.

+1


> 
> >>> +			iova);
> >>> +		return;
> >>> +	}
> >>> +
> >>> +	gpa = hva_to_gpa(dev, hva, len);
> >>> +	if (gpa)
> >>> +		__vhost_log_cache_write(dev, vq, gpa, len);
> >>> +}
> >>> +
> > 

Re: [dpdk-dev] [PATCH v3 2/3] vhost: convert buffer addresses to GPA for logging

[Adrian Moreno]

On 10/8/19 10:01 AM, Tiwei Bie wrote:
> On Tue, Oct 08, 2019 at 09:43:48AM +0200, Maxime Coquelin wrote:
>> On 10/8/19 9:38 AM, Adrian Moreno wrote:
>>> On 10/8/19 8:30 AM, Tiwei Bie wrote:
>>>> On Fri, Sep 27, 2019 at 08:37:58PM +0200, Adrian Moreno wrote:
>>>>> Note that the API facing rte_vhost_log_write is not modified.
>>>>> So, make explicit that it expects the address in GPA space.
>>>>>
>>>>> Fixes: 69c90e98f483 ("vhost: enable IOMMU support")
>>>>> Cc: maxime.coquelin@redhat.com
>>>>> Cc: stable@dpdk.org
>>>>>
>>>>> Signed-off-by: Adrian Moreno <amorenoz@redhat.com>
>>>>> ---
>>>>>  lib/librte_vhost/rte_vhost.h  |  2 +-
>>>>>  lib/librte_vhost/vdpa.c       |  4 +++-
>>>>>  lib/librte_vhost/vhost.c      | 21 +++++++++++++++++++++
>>>>>  lib/librte_vhost/vhost.h      | 16 ++++++++++++++++
>>>>>  lib/librte_vhost/virtio_net.c | 12 +++++++-----
>>>>>  5 files changed, 48 insertions(+), 7 deletions(-)
>>>>>
>>>>> diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
>>>>> index 7fb172912..16c7e4243 100644
>>>>> --- a/lib/librte_vhost/rte_vhost.h
>>>>> +++ b/lib/librte_vhost/rte_vhost.h
>>>>> @@ -265,7 +265,7 @@ rte_vhost_va_from_guest_pa(struct rte_vhost_memory *mem,
>>>>>   * @param vid
>>>>>   *  vhost device ID
>>>>>   * @param addr
>>>>> - *  the starting address for write
>>>>> + *  the starting address for write (in guest physical address space)
>>>>>   * @param len
>>>>>   *  the length to write
>>>>>   */
>>>>> diff --git a/lib/librte_vhost/vdpa.c b/lib/librte_vhost/vdpa.c
>>>>> index 8e45ce9f8..11fd5aede 100644
>>>>> --- a/lib/librte_vhost/vdpa.c
>>>>> +++ b/lib/librte_vhost/vdpa.c
>>>>> @@ -201,7 +201,9 @@ rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
>>>>>  				goto fail;
>>>>>  			desc = desc_ring[desc_id];
>>>>>  			if (desc.flags & VRING_DESC_F_WRITE)
>>>>> -				vhost_log_write(dev, desc.addr, desc.len);
>>>>> +				vhost_log_cache_write_iova(dev, vq,
>>>>> +							   desc.addr,
>>>>> +							   desc.len);
>>>>
>>>> This will only log the dirty pages in cache.
>>> You're right. Thanks for spotting it. I will change it in the next version.
>>>>
>>>>
>>>>>  			desc_id = desc.next;
>>>>>  		} while (desc.flags & VRING_DESC_F_NEXT);
>>>>>  
>>>>> diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
>>>>> index e57dda22f..78457a77c 100644
>>>>> --- a/lib/librte_vhost/vhost.c
>>>>> +++ b/lib/librte_vhost/vhost.c
>>>>> @@ -200,6 +200,27 @@ __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>>>>  	}
>>>>>  }
>>>>>  
>>>>> +void
>>>>> +__vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>>>> +			     uint64_t iova, uint64_t len)
>>>>> +{
>>>>> +	uint64_t hva, gpa, map_len;
>>>>> +	map_len = len;
>>>>> +
>>>>> +	hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW);
>>>>> +	if (map_len != len) {
>>>>> +		RTE_LOG(ERR, VHOST_CONFIG,
>>>>> +			"Failed to write log for IOVA 0x%0lx."
>>>>
>>>> PRIx64 should be used here, otherwise the build will be
>>>> broken on i686.
>>>>
>>>> It's better to add a space after '.'
>>> Thank you. Will do.
>>>>
>>>>
>>>>> +			"No IOTLB entry found\n",
>>
>> I personally prefer not to split error messages even it is is slightly
>> above 80 chars. It makes grepping for error messages easier.
> 
> +1
> 
Makes perfect sense. Addressed in the new version.
Thanks!

> 
>>
>>>>> +			iova);
>>>>> +		return;
>>>>> +	}
>>>>> +
>>>>> +	gpa = hva_to_gpa(dev, hva, len);
>>>>> +	if (gpa)
>>>>> +		__vhost_log_cache_write(dev, vq, gpa, len);
>>>>> +}
>>>>> +
>>>

-Adrián