Re: [dpdk-dev] [PATCH v5 03/15] vhost: vring address setup for packed queues

[Maxime Coquelin <maxime.coquelin@redhat.com>]

On 06/29/2018 05:59 PM, Tiwei Bie wrote:
> On Fri, Jun 22, 2018 at 03:43:15PM +0200, Maxime Coquelin wrote:
>> From: Yuanhan Liu <yuanhan.liu@linux.intel.com>
>>
>> Add code to set up packed queues when enabled.
>>
>> Signed-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
>> Signed-off-by: Jens Freimann <jfreiman@redhat.com>
>> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
>> ---
>>   lib/librte_vhost/vhost.c      | 44 ++++++++++++++++++++++++++++++++++++++-----
>>   lib/librte_vhost/vhost.h      |  7 ++++++-
>>   lib/librte_vhost/vhost_user.c | 29 +++++++++++++++++++++++++++-
>>   3 files changed, 73 insertions(+), 7 deletions(-)
>>
>> diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
>> index afded4952..a85c6646f 100644
>> --- a/lib/librte_vhost/vhost.c
>> +++ b/lib/librte_vhost/vhost.c
>> @@ -23,6 +23,7 @@
>>   #include "iotlb.h"
>>   #include "vhost.h"
>>   #include "vhost_user.h"
>> +#include "virtio-packed.h"
>>   
>>   struct virtio_net *vhost_devices[MAX_VHOST_DEVICE];
>>   
>> @@ -115,14 +116,11 @@ free_device(struct virtio_net *dev)
>>   	rte_free(dev);
>>   }
>>   
>> -int
>> -vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq)
>> +static int
>> +vring_translate_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
>>   {
>>   	uint64_t req_size, size;
>>   
>> -	if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)))
>> -		goto out;
>> -
>>   	req_size = sizeof(struct vring_desc) * vq->size;
>>   	size = req_size;
>>   	vq->desc = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, vq,
>> @@ -153,6 +151,40 @@ vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq)
>>   	if (!vq->used || size != req_size)
>>   		return -1;
>>   
>> +	return 0;
>> +}
>> +
>> +static int
>> +vring_translate_packed(struct virtio_net *dev, struct vhost_virtqueue *vq)
>> +{
>> +	uint64_t req_size, size;
>> +
>> +	req_size = sizeof(struct vring_desc_packed) * vq->size;
>> +	size = req_size;
>> +	vq->desc_packed =
>> +		(struct vring_desc_packed *)(uintptr_t)vhost_iova_to_vva(dev,
>> +					vq, vq->ring_addrs.desc_user_addr,
>> +					&size, VHOST_ACCESS_RW);
>> +	if (!vq->desc || size != req_size)
> 
> Should check vq->desc_packed instead of vq->desc.

Yes

>> +		return -1;
> 
> Also need to get vq->driver_event and vq->device_event.
> Maybe shouldn't do it in this patch. But it's missing
> in the whole patch set.

I'll add it in the notif patch.

Thanks,
Maxime

>> +
>> +	return 0;
>> +}
>> +
>> +int
>> +vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq)
>> +{
>> +
>> +	if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)))
>> +		goto out;
>> +
>> +	if (vq_is_packed(dev)) {
>> +		if (vring_translate_packed(dev, vq) < 0)
>> +			return -1;
>> +	} else {
>> +		if (vring_translate_split(dev, vq) < 0)
>> +			return -1;
>> +	}
>>   out:
>>   	vq->access_ok = 1;
>>   
>> @@ -234,6 +266,8 @@ alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
>>   	dev->virtqueue[vring_idx] = vq;
>>   	init_vring_queue(dev, vring_idx);
>>   	rte_spinlock_init(&vq->access_lock);
>> +	vq->avail_wrap_counter = 1;
>> +	vq->used_wrap_counter = 1;
>>   
>>   	dev->nr_vring += 1;
>>   
>> diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
>> index 34a614c97..671b4b3bf 100644
>> --- a/lib/librte_vhost/vhost.h
>> +++ b/lib/librte_vhost/vhost.h
>> @@ -84,7 +84,10 @@ struct log_cache_entry {
>>    * Structure contains variables relevant to RX/TX virtqueues.
>>    */
>>   struct vhost_virtqueue {
>> -	struct vring_desc	*desc;
>> +	union {
>> +		struct vring_desc	*desc;
>> +		struct vring_desc_packed   *desc_packed;
>> +	};
>>   	struct vring_avail	*avail;
>>   	struct vring_used	*used;
>>   	uint32_t		size;
>> @@ -122,6 +125,8 @@ struct vhost_virtqueue {
>>   
>>   	struct batch_copy_elem	*batch_copy_elems;
>>   	uint16_t		batch_copy_nb_elems;
>> +	uint16_t		used_wrap_counter;
>> +	uint16_t		avail_wrap_counter;
> 
> Not quite sure about this. Do you think it will be
> better to define wrap counters as bool (as only 0 or
> 1 are available)?

I think it should work, but not sure this is cleaner.

When defining something as bool, I expect to use true or false
for its assignment, but maybe that's me.

But I could certainly use uint8_t instead to reduce the size of the
struct.

What do you think?

>>   
>>   	struct log_cache_entry log_cache[VHOST_LOG_CACHE_NR];
>>   	uint16_t log_cache_nb_elem;
>> diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
>> index 947290fc3..b6097c085 100644
>> --- a/lib/librte_vhost/vhost_user.c
>> +++ b/lib/librte_vhost/vhost_user.c
>> @@ -39,6 +39,7 @@
>>   #include "iotlb.h"
>>   #include "vhost.h"
>>   #include "vhost_user.h"
>> +#include "virtio-packed.h"
>>   
>>   #define VIRTIO_MIN_MTU 68
>>   #define VIRTIO_MAX_MTU 65535
>> @@ -477,6 +478,30 @@ translate_ring_addresses(struct virtio_net *dev, int vq_index)
>>   	struct vhost_vring_addr *addr = &vq->ring_addrs;
>>   	uint64_t len;
>>   
>> +	if (vq_is_packed(dev)) {
>> +		len = sizeof(struct vring_desc_packed) * vq->size;
>> +		vq->desc_packed = (struct vring_desc_packed *) ring_addr_to_vva
>> +			(dev, vq, addr->desc_user_addr, &len);
>> +		vq->desc = NULL;
> 
> `vq->desc = NULL` will set vq->desc_packed to NULL.

Indeed, I was sure to have remove that line, as it makes
translate_ring_addresses to fail systematically...

Ha, I squashed the fix in the wrong patch (PATCH 14).

>> +		vq->avail = NULL;
>> +		vq->used = NULL;
>> +		vq->log_guest_addr = 0;
>> +		if (vq->desc_packed == 0 ||
> 
> It's better to compare with NULL.

I agree.

>> +				len != sizeof(struct vring_desc_packed) *
>> +				vq->size) {
>> +			RTE_LOG(DEBUG, VHOST_CONFIG,
>> +				"(%d) failed to map desc_packed ring.\n",
>> +				dev->vid);
>> +			return dev;
>> +		}
>> +
>> +		dev = numa_realloc(dev, vq_index);
>> +		vq = dev->virtqueue[vq_index];
>> +		addr = &vq->ring_addrs;
>> +
>> +		return dev;
>> +	}
>> +
>>   	/* The addresses are converted from QEMU virtual to Vhost virtual. */
>>   	if (vq->desc && vq->avail && vq->used)
>>   		return dev;
>> @@ -490,6 +515,7 @@ translate_ring_addresses(struct virtio_net *dev, int vq_index)
>>   			dev->vid);
>>   		return dev;
>>   	}
>> +	vq->desc_packed = NULL;

Same mistake, I squashed the change in patch 14.

> It seems that above assignment isn't needed.
> 
>>   
>>   	dev = numa_realloc(dev, vq_index);
>>   	vq = dev->virtqueue[vq_index];
>> @@ -888,7 +914,8 @@ vhost_user_set_mem_table(struct virtio_net **pdev, struct VhostUserMsg *pmsg)
>>   static int
>>   vq_is_ready(struct vhost_virtqueue *vq)
>>   {
>> -	return vq && vq->desc && vq->avail && vq->used &&
>> +	return vq &&
>> +	       (vq->desc_packed || (vq->desc && vq->avail && vq->used)) &&
>>   	       vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD &&
>>   	       vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD;
>>   }
>> -- 
>> 2.14.4
>>