[dpdk-dev] [PATCH] vhost: avoid iotlb mempool allocation while IOMMU disabled

[dpdk-dev] [PATCH] vhost: avoid iotlb mempool allocation while IOMMU disabled

[Wan Junjie]

If vhost device's IOMMU feature is disabled, iotlb mempool allocation
is unnecessary.

Reported-by: Peng He <hepeng.0320@bytedance.com>
Signed-off-by: Wan Junjie <wanjunjie@bytedance.com>
Reviewed-by: Zhihong Wang <wangzhihong.wzh@bytedance.com>
---
 lib/librte_vhost/vhost.c      | 6 ++++--
 lib/librte_vhost/vhost_user.c | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index efb136edd..00c5040e2 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -352,7 +352,8 @@ free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq)
 		vhost_free_async_mem(vq);
 	}
 	rte_free(vq->batch_copy_elems);
-	rte_mempool_free(vq->iotlb_pool);
+	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
+		rte_mempool_free(vq->iotlb_pool);
 	rte_free(vq);
 }
 
@@ -556,7 +557,8 @@ init_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 	vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
 	vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF;
 
-	vhost_user_iotlb_init(dev, vring_idx);
+	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
+		vhost_user_iotlb_init(dev, vring_idx);
 	/* Backends are set to -1 indicating an inactive device. */
 	vq->backend = -1;
 }
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index a60bb945a..d415607d7 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -575,7 +575,7 @@ numa_realloc(struct virtio_net *dev, int index)
 	dev->virtqueue[index] = vq;
 	vhost_devices[dev->vid] = dev;
 
-	if (old_vq != vq)
+	if (old_vq != vq && (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)))
 		vhost_user_iotlb_init(dev, index);
 
 	return dev;
-- 
2.29.2

Re: [dpdk-dev] [PATCH] vhost: avoid iotlb mempool allocation while IOMMU disabled

[Xia, Chenbo]

Hi Junjie,

> -----Original Message-----
> From: Wan Junjie <wanjunjie@bytedance.com>
> Sent: Saturday, January 30, 2021 12:39 AM
> To: maxime.coquelin@redhat.com; Xia, Chenbo <chenbo.xia@intel.com>
> Cc: dev@dpdk.org; Wan Junjie <wanjunjie@bytedance.com>; Peng He
> <hepeng.0320@bytedance.com>; Zhihong Wang <wangzhihong.wzh@bytedance.com>
> Subject: [PATCH] vhost: avoid iotlb mempool allocation while IOMMU disabled
> 
> If vhost device's IOMMU feature is disabled, iotlb mempool allocation
> is unnecessary.
> 
> Reported-by: Peng He <hepeng.0320@bytedance.com>
> Signed-off-by: Wan Junjie <wanjunjie@bytedance.com>
> Reviewed-by: Zhihong Wang <wangzhihong.wzh@bytedance.com>
> ---
>  lib/librte_vhost/vhost.c      | 6 ++++--
>  lib/librte_vhost/vhost_user.c | 2 +-
>  2 files changed, 5 insertions(+), 3 deletions(-)
> 
> diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
> index efb136edd..00c5040e2 100644
> --- a/lib/librte_vhost/vhost.c
> +++ b/lib/librte_vhost/vhost.c
> @@ -352,7 +352,8 @@ free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq)
>  		vhost_free_async_mem(vq);
>  	}
>  	rte_free(vq->batch_copy_elems);
> -	rte_mempool_free(vq->iotlb_pool);
> +	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
> +		rte_mempool_free(vq->iotlb_pool);

We could make it simpler, check vq->iotlb_pool is not NULL and then free the
mempool, which ignores the problem that features may not be set yet (See below
comment). Although we could also keep the original because rte_mempool_free
will do the check, let's do it outside to avoid useless func call.

>  	rte_free(vq);
>  }
> 
> @@ -556,7 +557,8 @@ init_vring_queue(struct virtio_net *dev, uint32_t
> vring_idx)
>  	vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
>  	vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF;
> 
> -	vhost_user_iotlb_init(dev, vring_idx);
> +	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
> +		vhost_user_iotlb_init(dev, vring_idx);

Note that when this func is called, dev->features are not set. So it's meaningless
to do this. So, we should do it after the features are set.

Thanks,
Chenbo

>  	/* Backends are set to -1 indicating an inactive device. */
>  	vq->backend = -1;
>  }
> diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
> index a60bb945a..d415607d7 100644
> --- a/lib/librte_vhost/vhost_user.c
> +++ b/lib/librte_vhost/vhost_user.c
> @@ -575,7 +575,7 @@ numa_realloc(struct virtio_net *dev, int index)
>  	dev->virtqueue[index] = vq;
>  	vhost_devices[dev->vid] = dev;
> 
> -	if (old_vq != vq)
> +	if (old_vq != vq && (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)))
>  		vhost_user_iotlb_init(dev, index);
> 
>  	return dev;
> --
> 2.29.2

[dpdk-dev] [PATCH v2] vhost: avoid iotlb mempool allocation while IOMMU disabled

[Wan Junjie]

If vhost device's IOMMU feature is disabled, iotlb mempool allocation
is unnecessary.

Reported-by: Peng He <hepeng.0320@bytedance.com>
Signed-off-by: Wan Junjie <wanjunjie@bytedance.com>
Reviewed-by: Zhihong Wang <wangzhihong.wzh@bytedance.com>
---
 lib/librte_vhost/vhost.c      | 4 ++--
 lib/librte_vhost/vhost_user.c | 6 +++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index b83cf639e..e5341ab98 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -352,7 +352,8 @@ free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq)
 		vhost_free_async_mem(vq);
 	}
 	rte_free(vq->batch_copy_elems);
-	rte_mempool_free(vq->iotlb_pool);
+	if (vq->iotlb_pool)
+		rte_mempool_free(vq->iotlb_pool);
 	rte_free(vq);
 }
 
@@ -556,7 +557,6 @@ init_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 	vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
 	vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF;
 
-	vhost_user_iotlb_init(dev, vring_idx);
 	/* Backends are set to -1 indicating an inactive device. */
 	vq->backend = -1;
 }
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index 45c8ac09d..81cbb9797 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -470,6 +470,10 @@ vhost_user_set_vring_num(struct virtio_net **pdev,
 		return RTE_VHOST_MSG_RESULT_ERR;
 	}
 
+	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
+		if (vhost_user_iotlb_init(dev, msg->payload.state.index))
+			return RTE_VHOST_MSG_RESULT_ERR;
+	}
 	return RTE_VHOST_MSG_RESULT_OK;
 }
 
@@ -575,7 +579,7 @@ numa_realloc(struct virtio_net *dev, int index)
 	dev->virtqueue[index] = vq;
 	vhost_devices[dev->vid] = dev;
 
-	if (old_vq != vq)
+	if (old_vq != vq && (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)))
 		vhost_user_iotlb_init(dev, index);
 
 	return dev;
-- 
2.11.0

Re: [dpdk-dev] [PATCH v2] vhost: avoid iotlb mempool allocation while IOMMU disabled

[Xia, Chenbo]

> -----Original Message-----
> From: Wan Junjie <wanjunjie@bytedance.com>
> Sent: Tuesday, February 2, 2021 4:39 PM
> To: Xia, Chenbo <chenbo.xia@intel.com>; Maxime Coquelin
> <maxime.coquelin@redhat.com>
> Cc: dev@dpdk.org; Wan Junjie <wanjunjie@bytedance.com>; Peng He
> <hepeng.0320@bytedance.com>; Zhihong Wang <wangzhihong.wzh@bytedance.com>
> Subject: [PATCH v2] vhost: avoid iotlb mempool allocation while IOMMU disabled
> 
> If vhost device's IOMMU feature is disabled, iotlb mempool allocation
> is unnecessary.
> 
> Reported-by: Peng He <hepeng.0320@bytedance.com>
> Signed-off-by: Wan Junjie <wanjunjie@bytedance.com>
> Reviewed-by: Zhihong Wang <wangzhihong.wzh@bytedance.com>
> ---
>  lib/librte_vhost/vhost.c      | 4 ++--
>  lib/librte_vhost/vhost_user.c | 6 +++++-
>  2 files changed, 7 insertions(+), 3 deletions(-)
> 
> diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
> index b83cf639e..e5341ab98 100644
> --- a/lib/librte_vhost/vhost.c
> +++ b/lib/librte_vhost/vhost.c
> @@ -352,7 +352,8 @@ free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq)
>  		vhost_free_async_mem(vq);
>  	}
>  	rte_free(vq->batch_copy_elems);
> -	rte_mempool_free(vq->iotlb_pool);
> +	if (vq->iotlb_pool)
> +		rte_mempool_free(vq->iotlb_pool);
>  	rte_free(vq);
>  }
> 
> @@ -556,7 +557,6 @@ init_vring_queue(struct virtio_net *dev, uint32_t
> vring_idx)
>  	vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
>  	vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF;
> 
> -	vhost_user_iotlb_init(dev, vring_idx);
>  	/* Backends are set to -1 indicating an inactive device. */
>  	vq->backend = -1;
>  }
> diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
> index 45c8ac09d..81cbb9797 100644
> --- a/lib/librte_vhost/vhost_user.c
> +++ b/lib/librte_vhost/vhost_user.c
> @@ -470,6 +470,10 @@ vhost_user_set_vring_num(struct virtio_net **pdev,
>  		return RTE_VHOST_MSG_RESULT_ERR;
>  	}
> 
> +	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
> +		if (vhost_user_iotlb_init(dev, msg->payload.state.index))
> +			return RTE_VHOST_MSG_RESULT_ERR;
> +	}
>  	return RTE_VHOST_MSG_RESULT_OK;
>  }
> 
> @@ -575,7 +579,7 @@ numa_realloc(struct virtio_net *dev, int index)
>  	dev->virtqueue[index] = vq;
>  	vhost_devices[dev->vid] = dev;
> 
> -	if (old_vq != vq)
> +	if (old_vq != vq && (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)))
>  		vhost_user_iotlb_init(dev, index);
> 
>  	return dev;
> --
> 2.11.0

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>

Re: [dpdk-dev] [PATCH v2] vhost: avoid iotlb mempool allocation while IOMMU disabled

[Maxime Coquelin]

Hi,

On 2/2/21 9:38 AM, Wan Junjie wrote:
> If vhost device's IOMMU feature is disabled, iotlb mempool allocation
> is unnecessary.
> 
> Reported-by: Peng He <hepeng.0320@bytedance.com>
> Signed-off-by: Wan Junjie <wanjunjie@bytedance.com>

Welcome to the community, looking forward for more contributions!

> Reviewed-by: Zhihong Wang <wangzhihong.wzh@bytedance.com>

And welcome back Zhihong! :)

> ---
>  lib/librte_vhost/vhost.c      | 4 ++--
>  lib/librte_vhost/vhost_user.c | 6 +++++-
>  2 files changed, 7 insertions(+), 3 deletions(-)
> 

The patch looks good to me, however this is more an optimization than a
fix. As we are late in the release cycle, I will postpone it to next
release.

Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>

Thanks,
Maxime

Re: [dpdk-dev] [PATCH v2] vhost: avoid iotlb mempool allocation while IOMMU disabled

[Xia, Chenbo]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Thursday, May 13, 2021 4:39 PM
> To: Xia, Chenbo <chenbo.xia@intel.com>; Maxime Coquelin
> <maxime.coquelin@redhat.com>
> Cc: Wan Junjie <wanjunjie@bytedance.com>; dev@dpdk.org; Peng He
> <hepeng.0320@bytedance.com>; Zhihong Wang <wangzhihong.wzh@bytedance.com>
> Subject: Re: [dpdk-dev] [PATCH v2] vhost: avoid iotlb mempool allocation while
> IOMMU disabled
> 
> On Wed, Apr 7, 2021 at 9:10 AM Xia, Chenbo <chenbo.xia@intel.com> wrote:
> >
> > > -----Original Message-----
> > > From: Wan Junjie <wanjunjie@bytedance.com>
> > > Sent: Tuesday, February 2, 2021 4:15 PM
> > > To: Xia, Chenbo <chenbo.xia@intel.com>; Maxime Coquelin
> > > <maxime.coquelin@redhat.com>
> > > Cc: dev@dpdk.org; Wan Junjie <wanjunjie@bytedance.com>; Peng He
> > > <hepeng.0320@bytedance.com>; Zhihong Wang <wangzhihong.wzh@bytedance.com>
> > > Subject: [PATCH v2] vhost: avoid iotlb mempool allocation while IOMMU
> disabled
> > >
> > > If vhost device's IOMMU feature is disabled, iotlb mempool allocation
> > > is unnecessary.
> > >
> > > Reported-by: Peng He <hepeng.0320@bytedance.com>
> > > Signed-off-by: Wan Junjie <wanjunjie@bytedance.com>
> > > Reviewed-by: Zhihong Wang <wangzhihong.wzh@bytedance.com>
> > > ---
> > > 2.11.0
> >
> > Patch applied to next-virtio/main with conflict resolved, Thanks
> 
> This patch triggered a regression reported by Red Hat QE.
> https://bugs.dpdk.org/show_bug.cgi?id=703
> 
> Reverting the patch is enough to fix the regression.
> 
> I'll let you guys decide on the next step for 21.05.
> 
> 
> --
> David Marchand

I cannot reproduce this issue on my machine.

@Wan Junjie Could you reproduce this? I prefer to revert the patch if things
are not very clear as it is only a minor optimization.

Thanks,
Chenbo 

Re: [dpdk-dev] [PATCH v2] vhost: avoid iotlb mempool allocation while IOMMU disabled

[David Marchand]

On Wed, Apr 7, 2021 at 9:10 AM Xia, Chenbo <chenbo.xia@intel.com> wrote:
>
> > -----Original Message-----
> > From: Wan Junjie <wanjunjie@bytedance.com>
> > Sent: Tuesday, February 2, 2021 4:15 PM
> > To: Xia, Chenbo <chenbo.xia@intel.com>; Maxime Coquelin
> > <maxime.coquelin@redhat.com>
> > Cc: dev@dpdk.org; Wan Junjie <wanjunjie@bytedance.com>; Peng He
> > <hepeng.0320@bytedance.com>; Zhihong Wang <wangzhihong.wzh@bytedance.com>
> > Subject: [PATCH v2] vhost: avoid iotlb mempool allocation while IOMMU disabled
> >
> > If vhost device's IOMMU feature is disabled, iotlb mempool allocation
> > is unnecessary.
> >
> > Reported-by: Peng He <hepeng.0320@bytedance.com>
> > Signed-off-by: Wan Junjie <wanjunjie@bytedance.com>
> > Reviewed-by: Zhihong Wang <wangzhihong.wzh@bytedance.com>
> > ---
> > 2.11.0
>
> Patch applied to next-virtio/main with conflict resolved, Thanks

This patch triggered a regression reported by Red Hat QE.
https://bugs.dpdk.org/show_bug.cgi?id=703

Reverting the patch is enough to fix the regression.

I'll let you guys decide on the next step for 21.05.


-- 
David Marchand

Re: [dpdk-dev] [PATCH v2] vhost: avoid iotlb mempool allocation while IOMMU disabled

[Xia, Chenbo]

> -----Original Message-----
> From: Wan Junjie <wanjunjie@bytedance.com>
> Sent: Tuesday, February 2, 2021 4:15 PM
> To: Xia, Chenbo <chenbo.xia@intel.com>; Maxime Coquelin
> <maxime.coquelin@redhat.com>
> Cc: dev@dpdk.org; Wan Junjie <wanjunjie@bytedance.com>; Peng He
> <hepeng.0320@bytedance.com>; Zhihong Wang <wangzhihong.wzh@bytedance.com>
> Subject: [PATCH v2] vhost: avoid iotlb mempool allocation while IOMMU disabled
> 
> If vhost device's IOMMU feature is disabled, iotlb mempool allocation
> is unnecessary.
> 
> Reported-by: Peng He <hepeng.0320@bytedance.com>
> Signed-off-by: Wan Junjie <wanjunjie@bytedance.com>
> Reviewed-by: Zhihong Wang <wangzhihong.wzh@bytedance.com>
> ---
> 2.11.0

Patch applied to next-virtio/main with conflict resolved, Thanks

[dpdk-dev] [PATCH v2] vhost: avoid iotlb mempool allocation while IOMMU disabled

[Wan Junjie]

If vhost device's IOMMU feature is disabled, iotlb mempool allocation
is unnecessary.

Reported-by: Peng He <hepeng.0320@bytedance.com>
Signed-off-by: Wan Junjie <wanjunjie@bytedance.com>
Reviewed-by: Zhihong Wang <wangzhihong.wzh@bytedance.com>
---
 lib/librte_vhost/vhost.c      | 4 ++--
 lib/librte_vhost/vhost_user.c | 6 +++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index b83cf639e..e5341ab98 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -352,7 +352,8 @@ free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq)
 		vhost_free_async_mem(vq);
 	}
 	rte_free(vq->batch_copy_elems);
-	rte_mempool_free(vq->iotlb_pool);
+	if (vq->iotlb_pool)
+		rte_mempool_free(vq->iotlb_pool);
 	rte_free(vq);
 }
 
@@ -556,7 +557,6 @@ init_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 	vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
 	vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF;
 
-	vhost_user_iotlb_init(dev, vring_idx);
 	/* Backends are set to -1 indicating an inactive device. */
 	vq->backend = -1;
 }
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index 45c8ac09d..81cbb9797 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -470,6 +470,10 @@ vhost_user_set_vring_num(struct virtio_net **pdev,
 		return RTE_VHOST_MSG_RESULT_ERR;
 	}
 
+	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
+		if (vhost_user_iotlb_init(dev, msg->payload.state.index))
+			return RTE_VHOST_MSG_RESULT_ERR;
+	}
 	return RTE_VHOST_MSG_RESULT_OK;
 }
 
@@ -575,7 +579,7 @@ numa_realloc(struct virtio_net *dev, int index)
 	dev->virtqueue[index] = vq;
 	vhost_devices[dev->vid] = dev;
 
-	if (old_vq != vq)
+	if (old_vq != vq && (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)))
 		vhost_user_iotlb_init(dev, index);
 
 	return dev;
-- 
2.11.0