[dpdk-dev] [PATCH v3 0/2] vhost: IOTLB fixes

[dpdk-dev] [PATCH v3 0/2] vhost: IOTLB fixes

[Maxime Coquelin]

First patch of the series fixes OOM handling from the IOTLB
mempool, the second one removes pending IOTLB entry when the
IOTLB miss request sending failed.

Changes since v2:
-----------------
- patch 2: Fix error message with correct IOVA

Changes since v1:
-----------------
- Make log levels consistent (Tiwei)
- Remove pending IOTLB entry of miss request seding failed (Tiwei)

Maxime Coquelin (2):
  vhost: fix iotlb pool out-of-memory handling
  vhost: remove pending IOTLB entry if IOTLB MISS request sending failed

 lib/librte_vhost/iotlb.c | 20 ++++++++++++++------
 lib/librte_vhost/iotlb.h |  3 +++
 lib/librte_vhost/vhost.c | 13 ++++++++++---
 3 files changed, 27 insertions(+), 9 deletions(-)

-- 
2.14.3

[dpdk-dev] [PATCH v3 1/2] vhost: fix iotlb pool out-of-memory handling

[Maxime Coquelin]

In the unlikely case the IOTLB memory pool runs out of memory,
an issue may happen if all entries are used by the IOTLB cache,
and an IOTLB miss happen. If the iotlb pending list is empty,
then no memory is freed and allocation fails a second time.

This patch fixes this by doing an IOTLB cache random evict if
the IOTLB pending list is empty, ensuring the second allocation
try will succeed.

In the same spirit, the opposite is done when inserting an
IOTLB entry in the IOTLB cache fails due to out of memory. In
this case, the IOTLB pending is flushed if the IOTLB cache is
empty to ensure the new entry can be inserted.

Fixes: d012d1f293f4 ("vhost: add IOTLB helper functions")
Fixes: f72c2ad63aeb ("vhost: add pending IOTLB miss request list and helpers")

Cc: stable@dpdk.org
Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 lib/librte_vhost/iotlb.c | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/lib/librte_vhost/iotlb.c b/lib/librte_vhost/iotlb.c
index b74cc6a78..72cd27df8 100644
--- a/lib/librte_vhost/iotlb.c
+++ b/lib/librte_vhost/iotlb.c
@@ -50,6 +50,9 @@ struct vhost_iotlb_entry {
 
 #define IOTLB_CACHE_SIZE 2048
 
+static void
+vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq);
+
 static void
 vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq)
 {
@@ -95,9 +98,11 @@ vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq,
 
 	ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
 	if (ret) {
-		RTE_LOG(INFO, VHOST_CONFIG,
-				"IOTLB pool empty, clear pending misses\n");
-		vhost_user_iotlb_pending_remove_all(vq);
+		RTE_LOG(DEBUG, VHOST_CONFIG, "IOTLB pool empty, clear entries\n");
+		if (!TAILQ_EMPTY(&vq->iotlb_pending_list))
+			vhost_user_iotlb_pending_remove_all(vq);
+		else
+			vhost_user_iotlb_cache_random_evict(vq);
 		ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
 		if (ret) {
 			RTE_LOG(ERR, VHOST_CONFIG, "IOTLB pool still empty, failure\n");
@@ -186,8 +191,11 @@ vhost_user_iotlb_cache_insert(struct vhost_virtqueue *vq, uint64_t iova,
 
 	ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
 	if (ret) {
-		RTE_LOG(DEBUG, VHOST_CONFIG, "IOTLB pool empty, evict one entry\n");
-		vhost_user_iotlb_cache_random_evict(vq);
+		RTE_LOG(DEBUG, VHOST_CONFIG, "IOTLB pool empty, clear entries\n");
+		if (!TAILQ_EMPTY(&vq->iotlb_list))
+			vhost_user_iotlb_cache_random_evict(vq);
+		else
+			vhost_user_iotlb_pending_remove_all(vq);
 		ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
 		if (ret) {
 			RTE_LOG(ERR, VHOST_CONFIG, "IOTLB pool still empty, failure\n");
-- 
2.14.3

[dpdk-dev] [PATCH v3 2/2] vhost: remove pending IOTLB entry if IOTLB MISS request sending failed

[Maxime Coquelin]

In case vhost_user_iotlb_miss returns an error, the pending IOTLB
entry has to be removed from the list as no IOTLB update will be
received.

Fixes: fed67a20ac94 ("vhost: introduce guest IOVA to backend VA helper")

Cc: stable@dpdk.org
Suggested-by: Tiwei Bie <tiwei.bie@intel.com>
Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 lib/librte_vhost/iotlb.c |  2 +-
 lib/librte_vhost/iotlb.h |  3 +++
 lib/librte_vhost/vhost.c | 13 ++++++++++---
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/lib/librte_vhost/iotlb.c b/lib/librte_vhost/iotlb.c
index 72cd27df8..c11ebcaac 100644
--- a/lib/librte_vhost/iotlb.c
+++ b/lib/librte_vhost/iotlb.c
@@ -120,7 +120,7 @@ vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq,
 	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
 }
 
-static void
+void
 vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq,
 				uint64_t iova, uint64_t size, uint8_t perm)
 {
diff --git a/lib/librte_vhost/iotlb.h b/lib/librte_vhost/iotlb.h
index f1a050e44..e7083e37b 100644
--- a/lib/librte_vhost/iotlb.h
+++ b/lib/librte_vhost/iotlb.h
@@ -71,6 +71,9 @@ bool vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova,
 						uint8_t perm);
 void vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq, uint64_t iova,
 						uint8_t perm);
+void vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq, uint64_t iova,
+						uint64_t size, uint8_t perm);
+
 int vhost_user_iotlb_init(struct virtio_net *dev, int vq_index);
 
 #endif /* _VHOST_IOTLB_H_ */
diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index 1dd9adbc7..6ac6cacff 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -42,7 +42,9 @@ __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (tmp_size == size)
 		return vva;
 
-	if (!vhost_user_iotlb_pending_miss(vq, iova + tmp_size, perm)) {
+	iova += tmp_size;
+
+	if (!vhost_user_iotlb_pending_miss(vq, iova, perm)) {
 		/*
 		 * iotlb_lock is read-locked for a full burst,
 		 * but it only protects the iotlb cache.
@@ -52,8 +54,13 @@ __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		 */
 		vhost_user_iotlb_rd_unlock(vq);
 
-		vhost_user_iotlb_pending_insert(vq, iova + tmp_size, perm);
-		vhost_user_iotlb_miss(dev, iova + tmp_size, perm);
+		vhost_user_iotlb_pending_insert(vq, iova, perm);
+		if (vhost_user_iotlb_miss(dev, iova, perm)) {
+			RTE_LOG(ERR, VHOST_CONFIG,
+					"Failed to send IOTLB miss request for IOVA %lx\n",
+					iova);
+			vhost_user_iotlb_pending_remove(vq, iova, 1, perm);
+		}
 
 		vhost_user_iotlb_rd_lock(vq);
 	}
-- 
2.14.3

Re: [dpdk-dev] [PATCH v3 0/2] vhost: IOTLB fixes

[Yuanhan Liu]

On Mon, Jan 29, 2018 at 05:30:38PM +0100, Maxime Coquelin wrote:
> First patch of the series fixes OOM handling from the IOTLB
> mempool, the second one removes pending IOTLB entry when the
> IOTLB miss request sending failed.
> 
> Changes since v2:
> -----------------
> - patch 2: Fix error message with correct IOVA
> 
> Changes since v1:
> -----------------
> - Make log levels consistent (Tiwei)
> - Remove pending IOTLB entry of miss request seding failed (Tiwei)
> 
> Maxime Coquelin (2):
>   vhost: fix iotlb pool out-of-memory handling
>   vhost: remove pending IOTLB entry if IOTLB MISS request sending failed

Series Acked-by: Yuanhan Liu <yliu@fridaylinux.org>

Thanks.

	--yliu
> 
>  lib/librte_vhost/iotlb.c | 20 ++++++++++++++------
>  lib/librte_vhost/iotlb.h |  3 +++
>  lib/librte_vhost/vhost.c | 13 ++++++++++---
>  3 files changed, 27 insertions(+), 9 deletions(-)
> 
> -- 
> 2.14.3