[PATCH] gpu/cuda: GPU_REGISTERED to distinguish GPU memory CPU mapped

[<eagostini@nvidia.com>]

From: Elena Agostini <eagostini@nvidia.com>

Enable GPU_REGISTERED flag in gpu/cuda driver in the memory list.
If a GPU memory address CPU mapped is freed before being
unmapped, CUDA driver unmaps it before freeing the memory.

Signed-off-by: Elena Agostini <eagostini@nvidia.com>
---
 drivers/gpu/cuda/cuda.c | 78 +++++++++++++++++++++++++----------------
 1 file changed, 47 insertions(+), 31 deletions(-)

diff --git a/drivers/gpu/cuda/cuda.c b/drivers/gpu/cuda/cuda.c
index 5b8476ac20..c6bf54c130 100644
--- a/drivers/gpu/cuda/cuda.c
+++ b/drivers/gpu/cuda/cuda.c
@@ -257,7 +257,7 @@ struct cuda_info {
 enum mem_type {
 	GPU_MEM = 0,
 	CPU_REGISTERED,
-	GPU_REGISTERED /* Not used yet */
+	GPU_REGISTERED
 };
 
 /* key associated to a memory address */
@@ -953,13 +953,14 @@ cuda_mem_cpu_map(struct rte_gpu *dev, __rte_unused size_t size, void *ptr_in, vo
 		return -rte_errno;
 	}
 
+	mem_item->mtype = GPU_REGISTERED;
 	*ptr_out = mem_item->ptr_h;
 
 	return 0;
 }
 
 static int
-cuda_mem_free(struct rte_gpu *dev, void *ptr)
+cuda_mem_unregister(struct rte_gpu *dev, void *ptr)
 {
 	CUresult res;
 	struct mem_entry *mem_item;
@@ -978,11 +979,11 @@ cuda_mem_free(struct rte_gpu *dev, void *ptr)
 		return -rte_errno;
 	}
 
-	if (mem_item->mtype == GPU_MEM) {
-		res = pfn_cuMemFree(mem_item->ptr_orig_d);
+	if (mem_item->mtype == CPU_REGISTERED) {
+		res = pfn_cuMemHostUnregister(ptr);
 		if (res != 0) {
 			pfn_cuGetErrorString(res, &(err_string));
-			rte_cuda_log(ERR, "cuMemFree current failed with %s",
+			rte_cuda_log(ERR, "cuMemHostUnregister current failed with %s",
 					err_string);
 			rte_errno = EPERM;
 			return -rte_errno;
@@ -993,74 +994,89 @@ cuda_mem_free(struct rte_gpu *dev, void *ptr)
 
 	rte_cuda_log(ERR, "Memory type %d not supported", mem_item->mtype);
 
-	return -EPERM;
+	rte_errno = EPERM;
+	return -rte_errno;
 }
 
 static int
-cuda_mem_unregister(struct rte_gpu *dev, void *ptr)
+cuda_mem_cpu_unmap(struct rte_gpu *dev, void *ptr_in)
 {
-	CUresult res;
 	struct mem_entry *mem_item;
-	const char *err_string;
 	cuda_ptr_key hk;
 
 	if (dev == NULL)
 		return -ENODEV;
 
-	hk = get_hash_from_ptr((void *)ptr);
+	hk = get_hash_from_ptr((void *)ptr_in);
 
 	mem_item = mem_list_find_item(hk);
 	if (mem_item == NULL) {
-		rte_cuda_log(ERR, "Memory address 0x%p not found in driver memory", ptr);
+		rte_cuda_log(ERR, "Memory address 0x%p not found in driver memory.", ptr_in);
 		rte_errno = EPERM;
 		return -rte_errno;
 	}
 
-	if (mem_item->mtype == CPU_REGISTERED) {
-		res = pfn_cuMemHostUnregister(ptr);
-		if (res != 0) {
-			pfn_cuGetErrorString(res, &(err_string));
-			rte_cuda_log(ERR, "cuMemHostUnregister current failed with %s",
-					err_string);
+	if (mem_item->mtype == GPU_REGISTERED) {
+		if (gdrcopy_unpin(gdrc_h, mem_item->mh, (void *)mem_item->ptr_d,
+				mem_item->size)) {
+			rte_cuda_log(ERR, "Error unexposing GPU memory address 0x%p.", ptr_in);
 			rte_errno = EPERM;
 			return -rte_errno;
 		}
 
-		return mem_list_del_item(hk);
+		mem_item->mtype = GPU_MEM;
+	} else {
+		rte_errno = EPERM;
+		return -rte_errno;
 	}
 
-	rte_cuda_log(ERR, "Memory type %d not supported", mem_item->mtype);
-
-	rte_errno = EPERM;
-	return -rte_errno;
+	return 0;
 }
 
 static int
-cuda_mem_cpu_unmap(struct rte_gpu *dev, void *ptr_in)
+cuda_mem_free(struct rte_gpu *dev, void *ptr)
 {
+	CUresult res;
 	struct mem_entry *mem_item;
+	const char *err_string;
 	cuda_ptr_key hk;
 
 	if (dev == NULL)
 		return -ENODEV;
 
-	hk = get_hash_from_ptr((void *)ptr_in);
+	hk = get_hash_from_ptr((void *)ptr);
 
 	mem_item = mem_list_find_item(hk);
 	if (mem_item == NULL) {
-		rte_cuda_log(ERR, "Memory address 0x%p not found in driver memory.", ptr_in);
+		rte_cuda_log(ERR, "Memory address 0x%p not found in driver memory", ptr);
 		rte_errno = EPERM;
 		return -rte_errno;
 	}
 
-	if (gdrcopy_unpin(gdrc_h, mem_item->mh, (void *)mem_item->ptr_d,
-			mem_item->size)) {
-		rte_cuda_log(ERR, "Error unexposing GPU memory address 0x%p.", ptr_in);
-		rte_errno = EPERM;
-		return -rte_errno;
+	/*
+	 * If a GPU memory area that's CPU mapped is being freed
+	 * without calling cpu_unmap, force the unmapping.
+	*/
+	if (mem_item->mtype == GPU_REGISTERED) {
+		cuda_mem_cpu_unmap(dev, ptr);
 	}
 
-	return 0;
+	if (mem_item->mtype == GPU_MEM) {
+		res = pfn_cuMemFree(mem_item->ptr_orig_d);
+		if (res != 0) {
+			pfn_cuGetErrorString(res, &(err_string));
+			rte_cuda_log(ERR, "cuMemFree current failed with %s",
+					err_string);
+			rte_errno = EPERM;
+			return -rte_errno;
+		}
+
+		return mem_list_del_item(hk);
+	}
+
+	rte_cuda_log(ERR, "Memory type %d not supported", mem_item->mtype);
+
+	return -EPERM;
 }
 
 static int
-- 
2.25.1