Re: [dpdk-dev] [PATCH 4/4] mem: allow usage of non-heap external memory in multiprocess

[Yongseok Koh <yskoh@mellanox.com>]

On Thu, Nov 29, 2018 at 01:48:35PM +0000, Anatoly Burakov wrote:
> Add multiprocess support for externally allocated memory areas that
> are not added to DPDK heap (and add relevant doc sections).
> 
> Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>
> ---

Acked-by: Yongseok Koh <yskoh@mellanox.com>

Thanks

>  .../prog_guide/env_abstraction_layer.rst      |  3 +
>  lib/librte_eal/common/eal_common_memory.c     | 42 +++++++++++++
>  lib/librte_eal/common/include/rte_memory.h    | 59 +++++++++++++++++++
>  lib/librte_eal/rte_eal_version.map            |  2 +
>  4 files changed, 106 insertions(+)
> 
> diff --git a/doc/guides/prog_guide/env_abstraction_layer.rst b/doc/guides/prog_guide/env_abstraction_layer.rst
> index d7799b626..b0491bf2d 100644
> --- a/doc/guides/prog_guide/env_abstraction_layer.rst
> +++ b/doc/guides/prog_guide/env_abstraction_layer.rst
> @@ -276,11 +276,14 @@ The expected workflow is as follows:
>  * Register memory within DPDK
>      - If IOVA table is not specified, IOVA addresses will be assumed to be
>        unavailable
> +    - Other processes must attach to the memory area before they can use it
>  * Perform DMA mapping with ``rte_vfio_dma_map`` if needed
>  * Use the memory area in your application
>  * If memory area is no longer needed, it can be unregistered
>      - If the area was mapped for DMA, unmapping must be performed before
>        unregistering memory
> +    - Other processes must detach from the memory area before it can be
> +      unregistered
>  
>  Since these externally allocated memory areas will not be managed by DPDK, it is
>  therefore up to the user application to decide how to use them and what to do
> diff --git a/lib/librte_eal/common/eal_common_memory.c b/lib/librte_eal/common/eal_common_memory.c
> index a2e085ae8..67b445c31 100644
> --- a/lib/librte_eal/common/eal_common_memory.c
> +++ b/lib/librte_eal/common/eal_common_memory.c
> @@ -849,6 +849,48 @@ rte_extmem_unregister(void *va_addr, size_t len)
>  	return ret;
>  }
>  
> +static int
> +sync_memory(void *va_addr, size_t len, bool attach)
> +{
> +	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
> +	struct rte_memseg_list *msl;
> +	int ret = 0;
> +
> +	if (va_addr == NULL || len == 0) {
> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	rte_rwlock_write_lock(&mcfg->memory_hotplug_lock);
> +
> +	/* find our segment */
> +	msl = malloc_heap_find_external_seg(va_addr, len);
> +	if (msl == NULL) {
> +		rte_errno = ENOENT;
> +		ret = -1;
> +		goto unlock;
> +	}
> +	if (attach)
> +		ret = rte_fbarray_attach(&msl->memseg_arr);
> +	else
> +		ret = rte_fbarray_detach(&msl->memseg_arr);
> +
> +unlock:
> +	rte_rwlock_write_unlock(&mcfg->memory_hotplug_lock);
> +	return ret;
> +}
> +
> +int __rte_experimental
> +rte_extmem_attach(void *va_addr, size_t len)
> +{
> +	return sync_memory(va_addr, len, true);
> +}
> +
> +int __rte_experimental
> +rte_extmem_detach(void *va_addr, size_t len)
> +{
> +	return sync_memory(va_addr, len, false);
> +}
> +
>  /* init memory subsystem */
>  int
>  rte_eal_memory_init(void)
> diff --git a/lib/librte_eal/common/include/rte_memory.h b/lib/librte_eal/common/include/rte_memory.h
> index 4a43c1a9e..050bb6d8e 100644
> --- a/lib/librte_eal/common/include/rte_memory.h
> +++ b/lib/librte_eal/common/include/rte_memory.h
> @@ -435,6 +435,10 @@ rte_memseg_get_fd_offset_thread_unsafe(const struct rte_memseg *ms,
>   * @note This API will not perform any DMA mapping. It is expected that user
>   *   will do that themselves.
>   *
> + * @note Before accessing this memory in other processes, it needs to be
> + *   attached in each of those processes by calling ``rte_extmem_attach`` in
> + *   each other process.
> + *
>   * @param va_addr
>   *   Start of virtual area to register
>   * @param len
> @@ -472,6 +476,9 @@ rte_extmem_register(void *va_addr, size_t len, rte_iova_t iova_addrs[],
>   * @note This API will not perform any DMA unmapping. It is expected that user
>   *   will do that themselves.
>   *
> + * @note Before calling this function, all other processes must call
> + *   ``rte_extmem_detach`` to detach from the memory area.
> + *
>   * @param va_addr
>   *   Start of virtual area to unregister
>   * @param len
> @@ -486,6 +493,58 @@ rte_extmem_register(void *va_addr, size_t len, rte_iova_t iova_addrs[],
>  int __rte_experimental
>  rte_extmem_unregister(void *va_addr, size_t len);
>  
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Attach to external memory chunk registered in another process.
> + *
> + * @note Using this API is mutually exclusive with ``rte_malloc`` family of
> + *   API's.
> + *
> + * @note This API will not perform any DMA mapping. It is expected that user
> + *   will do that themselves.
> + *
> + * @param va_addr
> + *   Start of virtual area to register
> + * @param len
> + *   Length of virtual area to register
> + *
> + * @return
> + *   - 0 on success
> + *   - -1 in case of error, with rte_errno set to one of the following:
> + *     EINVAL - one of the parameters was invalid
> + *     ENOENT - memory chunk was not found
> + */
> +int __rte_experimental
> +rte_extmem_attach(void *va_addr, size_t len);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Detach from external memory chunk registered in another process.
> + *
> + * @note Using this API is mutually exclusive with ``rte_malloc`` family of
> + *   API's.
> + *
> + * @note This API will not perform any DMA unmapping. It is expected that user
> + *   will do that themselves.
> + *
> + * @param va_addr
> + *   Start of virtual area to unregister
> + * @param len
> + *   Length of virtual area to unregister
> + *
> + * @return
> + *   - 0 on success
> + *   - -1 in case of error, with rte_errno set to one of the following:
> + *     EINVAL - one of the parameters was invalid
> + *     ENOENT - memory chunk was not found
> + */
> +int __rte_experimental
> +rte_extmem_detach(void *va_addr, size_t len);
> +
>  /**
>   * Dump the physical memory layout to a file.
>   *
> diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
> index 593691a14..eb5f7b9cb 100644
> --- a/lib/librte_eal/rte_eal_version.map
> +++ b/lib/librte_eal/rte_eal_version.map
> @@ -296,6 +296,8 @@ EXPERIMENTAL {
>  	rte_devargs_remove;
>  	rte_devargs_type_count;
>  	rte_eal_cleanup;
> +	rte_extmem_attach;
> +	rte_extmem_detach;
>  	rte_extmem_register;
>  	rte_extmem_unregister;
>  	rte_fbarray_attach;
> -- 
> 2.17.1