RE: [EXTERNAL] Re: [PATCH v2 1/1] lib/dma: introduce inter-process and inter-OS DMA

[Vamsi Krishna Attunuru <vattunuru@marvell.com>]

>On 2025/10/10 22:46, Vamsi Krishna wrote:
>> From: Vamsi Attunuru <vattunuru@marvell.com>
>>
>> Modern DMA hardware supports data transfers between multiple DMA
>> devices, facilitating data communication across isolated domains,
>> containers, or operating systems. These DMA transfers function as
>> standard memory-to-memory operations, but with source or destination
>> addresses residing in different process or OS address space. The
>> exchange of these addresses between processes is handled through
>> private driver mechanism, which are beyond the scope of this
>> specification change.
>>
>> This commit introduces new capability flags to advertise driver
>> support for inter-process and inter-OS domain DMA transfers. It adds a
>> mechanism to specify source and destination handlers via the vchan
>configuration.
>> Until standardized control-plane APIs are defined for various
>> categories of DMA devices, these handler details can be exchanged
>> through private driver mechanisms.
>>
>> Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>
>> ---
>> V2 changes:
>> * Seperate out the control-plane APIs
>> * Address v0 review comments
>> * Add validation checks
>> * Rename the enums
>>
>>   lib/dmadev/rte_dmadev.c       | 19 ++++++++++++
>>   lib/dmadev/rte_dmadev.h       | 56
>+++++++++++++++++++++++++++++++++++
>>   lib/dmadev/rte_dmadev_trace.h |  3 ++
>>   3 files changed, 78 insertions(+)
>>
>> diff --git a/lib/dmadev/rte_dmadev.c b/lib/dmadev/rte_dmadev.c index
>> 17ee0808a9..23864dcb00 100644
>> --- a/lib/dmadev/rte_dmadev.c
>> +++ b/lib/dmadev/rte_dmadev.c
>> @@ -659,6 +659,21 @@ rte_dma_vchan_setup(int16_t dev_id, uint16_t
>vchan,
>>   		RTE_DMA_LOG(ERR, "Device %d vchan out range!", dev_id);
>>   		return -EINVAL;
>>   	}
>> +	if (conf->direction != RTE_DMA_DIR_MEM_TO_MEM &&
>> +	    conf->domain.domain_type != RTE_DMA_INTER_DOMAIN_NONE)
>{
>> +		RTE_DMA_LOG(ERR, "Device %d direction and inter domain
>are
>> +invalid!", dev_id);
>
>how about "Device %d inter domain only support memory-to-memory
>transfer" ?

Ack, will address all comments in V3. Thanks

>
>so could be:
>
>if (conf->domain.type != RTE_DMA_INTER_DOMAIN_NONE &&
>
>     conf->direction != RTE_DMA_DIR_MEM_TO_MEM)
>
>
>> +		return -EINVAL;
>> +	}
>> +	if (conf->domain.domain_type == RTE_DMA_INTER_OS_DOMAIN &&
>> +	    !(dev_info.dev_capa & RTE_DMA_CAPA_INTER_OS_DOMAIN)) {
>> +		RTE_DMA_LOG(ERR, "Device %d does not support inter os
>domain", dev_id);
>> +		return -EINVAL;
>> +	}
>> +	if (conf->domain.domain_type ==
>RTE_DMA_INTER_PROCESS_DOMAIN &&
>> +	    !(dev_info.dev_capa &
>RTE_DMA_CAPA_INTER_PROCESS_DOMAIN)) {
>> +		RTE_DMA_LOG(ERR, "Device %d does not support inter
>process domain", dev_id);
>> +		return -EINVAL;
>> +	}
>>   	if (conf->direction != RTE_DMA_DIR_MEM_TO_MEM &&
>>   	    conf->direction != RTE_DMA_DIR_MEM_TO_DEV &&
>>   	    conf->direction != RTE_DMA_DIR_DEV_TO_MEM && @@ -805,6
>+820,8
>> @@ dma_capability_name(uint64_t capability)
>>   		{ RTE_DMA_CAPA_HANDLES_ERRORS, "handles_errors" },
>>   		{ RTE_DMA_CAPA_M2D_AUTO_FREE,  "m2d_auto_free"  },
>>   		{ RTE_DMA_CAPA_PRI_POLICY_SP,  "pri_policy_sp" },
>> +		{ RTE_DMA_CAPA_INTER_PROCESS_DOMAIN,
>"inter_process_domain" },
>> +		{ RTE_DMA_CAPA_INTER_OS_DOMAIN, "inter_os_domain" },
>>   		{ RTE_DMA_CAPA_OPS_COPY,    "copy"    },
>>   		{ RTE_DMA_CAPA_OPS_COPY_SG, "copy_sg" },
>>   		{ RTE_DMA_CAPA_OPS_FILL,    "fill"    },
>> @@ -1014,6 +1031,8 @@ dmadev_handle_dev_info(const char *cmd
>__rte_unused,
>>   	ADD_CAPA(dma_caps, dev_capa,
>RTE_DMA_CAPA_HANDLES_ERRORS);
>>   	ADD_CAPA(dma_caps, dev_capa,
>RTE_DMA_CAPA_M2D_AUTO_FREE);
>>   	ADD_CAPA(dma_caps, dev_capa, RTE_DMA_CAPA_PRI_POLICY_SP);
>> +	ADD_CAPA(dma_caps, dev_capa,
>RTE_DMA_CAPA_INTER_PROCESS_DOMAIN);
>> +	ADD_CAPA(dma_caps, dev_capa,
>RTE_DMA_CAPA_INTER_OS_DOMAIN);
>>   	ADD_CAPA(dma_caps, dev_capa, RTE_DMA_CAPA_OPS_COPY);
>>   	ADD_CAPA(dma_caps, dev_capa, RTE_DMA_CAPA_OPS_COPY_SG);
>>   	ADD_CAPA(dma_caps, dev_capa, RTE_DMA_CAPA_OPS_FILL); diff --
>git
>> a/lib/dmadev/rte_dmadev.h b/lib/dmadev/rte_dmadev.h index
>> 550dbfbf75..12c249ec00 100644
>> --- a/lib/dmadev/rte_dmadev.h
>> +++ b/lib/dmadev/rte_dmadev.h
>> @@ -265,6 +265,18 @@ int16_t rte_dma_next_dev(int16_t start_dev_id);
>>    * known from 'nb_priorities' field in struct rte_dma_info.
>>    */
>>   #define RTE_DMA_CAPA_PRI_POLICY_SP	RTE_BIT64(8)
>> +/** Support inter-process DMA transfers.
>> + *
>> + * When this bit is set, the DMA device can perform memory transfers
>> +between
>> + * different process memory spaces.
>> + */
>> +#define RTE_DMA_CAPA_INTER_PROCESS_DOMAIN	RTE_BIT64(9)
>> +/** Support inter-OS domain DMA transfers.
>> + *
>> + * The DMA device can perform memory transfers across different
>> +operating
>> + * system domains.
>> + */
>> +#define RTE_DMA_CAPA_INTER_OS_DOMAIN		RTE_BIT64(10)
>>
>>   /** Support copy operation.
>>    * This capability start with index of 32, so that it could leave
>> gap between @@ -418,8 +430,13 @@ int rte_dma_close(int16_t dev_id);
>>    */
>>   enum rte_dma_direction {
>>   	/** DMA transfer direction - from memory to memory.
>> +	 * When the device supports inter-process or inter-OS domain
>transfers,
>> +	 * the field `domain_type` in `struct rte_dma_vchan_conf::domain`
>specifies
>> +	 * the type of domain. For memory-to-memory transfers within the
>same domain
>> +	 * or process, `domain_type` should be set to
>`RTE_DMA_INTER_DOMAIN_NONE`.
>>   	 *
>>   	 * @see struct rte_dma_vchan_conf::direction
>> +	 * @see struct rte_dma_inter_domain_param::domain_type
>>   	 */
>>   	RTE_DMA_DIR_MEM_TO_MEM,
>>   	/** DMA transfer direction - from memory to device.
>> @@ -564,6 +581,36 @@ struct rte_dma_auto_free_param {
>>   	uint64_t reserved[2];
>>   };
>>
>> +/**
>> + * Inter-DMA transfer domain type.
>> + *
>> + * This enum defines the types of transfer domains applicable to DMA
>operations.
>> + * It helps categorize whether a DMA transfer is occurring within the
>> +same domain,
>> + * across different processes, or between distinct operating system
>domains.
>> + *
>> + * @see struct rte_dma_inter_domain_param:domain_type
>> + */
>> +enum rte_dma_inter_domain_type {
>> +	RTE_DMA_INTER_DOMAIN_NONE, /**< No inter-domain transfer;
>standard DMA within same domain */
>> +	RTE_DMA_INTER_PROCESS_DOMAIN, /**< Transfer occurs between
>different user-space processes */
>> +	RTE_DMA_INTER_OS_DOMAIN, /**< Transfer spans across different
>> +operating system domains. */
>
>please place the comment before the definition.
>
>> +};
>> +
>> +/**
>> + * Parameters for inter-process or inter-OS DMA transfers.
>> + *
>> + * This structure defines the parameters required to perform DMA
>> +transfers
>> + * across different domains, such as between processes or operating
>systems.
>> + * It includes the domain type and handler identifiers for both the
>> +source
>> + * and destination domains.
>> + */
>> +struct rte_dma_inter_domain_param {
>> +	enum rte_dma_inter_domain_type domain_type; /**< Type of
>> +inter-domain. */
>
>please rename domian_type to type because we are in domain context.
>
>> +	uint16_t src_handler; /**< Source domain handler identifier. */
>> +	uint16_t dst_handler; /**< Destination domain handler identifier. */
>> +	uint64_t reserved[2]; /**< Reserved for future fields. */ };
>> +
>>   /**
>>    * A structure used to configure a virtual DMA channel.
>>    *
>> @@ -601,6 +648,15 @@ struct rte_dma_vchan_conf {
>>   	 * @see struct rte_dma_auto_free_param
>>   	 */
>>   	struct rte_dma_auto_free_param auto_free;
>> +	/** Parameters for inter-process or inter-OS domain DMA transfers.
>This field
>> +	 * specifies the source and destination domain handlers required
>> +for DMA
>
>two spaces before 'for DMA'
>
>> +	 * operations that span across different processes or operating system
>domains.
>> +	 *
>> +	 * @see RTE_DMA_CAPA_INTER_PROCESS_DOMAIN
>> +	 * @see RTE_DMA_CAPA_INTER_OS_DOMAIN
>> +	 * @see struct rte_dma_inter_domain_param
>> +	 */
>> +	struct rte_dma_inter_domain_param domain;
>>   };
>>
>>   /**
>> diff --git a/lib/dmadev/rte_dmadev_trace.h
>> b/lib/dmadev/rte_dmadev_trace.h index 1de92655f2..12ea9d53f6 100644
>> --- a/lib/dmadev/rte_dmadev_trace.h
>> +++ b/lib/dmadev/rte_dmadev_trace.h
>> @@ -79,6 +79,9 @@ RTE_TRACE_POINT(
>>   	rte_trace_point_emit_int(conf->dst_port.port_type);
>>   	rte_trace_point_emit_u64(conf->dst_port.pcie.val);
>>   	rte_trace_point_emit_ptr(conf->auto_free.m2d.pool);
>> +	rte_trace_point_emit_int(conf->domain.domain_type);
>> +	rte_trace_point_emit_u16(conf->domain.src_handler);
>> +	rte_trace_point_emit_u16(conf->domain.dst_handler);
>>   	rte_trace_point_emit_int(ret);
>>   )
>>