Re: [EXTERNAL] Re: [PATCH v3 1/1] lib/dma: introduce inter-process and inter-OS DMA

[fengchengwen <fengchengwen@huawei.com>]

On 10/14/2025 2:31 AM, Vamsi Krishna Attunuru wrote:
>>
>> ZjQcmQRYFpfptBannerEnd
>> with below comment fixed:
>> Acked-by: Chengwen Feng <fengchengwen@huawei.com>
>>
>> On 10/13/2025 12:21 PM, Vamsi Krishna wrote:
>>> From: Vamsi Attunuru <vattunuru@marvell.com>
>>>
>>> Modern DMA hardware supports data transfers between multiple DMA
>>> devices, facilitating data communication across isolated domains,
>>> containers, or operating systems. These DMA transfers function as
>>> standard memory-to-memory operations, but with source or destination
>>> addresses residing in different process or OS address space. The
>>> exchange of these addresses between processes is handled through
>>> private driver mechanism, which are beyond the scope of this
>>> specification change.
>>>
>>> This commit introduces new capability flags to advertise driver
>>> support for inter-process and inter-OS domain DMA transfers. It adds a
>>> mechanism to specify source and destination handlers via the vchan
>> configuration.
>>> Until standardized control-plane APIs are defined for various
>>> categories of DMA devices, these handler details can be exchanged
>>> through private driver mechanisms.
>>>
>>> Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>
>>> ---
>>> V3 changes:
>>> * Fix v2 review comments
>>> V2 changes:
>>> * Seperate out the control-plane APIs
>>> * Address v0 review comments
>>> * Add validation checks
>>> * Rename the enums
>>>
>>>  lib/dmadev/rte_dmadev.c       | 19 +++++++++++
>>>  lib/dmadev/rte_dmadev.h       | 59
>> +++++++++++++++++++++++++++++++++++
>>>  lib/dmadev/rte_dmadev_trace.h |  3 ++
>>>  3 files changed, 81 insertions(+)
>>>
>>> diff --git a/lib/dmadev/rte_dmadev.c b/lib/dmadev/rte_dmadev.c index
>>> 17ee0808a9..d6792449bf 100644
>>> --- a/lib/dmadev/rte_dmadev.c
>>> +++ b/lib/dmadev/rte_dmadev.c
>>> @@ -659,6 +659,21 @@ rte_dma_vchan_setup(int16_t dev_id, uint16_t
>> vchan,
>>>  		RTE_DMA_LOG(ERR, "Device %d vchan out range!", dev_id);
>>>  		return -EINVAL;
>>>  	}
>>> +	if (conf->domain.type != RTE_DMA_INTER_DOMAIN_NONE &&
>>> +	    conf->direction != RTE_DMA_DIR_MEM_TO_MEM) {
>>> +		RTE_DMA_LOG(ERR, "Device %d inter domain only support
>> mem-to-mem transfer", dev_id);
>>> +		return -EINVAL;
>>> +	}
>>> +	if (conf->domain.type == RTE_DMA_INTER_OS_DOMAIN &&
>>> +	    !(dev_info.dev_capa & RTE_DMA_CAPA_INTER_OS_DOMAIN)) {
>>> +		RTE_DMA_LOG(ERR, "Device %d does not support inter os
>> domain", dev_id);
>>> +		return -EINVAL;
>>> +	}
>>> +	if (conf->domain.type == RTE_DMA_INTER_PROCESS_DOMAIN &&
>>> +	    !(dev_info.dev_capa &
>> RTE_DMA_CAPA_INTER_PROCESS_DOMAIN)) {
>>> +		RTE_DMA_LOG(ERR, "Device %d does not support inter
>> process domain", dev_id);
>>> +		return -EINVAL;
>>> +	}
>>>  	if (conf->direction != RTE_DMA_DIR_MEM_TO_MEM &&
>>>  	    conf->direction != RTE_DMA_DIR_MEM_TO_DEV &&
>>>  	    conf->direction != RTE_DMA_DIR_DEV_TO_MEM && @@ -805,6
>> +820,8 @@
>>> dma_capability_name(uint64_t capability)
>>>  		{ RTE_DMA_CAPA_HANDLES_ERRORS, "handles_errors" },
>>>  		{ RTE_DMA_CAPA_M2D_AUTO_FREE,  "m2d_auto_free"  },
>>>  		{ RTE_DMA_CAPA_PRI_POLICY_SP,  "pri_policy_sp" },
>>> +		{ RTE_DMA_CAPA_INTER_PROCESS_DOMAIN,
>> "inter_process_domain" },
>>> +		{ RTE_DMA_CAPA_INTER_OS_DOMAIN, "inter_os_domain" },
>>>  		{ RTE_DMA_CAPA_OPS_COPY,    "copy"    },
>>>  		{ RTE_DMA_CAPA_OPS_COPY_SG, "copy_sg" },
>>>  		{ RTE_DMA_CAPA_OPS_FILL,    "fill"    },
>>> @@ -1014,6 +1031,8 @@ dmadev_handle_dev_info(const char *cmd
>> __rte_unused,
>>>  	ADD_CAPA(dma_caps, dev_capa,
>> RTE_DMA_CAPA_HANDLES_ERRORS);
>>>  	ADD_CAPA(dma_caps, dev_capa,
>> RTE_DMA_CAPA_M2D_AUTO_FREE);
>>>  	ADD_CAPA(dma_caps, dev_capa, RTE_DMA_CAPA_PRI_POLICY_SP);
>>> +	ADD_CAPA(dma_caps, dev_capa,
>> RTE_DMA_CAPA_INTER_PROCESS_DOMAIN);
>>> +	ADD_CAPA(dma_caps, dev_capa,
>> RTE_DMA_CAPA_INTER_OS_DOMAIN);
>>>  	ADD_CAPA(dma_caps, dev_capa, RTE_DMA_CAPA_OPS_COPY);
>>>  	ADD_CAPA(dma_caps, dev_capa, RTE_DMA_CAPA_OPS_COPY_SG);
>>>  	ADD_CAPA(dma_caps, dev_capa, RTE_DMA_CAPA_OPS_FILL); diff --
>> git
>>> a/lib/dmadev/rte_dmadev.h b/lib/dmadev/rte_dmadev.h index
>>> 550dbfbf75..4f735bb0c9 100644
>>> --- a/lib/dmadev/rte_dmadev.h
>>> +++ b/lib/dmadev/rte_dmadev.h
>>> @@ -265,6 +265,18 @@ int16_t rte_dma_next_dev(int16_t start_dev_id);
>>>   * known from 'nb_priorities' field in struct rte_dma_info.
>>>   */
>>>  #define RTE_DMA_CAPA_PRI_POLICY_SP	RTE_BIT64(8)
>>> +/** Support inter-process DMA transfers.
>>> + *
>>> + * When this bit is set, the DMA device can perform memory transfers
>>> +between
>>> + * different process memory spaces.
>>> + */
>>> +#define RTE_DMA_CAPA_INTER_PROCESS_DOMAIN	RTE_BIT64(9)
>>> +/** Support inter-OS domain DMA transfers.
>>> + *
>>> + * The DMA device can perform memory transfers across different
>>> +operating
>>> + * system domains.
>>> + */
>>> +#define RTE_DMA_CAPA_INTER_OS_DOMAIN		RTE_BIT64(10)
>>>
>>>  /** Support copy operation.
>>>   * This capability start with index of 32, so that it could leave gap
>>> between @@ -418,8 +430,13 @@ int rte_dma_close(int16_t dev_id);
>>>   */
>>>  enum rte_dma_direction {
>>>  	/** DMA transfer direction - from memory to memory.
>>> +	 * When the device supports inter-process or inter-OS domain
>> transfers,
>>> +	 * the field `type` in `struct rte_dma_vchan_conf::domain` specifies
>> the
>>> +	 * type of domain. For memory-to-memory transfers within the same
>> domain
>>> +	 * or process, `type` should be set to
>> `RTE_DMA_INTER_DOMAIN_NONE`.
>>>  	 *
>>>  	 * @see struct rte_dma_vchan_conf::direction
>>> +	 * @see struct rte_dma_inter_domain_param::type
>>>  	 */
>>>  	RTE_DMA_DIR_MEM_TO_MEM,
>>>  	/** DMA transfer direction - from memory to device.
>>> @@ -564,6 +581,39 @@ struct rte_dma_auto_free_param {
>>>  	uint64_t reserved[2];
>>>  };
>>>
>>> +/**
>>> + * Inter-DMA transfer domain type.
>>> + *
>>> + * This enum defines the types of transfer domains applicable to DMA
>> operations.
>>> + * It helps categorize whether a DMA transfer is occurring within the
>>> +same domain,
>>> + * across different processes, or between distinct operating system
>> domains.
>>> + *
>>> + * @see struct rte_dma_inter_domain_param:type  */ enum
>>> +rte_dma_inter_domain_type {
>>> +	/**< No inter-domain transfer; standard DMA within same domaini.
>> */
>>> +	RTE_DMA_INTER_DOMAIN_NONE,
>>> +	/**< Transfer occurs between different user-space processes. */
>>> +	RTE_DMA_INTER_PROCESS_DOMAIN,
>>> +	/**< Transfer spans across different operating system domains. */
>>> +	RTE_DMA_INTER_OS_DOMAIN,
>>
>> If comment located before define, it should prefix with ** not **<
> 
> Ack, will fix it.
>>
>>
>>> +};
>>> +
>>> +/**
>>> + * Parameters for inter-process or inter-OS DMA transfers.
>>> + *
>>> + * This structure defines the parameters required to perform DMA
>>> +transfers
>>> + * across different domains, such as between processes or operating
>> systems.
>>> + * It includes the domain type and handler identifiers for both the
>>> +source
>>> + * and destination domains.
>>
>> I think we could add more comment about how to config use the src/dst
>> handler.
>> as I comment in before mail:
> 
> Sure, will add more comments. I believe a handler value of zero is still valid
> for both source and destination. It typically represents the local process when
> the transfer direction is mem-to-mem and the inter-domain type if NONE.

If the inter-domain type is None, the src/dst handler should be ignore.

As I said in comment of "control-api commit", if the group has multiple (>2) process,
and the device could perform three type mem-to-mem copy:
1\ local to remote
2\ remote to local
3\ remote-A to remote-B or event remote-A to remote-A

the remote handler could get by rte_dma_access_pair_group_handle_get()
does the local handler also get by rte_dma_access_pair_group_handle_get() ?


> Handler values --whether zero or non-zero -- can be used to represent transfers
>  From local process to another OS domain, or vice versa. 
> 
> Do you have any thoughts on this.?, Specifically, do you foresee any issues with
> using handler value of zero.
> 
> Thanks
>>
>> It's mean local process if the src/dst_handler is zero?
>> for example, I setup a inter-os domain param, if the src_handle is none zero
>> and dst_handle is zero, then all DMA copy will from another OS to local
>> process.
>>     all DMA fill will write to local process.
>>
>> If the src_handle is zero and dst_handle is non-zero, then all DMA copy will
>>from local process to another OS domain
>>     all DMA fill will write to another OS domain.
>>
>> If both src_handle and dst_handle is non-zero, then all DMA copy will from
>> another OS to another OS domain
>>     all DMA fill will write to another OS domain.
>>
>>> + */
>>> +struct rte_dma_inter_domain_param {
>>> +	enum rte_dma_inter_domain_type type; /**< Type of inter-domain.
>> */
>>> +	uint16_t src_handler; /**< Source domain handler identifier. */
>>> +	uint16_t dst_handler; /**< Destination domain handler identifier. */
>>> +	uint64_t reserved[2]; /**< Reserved for future fields. */
>>
>> Please move above comment before define to keep the whole header-file
>> consistency.
>>
>>> +};
>>> +
>>>  /**
>>>   * A structure used to configure a virtual DMA channel.
>>>   *
>>> @@ -601,6 +651,15 @@ struct rte_dma_vchan_conf {
>>>  	 * @see struct rte_dma_auto_free_param
>>>  	 */
>>>  	struct rte_dma_auto_free_param auto_free;
>>> +	/** Parameters for inter-process or inter-OS domain DMA transfers.
>> This field
>>> +	 * specifies the source and destination domain handlers required for
>> DMA
>>> +	 * operations that span across different processes or operating system
>> domains.
>>> +	 *
>>> +	 * @see RTE_DMA_CAPA_INTER_PROCESS_DOMAIN
>>> +	 * @see RTE_DMA_CAPA_INTER_OS_DOMAIN
>>> +	 * @see struct rte_dma_inter_domain_param
>>> +	 */
>>> +	struct rte_dma_inter_domain_param domain;

BTW: I remember the originl rfc only support inter-os domain, because Intel has
inter-process rfc, so we add it. now Intel has not follow this feature, I think
we could drop inter-process support (such as RTE_DMA_CAPA_INTER_PROCESS_XXX )
but keep the extend ability.

>>>  };
>>>
>>>  /**
>>> diff --git a/lib/dmadev/rte_dmadev_trace.h
>>> b/lib/dmadev/rte_dmadev_trace.h index 1de92655f2..fa619c4090 100644
>>> --- a/lib/dmadev/rte_dmadev_trace.h
>>> +++ b/lib/dmadev/rte_dmadev_trace.h
>>> @@ -79,6 +79,9 @@ RTE_TRACE_POINT(
>>>  	rte_trace_point_emit_int(conf->dst_port.port_type);
>>>  	rte_trace_point_emit_u64(conf->dst_port.pcie.val);
>>>  	rte_trace_point_emit_ptr(conf->auto_free.m2d.pool);
>>> +	rte_trace_point_emit_int(conf->domain.type);
>>> +	rte_trace_point_emit_u16(conf->domain.src_handler);
>>> +	rte_trace_point_emit_u16(conf->domain.dst_handler);
>>>  	rte_trace_point_emit_int(ret);
>>>  )
>>>
>>
>> BTW: please add the UT (test_dmadev_api.c) for above vchan_setup modify,
>> this could be another commit I think.
>>
> Yes, will add it.
>