Re: [PATCH v2] app/dma-perf: support bi-directional transfer

[fengchengwen <fengchengwen@huawei.com>]

Hi Amit and Gowrishankar,

It's nature to support multiple dmadev test in one testcase, and the original framework supports it.
But it seem we both complicated it when adding support for non-mem2mem dma test.

The new added "direction" and "vchan_dev" could treat as the dmadev's private configure,
some thing like:
  lcore_dma=lcore10@0000:00:04.2,vchan=0,dir=mem2dev,devtype=pcie,raddr=xxx,coreid=1,pfid=2,vfid=3

then this bi-directional could impl only with config:
  lcore_dma=lcore10@0000:00:04.2,dir=mem2dev,devtype=pcie,raddr=xxx,coreid=1,pfid=2,vfid=3, lcore11@0000:00:04.3,dir=dev2mem,devtype=pcie,raddr=xxx,coreid=1,pfid=2,vfid=3
so that the lcore10 will do mem2dev with device 0000:00:04.2, while lcore11 will do dev2mem with device 0000:00:04.3.

Thanks

On 2024/2/28 3:26, Amit Prakash Shukla wrote:
> Adds bi-directional DMA transfer support to test performance.
> One DMA device on one core will do mem2dev transfer and another
> DMA device on another core will do dev2mem transfer.
> 
> Depends-on: series-31252 ("PCI Dev and SG copy support")
> 
> Signed-off-by: Amit Prakash Shukla <amitprakashs@marvell.com>
> ---
> v2:
> - Fixed depends on series.
> 
>  app/test-dma-perf/benchmark.c | 64 +++++++++++++++++++++++++++--------
>  app/test-dma-perf/config.ini  |  5 +++
>  app/test-dma-perf/main.c      | 18 +++++++++-
>  app/test-dma-perf/main.h      |  1 +
>  4 files changed, 73 insertions(+), 15 deletions(-)
> 
> diff --git a/app/test-dma-perf/benchmark.c b/app/test-dma-perf/benchmark.c
> index 25ed6fa6d0..8a23944763 100644
> --- a/app/test-dma-perf/benchmark.c
> +++ b/app/test-dma-perf/benchmark.c
> @@ -144,12 +144,19 @@ cache_flush_buf(__rte_unused struct rte_mbuf **array,
>  
>  static int
>  vchan_data_populate(uint32_t dev_id, struct rte_dma_vchan_conf *qconf,
> -		    struct test_configure *cfg)
> +		    struct test_configure *cfg, uint16_t dev_num)
>  {
>  	struct rte_dma_info info;
>  
>  	qconf->direction = cfg->transfer_dir;
>  
> +	/* If its a bi-directional test, configure odd device for inbound dma
> +	 * transfer and even device for outbound dma transfer.
> +	 */
> +	if (cfg->is_bidir)
> +		qconf->direction = (dev_num % 2) ? RTE_DMA_DIR_MEM_TO_DEV :
> +				   RTE_DMA_DIR_DEV_TO_MEM;
> +
>  	rte_dma_info_get(dev_id, &info);
>  	if (!(RTE_BIT64(qconf->direction) & info.dev_capa))
>  		return -1;
> @@ -181,14 +188,15 @@ vchan_data_populate(uint32_t dev_id, struct rte_dma_vchan_conf *qconf,
>  
>  /* Configuration of device. */
>  static void
> -configure_dmadev_queue(uint32_t dev_id, struct test_configure *cfg, uint8_t ptrs_max)
> +configure_dmadev_queue(uint32_t dev_id, struct test_configure *cfg, uint8_t ptrs_max,
> +		       uint16_t dev_num)
>  {
>  	uint16_t vchan = 0;
>  	struct rte_dma_info info;
>  	struct rte_dma_conf dev_config = { .nb_vchans = 1 };
>  	struct rte_dma_vchan_conf qconf = { 0 };
>  
> -	if (vchan_data_populate(dev_id, &qconf, cfg) != 0)
> +	if (vchan_data_populate(dev_id, &qconf, cfg, dev_num) != 0)
>  		rte_exit(EXIT_FAILURE, "Error with vchan data populate.\n");
>  
>  	if (rte_dma_configure(dev_id, &dev_config) != 0)
> @@ -235,7 +243,7 @@ config_dmadevs(struct test_configure *cfg)
>  		}
>  
>  		ldm->dma_ids[i] = dev_id;
> -		configure_dmadev_queue(dev_id, cfg, ptrs_max);
> +		configure_dmadev_queue(dev_id, cfg, ptrs_max, nb_dmadevs);
>  		++nb_dmadevs;
>  	}
>  
> @@ -504,7 +512,7 @@ setup_memory_env(struct test_configure *cfg,
>  		}
>  	}
>  
> -	if (cfg->transfer_dir == RTE_DMA_DIR_DEV_TO_MEM) {
> +	if (cfg->transfer_dir == RTE_DMA_DIR_DEV_TO_MEM && !cfg->is_bidir) {
>  		ext_buf_info->free_cb = dummy_free_ext_buf;
>  		ext_buf_info->fcb_opaque = NULL;
>  		for (i = 0; i < nr_buf; i++) {
> @@ -516,7 +524,7 @@ setup_memory_env(struct test_configure *cfg,
>  		}
>  	}
>  
> -	if (cfg->transfer_dir == RTE_DMA_DIR_MEM_TO_DEV) {
> +	if (cfg->transfer_dir == RTE_DMA_DIR_MEM_TO_DEV && !cfg->is_bidir) {
>  		ext_buf_info->free_cb = dummy_free_ext_buf;
>  		ext_buf_info->fcb_opaque = NULL;
>  		for (i = 0; i < nr_buf; i++) {
> @@ -528,6 +536,18 @@ setup_memory_env(struct test_configure *cfg,
>  		}
>  	}
>  
> +	if (cfg->is_bidir) {
> +		ext_buf_info->free_cb = dummy_free_ext_buf;
> +		ext_buf_info->fcb_opaque = NULL;
> +		for (i = 0; i < nr_buf; i++) {
> +			/* Using mbuf structure to hold remote iova address. */
> +			rte_pktmbuf_attach_extbuf((*srcs)[i], (void *)(cfg->vchan_dev.raddr +
> +						 (i * buf_size)), (rte_iova_t)(cfg->vchan_dev.raddr +
> +						 (i * buf_size)), 0, ext_buf_info);
> +			rte_mbuf_ext_refcnt_update(ext_buf_info, 1);
> +		}
> +	}
> +
>  	if (cfg->is_sg) {
>  		uint8_t src_ptrs = cfg->src_ptrs;
>  		uint8_t dst_ptrs = cfg->dst_ptrs;
> @@ -649,16 +669,30 @@ mem_copy_benchmark(struct test_configure *cfg)
>  		lcores[i]->nr_buf = (uint32_t)(nr_buf / nb_workers);
>  		lcores[i]->buf_size = buf_size;
>  		lcores[i]->test_secs = test_secs;
> -		lcores[i]->srcs = srcs + offset;
> -		lcores[i]->dsts = dsts + offset;
>  		lcores[i]->scenario_id = cfg->scenario_id;
>  		lcores[i]->lcore_id = lcore_id;
>  
> -		if (cfg->is_sg) {
> -			lcores[i]->src_ptrs = cfg->src_ptrs;
> -			lcores[i]->dst_ptrs = cfg->dst_ptrs;
> -			lcores[i]->src_sges = src_sges + (nr_sgsrc / nb_workers * i);
> -			lcores[i]->dst_sges = dst_sges + (nr_sgdst / nb_workers * i);
> +		/* Number of workers is equal to number of devices. In case of bi-directional
> +		 * dma, use 1 device for mem-to-dev and 1 device for dev-to-mem.
> +		 */
> +		if (cfg->is_dma && cfg->is_bidir && (i % 2 != 0)) {
> +			lcores[i]->dsts = srcs + offset;
> +			lcores[i]->srcs = dsts + offset;
> +			if (cfg->is_sg) {
> +				lcores[i]->dst_ptrs = cfg->src_ptrs;
> +				lcores[i]->src_ptrs = cfg->dst_ptrs;
> +				lcores[i]->dst_sges = src_sges + (nr_sgsrc / nb_workers * i);
> +				lcores[i]->src_sges = dst_sges + (nr_sgdst / nb_workers * i);
> +			}
> +		} else {
> +			lcores[i]->srcs = srcs + offset;
> +			lcores[i]->dsts = dsts + offset;
> +			if (cfg->is_sg) {
> +				lcores[i]->src_ptrs = cfg->src_ptrs;
> +				lcores[i]->dst_ptrs = cfg->dst_ptrs;
> +				lcores[i]->src_sges = src_sges + (nr_sgsrc / nb_workers * i);
> +				lcores[i]->dst_sges = dst_sges + (nr_sgdst / nb_workers * i);
> +			}
>  		}
>  
>  		if (cfg->is_dma) {
> @@ -759,6 +793,8 @@ mem_copy_benchmark(struct test_configure *cfg)
>  		calc_result(buf_size, nr_buf, nb_workers, test_secs,
>  			lcores[i]->worker_info.test_cpl,
>  			&memory, &avg_cycles, &bandwidth, &mops);
> +		if (cfg->is_bidir)
> +			printf("%s direction\n", i % 2 ? "MEM-to-DEV" : "DEV-to-MEM");
>  		output_result(cfg, lcores[i], kick_batch, avg_cycles, buf_size,
>  			nr_buf / nb_workers, memory, bandwidth, mops);
>  		mops_total += mops;
> @@ -772,7 +808,7 @@ mem_copy_benchmark(struct test_configure *cfg)
>  
>  out:
>  
> -	if (cfg->transfer_dir == RTE_DMA_DIR_DEV_TO_MEM)
> +	if (cfg->transfer_dir == RTE_DMA_DIR_DEV_TO_MEM || cfg->is_bidir)
>  		m = srcs;
>  	else if (cfg->transfer_dir == RTE_DMA_DIR_MEM_TO_DEV)
>  		m = dsts;
> diff --git a/app/test-dma-perf/config.ini b/app/test-dma-perf/config.ini
> index 28f6c9d1db..7814ddcdc3 100644
> --- a/app/test-dma-perf/config.ini
> +++ b/app/test-dma-perf/config.ini
> @@ -61,6 +61,10 @@
>  ;    "pfid" denotes PF-id to be used for data transfer
>  ;    "vfid" denotes VF-id of PF-id to be used for data transfer.
>  
> +; "xfer_mode" denotes mode of data transfer. It can take 2 values:
> +;    0 - unidirection transfer based on direction configured (default).
> +;    1 - Bi-directional transfer based on direction configured (mem-to-dev and dev-to-mem).
> +
>  ; =========== End of "mem2dev" and "dev2mem" config parameters. ==============
>  
>  [case1]
> @@ -95,6 +99,7 @@ eal_args=--in-memory --file-prefix=test
>  skip=1
>  type=DMA_MEM_COPY
>  direction=dev2mem
> +xfer_mode=0
>  vchan_dev=raddr=0x200000000,coreid=1,pfid=2,vfid=3
>  mem_size=10
>  buf_size=64,4096,2,MUL
> diff --git a/app/test-dma-perf/main.c b/app/test-dma-perf/main.c
> index a27e4c9429..4488890697 100644
> --- a/app/test-dma-perf/main.c
> +++ b/app/test-dma-perf/main.c
> @@ -368,6 +368,7 @@ load_configs(const char *path)
>  	const char *skip;
>  	struct rte_kvargs *kvlist;
>  	const char *vchan_dev;
> +	const char *xfer_mode;
>  	int args_nr, nb_vp;
>  	bool is_dma;
>  
> @@ -421,6 +422,21 @@ load_configs(const char *path)
>  					test_case->transfer_dir = RTE_DMA_DIR_MEM_TO_MEM;
>  				}
>  			}
> +
> +			xfer_mode = rte_cfgfile_get_entry(cfgfile, section_name, "xfer_mode");
> +			if (xfer_mode) {
> +				int xmode = atoi(xfer_mode);
> +				if (xmode == 1) {
> +					if (test_case->transfer_dir == RTE_DMA_DIR_MEM_TO_MEM) {
> +						printf("Error: Invalid configuration. For mem to"
> +						       " mem dma transfer bi-directional cannot be"
> +						       " configured.\n");
> +						test_case->is_valid = false;
> +						continue;
> +					}
> +					test_case->is_bidir = true;
> +				}
> +			}
>  			is_dma = true;
>  		} else if (strcmp(case_type, CPU_MEM_COPY) == 0) {
>  			test_case->test_type = TEST_TYPE_CPU_MEM_COPY;
> @@ -433,7 +449,7 @@ load_configs(const char *path)
>  		}
>  
>  		if (test_case->transfer_dir == RTE_DMA_DIR_MEM_TO_DEV ||
> -			test_case->transfer_dir == RTE_DMA_DIR_DEV_TO_MEM) {
> +		    test_case->transfer_dir == RTE_DMA_DIR_DEV_TO_MEM) {
>  			vchan_dev = rte_cfgfile_get_entry(cfgfile, section_name, "vchan_dev");
>  			if (vchan_dev == NULL) {
>  				printf("Transfer direction mem2dev and dev2mem"
> diff --git a/app/test-dma-perf/main.h b/app/test-dma-perf/main.h
> index baf149b72b..70f6c393c2 100644
> --- a/app/test-dma-perf/main.h
> +++ b/app/test-dma-perf/main.h
> @@ -67,6 +67,7 @@ struct test_configure {
>  	const char *eal_args;
>  	uint8_t scenario_id;
>  	struct test_vchan_dev_config vchan_dev;
> +	bool is_bidir;
>  };
>  
>  int mem_copy_benchmark(struct test_configure *cfg);
>