[PATCH] app/testeventdev: add vector worker to perf test

[PATCH] app/testeventdev: add vector worker to perf test

[Volodymyr Fialko]

Add worker for handling vector events to perf tests, vector events could
be generated by crypto adapter producer.

Example:
    ./dpdk-test-eventdev -l 0-2 -a <EVENTDEV> -a <CRYPTODEV> -- \
    --prod_type_cryptodev --crypto_adptr_mode 1 --test=perf_queue \
    --stlist=a --wlcores 1 --plcores 2 --prod_enq_burst_sz 32 \
    --enable_vector --vector_tmo_ns 0 --nb_flows 2

Signed-off-by: Volodymyr Fialko <vfialko@marvell.com>
---
 app/test-eventdev/test_perf_atq.c    |  62 ++++++++++++++--
 app/test-eventdev/test_perf_common.c |  68 +++++++++++++++---
 app/test-eventdev/test_perf_common.h | 102 ++++++++++++++++++++++++++-
 app/test-eventdev/test_perf_queue.c  |  63 +++++++++++++++--
 doc/guides/tools/testeventdev.rst    |  12 ++--
 5 files changed, 279 insertions(+), 28 deletions(-)

diff --git a/app/test-eventdev/test_perf_atq.c b/app/test-eventdev/test_perf_atq.c
index 9d30081117..4ac60cc38b 100644
--- a/app/test-eventdev/test_perf_atq.c
+++ b/app/test-eventdev/test_perf_atq.c
@@ -24,14 +24,22 @@ atq_fwd_event(struct rte_event *const ev, uint8_t *const sched_type_list,
 	ev->event_type = RTE_EVENT_TYPE_CPU;
 }
 
+static __rte_always_inline void
+atq_fwd_event_vector(struct rte_event *const ev, uint8_t *const sched_type_list,
+		const uint8_t nb_stages)
+{
+	ev->sub_event_type++;
+	ev->sched_type = sched_type_list[ev->sub_event_type % nb_stages];
+	ev->op = RTE_EVENT_OP_FORWARD;
+	ev->event_type = RTE_EVENT_TYPE_CPU_VECTOR;
+}
+
 static int
 perf_atq_worker(void *arg, const int enable_fwd_latency)
 {
-	struct perf_elt *pe = NULL;
 	uint16_t enq = 0, deq = 0;
 	struct rte_event ev;
 	PERF_WORKER_INIT;
-	uint8_t stage;
 
 	while (t->done == false) {
 		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
@@ -79,9 +87,7 @@ perf_atq_worker_burst(void *arg, const int enable_fwd_latency)
 	/* +1 to avoid prefetch out of array check */
 	struct rte_event ev[BURST_SIZE + 1];
 	uint16_t enq = 0, nb_rx = 0;
-	struct perf_elt *pe = NULL;
 	PERF_WORKER_INIT;
-	uint8_t stage;
 	uint16_t i;
 
 	while (t->done == false) {
@@ -134,6 +140,50 @@ perf_atq_worker_burst(void *arg, const int enable_fwd_latency)
 	return 0;
 }
 
+static int
+perf_atq_worker_vector(void *arg, const int enable_fwd_latency)
+{
+	uint16_t enq = 0, deq = 0;
+	struct rte_event ev;
+	PERF_WORKER_INIT;
+
+	RTE_SET_USED(sz);
+	RTE_SET_USED(cnt);
+	RTE_SET_USED(prod_crypto_type);
+
+	while (t->done == false) {
+		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
+
+		if (!deq)
+			continue;
+
+		if (ev.event_type == RTE_EVENT_TYPE_CRYPTODEV_VECTOR) {
+			if (perf_handle_crypto_vector_ev(&ev, &pe, enable_fwd_latency))
+				continue;
+		}
+
+		stage = ev.sub_event_type % nb_stages;
+		/* First q in pipeline, mark timestamp to compute fwd latency */
+		if (enable_fwd_latency && !prod_timer_type && stage == 0)
+			perf_mark_fwd_latency(pe);
+
+		/* Last stage in pipeline */
+		if (unlikely(stage == laststage)) {
+			perf_process_vector_last_stage(pool, t->ca_op_pool, &ev, w,
+							enable_fwd_latency);
+		} else {
+			atq_fwd_event_vector(&ev, sched_type_list, nb_stages);
+			do {
+				enq = rte_event_enqueue_burst(dev, port, &ev, 1);
+			} while (!enq && !t->done);
+		}
+	}
+
+	perf_worker_cleanup(pool, dev, port, &ev, enq, deq);
+
+	return 0;
+}
+
 static int
 worker_wrapper(void *arg)
 {
@@ -144,7 +194,9 @@ worker_wrapper(void *arg)
 	const int fwd_latency = opt->fwd_latency;
 
 	/* allow compiler to optimize */
-	if (!burst && !fwd_latency)
+	if (opt->ena_vector && opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR)
+		return perf_atq_worker_vector(arg, fwd_latency);
+	else if (!burst && !fwd_latency)
 		return perf_atq_worker(arg, 0);
 	else if (!burst && fwd_latency)
 		return perf_atq_worker(arg, 1);
diff --git a/app/test-eventdev/test_perf_common.c b/app/test-eventdev/test_perf_common.c
index 140c0c2dc3..8d7e483c55 100644
--- a/app/test-eventdev/test_perf_common.c
+++ b/app/test-eventdev/test_perf_common.c
@@ -827,10 +827,13 @@ perf_event_timer_adapter_setup(struct test_perf *t)
 static int
 perf_event_crypto_adapter_setup(struct test_perf *t, struct prod_data *p)
 {
+	struct rte_event_crypto_adapter_queue_conf conf;
 	struct evt_options *opt = t->opt;
 	uint32_t cap;
 	int ret;
 
+	memset(&conf, 0, sizeof(conf));
+
 	ret = rte_event_crypto_adapter_caps_get(p->dev_id, p->ca.cdev_id, &cap);
 	if (ret) {
 		evt_err("Failed to get crypto adapter capabilities");
@@ -849,19 +852,53 @@ perf_event_crypto_adapter_setup(struct test_perf *t, struct prod_data *p)
 		return -ENOTSUP;
 	}
 
-	if (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) {
-		struct rte_event_crypto_adapter_queue_conf conf;
+	if (opt->ena_vector) {
+		struct rte_event_crypto_adapter_vector_limits limits;
+
+		if (!(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_EVENT_VECTOR)) {
+			evt_err("Crypto adapter doesn't support event vector");
+			return -EINVAL;
+		}
+
+		ret = rte_event_crypto_adapter_vector_limits_get(p->dev_id, p->ca.cdev_id, &limits);
+		if (ret) {
+			evt_err("Failed to get crypto adapter's vector limits");
+			return ret;
+		}
 
-		memset(&conf, 0, sizeof(conf));
+		if (opt->vector_size < limits.min_sz || opt->vector_size > limits.max_sz) {
+			evt_err("Vector size [%d] not within limits max[%d] min[%d]",
+				opt->vector_size, limits.max_sz, limits.min_sz);
+			return -EINVAL;
+		}
+
+		if (limits.log2_sz && !rte_is_power_of_2(opt->vector_size)) {
+			evt_err("Vector size [%d] not power of 2", opt->vector_size);
+			return -EINVAL;
+		}
+
+		if (opt->vector_tmo_nsec > limits.max_timeout_ns ||
+			opt->vector_tmo_nsec < limits.min_timeout_ns) {
+			evt_err("Vector timeout [%" PRIu64 "] not within limits "
+				"max[%" PRIu64 "] min[%" PRIu64 "]",
+				opt->vector_tmo_nsec, limits.max_timeout_ns, limits.min_timeout_ns);
+			return -EINVAL;
+		}
+
+		conf.vector_mp = t->ca_vector_pool;
+		conf.vector_sz = opt->vector_size;
+		conf.vector_timeout_ns = opt->vector_tmo_nsec;
+		conf.flags |= RTE_EVENT_CRYPTO_ADAPTER_EVENT_VECTOR;
+	}
+
+	if (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) {
 		conf.ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
 		conf.ev.queue_id = p->queue_id;
-		ret = rte_event_crypto_adapter_queue_pair_add(
-			TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id, &conf);
-	} else {
-		ret = rte_event_crypto_adapter_queue_pair_add(
-			TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id, NULL);
 	}
 
+	ret = rte_event_crypto_adapter_queue_pair_add(
+		TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id, &conf);
+
 	return ret;
 }
 
@@ -1411,6 +1448,19 @@ perf_cryptodev_setup(struct evt_test *test, struct evt_options *opt)
 		goto err;
 	}
 
+	if (opt->ena_vector) {
+		unsigned int nb_elem = (opt->pool_sz / opt->vector_size) * 2;
+		nb_elem = RTE_MAX(512U, nb_elem);
+		nb_elem += evt_nr_active_lcores(opt->wlcores) * 32;
+		t->ca_vector_pool = rte_event_vector_pool_create("vector_pool", nb_elem, 32,
+				opt->vector_size, opt->socket_id);
+		if (t->ca_vector_pool == NULL) {
+			evt_err("Failed to create event vector pool");
+			ret = -ENOMEM;
+			goto err;
+		}
+	}
+
 	/*
 	 * Calculate number of needed queue pairs, based on the amount of
 	 * available number of logical cores and crypto devices. For instance,
@@ -1467,6 +1517,7 @@ perf_cryptodev_setup(struct evt_test *test, struct evt_options *opt)
 	rte_mempool_free(t->ca_op_pool);
 	rte_mempool_free(t->ca_sess_pool);
 	rte_mempool_free(t->ca_asym_sess_pool);
+	rte_mempool_free(t->ca_vector_pool);
 
 	return ret;
 }
@@ -1507,6 +1558,7 @@ perf_cryptodev_destroy(struct evt_test *test, struct evt_options *opt)
 	rte_mempool_free(t->ca_op_pool);
 	rte_mempool_free(t->ca_sess_pool);
 	rte_mempool_free(t->ca_asym_sess_pool);
+	rte_mempool_free(t->ca_vector_pool);
 }
 
 int
diff --git a/app/test-eventdev/test_perf_common.h b/app/test-eventdev/test_perf_common.h
index 503b6aa1db..faedd471c6 100644
--- a/app/test-eventdev/test_perf_common.h
+++ b/app/test-eventdev/test_perf_common.h
@@ -71,6 +71,7 @@ struct test_perf {
 	struct rte_mempool *ca_op_pool;
 	struct rte_mempool *ca_sess_pool;
 	struct rte_mempool *ca_asym_sess_pool;
+	struct rte_mempool *ca_vector_pool;
 } __rte_cache_aligned;
 
 struct perf_elt {
@@ -103,6 +104,8 @@ struct perf_elt {
 	uint8_t cnt = 0;\
 	void *bufs[16] __rte_cache_aligned;\
 	int const sz = RTE_DIM(bufs);\
+	uint8_t stage;\
+	struct perf_elt *pe = NULL;\
 	if (opt->verbose_level > 1)\
 		printf("%s(): lcore %d dev_id %d port=%d\n", __func__,\
 				rte_lcore_id(), dev, port)
@@ -143,6 +146,64 @@ perf_handle_crypto_ev(struct rte_event *ev, struct perf_elt **pe, int enable_fwd
 	return 0;
 }
 
+static __rte_always_inline struct perf_elt *
+perf_elt_from_vec_get(struct rte_event_vector *vec)
+{
+	/* Timestamp for vector event stored in first element */
+	struct rte_crypto_op *cop = vec->ptrs[0];
+	struct rte_mbuf *m;
+
+	if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
+		m = cop->sym->m_dst == NULL ? cop->sym->m_src : cop->sym->m_dst;
+		return rte_pktmbuf_mtod(m, struct perf_elt *);
+	} else {
+		return RTE_PTR_ADD(cop->asym->modex.result.data, cop->asym->modex.result.length);
+	}
+}
+
+static __rte_always_inline int
+perf_handle_crypto_vector_ev(struct rte_event *ev, struct perf_elt **pe,
+		const int enable_fwd_latency)
+{
+	struct rte_event_vector *vec = ev->vec;
+	struct rte_crypto_op *cop;
+	struct rte_mbuf *m;
+	int i, n = 0;
+	void *data;
+
+	for (i = 0; i < vec->nb_elem; i++) {
+		cop = vec->ptrs[i];
+		if (unlikely(cop->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) {
+			if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
+				m = cop->sym->m_dst == NULL ? cop->sym->m_src : cop->sym->m_dst;
+				rte_pktmbuf_free(m);
+			} else {
+				data = cop->asym->modex.result.data;
+				rte_mempool_put(rte_mempool_from_obj(data), data);
+			}
+			rte_crypto_op_free(cop);
+			continue;
+		}
+		vec->ptrs[n++] = cop;
+	}
+
+	/* All cops failed, free the vector */
+	if (n == 0) {
+		rte_mempool_put(rte_mempool_from_obj(vec), vec);
+		return -ENOENT;
+	}
+
+	vec->nb_elem = n;
+
+	/* Forward latency not enabled - perf data will be not accessed */
+	if (!enable_fwd_latency)
+		return 0;
+
+	/* Get pointer to perf data */
+	*pe = perf_elt_from_vec_get(vec);
+
+	return 0;
+}
 
 static __rte_always_inline int
 perf_process_last_stage(struct rte_mempool *const pool, uint8_t prod_crypto_type,
@@ -195,9 +256,8 @@ perf_process_last_stage_latency(struct rte_mempool *const pool, uint8_t prod_cry
 	struct perf_elt *pe;
 	void *to_free_in_bulk;
 
-	/* release fence here ensures event_prt is
-	 * stored before updating the number of
-	 * processed packets for worker lcores
+	/* Release fence here ensures event_prt is stored before updating the number of processed
+	 * packets for worker lcores.
 	 */
 	rte_atomic_thread_fence(__ATOMIC_RELEASE);
 	w->processed_pkts++;
@@ -237,6 +297,42 @@ perf_process_last_stage_latency(struct rte_mempool *const pool, uint8_t prod_cry
 	return count;
 }
 
+static __rte_always_inline void
+perf_process_vector_last_stage(struct rte_mempool *const pool,
+		struct rte_mempool *const ca_pool, struct rte_event *const ev,
+		struct worker_data *const w, const bool enable_fwd_latency)
+{
+	struct rte_event_vector *vec = ev->vec;
+	struct rte_crypto_op *cop;
+	void *bufs[vec->nb_elem];
+	struct perf_elt *pe;
+	uint64_t latency;
+	int i;
+
+	/* Release fence here ensures event_prt is stored before updating the number of processed
+	 * packets for worker lcores.
+	 */
+	rte_atomic_thread_fence(__ATOMIC_RELEASE);
+	w->processed_pkts += vec->nb_elem;
+
+	if (enable_fwd_latency) {
+		pe = perf_elt_from_vec_get(vec);
+		latency = rte_get_timer_cycles() - pe->timestamp;
+		w->latency += latency;
+	}
+
+	for (i = 0; i < vec->nb_elem; i++) {
+		cop = vec->ptrs[i];
+		if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC)
+			bufs[i] = cop->sym->m_dst == NULL ? cop->sym->m_src : cop->sym->m_dst;
+		else
+			bufs[i] = cop->asym->modex.result.data;
+	}
+
+	rte_mempool_put_bulk(pool, bufs, vec->nb_elem);
+	rte_mempool_put_bulk(ca_pool, (void * const *)vec->ptrs, vec->nb_elem);
+	rte_mempool_put(rte_mempool_from_obj(vec), vec);
+}
 
 static inline int
 perf_nb_event_ports(struct evt_options *opt)
diff --git a/app/test-eventdev/test_perf_queue.c b/app/test-eventdev/test_perf_queue.c
index 69ef0ebbac..2399cfb69b 100644
--- a/app/test-eventdev/test_perf_queue.c
+++ b/app/test-eventdev/test_perf_queue.c
@@ -25,15 +25,22 @@ fwd_event(struct rte_event *const ev, uint8_t *const sched_type_list,
 	ev->event_type = RTE_EVENT_TYPE_CPU;
 }
 
+static __rte_always_inline void
+fwd_event_vector(struct rte_event *const ev, uint8_t *const sched_type_list,
+		const uint8_t nb_stages)
+{
+	ev->queue_id++;
+	ev->sched_type = sched_type_list[ev->queue_id % nb_stages];
+	ev->op = RTE_EVENT_OP_FORWARD;
+	ev->event_type = RTE_EVENT_TYPE_CPU_VECTOR;
+}
+
 static int
 perf_queue_worker(void *arg, const int enable_fwd_latency)
 {
-	struct perf_elt *pe = NULL;
 	uint16_t enq = 0, deq = 0;
 	struct rte_event ev;
 	PERF_WORKER_INIT;
-	uint8_t stage;
-
 
 	while (t->done == false) {
 		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
@@ -82,9 +89,7 @@ perf_queue_worker_burst(void *arg, const int enable_fwd_latency)
 	/* +1 to avoid prefetch out of array check */
 	struct rte_event ev[BURST_SIZE + 1];
 	uint16_t enq = 0, nb_rx = 0;
-	struct perf_elt *pe = NULL;
 	PERF_WORKER_INIT;
-	uint8_t stage;
 	uint16_t i;
 
 	while (t->done == false) {
@@ -137,6 +142,50 @@ perf_queue_worker_burst(void *arg, const int enable_fwd_latency)
 	return 0;
 }
 
+static int
+perf_queue_worker_vector(void *arg, const int enable_fwd_latency)
+{
+	uint16_t enq = 0, deq = 0;
+	struct rte_event ev;
+	PERF_WORKER_INIT;
+
+	RTE_SET_USED(sz);
+	RTE_SET_USED(cnt);
+	RTE_SET_USED(prod_crypto_type);
+
+	while (t->done == false) {
+		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
+
+		if (!deq)
+			continue;
+
+		if (ev.event_type == RTE_EVENT_TYPE_CRYPTODEV_VECTOR) {
+			if (perf_handle_crypto_vector_ev(&ev, &pe, enable_fwd_latency))
+				continue;
+		}
+
+		stage = ev.queue_id % nb_stages;
+		/* First q in pipeline, mark timestamp to compute fwd latency */
+		if (enable_fwd_latency && !prod_timer_type && stage == 0)
+			perf_mark_fwd_latency(pe);
+
+		/* Last stage in pipeline */
+		if (unlikely(stage == laststage)) {
+			perf_process_vector_last_stage(pool, t->ca_op_pool, &ev, w,
+						       enable_fwd_latency);
+		} else {
+			fwd_event_vector(&ev, sched_type_list, nb_stages);
+			do {
+				enq = rte_event_enqueue_burst(dev, port, &ev, 1);
+			} while (!enq && !t->done);
+		}
+	}
+
+	perf_worker_cleanup(pool, dev, port, &ev, enq, deq);
+
+	return 0;
+}
+
 static int
 worker_wrapper(void *arg)
 {
@@ -147,7 +196,9 @@ worker_wrapper(void *arg)
 	const int fwd_latency = opt->fwd_latency;
 
 	/* allow compiler to optimize */
-	if (!burst && !fwd_latency)
+	if (opt->ena_vector && opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR)
+		return perf_queue_worker_vector(arg, fwd_latency);
+	else if (!burst && !fwd_latency)
 		return perf_queue_worker(arg, 0);
 	else if (!burst && fwd_latency)
 		return perf_queue_worker(arg, 1);
diff --git a/doc/guides/tools/testeventdev.rst b/doc/guides/tools/testeventdev.rst
index cd278e8998..6f065b9752 100644
--- a/doc/guides/tools/testeventdev.rst
+++ b/doc/guides/tools/testeventdev.rst
@@ -185,18 +185,18 @@ The following are the application command-line options:
 
 * ``--enable_vector``
 
-       Enable event vector for Rx/Tx adapters.
-       Only applicable for `pipeline_atq` and `pipeline_queue` tests.
+       Enable event vector for Rx/Tx/crypto adapters.
+       Only applicable for `pipeline_*` and `perf_*` tests.
 
 * ``--vector_size``
 
-       Vector size to configure for the Rx adapter.
-       Only applicable for `pipeline_atq` and `pipeline_queue` tests.
+       Vector size to configure for the Rx/crypto adapter.
+       Only applicable for `pipeline_*` and `perf_*` tests.
 
 * ``--vector_tmo_ns``
 
-       Vector timeout nanoseconds to be configured for the Rx adapter.
-       Only applicable for `pipeline_atq` and `pipeline_queue` tests.
+       Vector timeout nanoseconds to be configured for the Rx/crypto adapter.
+       Only applicable for `pipeline_*` and `perf_*` tests.
 
 * ``--per_port_pool``
 
-- 
2.25.1

RE: [PATCH] app/testeventdev: add vector worker to perf test

[Volodymyr Fialko]

> -----Original Message-----
> From: Volodymyr Fialko <vfialko@marvell.com>
> Sent: Friday, December 2, 2022 11:12 AM
> To: dev@dpdk.org; Jerin Jacob Kollanukkaran <jerinj@marvell.com>
> Cc: Anoob Joseph <anoobj@marvell.com>; Akhil Goyal <gakhil@marvell.com>; Volodymyr Fialko
> <vfialko@marvell.com>
> Subject: [PATCH] app/testeventdev: add vector worker to perf test
> 
> Add worker for handling vector events to perf tests, vector events could be generated by crypto adapter
> producer.
> 
> Example:
>     ./dpdk-test-eventdev -l 0-2 -a <EVENTDEV> -a <CRYPTODEV> -- \
>     --prod_type_cryptodev --crypto_adptr_mode 1 --test=perf_queue \
>     --stlist=a --wlcores 1 --plcores 2 --prod_enq_burst_sz 32 \
>     --enable_vector --vector_tmo_ns 0 --nb_flows 2
> 
> Signed-off-by: Volodymyr Fialko <vfialko@marvell.com>
> ---
Depens-on: series-26008 ("build: fix missing crypto vec limits in version")

RE: [EXT] [PATCH] app/testeventdev: add vector worker to perf test

[Shijith Thotton]

>Add worker for handling vector events to perf tests, vector events could
>be generated by crypto adapter producer.
>
>Example:
>    ./dpdk-test-eventdev -l 0-2 -a <EVENTDEV> -a <CRYPTODEV> -- \
>    --prod_type_cryptodev --crypto_adptr_mode 1 --test=perf_queue \
>    --stlist=a --wlcores 1 --plcores 2 --prod_enq_burst_sz 32 \
>    --enable_vector --vector_tmo_ns 0 --nb_flows 2
>
>Signed-off-by: Volodymyr Fialko <vfialko@marvell.com>
 
Acked-by: Shijith Thotton <sthotton@marvell.com>

>---
> app/test-eventdev/test_perf_atq.c    |  62 ++++++++++++++--
> app/test-eventdev/test_perf_common.c |  68 +++++++++++++++---
> app/test-eventdev/test_perf_common.h | 102
>++++++++++++++++++++++++++-
> app/test-eventdev/test_perf_queue.c  |  63 +++++++++++++++--
> doc/guides/tools/testeventdev.rst    |  12 ++--
> 5 files changed, 279 insertions(+), 28 deletions(-)
>
>diff --git a/app/test-eventdev/test_perf_atq.c b/app/test-
>eventdev/test_perf_atq.c
>index 9d30081117..4ac60cc38b 100644
>--- a/app/test-eventdev/test_perf_atq.c
>+++ b/app/test-eventdev/test_perf_atq.c
>@@ -24,14 +24,22 @@ atq_fwd_event(struct rte_event *const ev, uint8_t *const
>sched_type_list,
> 	ev->event_type = RTE_EVENT_TYPE_CPU;
> }
>
>+static __rte_always_inline void
>+atq_fwd_event_vector(struct rte_event *const ev, uint8_t *const
>sched_type_list,
>+		const uint8_t nb_stages)
>+{
>+	ev->sub_event_type++;
>+	ev->sched_type = sched_type_list[ev->sub_event_type % nb_stages];
>+	ev->op = RTE_EVENT_OP_FORWARD;
>+	ev->event_type = RTE_EVENT_TYPE_CPU_VECTOR;
>+}
>+
> static int
> perf_atq_worker(void *arg, const int enable_fwd_latency)
> {
>-	struct perf_elt *pe = NULL;
> 	uint16_t enq = 0, deq = 0;
> 	struct rte_event ev;
> 	PERF_WORKER_INIT;
>-	uint8_t stage;
>
> 	while (t->done == false) {
> 		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
>@@ -79,9 +87,7 @@ perf_atq_worker_burst(void *arg, const int
>enable_fwd_latency)
> 	/* +1 to avoid prefetch out of array check */
> 	struct rte_event ev[BURST_SIZE + 1];
> 	uint16_t enq = 0, nb_rx = 0;
>-	struct perf_elt *pe = NULL;
> 	PERF_WORKER_INIT;
>-	uint8_t stage;
> 	uint16_t i;
>
> 	while (t->done == false) {
>@@ -134,6 +140,50 @@ perf_atq_worker_burst(void *arg, const int
>enable_fwd_latency)
> 	return 0;
> }
>
>+static int
>+perf_atq_worker_vector(void *arg, const int enable_fwd_latency)
>+{
>+	uint16_t enq = 0, deq = 0;
>+	struct rte_event ev;
>+	PERF_WORKER_INIT;
>+
>+	RTE_SET_USED(sz);
>+	RTE_SET_USED(cnt);
>+	RTE_SET_USED(prod_crypto_type);
>+
>+	while (t->done == false) {
>+		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
>+
>+		if (!deq)
>+			continue;
>+
>+		if (ev.event_type == RTE_EVENT_TYPE_CRYPTODEV_VECTOR) {
>+			if (perf_handle_crypto_vector_ev(&ev, &pe,
>enable_fwd_latency))
>+				continue;
>+		}
>+
>+		stage = ev.sub_event_type % nb_stages;
>+		/* First q in pipeline, mark timestamp to compute fwd latency */
>+		if (enable_fwd_latency && !prod_timer_type && stage == 0)
>+			perf_mark_fwd_latency(pe);
>+
>+		/* Last stage in pipeline */
>+		if (unlikely(stage == laststage)) {
>+			perf_process_vector_last_stage(pool, t->ca_op_pool,
>&ev, w,
>+							enable_fwd_latency);
>+		} else {
>+			atq_fwd_event_vector(&ev, sched_type_list, nb_stages);
>+			do {
>+				enq = rte_event_enqueue_burst(dev, port, &ev,
>1);
>+			} while (!enq && !t->done);
>+		}
>+	}
>+
>+	perf_worker_cleanup(pool, dev, port, &ev, enq, deq);
>+
>+	return 0;
>+}
>+
> static int
> worker_wrapper(void *arg)
> {
>@@ -144,7 +194,9 @@ worker_wrapper(void *arg)
> 	const int fwd_latency = opt->fwd_latency;
>
> 	/* allow compiler to optimize */
>-	if (!burst && !fwd_latency)
>+	if (opt->ena_vector && opt->prod_type ==
>EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR)
>+		return perf_atq_worker_vector(arg, fwd_latency);
>+	else if (!burst && !fwd_latency)
> 		return perf_atq_worker(arg, 0);
> 	else if (!burst && fwd_latency)
> 		return perf_atq_worker(arg, 1);
>diff --git a/app/test-eventdev/test_perf_common.c b/app/test-
>eventdev/test_perf_common.c
>index 140c0c2dc3..8d7e483c55 100644
>--- a/app/test-eventdev/test_perf_common.c
>+++ b/app/test-eventdev/test_perf_common.c
>@@ -827,10 +827,13 @@ perf_event_timer_adapter_setup(struct test_perf *t)
> static int
> perf_event_crypto_adapter_setup(struct test_perf *t, struct prod_data *p)
> {
>+	struct rte_event_crypto_adapter_queue_conf conf;
> 	struct evt_options *opt = t->opt;
> 	uint32_t cap;
> 	int ret;
>
>+	memset(&conf, 0, sizeof(conf));
>+
> 	ret = rte_event_crypto_adapter_caps_get(p->dev_id, p->ca.cdev_id,
>&cap);
> 	if (ret) {
> 		evt_err("Failed to get crypto adapter capabilities");
>@@ -849,19 +852,53 @@ perf_event_crypto_adapter_setup(struct test_perf *t,
>struct prod_data *p)
> 		return -ENOTSUP;
> 	}
>
>-	if (cap &
>RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) {
>-		struct rte_event_crypto_adapter_queue_conf conf;
>+	if (opt->ena_vector) {
>+		struct rte_event_crypto_adapter_vector_limits limits;
>+
>+		if (!(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_EVENT_VECTOR))
>{
>+			evt_err("Crypto adapter doesn't support event vector");
>+			return -EINVAL;
>+		}
>+
>+		ret = rte_event_crypto_adapter_vector_limits_get(p->dev_id, p-
>>ca.cdev_id, &limits);
>+		if (ret) {
>+			evt_err("Failed to get crypto adapter's vector limits");
>+			return ret;
>+		}
>
>-		memset(&conf, 0, sizeof(conf));
>+		if (opt->vector_size < limits.min_sz || opt->vector_size >
>limits.max_sz) {
>+			evt_err("Vector size [%d] not within limits max[%d]
>min[%d]",
>+				opt->vector_size, limits.max_sz, limits.min_sz);
>+			return -EINVAL;
>+		}
>+
>+		if (limits.log2_sz && !rte_is_power_of_2(opt->vector_size)) {
>+			evt_err("Vector size [%d] not power of 2", opt-
>>vector_size);
>+			return -EINVAL;
>+		}
>+
>+		if (opt->vector_tmo_nsec > limits.max_timeout_ns ||
>+			opt->vector_tmo_nsec < limits.min_timeout_ns) {
>+			evt_err("Vector timeout [%" PRIu64 "] not within limits "
>+				"max[%" PRIu64 "] min[%" PRIu64 "]",
>+				opt->vector_tmo_nsec, limits.max_timeout_ns,
>limits.min_timeout_ns);
>+			return -EINVAL;
>+		}
>+
>+		conf.vector_mp = t->ca_vector_pool;
>+		conf.vector_sz = opt->vector_size;
>+		conf.vector_timeout_ns = opt->vector_tmo_nsec;
>+		conf.flags |= RTE_EVENT_CRYPTO_ADAPTER_EVENT_VECTOR;
>+	}
>+
>+	if (cap &
>RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) {
> 		conf.ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
> 		conf.ev.queue_id = p->queue_id;
>-		ret = rte_event_crypto_adapter_queue_pair_add(
>-			TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id,
>&conf);
>-	} else {
>-		ret = rte_event_crypto_adapter_queue_pair_add(
>-			TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id,
>NULL);
> 	}
>
>+	ret = rte_event_crypto_adapter_queue_pair_add(
>+		TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id, &conf);
>+
> 	return ret;
> }
>
>@@ -1411,6 +1448,19 @@ perf_cryptodev_setup(struct evt_test *test, struct
>evt_options *opt)
> 		goto err;
> 	}
>
>+	if (opt->ena_vector) {
>+		unsigned int nb_elem = (opt->pool_sz / opt->vector_size) * 2;
>+		nb_elem = RTE_MAX(512U, nb_elem);
>+		nb_elem += evt_nr_active_lcores(opt->wlcores) * 32;
>+		t->ca_vector_pool =
>rte_event_vector_pool_create("vector_pool", nb_elem, 32,
>+				opt->vector_size, opt->socket_id);
>+		if (t->ca_vector_pool == NULL) {
>+			evt_err("Failed to create event vector pool");
>+			ret = -ENOMEM;
>+			goto err;
>+		}
>+	}
>+
> 	/*
> 	 * Calculate number of needed queue pairs, based on the amount of
> 	 * available number of logical cores and crypto devices. For instance,
>@@ -1467,6 +1517,7 @@ perf_cryptodev_setup(struct evt_test *test, struct
>evt_options *opt)
> 	rte_mempool_free(t->ca_op_pool);
> 	rte_mempool_free(t->ca_sess_pool);
> 	rte_mempool_free(t->ca_asym_sess_pool);
>+	rte_mempool_free(t->ca_vector_pool);
>
> 	return ret;
> }
>@@ -1507,6 +1558,7 @@ perf_cryptodev_destroy(struct evt_test *test, struct
>evt_options *opt)
> 	rte_mempool_free(t->ca_op_pool);
> 	rte_mempool_free(t->ca_sess_pool);
> 	rte_mempool_free(t->ca_asym_sess_pool);
>+	rte_mempool_free(t->ca_vector_pool);
> }
>
> int
>diff --git a/app/test-eventdev/test_perf_common.h b/app/test-
>eventdev/test_perf_common.h
>index 503b6aa1db..faedd471c6 100644
>--- a/app/test-eventdev/test_perf_common.h
>+++ b/app/test-eventdev/test_perf_common.h
>@@ -71,6 +71,7 @@ struct test_perf {
> 	struct rte_mempool *ca_op_pool;
> 	struct rte_mempool *ca_sess_pool;
> 	struct rte_mempool *ca_asym_sess_pool;
>+	struct rte_mempool *ca_vector_pool;
> } __rte_cache_aligned;
>
> struct perf_elt {
>@@ -103,6 +104,8 @@ struct perf_elt {
> 	uint8_t cnt = 0;\
> 	void *bufs[16] __rte_cache_aligned;\
> 	int const sz = RTE_DIM(bufs);\
>+	uint8_t stage;\
>+	struct perf_elt *pe = NULL;\
> 	if (opt->verbose_level > 1)\
> 		printf("%s(): lcore %d dev_id %d port=%d\n", __func__,\
> 				rte_lcore_id(), dev, port)
>@@ -143,6 +146,64 @@ perf_handle_crypto_ev(struct rte_event *ev, struct
>perf_elt **pe, int enable_fwd
> 	return 0;
> }
>
>+static __rte_always_inline struct perf_elt *
>+perf_elt_from_vec_get(struct rte_event_vector *vec)
>+{
>+	/* Timestamp for vector event stored in first element */
>+	struct rte_crypto_op *cop = vec->ptrs[0];
>+	struct rte_mbuf *m;
>+
>+	if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
>+		m = cop->sym->m_dst == NULL ? cop->sym->m_src : cop->sym-
>>m_dst;
>+		return rte_pktmbuf_mtod(m, struct perf_elt *);
>+	} else {
>+		return RTE_PTR_ADD(cop->asym->modex.result.data, cop-
>>asym->modex.result.length);
>+	}
>+}
>+
>+static __rte_always_inline int
>+perf_handle_crypto_vector_ev(struct rte_event *ev, struct perf_elt **pe,
>+		const int enable_fwd_latency)
>+{
>+	struct rte_event_vector *vec = ev->vec;
>+	struct rte_crypto_op *cop;
>+	struct rte_mbuf *m;
>+	int i, n = 0;
>+	void *data;
>+
>+	for (i = 0; i < vec->nb_elem; i++) {
>+		cop = vec->ptrs[i];
>+		if (unlikely(cop->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) {
>+			if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
>+				m = cop->sym->m_dst == NULL ? cop->sym-
>>m_src : cop->sym->m_dst;
>+				rte_pktmbuf_free(m);
>+			} else {
>+				data = cop->asym->modex.result.data;
>+
>	rte_mempool_put(rte_mempool_from_obj(data), data);
>+			}
>+			rte_crypto_op_free(cop);
>+			continue;
>+		}
>+		vec->ptrs[n++] = cop;
>+	}
>+
>+	/* All cops failed, free the vector */
>+	if (n == 0) {
>+		rte_mempool_put(rte_mempool_from_obj(vec), vec);
>+		return -ENOENT;
>+	}
>+
>+	vec->nb_elem = n;
>+
>+	/* Forward latency not enabled - perf data will be not accessed */
>+	if (!enable_fwd_latency)
>+		return 0;
>+
>+	/* Get pointer to perf data */
>+	*pe = perf_elt_from_vec_get(vec);
>+
>+	return 0;
>+}
>
> static __rte_always_inline int
> perf_process_last_stage(struct rte_mempool *const pool, uint8_t
>prod_crypto_type,
>@@ -195,9 +256,8 @@ perf_process_last_stage_latency(struct rte_mempool
>*const pool, uint8_t prod_cry
> 	struct perf_elt *pe;
> 	void *to_free_in_bulk;
>
>-	/* release fence here ensures event_prt is
>-	 * stored before updating the number of
>-	 * processed packets for worker lcores
>+	/* Release fence here ensures event_prt is stored before updating the
>number of processed
>+	 * packets for worker lcores.
> 	 */
> 	rte_atomic_thread_fence(__ATOMIC_RELEASE);
> 	w->processed_pkts++;
>@@ -237,6 +297,42 @@ perf_process_last_stage_latency(struct rte_mempool
>*const pool, uint8_t prod_cry
> 	return count;
> }
>
>+static __rte_always_inline void
>+perf_process_vector_last_stage(struct rte_mempool *const pool,
>+		struct rte_mempool *const ca_pool, struct rte_event *const ev,
>+		struct worker_data *const w, const bool enable_fwd_latency)
>+{
>+	struct rte_event_vector *vec = ev->vec;
>+	struct rte_crypto_op *cop;
>+	void *bufs[vec->nb_elem];
>+	struct perf_elt *pe;
>+	uint64_t latency;
>+	int i;
>+
>+	/* Release fence here ensures event_prt is stored before updating the
>number of processed
>+	 * packets for worker lcores.
>+	 */
>+	rte_atomic_thread_fence(__ATOMIC_RELEASE);
>+	w->processed_pkts += vec->nb_elem;
>+
>+	if (enable_fwd_latency) {
>+		pe = perf_elt_from_vec_get(vec);
>+		latency = rte_get_timer_cycles() - pe->timestamp;
>+		w->latency += latency;
>+	}
>+
>+	for (i = 0; i < vec->nb_elem; i++) {
>+		cop = vec->ptrs[i];
>+		if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC)
>+			bufs[i] = cop->sym->m_dst == NULL ? cop->sym->m_src :
>cop->sym->m_dst;
>+		else
>+			bufs[i] = cop->asym->modex.result.data;
>+	}
>+
>+	rte_mempool_put_bulk(pool, bufs, vec->nb_elem);
>+	rte_mempool_put_bulk(ca_pool, (void * const *)vec->ptrs, vec-
>>nb_elem);
>+	rte_mempool_put(rte_mempool_from_obj(vec), vec);
>+}
>
> static inline int
> perf_nb_event_ports(struct evt_options *opt)
>diff --git a/app/test-eventdev/test_perf_queue.c b/app/test-
>eventdev/test_perf_queue.c
>index 69ef0ebbac..2399cfb69b 100644
>--- a/app/test-eventdev/test_perf_queue.c
>+++ b/app/test-eventdev/test_perf_queue.c
>@@ -25,15 +25,22 @@ fwd_event(struct rte_event *const ev, uint8_t *const
>sched_type_list,
> 	ev->event_type = RTE_EVENT_TYPE_CPU;
> }
>
>+static __rte_always_inline void
>+fwd_event_vector(struct rte_event *const ev, uint8_t *const sched_type_list,
>+		const uint8_t nb_stages)
>+{
>+	ev->queue_id++;
>+	ev->sched_type = sched_type_list[ev->queue_id % nb_stages];
>+	ev->op = RTE_EVENT_OP_FORWARD;
>+	ev->event_type = RTE_EVENT_TYPE_CPU_VECTOR;
>+}
>+
> static int
> perf_queue_worker(void *arg, const int enable_fwd_latency)
> {
>-	struct perf_elt *pe = NULL;
> 	uint16_t enq = 0, deq = 0;
> 	struct rte_event ev;
> 	PERF_WORKER_INIT;
>-	uint8_t stage;
>-
>
> 	while (t->done == false) {
> 		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
>@@ -82,9 +89,7 @@ perf_queue_worker_burst(void *arg, const int
>enable_fwd_latency)
> 	/* +1 to avoid prefetch out of array check */
> 	struct rte_event ev[BURST_SIZE + 1];
> 	uint16_t enq = 0, nb_rx = 0;
>-	struct perf_elt *pe = NULL;
> 	PERF_WORKER_INIT;
>-	uint8_t stage;
> 	uint16_t i;
>
> 	while (t->done == false) {
>@@ -137,6 +142,50 @@ perf_queue_worker_burst(void *arg, const int
>enable_fwd_latency)
> 	return 0;
> }
>
>+static int
>+perf_queue_worker_vector(void *arg, const int enable_fwd_latency)
>+{
>+	uint16_t enq = 0, deq = 0;
>+	struct rte_event ev;
>+	PERF_WORKER_INIT;
>+
>+	RTE_SET_USED(sz);
>+	RTE_SET_USED(cnt);
>+	RTE_SET_USED(prod_crypto_type);
>+
>+	while (t->done == false) {
>+		deq = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
>+
>+		if (!deq)
>+			continue;
>+
>+		if (ev.event_type == RTE_EVENT_TYPE_CRYPTODEV_VECTOR) {
>+			if (perf_handle_crypto_vector_ev(&ev, &pe,
>enable_fwd_latency))
>+				continue;
>+		}
>+
>+		stage = ev.queue_id % nb_stages;
>+		/* First q in pipeline, mark timestamp to compute fwd latency */
>+		if (enable_fwd_latency && !prod_timer_type && stage == 0)
>+			perf_mark_fwd_latency(pe);
>+
>+		/* Last stage in pipeline */
>+		if (unlikely(stage == laststage)) {
>+			perf_process_vector_last_stage(pool, t->ca_op_pool,
>&ev, w,
>+						       enable_fwd_latency);
>+		} else {
>+			fwd_event_vector(&ev, sched_type_list, nb_stages);
>+			do {
>+				enq = rte_event_enqueue_burst(dev, port, &ev,
>1);
>+			} while (!enq && !t->done);
>+		}
>+	}
>+
>+	perf_worker_cleanup(pool, dev, port, &ev, enq, deq);
>+
>+	return 0;
>+}
>+
> static int
> worker_wrapper(void *arg)
> {
>@@ -147,7 +196,9 @@ worker_wrapper(void *arg)
> 	const int fwd_latency = opt->fwd_latency;
>
> 	/* allow compiler to optimize */
>-	if (!burst && !fwd_latency)
>+	if (opt->ena_vector && opt->prod_type ==
>EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR)
>+		return perf_queue_worker_vector(arg, fwd_latency);
>+	else if (!burst && !fwd_latency)
> 		return perf_queue_worker(arg, 0);
> 	else if (!burst && fwd_latency)
> 		return perf_queue_worker(arg, 1);
>diff --git a/doc/guides/tools/testeventdev.rst
>b/doc/guides/tools/testeventdev.rst
>index cd278e8998..6f065b9752 100644
>--- a/doc/guides/tools/testeventdev.rst
>+++ b/doc/guides/tools/testeventdev.rst
>@@ -185,18 +185,18 @@ The following are the application command-line options:
>
> * ``--enable_vector``
>
>-       Enable event vector for Rx/Tx adapters.
>-       Only applicable for `pipeline_atq` and `pipeline_queue` tests.
>+       Enable event vector for Rx/Tx/crypto adapters.
>+       Only applicable for `pipeline_*` and `perf_*` tests.
>
> * ``--vector_size``
>
>-       Vector size to configure for the Rx adapter.
>-       Only applicable for `pipeline_atq` and `pipeline_queue` tests.
>+       Vector size to configure for the Rx/crypto adapter.
>+       Only applicable for `pipeline_*` and `perf_*` tests.
>
> * ``--vector_tmo_ns``
>
>-       Vector timeout nanoseconds to be configured for the Rx adapter.
>-       Only applicable for `pipeline_atq` and `pipeline_queue` tests.
>+       Vector timeout nanoseconds to be configured for the Rx/crypto adapter.
>+       Only applicable for `pipeline_*` and `perf_*` tests.
>
> * ``--per_port_pool``
>
>--
>2.25.1

Re: [EXT] [PATCH] app/testeventdev: add vector worker to perf test

[Jerin Jacob]

On Tue, Jan 17, 2023 at 8:59 PM Shijith Thotton <sthotton@marvell.com> wrote:
>
>
> >Add worker for handling vector events to perf tests, vector events could
> >be generated by crypto adapter producer.
> >
> >Example:
> >    ./dpdk-test-eventdev -l 0-2 -a <EVENTDEV> -a <CRYPTODEV> -- \
> >    --prod_type_cryptodev --crypto_adptr_mode 1 --test=perf_queue \
> >    --stlist=a --wlcores 1 --plcores 2 --prod_enq_burst_sz 32 \
> >    --enable_vector --vector_tmo_ns 0 --nb_flows 2
> >
> >Signed-off-by: Volodymyr Fialko <vfialko@marvell.com>
>
> Acked-by: Shijith Thotton <sthotton@marvell.com>

Applied to dpdk-next-net-eventdev/for-main. Thanks