[dpdk-dev] [PATCH v8 08/10] examples/l2fwd-event: add eventdev main loop

[<pbhagavatula@marvell.com>]

From: Pavan Nikhilesh <pbhagavatula@marvell.com>

Add event dev main loop based on enabled l2fwd options and eventdev
capabilities.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
---
 examples/l2fwd-event/l2fwd_common.c |   6 +
 examples/l2fwd-event/l2fwd_common.h |   1 +
 examples/l2fwd-event/l2fwd_event.c  | 238 ++++++++++++++++++++++++++++
 examples/l2fwd-event/l2fwd_event.h  |   2 +
 examples/l2fwd-event/main.c         |  68 +++++++-
 examples/l2fwd-event/meson.build    |   5 +
 6 files changed, 312 insertions(+), 8 deletions(-)

diff --git a/examples/l2fwd-event/l2fwd_common.c b/examples/l2fwd-event/l2fwd_common.c
index c206415d0..4ba788550 100644
--- a/examples/l2fwd-event/l2fwd_common.c
+++ b/examples/l2fwd-event/l2fwd_common.c
@@ -18,6 +18,12 @@ l2fwd_event_init_ports(struct l2fwd_resources *rsrc)
 	uint16_t port_id;
 	int ret;
 
+	if (rsrc->event_mode) {
+		port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
+		port_conf.rx_adv_conf.rss_conf.rss_key = NULL;
+		port_conf.rx_adv_conf.rss_conf.rss_hf = ETH_RSS_IP;
+	}
+
 	/* Initialise each port */
 	RTE_ETH_FOREACH_DEV(port_id) {
 		struct rte_eth_conf local_port_conf = port_conf;
diff --git a/examples/l2fwd-event/l2fwd_common.h b/examples/l2fwd-event/l2fwd_common.h
index a4e17ab97..7e33ee749 100644
--- a/examples/l2fwd-event/l2fwd_common.h
+++ b/examples/l2fwd-event/l2fwd_common.h
@@ -114,6 +114,7 @@ l2fwd_get_rsrc(void)
 
 		memset(rsrc, 0, sizeof(struct l2fwd_resources));
 		rsrc->mac_updating = true;
+		rsrc->event_mode = true;
 		rsrc->rx_queue_per_lcore = 1;
 		rsrc->sched_type = RTE_SCHED_TYPE_ATOMIC;
 		rsrc->timer_period = 10 * rte_get_timer_hz();
diff --git a/examples/l2fwd-event/l2fwd_event.c b/examples/l2fwd-event/l2fwd_event.c
index 562d61292..c7782cbc5 100644
--- a/examples/l2fwd-event/l2fwd_event.c
+++ b/examples/l2fwd-event/l2fwd_event.c
@@ -17,6 +17,12 @@
 
 #include "l2fwd_event.h"
 
+#define L2FWD_EVENT_SINGLE	0x1
+#define L2FWD_EVENT_BURST	0x2
+#define L2FWD_EVENT_TX_DIRECT	0x4
+#define L2FWD_EVENT_TX_ENQ	0x8
+#define L2FWD_EVENT_UPDT_MAC	0x10
+
 static inline int
 l2fwd_event_service_enable(uint32_t service_id)
 {
@@ -122,11 +128,233 @@ l2fwd_event_capability_setup(struct l2fwd_event_resources *evt_rsrc)
 		l2fwd_event_set_internal_port_ops(&evt_rsrc->ops);
 }
 
+static __rte_noinline int
+l2fwd_get_free_event_port(struct l2fwd_event_resources *evt_rsrc)
+{
+	static int index;
+	int port_id;
+
+	rte_spinlock_lock(&evt_rsrc->evp.lock);
+	if (index >= evt_rsrc->evp.nb_ports) {
+		printf("No free event port is available\n");
+		return -1;
+	}
+
+	port_id = evt_rsrc->evp.event_p_id[index];
+	index++;
+	rte_spinlock_unlock(&evt_rsrc->evp.lock);
+
+	return port_id;
+}
+
+static  __rte_always_inline void
+l2fwd_event_fwd(struct l2fwd_resources *rsrc, struct rte_event *ev,
+		const uint8_t tx_q_id, const uint64_t timer_period,
+		const uint32_t flags)
+{
+	struct rte_mbuf *mbuf = ev->mbuf;
+	uint16_t dst_port;
+
+	rte_prefetch0(rte_pktmbuf_mtod(mbuf, void *));
+	dst_port = rsrc->dst_ports[mbuf->port];
+
+	if (timer_period > 0)
+		__atomic_fetch_add(&rsrc->port_stats[mbuf->port].rx,
+				1, __ATOMIC_RELAXED);
+	mbuf->port = dst_port;
+
+	if (flags & L2FWD_EVENT_UPDT_MAC)
+		l2fwd_mac_updating(mbuf, dst_port, &rsrc->eth_addr[dst_port]);
+
+	if (flags & L2FWD_EVENT_TX_ENQ) {
+		ev->queue_id = tx_q_id;
+		ev->op = RTE_EVENT_OP_FORWARD;
+	}
+
+	if (flags & L2FWD_EVENT_TX_DIRECT)
+		rte_event_eth_tx_adapter_txq_set(mbuf, 0);
+
+	if (timer_period > 0)
+		__atomic_fetch_add(&rsrc->port_stats[mbuf->port].tx,
+				1, __ATOMIC_RELAXED);
+}
+
+static __rte_always_inline void
+l2fwd_event_loop_single(struct l2fwd_resources *rsrc,
+			const uint32_t flags)
+{
+	struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
+	const int port_id = l2fwd_get_free_event_port(evt_rsrc);
+	const uint8_t tx_q_id = evt_rsrc->evq.event_q_id[
+					evt_rsrc->evq.nb_queues - 1];
+	const uint64_t timer_period = rsrc->timer_period;
+	const uint8_t event_d_id = evt_rsrc->event_d_id;
+	struct rte_event ev;
+
+	if (port_id < 0)
+		return;
+
+	printf("%s(): entering eventdev main loop on lcore %u\n", __func__,
+		rte_lcore_id());
+
+	while (!rsrc->force_quit) {
+		/* Read packet from eventdev */
+		if (!rte_event_dequeue_burst(event_d_id, port_id, &ev, 1, 0))
+			continue;
+
+		l2fwd_event_fwd(rsrc, &ev, tx_q_id, timer_period, flags);
+
+		if (flags & L2FWD_EVENT_TX_ENQ) {
+			while (rte_event_enqueue_burst(event_d_id, port_id,
+						       &ev, 1) &&
+					!rsrc->force_quit)
+				;
+		}
+
+		if (flags & L2FWD_EVENT_TX_DIRECT) {
+			while (!rte_event_eth_tx_adapter_enqueue(event_d_id,
+								port_id,
+								&ev, 1, 0) &&
+					!rsrc->force_quit)
+				;
+		}
+	}
+}
+
+static __rte_always_inline void
+l2fwd_event_loop_burst(struct l2fwd_resources *rsrc,
+		       const uint32_t flags)
+{
+	struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
+	const int port_id = l2fwd_get_free_event_port(evt_rsrc);
+	const uint8_t tx_q_id = evt_rsrc->evq.event_q_id[
+					evt_rsrc->evq.nb_queues - 1];
+	const uint64_t timer_period = rsrc->timer_period;
+	const uint8_t event_d_id = evt_rsrc->event_d_id;
+	const uint8_t deq_len = evt_rsrc->deq_depth;
+	struct rte_event ev[MAX_PKT_BURST];
+	uint16_t nb_rx, nb_tx;
+	uint8_t i;
+
+	if (port_id < 0)
+		return;
+
+	printf("%s(): entering eventdev main loop on lcore %u\n", __func__,
+		rte_lcore_id());
+
+	while (!rsrc->force_quit) {
+		/* Read packet from eventdev */
+		nb_rx = rte_event_dequeue_burst(event_d_id, port_id, ev,
+						deq_len, 0);
+		if (nb_rx == 0)
+			continue;
+
+		for (i = 0; i < nb_rx; i++) {
+			l2fwd_event_fwd(rsrc, &ev[i], tx_q_id, timer_period,
+					flags);
+		}
+
+		if (flags & L2FWD_EVENT_TX_ENQ) {
+			nb_tx = rte_event_enqueue_burst(event_d_id, port_id,
+							ev, nb_rx);
+			while (nb_tx < nb_rx && !rsrc->force_quit)
+				nb_tx += rte_event_enqueue_burst(event_d_id,
+						port_id, ev + nb_tx,
+						nb_rx - nb_tx);
+		}
+
+		if (flags & L2FWD_EVENT_TX_DIRECT) {
+			nb_tx = rte_event_eth_tx_adapter_enqueue(event_d_id,
+								 port_id, ev,
+								 nb_rx, 0);
+			while (nb_tx < nb_rx && !rsrc->force_quit)
+				nb_tx += rte_event_eth_tx_adapter_enqueue(
+						event_d_id, port_id,
+						ev + nb_tx, nb_rx - nb_tx, 0);
+		}
+	}
+}
+
+static __rte_always_inline void
+l2fwd_event_loop(struct l2fwd_resources *rsrc,
+			const uint32_t flags)
+{
+	if (flags & L2FWD_EVENT_SINGLE)
+		l2fwd_event_loop_single(rsrc, flags);
+	if (flags & L2FWD_EVENT_BURST)
+		l2fwd_event_loop_burst(rsrc, flags);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_d(struct l2fwd_resources *rsrc)
+{
+	l2fwd_event_loop(rsrc,
+			 L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_SINGLE);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_d_brst(struct l2fwd_resources *rsrc)
+{
+	l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_BURST);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_q(struct l2fwd_resources *rsrc)
+{
+	l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_SINGLE);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_q_brst(struct l2fwd_resources *rsrc)
+{
+	l2fwd_event_loop(rsrc, L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_BURST);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_d_mac(struct l2fwd_resources *rsrc)
+{
+	l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
+			L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_SINGLE);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_d_brst_mac(struct l2fwd_resources *rsrc)
+{
+	l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
+			L2FWD_EVENT_TX_DIRECT | L2FWD_EVENT_BURST);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_q_mac(struct l2fwd_resources *rsrc)
+{
+	l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
+			L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_SINGLE);
+}
+
+static void __rte_noinline
+l2fwd_event_main_loop_tx_q_brst_mac(struct l2fwd_resources *rsrc)
+{
+	l2fwd_event_loop(rsrc, L2FWD_EVENT_UPDT_MAC |
+			L2FWD_EVENT_TX_ENQ | L2FWD_EVENT_BURST);
+}
+
 void
 l2fwd_event_resource_setup(struct l2fwd_resources *rsrc)
 {
+	/* [MAC_UPDT][TX_MODE][BURST] */
+	const event_loop_cb event_loop[2][2][2] = {
+		[0][0][0] = l2fwd_event_main_loop_tx_d,
+		[0][0][1] = l2fwd_event_main_loop_tx_d_brst,
+		[0][1][0] = l2fwd_event_main_loop_tx_q,
+		[0][1][1] = l2fwd_event_main_loop_tx_q_brst,
+		[1][0][0] = l2fwd_event_main_loop_tx_d_mac,
+		[1][0][1] = l2fwd_event_main_loop_tx_d_brst_mac,
+		[1][1][0] = l2fwd_event_main_loop_tx_q_mac,
+		[1][1][1] = l2fwd_event_main_loop_tx_q_brst_mac,
+	};
 	struct l2fwd_event_resources *evt_rsrc;
 	uint32_t event_queue_cfg;
+	int ret;
 
 	if (!rte_event_dev_count())
 		rte_panic("No Eventdev found\n");
@@ -152,4 +380,14 @@ l2fwd_event_resource_setup(struct l2fwd_resources *rsrc)
 
 	/* Rx/Tx adapters configuration */
 	evt_rsrc->ops.adapter_setup(rsrc);
+
+	/* Start event device */
+	ret = rte_event_dev_start(evt_rsrc->event_d_id);
+	if (ret < 0)
+		rte_panic("Error in starting eventdev\n");
+
+	evt_rsrc->ops.l2fwd_event_loop = event_loop
+					[rsrc->mac_updating]
+					[evt_rsrc->tx_mode_q]
+					[evt_rsrc->has_burst];
 }
diff --git a/examples/l2fwd-event/l2fwd_event.h b/examples/l2fwd-event/l2fwd_event.h
index ebfbfe460..78f22e5f9 100644
--- a/examples/l2fwd-event/l2fwd_event.h
+++ b/examples/l2fwd-event/l2fwd_event.h
@@ -18,6 +18,7 @@ typedef void (*event_port_setup_cb)(struct l2fwd_resources *rsrc);
 typedef void (*event_queue_setup_cb)(struct l2fwd_resources *rsrc,
 				     uint32_t event_queue_cfg);
 typedef void (*adapter_setup_cb)(struct l2fwd_resources *rsrc);
+typedef void (*event_loop_cb)(struct l2fwd_resources *rsrc);
 
 struct event_queues {
 	uint8_t *event_q_id;
@@ -47,6 +48,7 @@ struct event_setup_ops {
 	event_queue_setup_cb event_queue_setup;
 	event_port_setup_cb event_port_setup;
 	adapter_setup_cb adapter_setup;
+	event_loop_cb l2fwd_event_loop;
 };
 
 struct l2fwd_event_resources {
diff --git a/examples/l2fwd-event/main.c b/examples/l2fwd-event/main.c
index 1ae18bb51..01eb19f3e 100644
--- a/examples/l2fwd-event/main.c
+++ b/examples/l2fwd-event/main.c
@@ -214,8 +214,12 @@ l2fwd_launch_one_lcore(void *args)
 {
 	struct l2fwd_resources *rsrc = args;
 	struct l2fwd_poll_resources *poll_rsrc = rsrc->poll_rsrc;
+	struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
 
-	poll_rsrc->poll_main_loop(rsrc);
+	if (rsrc->event_mode)
+		evt_rsrc->ops.l2fwd_event_loop(rsrc);
+	else
+		poll_rsrc->poll_main_loop(rsrc);
 
 	return 0;
 }
@@ -304,9 +308,9 @@ print_stats(struct l2fwd_resources *rsrc)
 		if ((rsrc->enabled_port_mask & (1 << port_id)) == 0)
 			continue;
 		printf("\nStatistics for port %u ------------------------------"
-			   "\nPackets sent: %24"PRIu64
-			   "\nPackets received: %20"PRIu64
-			   "\nPackets dropped: %21"PRIu64,
+			   "\nPackets sent: %29"PRIu64
+			   "\nPackets received: %25"PRIu64
+			   "\nPackets dropped: %26"PRIu64,
 			   port_id,
 			   rsrc->port_stats[port_id].tx,
 			   rsrc->port_stats[port_id].rx,
@@ -317,10 +321,58 @@ print_stats(struct l2fwd_resources *rsrc)
 		total_packets_tx += rsrc->port_stats[port_id].tx;
 		total_packets_rx += rsrc->port_stats[port_id].rx;
 	}
-	printf("\nAggregate statistics ==============================="
-		   "\nTotal packets sent: %18"PRIu64
-		   "\nTotal packets received: %14"PRIu64
-		   "\nTotal packets dropped: %15"PRIu64,
+
+	if (rsrc->event_mode) {
+		struct l2fwd_event_resources *evt_rsrc = rsrc->evt_rsrc;
+		struct rte_event_eth_rx_adapter_stats rx_adptr_stats;
+		struct rte_event_eth_tx_adapter_stats tx_adptr_stats;
+		int ret, i;
+
+		for (i = 0; i < evt_rsrc->rx_adptr.nb_rx_adptr; i++) {
+			ret = rte_event_eth_rx_adapter_stats_get(
+					evt_rsrc->rx_adptr.rx_adptr[i],
+					&rx_adptr_stats);
+			if (ret < 0)
+				continue;
+			printf("\nRx adapter[%d] statistics===================="
+				   "\nReceive queue poll count: %17"PRIu64
+				   "\nReceived packet count: %20"PRIu64
+				   "\nEventdev enqueue count: %19"PRIu64
+				   "\nEventdev enqueue retry count: %13"PRIu64
+				   "\nReceived packet dropped count: %12"PRIu64
+				   "\nRx enqueue start timestamp: %15"PRIu64
+				   "\nRx enqueue block cycles: %18"PRIu64
+				   "\nRx enqueue unblock timestamp: %13"PRIu64,
+				   evt_rsrc->rx_adptr.rx_adptr[i],
+				   rx_adptr_stats.rx_poll_count,
+				   rx_adptr_stats.rx_packets,
+				   rx_adptr_stats.rx_enq_count,
+				   rx_adptr_stats.rx_enq_retry,
+				   rx_adptr_stats.rx_dropped,
+				   rx_adptr_stats.rx_enq_start_ts,
+				   rx_adptr_stats.rx_enq_block_cycles,
+				   rx_adptr_stats.rx_enq_end_ts);
+		}
+		for (i = 0; i <  evt_rsrc->tx_adptr.nb_tx_adptr; i++) {
+			ret = rte_event_eth_tx_adapter_stats_get(
+					evt_rsrc->tx_adptr.tx_adptr[i],
+					&tx_adptr_stats);
+			if (ret < 0)
+				continue;
+			printf("\nTx adapter[%d] statistics===================="
+				   "\nNumber of transmit retries: %15"PRIu64
+				   "\nNumber of packets transmitted: %12"PRIu64
+				   "\nNumber of packets dropped: %16"PRIu64,
+				   evt_rsrc->tx_adptr.tx_adptr[i],
+				   tx_adptr_stats.tx_retry,
+				   tx_adptr_stats.tx_packets,
+				   tx_adptr_stats.tx_dropped);
+		}
+	}
+	printf("\nAggregate lcore statistics ========================="
+		   "\nTotal packets sent: %23"PRIu64
+		   "\nTotal packets received: %19"PRIu64
+		   "\nTotal packets dropped: %20"PRIu64,
 		   total_packets_tx,
 		   total_packets_rx,
 		   total_packets_dropped);
diff --git a/examples/l2fwd-event/meson.build b/examples/l2fwd-event/meson.build
index 4e9a069d6..c4664c3a3 100644
--- a/examples/l2fwd-event/meson.build
+++ b/examples/l2fwd-event/meson.build
@@ -16,3 +16,8 @@ sources = files(
 	'l2fwd_event_internal_port.c',
 	'l2fwd_event_generic.c'
 )
+
+# for clang 32-bit compiles we need libatomic for 64-bit atomic ops
+if cc.get_id() == 'clang' and dpdk_conf.get('RTE_ARCH_64') == false
+	ext_deps += cc.find_library('atomic')
+endif
-- 
2.17.1