[dpdk-dev] [RFC PATCH 01/13] examples/ipsec-secgw: add framework for eventmode helper

[Anoob Joseph <anoobj@marvell.com>]

Add framework for eventmode helper. Event mode would involve
initialization of multiple devices, like eventdev, ethdev etc.
Add routines to initialize and uninitialize event devices.
Generate a default config for event devices if it is not
specified in the configuration. The routine will iterate
over available event devices and their properties and will
set the config accordingly.

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Signed-off-by: Lukasz Bartosik <lbartosik@marvell.com>
---
 examples/ipsec-secgw/Makefile       |   1 +
 examples/ipsec-secgw/event_helper.c | 322 ++++++++++++++++++++++++++++++++++++
 examples/ipsec-secgw/event_helper.h | 115 +++++++++++++
 examples/ipsec-secgw/meson.build    |   4 +-
 4 files changed, 440 insertions(+), 2 deletions(-)
 create mode 100644 examples/ipsec-secgw/event_helper.c
 create mode 100644 examples/ipsec-secgw/event_helper.h

diff --git a/examples/ipsec-secgw/Makefile b/examples/ipsec-secgw/Makefile
index 851123b..2a76900 100644
--- a/examples/ipsec-secgw/Makefile
+++ b/examples/ipsec-secgw/Makefile
@@ -15,6 +15,7 @@ SRCS-y += sa.c
 SRCS-y += rt.c
 SRCS-y += ipsec_process.c
 SRCS-y += ipsec-secgw.c
+SRCS-y += event_helper.c
 
 CFLAGS += -gdwarf-2
 
diff --git a/examples/ipsec-secgw/event_helper.c b/examples/ipsec-secgw/event_helper.c
new file mode 100644
index 0000000..af7c7e2
--- /dev/null
+++ b/examples/ipsec-secgw/event_helper.c
@@ -0,0 +1,322 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C) 2019 Marvell International Ltd.
+ */
+#include <rte_eventdev.h>
+#include <rte_ethdev.h>
+
+#include "event_helper.h"
+
+static int
+eh_validate_user_params(struct eventmode_conf *em_conf)
+{
+	/* TODO */
+	/* Check sanity of the conf requested by user */
+
+	RTE_SET_USED(em_conf);
+
+	return 0;
+}
+
+static int
+eh_set_default_conf_eventdev(struct eventmode_conf *em_conf)
+{
+	int i, ret;
+	int nb_eventdev;
+	struct eventdev_params *eventdev_config;
+	struct rte_event_dev_info dev_info;
+
+	/* Get the number of event devices */
+	nb_eventdev = rte_event_dev_count();
+
+	if (nb_eventdev == 0) {
+		EH_LOG_ERR("No event devices detected");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < nb_eventdev; i++) {
+
+		/* Get the event dev conf */
+		eventdev_config = &(em_conf->eventdev_config[i]);
+
+		/* Read event device info */
+		ret = rte_event_dev_info_get(i, &dev_info);
+
+		if (ret < 0) {
+			EH_LOG_ERR("Failed reading event device info (err:%d)",
+				ret);
+			return ret;
+		}
+
+		/* Check if enough ports are available */
+		if (dev_info.max_event_ports < 2) {
+			EH_LOG_ERR("Not enough event ports available");
+			return -EINVAL;
+		}
+
+		/* Save number of queues & ports available */
+		eventdev_config->eventdev_id = i;
+		eventdev_config->nb_eventqueue = dev_info.max_event_queues;
+		eventdev_config->nb_eventport = dev_info.max_event_ports;
+		eventdev_config->ev_queue_mode =
+				RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+		/* One port is required for eth Rx adapter */
+		eventdev_config->nb_eventport -= 1;
+
+		/* One port is reserved for eth Tx adapter */
+		eventdev_config->nb_eventport -= 1;
+
+		/* Update the number of eventdevs */
+		em_conf->nb_eventdev++;
+	}
+
+	return 0;
+}
+
+static int
+eh_validate_conf(struct eventmode_conf *em_conf)
+{
+	int ret;
+
+	/* After parsing all args, verify that the conf can be allowed */
+	ret = eh_validate_user_params(em_conf);
+	if (ret != 0)
+		return ret;
+
+	/*
+	 * See if event devs are specified. Else probe the event devices
+	 * and initialize the conf with all ports & queues available
+	 */
+	if (em_conf->nb_eventdev == 0) {
+		ret = eh_set_default_conf_eventdev(em_conf);
+		if (ret != 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int
+eh_initialize_eventdev(struct eventmode_conf *em_conf)
+{
+	int ret = -1;
+	uint8_t i, j;
+	struct rte_event_dev_config eventdev_conf;
+	struct rte_event_dev_info evdev_default_conf;
+	struct rte_event_queue_conf eventq_conf = {0};
+	struct eventdev_params *eventdev_config;
+	int nb_eventdev = em_conf->nb_eventdev;
+	int nb_eventqueue;
+	uint8_t eventdev_id;
+
+	for (i = 0; i < nb_eventdev; i++) {
+
+		/* Get eventdev config */
+		eventdev_config = &(em_conf->eventdev_config[i]);
+
+		/* Get event dev ID */
+		eventdev_id = eventdev_config->eventdev_id;
+
+		/* Get the number of queues */
+		nb_eventqueue = eventdev_config->nb_eventqueue;
+
+		/* One queue is reserved for the final stage (doing eth tx) */
+		/* TODO handles only one Tx adapter. Fix this */
+		nb_eventqueue += 1;
+
+		/* Reset the default conf */
+		memset(&evdev_default_conf, 0,
+			sizeof(struct rte_event_dev_info));
+
+		/* Get default conf of eventdev */
+		ret = rte_event_dev_info_get(eventdev_id, &evdev_default_conf);
+		if (ret < 0) {
+			EH_LOG_ERR(
+				"Error in getting event device info[devID:%d]",
+				eventdev_id);
+			return ret;
+		}
+
+		memset(&eventdev_conf, 0, sizeof(struct rte_event_dev_config));
+		eventdev_conf.nb_events_limit =
+				evdev_default_conf.max_num_events;
+		eventdev_conf.nb_event_queues = nb_eventqueue;
+		eventdev_conf.nb_event_ports =
+				eventdev_config->nb_eventport;
+		eventdev_conf.nb_event_queue_flows =
+				evdev_default_conf.max_event_queue_flows;
+		eventdev_conf.nb_event_port_dequeue_depth =
+				evdev_default_conf.max_event_port_dequeue_depth;
+		eventdev_conf.nb_event_port_enqueue_depth =
+				evdev_default_conf.max_event_port_enqueue_depth;
+
+		/* Configure event device */
+		ret = rte_event_dev_configure(eventdev_id, &eventdev_conf);
+		if (ret < 0) {
+			EH_LOG_ERR("Error in configuring event device");
+			return ret;
+		}
+
+		/* Configure event queues */
+		for (j = 0; j < nb_eventqueue; j++) {
+
+			memset(&eventq_conf, 0,
+					sizeof(struct rte_event_queue_conf));
+
+			/* Read the requested conf */
+
+			/* Per event dev queues can be ATQ or SINGLE LINK */
+			eventq_conf.event_queue_cfg =
+					eventdev_config->ev_queue_mode;
+			/*
+			 * All queues need to be set with sched_type as
+			 * schedule type for the application stage. One queue
+			 * would be reserved for the final eth tx stage. This
+			 * will be an atomic queue.
+			 */
+			if (j == nb_eventqueue-1) {
+				eventq_conf.schedule_type =
+					RTE_SCHED_TYPE_ATOMIC;
+			} else {
+				eventq_conf.schedule_type =
+					em_conf->ext_params.sched_type;
+			}
+
+			/* Set max atomic flows to 1024 */
+			eventq_conf.nb_atomic_flows = 1024;
+			eventq_conf.nb_atomic_order_sequences = 1024;
+
+			/* Setup the queue */
+			ret = rte_event_queue_setup(eventdev_id, j,
+					&eventq_conf);
+			if (ret < 0) {
+				EH_LOG_ERR("Error in event queue setup");
+				return ret;
+			}
+		}
+
+		/* Configure event ports */
+		for (j = 0; j <  eventdev_config->nb_eventport; j++) {
+			ret = rte_event_port_setup(eventdev_id, j, NULL);
+			if (ret < 0) {
+				EH_LOG_ERR("Error setting up event port");
+				return ret;
+			}
+		}
+	}
+
+	/* Start event devices */
+	for (i = 0; i < nb_eventdev; i++) {
+
+		/* Get eventdev config */
+		eventdev_config = &(em_conf->eventdev_config[i]);
+
+		ret = rte_event_dev_start(eventdev_config->eventdev_id);
+		if (ret < 0) {
+			EH_LOG_ERR("Error in starting event device[devID: %d]",
+				eventdev_config->eventdev_id);
+			return ret;
+		}
+	}
+	return 0;
+}
+
+int32_t
+eh_devs_init(struct eh_conf *mode_conf)
+{
+	int ret;
+	uint16_t portid;
+	struct eventmode_conf *em_conf;
+
+	if (mode_conf == NULL) {
+		EH_LOG_ERR("Invalid conf");
+		return -EINVAL;
+	}
+
+	if (mode_conf->mode != EH_PKT_TRANSFER_MODE_EVENT)
+		return 0;
+
+	if (mode_conf->mode_params == NULL) {
+		EH_LOG_ERR("Invalid mode params");
+		return -EINVAL;
+	}
+
+	/* Get eventmode conf */
+	em_conf = (struct eventmode_conf *)(mode_conf->mode_params);
+
+	/* Validate the conf requested */
+	if (eh_validate_conf(em_conf) != 0) {
+		EH_LOG_ERR("Failed while validating the conf requested");
+		return -EINVAL;
+	}
+
+	/* Stop eth devices before setting up adapter */
+	RTE_ETH_FOREACH_DEV(portid) {
+
+		/* Use only the ports enabled */
+		if ((mode_conf->eth_portmask & (1 << portid)) == 0)
+			continue;
+
+		rte_eth_dev_stop(portid);
+	}
+
+	/* Setup eventdev */
+	ret = eh_initialize_eventdev(em_conf);
+	if (ret != 0)
+		return ret;
+
+	/* Start eth devices after setting up adapter */
+	RTE_ETH_FOREACH_DEV(portid) {
+
+		/* Use only the ports enabled */
+		if ((mode_conf->eth_portmask & (1 << portid)) == 0)
+			continue;
+
+		ret = rte_eth_dev_start(portid);
+		if (ret < 0) {
+			EH_LOG_ERR("Error starting eth dev %d", portid);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+int32_t
+eh_devs_uninit(struct eh_conf *mode_conf)
+{
+	int ret, i;
+	uint16_t id;
+	struct eventmode_conf *em_conf;
+
+	if (mode_conf == NULL) {
+		EH_LOG_ERR("Invalid conf");
+		return -EINVAL;
+	}
+
+	if (mode_conf->mode != EH_PKT_TRANSFER_MODE_EVENT)
+		return 0;
+
+	if (mode_conf->mode_params == NULL) {
+		EH_LOG_ERR("Invalid mode params");
+		return -EINVAL;
+	}
+
+	/* Get eventmode conf */
+	em_conf = (struct eventmode_conf *)(mode_conf->mode_params);
+
+	/* Stop and release event devices */
+	for (i = 0; i < em_conf->nb_eventdev; i++) {
+
+		id = em_conf->eventdev_config[i].eventdev_id;
+		rte_event_dev_stop(id);
+
+		ret = rte_event_dev_close(id);
+		if (ret < 0) {
+			EH_LOG_ERR("Error closing event dev %d", id);
+			return ret;
+		}
+	}
+
+	return 0;
+}
diff --git a/examples/ipsec-secgw/event_helper.h b/examples/ipsec-secgw/event_helper.h
new file mode 100644
index 0000000..71990f9
--- /dev/null
+++ b/examples/ipsec-secgw/event_helper.h
@@ -0,0 +1,115 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C) 2019 Marvell International Ltd.
+ */
+#ifndef _EVENT_HELPER_H_
+#define _EVENT_HELPER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <rte_log.h>
+
+#define RTE_LOGTYPE_EH  RTE_LOGTYPE_USER4
+
+#define EH_LOG_ERR(...) \
+	RTE_LOG(ERR, EH, \
+		RTE_FMT("%s() line %u: " RTE_FMT_HEAD(__VA_ARGS__ ,) "\n", \
+			__func__, __LINE__, RTE_FMT_TAIL(__VA_ARGS__ ,)))
+
+/* Max event devices supported */
+#define EVENT_MODE_MAX_EVENT_DEVS RTE_EVENT_MAX_DEVS
+
+/**
+ * Packet transfer mode of the application
+ */
+enum eh_pkt_transfer_mode {
+	EH_PKT_TRANSFER_MODE_POLL = 0,
+	EH_PKT_TRANSFER_MODE_EVENT,
+};
+
+/* Event dev params */
+struct eventdev_params {
+	uint8_t eventdev_id;
+	uint8_t nb_eventqueue;
+	uint8_t nb_eventport;
+	uint8_t ev_queue_mode;
+};
+
+/* Eventmode conf data */
+struct eventmode_conf {
+	int nb_eventdev;
+		/**< No of event devs */
+	struct eventdev_params eventdev_config[EVENT_MODE_MAX_EVENT_DEVS];
+		/**< Per event dev conf */
+	union {
+		RTE_STD_C11
+		struct {
+			uint64_t sched_type			: 2;
+		/**< Schedule type */
+		};
+		uint64_t u64;
+	} ext_params;
+		/**< 64 bit field to specify extended params */
+};
+
+/**
+ * Event helper configuration
+ */
+struct eh_conf {
+	enum eh_pkt_transfer_mode mode;
+		/**< Packet transfer mode of the application */
+	uint32_t eth_portmask;
+		/**<
+		 * Mask of the eth ports to be used. This portmask would be
+		 * checked while initializing devices using helper routines.
+		 */
+	void *mode_params;
+		/**< Mode specific parameters */
+};
+
+/**
+ * Initialize event mode devices
+ *
+ * Application can call this function to get the event devices, eth devices
+ * and eth rx & tx adapters initialized according to the default config or
+ * config populated using the command line args.
+ *
+ * Application is expected to initialize the eth devices and then the event
+ * mode helper subsystem will stop & start eth devices according to its
+ * requirement. Call to this function should be done after the eth devices
+ * are successfully initialized.
+ *
+ * @param mode_conf
+ *   Configuration of the mode in which app is doing packet handling
+ * @return
+ *  - 0 on success.
+ *  - (<0) on failure.
+ */
+int32_t
+eh_devs_init(struct eh_conf *mode_conf);
+
+/**
+ * Release event mode devices
+ *
+ * Application could call this function to release event devices,
+ * eth rx & tx adapters according to the conf.
+ *
+ * Call to this function should be done before application stops
+ * and closes eth devices. This function will not close and stop
+ * eth devices.
+ *
+ * @param mode_conf
+ *   Configuration of the mode in which app is doing packet handling
+ * @return
+ *  - 0 on success.
+ *  - (<0) on failure.
+ */
+int32_t
+eh_devs_uninit(struct eh_conf *mode_conf);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _EVENT_HELPER_H_ */
diff --git a/examples/ipsec-secgw/meson.build b/examples/ipsec-secgw/meson.build
index 9ece345..20f4064 100644
--- a/examples/ipsec-secgw/meson.build
+++ b/examples/ipsec-secgw/meson.build
@@ -6,9 +6,9 @@
 # To build this example as a standalone application with an already-installed
 # DPDK instance, use 'make'
 
-deps += ['security', 'lpm', 'acl', 'hash', 'ip_frag', 'ipsec']
+deps += ['security', 'lpm', 'acl', 'hash', 'ip_frag', 'ipsec', 'eventdev']
 allow_experimental_apis = true
 sources = files(
 	'esp.c', 'ipsec.c', 'ipsec_process.c', 'ipsec-secgw.c',
-	'parser.c', 'rt.c', 'sa.c', 'sp4.c', 'sp6.c'
+	'parser.c', 'rt.c', 'sa.c', 'sp4.c', 'sp6.c', 'event_helper.c'
 )
-- 
2.7.4