[RFC 1/3] eventdev: introduce event prefetching

[<pbhagavatula@marvell.com>]

From: Pavan Nikhilesh <pbhagavatula@marvell.com>

Event prefetching enhances scheduling performance by pre-scheduling
events to event ports when dequeues are issued.
The dequeue operation initiates the prefetch operation, which completes
in parallel without affecting the dequeued event flow contexts and
dequeue latency.

Event devices can indicate prefetching capabilities using
`RTE_EVENT_DEV_CAP_EVENT_PREFETCH` and
`RTE_EVENT_DEV_CAP_EVENT_INTELLIGENT_PREFETCH` via the event device info
function `info.event_dev_cap`.

Applications can select the prefetch type and configure it via
`rte_event_dev_config.prefetch_type` in the `rte_event_dev_configure` API.

The supported prefetch types are:
 * `RTE_EVENT_DEV_PREFETCH_NONE` - No prefetching.
 * `RTE_EVENT_DEV_PREFETCH` - Always issue a prefetch on dequeue.
 * `RTE_EVENT_DEV_PREFETCH_INTELLIGENT` - Delay issuing prefetch until
   there are no forward progress constraints with the held flow contexts.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
---
 app/test/test_eventdev.c                    | 62 +++++++++++++++++++++
 doc/guides/prog_guide/eventdev/eventdev.rst | 22 ++++++++
 lib/eventdev/rte_eventdev.h                 | 48 ++++++++++++++++
 3 files changed, 132 insertions(+)

diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
index e4e234dc98..1fd3d1fa69 100644
--- a/app/test/test_eventdev.c
+++ b/app/test/test_eventdev.c
@@ -1250,6 +1250,66 @@ test_eventdev_profile_switch(void)
 	return TEST_SUCCESS;
 }
 
+static int
+prefetch_test(rte_event_dev_prefetch_type_t prefetch_type, const char *prefetch_name)
+{
+#define NB_EVENTS     1024
+	uint64_t start, total;
+	struct rte_event ev;
+	int rc, cnt;
+
+	ev.event_type = RTE_EVENT_TYPE_CPU;
+	ev.queue_id = 0;
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.u64 = 0xBADF00D0;
+
+	for (cnt = 0; cnt < NB_EVENTS; cnt++) {
+		ev.flow_id = cnt;
+		rc = rte_event_enqueue_burst(TEST_DEV_ID, 0, &ev, 1);
+		TEST_ASSERT(rc == 1, "Failed to enqueue event");
+	}
+
+	RTE_SET_USED(prefetch_type);
+	total = 0;
+	while (cnt) {
+		start = rte_rdtsc_precise();
+		rc = rte_event_dequeue_burst(TEST_DEV_ID, 0, &ev, 1, 0);
+		if (rc) {
+			total += rte_rdtsc_precise() - start;
+			cnt--;
+		}
+	}
+	printf("Prefetch type : %s, avg cycles %" PRIu64 "\n", prefetch_name, total / NB_EVENTS);
+
+	return TEST_SUCCESS;
+}
+
+static int
+test_eventdev_prefetch_configure(void)
+{
+	struct rte_event_dev_config dev_conf;
+	struct rte_event_dev_info info;
+	int rc;
+
+	rte_event_dev_info_get(TEST_DEV_ID, &info);
+
+	if ((info.event_dev_cap & RTE_EVENT_DEV_CAP_EVENT_PREFETCH) == 0)
+		return TEST_SKIPPED;
+
+	devconf_set_default_sane_values(&dev_conf, &info);
+	dev_conf.prefetch_type = RTE_EVENT_DEV_PREFETCH;
+	rc = rte_event_dev_configure(TEST_DEV_ID, &dev_conf);
+	TEST_ASSERT_SUCCESS(rc, "Failed to configure eventdev");
+
+	rc = prefetch_test(RTE_EVENT_DEV_PREFETCH_NONE, "RTE_EVENT_DEV_PREFETCH_NONE");
+	rc |= prefetch_test(RTE_EVENT_DEV_PREFETCH, "RTE_EVENT_DEV_PREFETCH");
+	if (info.event_dev_cap & RTE_EVENT_DEV_CAP_EVENT_INTELLIGENT_PREFETCH)
+		rc |= prefetch_test(RTE_EVENT_DEV_PREFETCH_INTELLIGENT,
+				    "RTE_EVENT_DEV_PREFETCH_INTELLIGENT");
+
+	return rc;
+}
+
 static int
 test_eventdev_close(void)
 {
@@ -1310,6 +1370,8 @@ static struct unit_test_suite eventdev_common_testsuite  = {
 			test_eventdev_start_stop),
 		TEST_CASE_ST(eventdev_configure_setup, eventdev_stop_device,
 			test_eventdev_profile_switch),
+		TEST_CASE_ST(eventdev_configure_setup, NULL,
+			test_eventdev_prefetch_configure),
 		TEST_CASE_ST(eventdev_setup_device, eventdev_stop_device,
 			test_eventdev_link),
 		TEST_CASE_ST(eventdev_setup_device, eventdev_stop_device,
diff --git a/doc/guides/prog_guide/eventdev/eventdev.rst b/doc/guides/prog_guide/eventdev/eventdev.rst
index fb6dfce102..89064883b7 100644
--- a/doc/guides/prog_guide/eventdev/eventdev.rst
+++ b/doc/guides/prog_guide/eventdev/eventdev.rst
@@ -357,6 +357,28 @@ Worker path:
        // Process the event received.
    }
 
+Event Prefetching
+~~~~~~~~~~~~~~~~~
+
+Event prefetching enhances scheduling performance by pre-scheduling events to event ports
+when dequeues are issued.
+The `rte_event_dequeue_burst` operation initiates the prefetch operation, which completes
+in parallel without affecting the dequeued event flow contexts and dequeue latency.
+On the next dequeue operation, the prefetched events are dequeued and prefetch is initiated
+again.
+
+An application can use event prefetching if the event device supports it at either device
+level or at a individual port level.
+The application can check prefetch capability by checking if ``rte_event_dev_info.event_dev_cap``
+has the bit ``RTE_EVENT_DEV_CAP_PREFETCH`` set, if present prefetching can be enabled at device
+configuration time by setting appropriate prefetch type in ``rte_event_dev_config.prefetch``.
+
+Currently, the following prefetch types are supported:
+ * ``RTE_EVENT_DEV_PREFETCH_NONE`` - No prefetching.
+ * ``RTE_EVENT_DEV_PREFETCH`` - Always issue a prefetch when dequeue is issued.
+ * ``RTE_EVENT_DEV_PREFETCH_INTELLIGENT`` - Issue prefetch when dequeue is issued and there are
+   no forward progress constraints.
+
 Starting the EventDev
 ~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/lib/eventdev/rte_eventdev.h b/lib/eventdev/rte_eventdev.h
index 08e5f9320b..59c323c8ee 100644
--- a/lib/eventdev/rte_eventdev.h
+++ b/lib/eventdev/rte_eventdev.h
@@ -446,6 +446,30 @@ struct rte_event;
  * @see RTE_SCHED_TYPE_PARALLEL
  */
 
+#define RTE_EVENT_DEV_CAP_EVENT_PREFETCH (1ULL << 16)
+/**< Event device supports event prefetching.
+ *
+ * When this capability is available, the application can enable event prefetching on the event
+ * device to prefetch/pre-schedule events to a event port when `rte_event_dequeue_burst()`
+ * is issued.
+ * The prefetch process starts with the `rte_event_dequeue_burst()` call and the
+ * prefetched events are returned on the next `rte_event_dequeue_burst()` call.
+ *
+ * @see rte_event_dev_configure()
+ */
+
+#define RTE_EVENT_DEV_CAP_EVENT_INTELLIGENT_PREFETCH (1ULL << 17)
+/**< Event device supports intelligent event prefetching.
+ *
+ * When this capability is available, the application can enable intelligent prefetching
+ * on the event device where the events are prefetched/pre-scheduled when
+ * there are no forward progress constraints with the currently held flow contexts.
+ * The prefetch process starts with the `rte_event_dequeue_burst()` call and the
+ * prefetched events are returned on the next `rte_event_dequeue_burst()` call.
+ *
+ * @see rte_event_dev_configure()
+ */
+
 /* Event device priority levels */
 #define RTE_EVENT_DEV_PRIORITY_HIGHEST   0
 /**< Highest priority level for events and queues.
@@ -680,6 +704,25 @@ rte_event_dev_attr_get(uint8_t dev_id, uint32_t attr_id,
  *  @see rte_event_dequeue_timeout_ticks(), rte_event_dequeue_burst()
  */
 
+typedef enum {
+	RTE_EVENT_DEV_PREFETCH_NONE = 0,
+	/* Disable prefetch across the event device or on a given event port.
+	 * @ref rte_event_dev_config.prefetch_type
+	 */
+	RTE_EVENT_DEV_PREFETCH,
+	/* Enable prefetch always across the event device or a given event port.
+	 * @ref rte_event_dev_config.prefetch_type
+	 * @see RTE_EVENT_DEV_CAP_EVENT_PREFETCH
+	 */
+	RTE_EVENT_DEV_PREFETCH_INTELLIGENT,
+	/* Enable intelligent prefetch across the event device or a given event port.
+	 * Delay issuing prefetch until there are no forward progress constraints with
+	 * the held flow contexts.
+	 * @ref rte_event_dev_config.prefetch_type
+	 * @see RTE_EVENT_DEV_CAP_EVENT_INTELLIGENT_PREFETCH
+	 */
+} rte_event_dev_prefetch_type_t;
+
 /** Event device configuration structure */
 struct rte_event_dev_config {
 	uint32_t dequeue_timeout_ns;
@@ -752,6 +795,11 @@ struct rte_event_dev_config {
 	 * optimized for single-link usage, this field is a hint for how many
 	 * to allocate; otherwise, regular event ports and queues will be used.
 	 */
+	rte_event_dev_prefetch_type_t prefetch_type;
+	/**< Event prefetch type to use across the event device, if supported.
+	 * @see RTE_EVENT_DEV_CAP_EVENT_PREFETCH
+	 * @see RTE_EVENT_DEV_CAP_EVENT_INTELLIGENT_PREFETCH
+	 */
 };
 
 /**
-- 
2.25.1