[RFC 1/4] eventdev: add support for credit preallocation

["Mattias Rönnblom" <mattias.ronnblom@ericsson.com>]

Optionally split the enqueue operation for new events into two steps;
allocating a "slot" for the event in the event device, and the actual
enqueue operation.

Pre-allocating credits reduces the risk of enqueue failures (i.e.,
backpressure) for new events. This is useful for applications
performing expensive or effectively irreversible processing before the
enqueue operation. In such a scenario, efficiency may be improved and
code complexity reduced, in case the application can know ahead of
time, with some certainty, that the enqueue operation will succeed.

A new function rte_event_credit_alloc() is used to allocate credits.
A new function rte_event_credit_free() may be used, in case the
application decides to not use allocated credits.

A new operation type RTE_EVENT_NEW_PREALLOCED is added, which is the
equivalent to RTE_EVENT_NEW, only the event consumes one of the
pre-allocated credits when the event is successfully enqueued.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 lib/eventdev/eventdev_pmd.h          |   4 +
 lib/eventdev/eventdev_private.c      |  23 +++++
 lib/eventdev/eventdev_trace_points.c |   8 ++
 lib/eventdev/rte_eventdev.h          | 135 +++++++++++++++++++++++++++
 lib/eventdev/rte_eventdev_core.h     |  10 ++
 lib/eventdev/rte_eventdev_trace_fp.h |  19 ++++
 6 files changed, 199 insertions(+)

diff --git a/lib/eventdev/eventdev_pmd.h b/lib/eventdev/eventdev_pmd.h
index dda8ad82c9..84ec3ea555 100644
--- a/lib/eventdev/eventdev_pmd.h
+++ b/lib/eventdev/eventdev_pmd.h
@@ -172,6 +172,10 @@ struct __rte_cache_aligned rte_eventdev {
 	/**< Pointer to PMD dequeue burst function. */
 	event_maintain_t maintain;
 	/**< Pointer to PMD port maintenance function. */
+	event_credit_alloc_t credit_alloc;
+	/**< Pointer to PMD credit allocation function. */
+	event_credit_free_t credit_free;
+	/**< Pointer to PMD credit release function. */
 	event_tx_adapter_enqueue_t txa_enqueue_same_dest;
 	/**< Pointer to PMD eth Tx adapter burst enqueue function with
 	 * events destined to same Eth port & Tx queue.
diff --git a/lib/eventdev/eventdev_private.c b/lib/eventdev/eventdev_private.c
index dffd2c71d0..ec16125d83 100644
--- a/lib/eventdev/eventdev_private.c
+++ b/lib/eventdev/eventdev_private.c
@@ -34,6 +34,25 @@ dummy_event_maintain(__rte_unused void *port, __rte_unused int op)
 		"maintenance requested for unconfigured event device");
 }
 
+static int
+dummy_event_credit_alloc(__rte_unused void *port,
+			 __rte_unused unsigned int new_event_threshold,
+			 __rte_unused unsigned int num_credits)
+{
+	RTE_EDEV_LOG_ERR(
+		"credit allocation request for unconfigured event device");
+	return 0;
+}
+
+static int
+dummy_event_credit_free(__rte_unused void *port,
+			 __rte_unused unsigned int num_credits)
+{
+	RTE_EDEV_LOG_ERR(
+		"credit return request for unconfigured event device");
+	return 0;
+}
+
 static uint16_t
 dummy_event_tx_adapter_enqueue(__rte_unused void *port,
 			       __rte_unused struct rte_event ev[],
@@ -118,6 +137,8 @@ event_dev_fp_ops_reset(struct rte_event_fp_ops *fp_op)
 		.enqueue_forward_burst = dummy_event_enqueue_burst,
 		.dequeue_burst = dummy_event_dequeue_burst,
 		.maintain = dummy_event_maintain,
+		.credit_alloc = dummy_event_credit_alloc,
+		.credit_free = dummy_event_credit_free,
 		.txa_enqueue = dummy_event_tx_adapter_enqueue,
 		.txa_enqueue_same_dest = dummy_event_tx_adapter_enqueue_same_dest,
 		.ca_enqueue = dummy_event_crypto_adapter_enqueue,
@@ -141,6 +162,8 @@ event_dev_fp_ops_set(struct rte_event_fp_ops *fp_op,
 	fp_op->enqueue_forward_burst = dev->enqueue_forward_burst;
 	fp_op->dequeue_burst = dev->dequeue_burst;
 	fp_op->maintain = dev->maintain;
+	fp_op->credit_alloc = dev->credit_alloc;
+	fp_op->credit_free = dev->credit_free;
 	fp_op->txa_enqueue = dev->txa_enqueue;
 	fp_op->txa_enqueue_same_dest = dev->txa_enqueue_same_dest;
 	fp_op->ca_enqueue = dev->ca_enqueue;
diff --git a/lib/eventdev/eventdev_trace_points.c b/lib/eventdev/eventdev_trace_points.c
index ade6723b7b..c563f5cab1 100644
--- a/lib/eventdev/eventdev_trace_points.c
+++ b/lib/eventdev/eventdev_trace_points.c
@@ -50,6 +50,14 @@ RTE_EXPORT_SYMBOL(__rte_eventdev_trace_maintain)
 RTE_TRACE_POINT_REGISTER(rte_eventdev_trace_maintain,
 	lib.eventdev.maintain)
 
+RTE_EXPORT_SYMBOL(__rte_eventdev_credit_alloc)
+RTE_TRACE_POINT_REGISTER(rte_eventdev_trace_credit_alloc,
+	lib.eventdev.credit_alloc)
+
+RTE_EXPORT_SYMBOL(__rte_eventdev_credit_free)
+RTE_TRACE_POINT_REGISTER(rte_eventdev_trace_credit_free,
+	lib.eventdev.credit_free)
+
 RTE_EXPORT_EXPERIMENTAL_SYMBOL(__rte_eventdev_trace_port_profile_switch, 23.11)
 RTE_TRACE_POINT_REGISTER(rte_eventdev_trace_port_profile_switch,
 	lib.eventdev.port.profile.switch)
diff --git a/lib/eventdev/rte_eventdev.h b/lib/eventdev/rte_eventdev.h
index 3c7fcbf0be..812ed2705c 100644
--- a/lib/eventdev/rte_eventdev.h
+++ b/lib/eventdev/rte_eventdev.h
@@ -504,6 +504,16 @@ struct rte_event;
  * @see rte_event_port_preschedule()
  */
 
+#define RTE_EVENT_DEV_CAP_CREDIT_PREALLOCATION (1ULL << 21)
+/**< Event device supports credit preallocation for new events.
+ *
+ * The event device supports preallocation credits, which in turn allows
+ * the use of @ref RTE_EVENT_OP_NEW_PREALLOCED.
+ *
+ * @see rte_event_credit_alloc()
+ * @see rte_event_credit_free()
+ */
+
 /* Event device priority levels */
 #define RTE_EVENT_DEV_PRIORITY_HIGHEST   0
 /**< Highest priority level for events and queues.
@@ -1621,6 +1631,10 @@ struct __rte_aligned(16) rte_event_vector {
  * i.e. one not previously dequeued, into the event device, to be scheduled
  * for processing.
  */
+#define RTE_EVENT_OP_NEW_PREALLOCED     3
+/**< The @ref rte_event.op field must be set to this operation type to inject a new event
+ * for which a credit has already been allocated with rte_event_credit_alloc().
+ */
 #define RTE_EVENT_OP_FORWARD            1
 /**< The application must set the @ref rte_event.op field to this operation type to return a
  * previously dequeued event to the event device to be scheduled for further processing.
@@ -2933,6 +2947,127 @@ rte_event_maintain(uint8_t dev_id, uint8_t port_id, int op)
 	return 0;
 }
 
+/**
+ * Preallocate credits for new events.
+ *
+ * Preallocate credits for use with @ref RTE_EVENT_OP_NEW_PREALLOCED
+ * events. One credit gives the right to enqueue one such event. Upon
+ * successfully enqueuing an @ref RTE_EVENT_OP_NEW_PREALLOCED type
+ * event, one credit is considered spent.
+ *
+ * The credits are tied to the event port from which it was allocated.
+ * Thus, credit allocation and the enqueue operation must happen on
+ * the same port.
+ *
+ * The use of preallocated credits reduces the risk of enqueue
+ * failures, but does not guarantee that such will not occur.
+ *
+ * Besides using up credits by enqueuing @ref RTE_EVENT_OP_NEW_PREALLOCAD
+ * events, the application may also return credits using
+ * rte_event_credit_free().
+ *
+ * rte_event_credit_alloc() may also be used to pick a different @c
+ * new_event_threshold than is configured on the event port.
+ *
+ * This function will only succeed for event devices which have the
+ * @ref RTE_EVENT_DEV_CAP_CREDIT_PREALLOCATION flag set.
+ *
+ * The application may not attempt to enqueue @ref RTE_EVENT_OP_NEW_PREALLOCED
+ * events if it does not possess any credits (for that event port).
+ *
+ * Since credits are allocated against a certain @c new_event_threshold,
+ * and the number of in-flight events may change quickly (i.e., with
+ * an incoming burst of packets), storing credits for long durations
+ * of time may impact desired backpressure behavior. Also, since a
+ * credit represent an in-flight event, allocated but unused credits
+ * reduces the number of actual in-flight events allowed.
+ *
+ * @param dev_id
+ *   The identifier of the device.
+ * @param port_id
+ *   The identifier of the event port.
+ * @param new_event_threshold
+ *   The @c new_event_threshold to use for this allocation. If set to 0, the
+ *   event port's @c new_event_threshold will be used.
+ * @param num_credits
+ *   The number of credits the application wish to acquire.
+ * @return
+ *  - The number of credits allocated (<= @c num_credits).
+ *  - -EINVAL if *dev_id*,  *port_id*, or *op* is invalid.
+ *  - -ENOTSUP if event device does not support credit preallocation.
+ *
+ * @see RTE_EVENT_DEV_CAP_CREDIT_PREALLOCATION
+ */
+static inline int
+rte_event_credit_alloc(uint8_t dev_id, uint8_t port_id, unsigned int new_event_threshold,
+		       unsigned int num_credits)
+{
+	const struct rte_event_fp_ops *fp_ops;
+	void *port;
+
+	fp_ops = &rte_event_fp_ops[dev_id];
+	port = fp_ops->data[port_id];
+#ifdef RTE_LIBRTE_EVENTDEV_DEBUG
+	if (dev_id >= RTE_EVENT_MAX_DEVS ||
+	    port_id >= RTE_EVENT_MAX_PORTS_PER_DEV)
+		return -EINVAL;
+
+	if (port == NULL)
+		return -EINVAL;
+#endif
+	rte_eventdev_trace_credit_alloc(dev_id, port_id, new_event_threshold, num_credits);
+
+	if (unlikely(fp_ops->credit_alloc == NULL))
+		return -ENOTSUP;
+
+	return fp_ops->credit_alloc(port, new_event_threshold, num_credits);
+}
+
+/**
+ * Return preallocated credits for new events.
+ *
+ * Return unused credits allocated with rte_event_credit_alloc().
+ *
+ * This function will only succeed for event devices which have the
+ * @ref RTE_EVENT_DEV_CAP_CREDIT_PREALLOCATION flag set.
+ *
+ * @param dev_id
+ *   The identifier of the device.
+ * @param port_id
+ *   The identifier of the event port.
+ * @param num_credits
+ *   The number of credits the application wish to return.
+ * @return
+ *  - 0 on success.
+ *  - -EINVAL if *dev_id*,  *port_id*, or *op* is invalid.
+ *  - -ENOTSUP if event device does not support credit preallocation.
+ *
+ * @see RTE_EVENT_DEV_CAP_CREDIT_PREALLOCATION
+ */
+static inline int
+rte_event_credit_free(uint8_t dev_id, uint8_t port_id, unsigned int num_credits)
+{
+	const struct rte_event_fp_ops *fp_ops;
+	void *port;
+
+	fp_ops = &rte_event_fp_ops[dev_id];
+	port = fp_ops->data[port_id];
+#ifdef RTE_LIBRTE_EVENTDEV_DEBUG
+	if (dev_id >= RTE_EVENT_MAX_DEVS ||
+	    port_id >= RTE_EVENT_MAX_PORTS_PER_DEV)
+		return -EINVAL;
+
+	if (port == NULL)
+		return -EINVAL;
+#endif
+	rte_eventdev_trace_credit_free(dev_id, port_id, num_credits);
+
+	if (unlikely(fp_ops->credit_free == NULL))
+		return -ENOTSUP;
+
+	return fp_ops->credit_free(port, num_credits);
+}
+
 /**
  * Change the active profile on an event port.
  *
diff --git a/lib/eventdev/rte_eventdev_core.h b/lib/eventdev/rte_eventdev_core.h
index 1818483044..57a3ff4b67 100644
--- a/lib/eventdev/rte_eventdev_core.h
+++ b/lib/eventdev/rte_eventdev_core.h
@@ -25,6 +25,12 @@ typedef uint16_t (*event_dequeue_burst_t)(void *port, struct rte_event ev[],
 typedef void (*event_maintain_t)(void *port, int op);
 /**< @internal Maintains a port */
 
+typedef int (*event_credit_alloc_t)(void *port, unsigned int new_event_threshold, unsigned int num_credits);
+/**< @internal Allocates credits for new events */
+
+typedef int (*event_credit_free_t)(void *port, unsigned int num_credits);
+/**< @internal Returns credits for new events */
+
 typedef uint16_t (*event_tx_adapter_enqueue_t)(void *port,
 					       struct rte_event ev[],
 					       uint16_t nb_events);
@@ -63,6 +69,10 @@ struct __rte_cache_aligned rte_event_fp_ops {
 	/**< PMD dequeue burst function. */
 	event_maintain_t maintain;
 	/**< PMD port maintenance function. */
+	event_credit_alloc_t credit_alloc;
+	/**< PMD credit pre-allocation function. */
+	event_credit_free_t credit_free;
+	/**< PMD credit return function. */
 	event_tx_adapter_enqueue_t txa_enqueue;
 	/**< PMD Tx adapter enqueue function. */
 	event_tx_adapter_enqueue_t txa_enqueue_same_dest;
diff --git a/lib/eventdev/rte_eventdev_trace_fp.h b/lib/eventdev/rte_eventdev_trace_fp.h
index 8b794d1362..9a4b4f3f77 100644
--- a/lib/eventdev/rte_eventdev_trace_fp.h
+++ b/lib/eventdev/rte_eventdev_trace_fp.h
@@ -46,6 +46,25 @@ RTE_TRACE_POINT_FP(
 	rte_trace_point_emit_int(op);
 )
 
+RTE_TRACE_POINT_FP(
+	rte_eventdev_trace_credit_alloc,
+	RTE_TRACE_POINT_ARGS(uint8_t dev_id, uint8_t port_id,
+		unsigned int new_event_threshold, unsigned int num_credits),
+	rte_trace_point_emit_u8(dev_id);
+	rte_trace_point_emit_u8(port_id);
+	rte_trace_point_emit_int(new_event_threshold);
+	rte_trace_point_emit_int(num_credits);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_eventdev_trace_credit_free,
+	RTE_TRACE_POINT_ARGS(uint8_t dev_id, uint8_t port_id,
+		unsigned int num_credits),
+	rte_trace_point_emit_u8(dev_id);
+	rte_trace_point_emit_u8(port_id);
+	rte_trace_point_emit_int(num_credits);
+)
+
 RTE_TRACE_POINT_FP(
 	rte_eventdev_trace_port_profile_switch,
 	RTE_TRACE_POINT_ARGS(uint8_t dev_id, uint8_t port_id, uint8_t profile),
-- 
2.43.0