[dpdk-dev] [RFC] eventdev: event tx adapter APIs

[dpdk-dev] [RFC] eventdev: event tx adapter APIs

[Nikhil Rao]

The patch below introduces the event tx adapter APIs that encapsulate common
code for the tx stage of a packet processing pipeline. These APIs allow
the application to configure a rte_service function that reads mbuf events
from an event queue and sends mbufs on the transmit queue and ethernet
port specified in the mbuf private area.

Signed-off-by: Nikhil Rao <nikhil.rao@intel.com>
---
 lib/librte_eventdev/rte_event_eth_tx_adapter.h | 272 +++++++++++++++++++++++++
 1 file changed, 272 insertions(+)
 create mode 100644 lib/librte_eventdev/rte_event_eth_tx_adapter.h

diff --git a/lib/librte_eventdev/rte_event_eth_tx_adapter.h b/lib/librte_eventdev/rte_event_eth_tx_adapter.h
new file mode 100644
index 0000000..56218fe
--- /dev/null
+++ b/lib/librte_eventdev/rte_event_eth_tx_adapter.h
@@ -0,0 +1,272 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation.
+ */
+
+#ifndef _RTE_EVENT_ETH_TX_ADAPTER_
+#define _RTE_EVENT_ETH_TX_ADAPTER_
+
+/**
+ * @file
+ *
+ * RTE Event Ethernet Tx Adapter
+ *
+ * The event ethernet Tx adapter provides the transmit stage of a
+ * event driven packet processing application. It dequeues mbuf events from
+ * an event queue and sends mbufs on a ethernet device.
+ *
+ * The ethernet Tx event adapter's functions are:
+ *  - rte_event_eth_tx_adapter_create()
+ *  - rte_event_eth_tx_adapter_free()
+ *  - rte_event_eth_tx_adapter_start()
+ *  - rte_event_eth_tx_adapter_stop()
+ *  - rte_event_eth_tx_adapter_queue_start()
+ *  - rte_event_eth_tx_adapter_queue_stop()
+ *  - rte_event_eth_tx_adapter_stats_get()
+ *  - rte_event_eth_tx_adapter_stats_reset()
+ *
+ * The application creates the adapter using
+ * rte_event_eth_tx_adapter_create(). The application is required to
+ * have linked a queue to the event port specified in this call.
+ *
+ * The application sends mbufs to the adapter via this event queue. The
+ * ethernet port and transmit queue index to send the mbuf on are specified
+ * in the mbuf private area and are accessed using
+ * struct rte_event_eth_tx_adapter_meta. The offset to this structure within
+ * the private area is provided when creating the adapter.
+ *
+ * The application can start and stop the adapter using the
+ * rte_event_eth_tx_adapter_start/stop() calls.
+ *
+ * To support dynamic reconfiguration of Tx queues, the application can
+ * call rte_event_eth_tx_adapter_queue_start()/stop() to synchronize
+ * access to the Tx queue with the adapter. For example, if the application
+ * wants to reconfigure a Tx queue that could be concurrently
+ * being accessed by the adapter, it calls rte_event_eth_tx_adapter_queue_stop()
+ * first, reconfigures the queue and then calls
+ * rte_event_eth_tx_adapter_queue_start() which signals to the adapter
+ * that it is safe to resume access to the Tx queue.
+ *
+ * The adapter uses an EAL service function and its execution is controlled
+ * using the rte_service APIs. The rte_event_eth_tx_adapter_service_id_get()
+ * function can be used to retrieve the adapter's service function ID.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+#include "rte_eventdev.h"
+
+#define RTE_EVENT_ETH_TX_ADAPTER_MAX_INSTANCE 32
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Adapter configuration structure
+ */
+struct rte_event_eth_tx_adapter_conf {
+	uint8_t event_port_id;
+	/**< Event port identifier, the adapter dequeues mbuf events from this
+	 * port.
+	 */
+	uint16_t tx_metadata_off;
+	/**<  Offset of struct rte_event_eth_tx_adapter_meta in the private
+	 * area of the mbuf
+	 */
+	uint32_t max_nb_tx;
+	/**< The adapter can return early if it has processed at least
+	 * max_nb_tx mbufs. This isn't treated as a requirement; batching may
+	 * cause the adapter to process more than max_nb_tx mbufs.
+	 */
+};
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * A structure used to retrieve statistics for an eth tx adapter instance.
+ */
+struct rte_event_eth_tx_adapter_stats {
+	uint64_t event_poll_count;
+	/*< Event port poll count */
+	uint64_t tx_packets;
+	/*< Number of packets transmitted */
+	uint64_t tx_dropped;
+	/*< Number of packets dropped */
+};
+
+/*
+ * Tx adapter metadata used to specifiy the ethernet port id and queue id to be
+ * used for transmitting the mbuf
+ */
+struct rte_event_eth_tx_adapter_meta {
+	uint16_t eth_dev_id;
+	/**< Port identifier of the ethernet device */
+	uint16_t eth_queue_id;
+	/**< Transmit queue index */
+};
+	
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Create a new event ethernet Tx adapter with the specified identifier.
+ *
+ * @param id
+ *  The identifier of the event ethernet Tx adapter.
+ * @param dev_id
+ *  The event device identifier.
+ * @param conf
+ * Tx adapter configuration.
+ * @return
+ *   - 0: Success
+ *   - <0: Error code on failure
+ */
+int rte_event_eth_tx_adapter_create(uint8_t id, uint8_t dev_id,
+				struct rte_event_eth_tx_adapter_conf *conf);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Free an event adapter
+ *
+ * @param id
+ *  Adapter identifier.
+ * @return
+ *   - 0: Success
+ *   - <0: Error code on failure, If the adapter still has Tx queues
+ *      added to it, the function returns -EBUSY.
+ */
+int rte_event_eth_tx_adapter_free(uint8_t id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Start ethernet Tx event adapter
+ *
+ * @param id
+ *  Adapter identifier.
+ * @return
+ *  - 0: Success, Adapter started correctly.
+ *  - <0: Error code on failure.
+ */
+int rte_event_eth_tx_adapter_start(uint8_t id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Stop ethernet Tx event adapter
+ *
+ * @param id
+ *  Adapter identifier.
+ * @return
+ *  - 0: Success, Adapter started correctly.
+ *  - <0: Error code on failure.
+ */
+int rte_event_eth_tx_adapter_stop(uint8_t id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Signal the Tx adapter to start processing mbufs for a
+ * Tx queue. A queue value of -1 is used to indicate all
+ * queues within the device.
+ *
+ * @param id
+ *  Adapter identifier.
+ * @param eth_dev_id
+ *  Ethernet Port Identifier.
+ * @param queue
+ *  Tx queue index.
+ * @return
+ *  - 0: Success, Adapter started correctly.
+ *  - <0: Error code on failure.
+ */
+int rte_event_eth_tx_adapter_queue_start(uint8_t id, 
+					uint16_t eth_dev_id,
+					int32_t queue);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Signal the Tx adapter to stop processing mbufs for a
+ * Tx queue. A queue value of -1 is used to indicate all
+ * queues within the device.
+ *
+ * @param id
+ *  Adapter identifier.
+ * @param eth_dev_id
+ *  Ethernet Port Identifier.
+ * @param queue
+ *  Tx queue index.
+ * @return
+ *  - 0: Success, Adapter started correctly.
+ *  - <0: Error code on failure.
+ */
+int rte_event_eth_tx_adapter_queue_stop(uint8_t id, 
+					uint16_t eth_dev_id,
+					int32_t queue);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Retrieve statistics for an adapter
+ *
+ * @param id
+ *  Adapter identifier.
+ * @param [out] stats
+ *  A pointer to structure used to retrieve statistics for an adapter.
+ * @return
+ *  - 0: Success, retrieved successfully.
+ *  - <0: Error code on failure.
+ */
+int rte_event_eth_tx_adapter_stats_get(uint8_t id,
+				struct rte_event_eth_tx_adapter_stats *stats);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Reset statistics for an adapter.
+ *
+ * @param id
+ *  Adapter identifier.
+ * @return
+ *  - 0: Success, statistics reset successfully.
+ *  - <0: Error code on failure.
+ */
+int rte_event_eth_tx_adapter_stats_reset(uint8_t id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Retrieve the service ID of an adapter. If the adapter doesn't use
+ * a rte_service function, this function returns -ESRCH.
+ *
+ * @param id
+ *  Adapter identifier.
+ * @param [out] service_id
+ *  A pointer to a uint32_t, to be filled in with the service id.
+ * @return
+ *  - 0: Success
+ *  - <0: Error code on failure, if the adapter doesn't use a rte_service
+ * function, this function returns -ESRCH.
+ */
+int rte_event_eth_tx_adapter_service_id_get(uint8_t id, uint32_t *service_id);
+
+#ifdef __cplusplus
+}
+#endif
+#endif	/* _RTE_EVENT_ETH_TX_ADAPTER_ */
-- 
1.8.3.1

Re: [dpdk-dev] [RFC] eventdev: event tx adapter APIs

[Jerin Jacob]

-----Original Message-----
> Date: Fri, 25 May 2018 20:38:44 +0530
> From: Nikhil Rao <nikhil.rao@intel.com>
> To: jerin.jacob@caviumnetworks.com, hemant.agrawal@nxp.com
> CC: dev@dpdk.org, narender.vangati@intel.com, abhinandan.gujjar@intel.com,
>  gage.eads@intel.com, Nikhil Rao <nikhil.rao@intel.com>
> Subject: [RFC] eventdev: event tx adapter APIs
> X-Mailer: git-send-email 1.8.3.1
> 
> The patch below introduces the event tx adapter APIs that encapsulate common
> code for the tx stage of a packet processing pipeline. These APIs allow
> the application to configure a rte_service function that reads mbuf events
> from an event queue and sends mbufs on the transmit queue and ethernet
> port specified in the mbuf private area.
> 

Hi Nikhil,

I think, it is reasonable to have Tx adapter.

Some top level comments to starts with,

1) Slow path API looks fine. The only comment is, How about changing
rte_event_eth_tx_adapter_queue_add() instead of rte_event_eth_tx_adapter_queue_start()
to get align  with Rx adapter?

2) This my understanding of fastpath

a) Common code will have a extra port(given through adapter create)
where all workers invoke rte_event_enqueue_burst() to that port and then common code 
dequeue from that port and send packet
using rte_eth_tx_burst() and/or using tx buffering APIs

b) If the source queue or sched_type is ORDERED, When it enqueue to the extra port it
will change to atomic/direct to maintain the the ingress order if need.

Couple of issues and a minor change in RFC to fix the issues as a proposal.

Issue 1) I think, Using mbuf private data for such purpose will be a problem as
exiting application already may using for it own needs and there will be additional cache
miss etc on fastpath to get private data.

Issue 2) At least on Cavium, We can do so optimization by introducing
the same RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT schematics of Rx
adapter. i.e we don't need extra port to converge all traffic from
different workers and transmit.


Considering above two issues2, We would like propose an fastpath API,
which semantics is almost same rte_ethdev_tx_burst().
a) It will get ride of mbuf metadata need
b) New fastpath API gives driver enough possibilities of optimization if
possible.(address the issue (2))

The API can be like,

uint16_t rte_event_eth_tx_adapter_enqueue(uint16_t eth_port_id, uint16_t
eth_queue_id, uint8_t event_port_id, const struct rte_event ev[], uint16_t nb_events);

This API would land in driver as function pointer if device has
RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT capability.

If device does not have !RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT then 
common code can choose to ignore event_port_id in
rte_event_eth_tx_adapter_enqueue() and select the port given in adapter
create in common code.

So from the top-level, each worker lcore will look like

while(1)
{
	rte_event_dequeue_burst(..,worker[port],..);
	(process the packets)
	rte_event_eth_tx_adapter_enqueue(... worker[port],...);
}

Thoughts?

Re: [dpdk-dev] [RFC] eventdev: event tx adapter APIs

[Rao, Nikhil]

Hi Jerin,

On 5/30/2018 12:56 PM, Jerin Jacob wrote:
> -----Original Message-----
> Hi Nikhil,
> I think, it is reasonable to have Tx adapter.
>
> Some top level comments to starts with,
>
> 1) Slow path API looks fine. The only comment is, How about changing
> rte_event_eth_tx_adapter_queue_add() instead of rte_event_eth_tx_adapter_queue_start()
> to get align  with Rx adapter?
OK.
>
> 2) This my understanding of fastpath
>
> a) Common code will have a extra port(given through adapter create)
> where all workers invoke rte_event_enqueue_burst() to that port and then common code
> dequeue from that port and send packet
> using rte_eth_tx_burst() and/or using tx buffering APIs
The workers invoke rte_event_enqueue_burst() to their local port not to 
the extra port as you described. The queue ID specified when
enqueuing is linked to the the adapter's port, the adapter reads these 
events and transmits mbufs on the
ethernet port and queue specified in these mbufs. The diagram below 
illustrates what I just described.

+------+
|      |   +----+
|Worker+-->+port+--+
|      |   +----+  |                                         +----+
+------+           |                                     +-->+eth0|
                    |  +---------+            +-------+   |   +----+
                    +--+         |   +----+   |       +---+   +----+
                       |  Queue  +-->+port+-->+Adapter|------>+eth1|
                    +--+         |   +----+   |       +---+   +----+
+------+           |  +---------+            +-------+   |   +----+
|      |   +----+  |                                     +-->+eth2|
|Worker+-->+port+--+                                         +----+
|      |   +----+
+------+

> b) If the source queue or sched_type is ORDERED, When it enqueue to the extra port it
> will change to atomic/direct to maintain the the ingress order if need.
>
> Couple of issues and a minor change in RFC to fix the issues as a proposal.
>
> Issue 1) I think, Using mbuf private data for such purpose will be a problem as
> exiting application already may using for it own needs and there will be additional cache
> miss etc on fastpath to get private data.
Instead of using mbuf private data, the eth port and queue can be 
specified in
the mbuf itself using mbuf:port and mbuf:hash respectively.

> Issue 2) At least on Cavium, We can do so optimization by introducing
> the same RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT schematics of Rx
> adapter. i.e we don't need extra port to converge all traffic from
> different workers and transmit.
OK.
>
> Considering above two issues2, We would like propose an fastpath API,
> which semantics is almost same rte_ethdev_tx_burst().
> a) It will get ride of mbuf metadata need
> b) New fastpath API gives driver enough possibilities of optimization if
> possible.(address the issue (2))
>
> The API can be like,
>
> uint16_t rte_event_eth_tx_adapter_enqueue(uint16_t eth_port_id, uint16_t
> eth_queue_id, uint8_t event_port_id, const struct rte_event ev[], uint16_t nb_events);
The worker core will receive events pointing to mbufs that need to be 
transmitted to different
ports/queues, as described above. The port and the queue will be 
populated in the mbuf and the
API can be as below

uint16_t rte_event_eth_tx_adapter_enqueue(uint8_t instance_id, uint8_t event_port_id, const struct rte_event ev[], uint16_t nb_events);

Let me know if that works for you.

> This API would land in driver as function pointer if device has
> RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT capability.
Agreed.
> If device does not have !RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT then
> common code can choose to ignore event_port_id in
> rte_event_eth_tx_adapter_enqueue() and select the port given in adapter
> create in common code.
The common code will still use the local worker port.
> So from the top-level, each worker lcore will look like
>
> while(1)
> {
> 	rte_event_dequeue_burst(..,worker[port],..);
> 	(process the packets)
> 	rte_event_eth_tx_adapter_enqueue(... worker[port],...);
> }
Agreed, with the API modification above

Nikhil

Re: [dpdk-dev] [RFC] eventdev: event tx adapter APIs

[Jerin Jacob]

-----Original Message-----
> Date: Fri, 1 Jun 2018 23:47:00 +0530
> From: "Rao, Nikhil" <nikhil.rao@intel.com>
> To: Jerin Jacob <jerin.jacob@caviumnetworks.com>
> CC: hemant.agrawal@nxp.com, dev@dpdk.org, narender.vangati@intel.com,
>  abhinandan.gujjar@intel.com, gage.eads@intel.com, nikhil.rao@intel.com
> Subject: Re: [RFC] eventdev: event tx adapter APIs
> User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101
>  Thunderbird/52.8.0
> 
> 
> Hi Jerin,

Hi Nikhil,

> 
> On 5/30/2018 12:56 PM, Jerin Jacob wrote:
> > -----Original Message-----
> > Hi Nikhil,
> > I think, it is reasonable to have Tx adapter.
> > 
> > Some top level comments to starts with,
> > 
> > 1) Slow path API looks fine. The only comment is, How about changing
> > rte_event_eth_tx_adapter_queue_add() instead of rte_event_eth_tx_adapter_queue_start()
> > to get align  with Rx adapter?
> OK.
> > 
> > 2) This my understanding of fastpath
> > 
> > a) Common code will have a extra port(given through adapter create)
> > where all workers invoke rte_event_enqueue_burst() to that port and then common code
> > dequeue from that port and send packet
> > using rte_eth_tx_burst() and/or using tx buffering APIs
> The workers invoke rte_event_enqueue_burst() to their local port not to the
> extra port as you described. The queue ID specified when
> enqueuing is linked to the the adapter's port, the adapter reads these
> events and transmits mbufs on the
> ethernet port and queue specified in these mbufs. The diagram below
> illustrates what I just described.
> 
> +------+
> |      |   +----+
> |Worker+-->+port+--+
> |      |   +----+  |                                         +----+
> +------+           |                                     +-->+eth0|
>                    |  +---------+            +-------+   |   +----+
>                    +--+         |   +----+   |       +---+   +----+
>                       |  Queue  +-->+port+-->+Adapter|------>+eth1|
>                    +--+         |   +----+   |       +---+   +----+
> +------+           |  +---------+            +-------+   |   +----+
> |      |   +----+  |                                     +-->+eth2|
> |Worker+-->+port+--+                                         +----+
> |      |   +----+
> +------+


Makes sense. One suggestion here, Since we have ALL type queue and
normal queues, Can we move the queue change or sched_type change code
from the application and move that down to function pointer abstraction(any
way adapter knows which queues to enqueue for), that way we can have same
final stage code for ALL type queues and Normal queues.

> 
> > b) If the source queue or sched_type is ORDERED, When it enqueue to the extra port it
> > will change to atomic/direct to maintain the the ingress order if need.
> > 
> > Couple of issues and a minor change in RFC to fix the issues as a proposal.
> > 
> > Issue 1) I think, Using mbuf private data for such purpose will be a problem as
> > exiting application already may using for it own needs and there will be additional cache
> > miss etc on fastpath to get private data.
> Instead of using mbuf private data, the eth port and queue can be specified
> in
> the mbuf itself using mbuf:port and mbuf:hash respectively.


Looks good to me.

> 
> > Issue 2) At least on Cavium, We can do so optimization by introducing
> > the same RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT schematics of Rx
> > adapter. i.e we don't need extra port to converge all traffic from
> > different workers and transmit.
> OK.
> > 
> > Considering above two issues2, We would like propose an fastpath API,
> > which semantics is almost same rte_ethdev_tx_burst().
> > a) It will get ride of mbuf metadata need
> > b) New fastpath API gives driver enough possibilities of optimization if
> > possible.(address the issue (2))
> > 
> > The API can be like,
> > 
> > uint16_t rte_event_eth_tx_adapter_enqueue(uint16_t eth_port_id, uint16_t
> > eth_queue_id, uint8_t event_port_id, const struct rte_event ev[], uint16_t nb_events);
> The worker core will receive events pointing to mbufs that need to be
> transmitted to different
> ports/queues, as described above. The port and the queue will be populated
> in the mbuf and the
> API can be as below
> 
> uint16_t rte_event_eth_tx_adapter_enqueue(uint8_t instance_id, uint8_t event_port_id, const struct rte_event ev[], uint16_t nb_events);
> 
> Let me know if that works for you.

Yes. That API works for me. I think, we can leverage "struct
rte_eventdev" area for adding new function pointer. Just like 
enqueue_new_burst, enqueue_forward_burst variants, we can add one more
there, so that we can reuse that hot cacheline for all fastpath function pointer case.
That would translate to adding "uint8_t dev_id" on the above API.


> 
> > This API would land in driver as function pointer if device has
> > RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT capability.
> Agreed.
> > If device does not have !RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT then
> > common code can choose to ignore event_port_id in
> > rte_event_eth_tx_adapter_enqueue() and select the port given in adapter
> > create in common code.
> The common code will still use the local worker port.
> > So from the top-level, each worker lcore will look like
> > 
> > while(1)
> > {
> > 	rte_event_dequeue_burst(..,worker[port],..);
> > 	(process the packets)
> > 	rte_event_eth_tx_adapter_enqueue(... worker[port],...);
> > }
> Agreed, with the API modification above

Great.

> 
> Nikhil

Re: [dpdk-dev] [RFC] eventdev: event tx adapter APIs

[Rao, Nikhil]

On 6/4/2018 10:41 AM, Jerin Jacob wrote:
> -----Original Message-----
>> Date: Fri, 1 Jun 2018 23:47:00 +0530
>> From: "Rao, Nikhil" <nikhil.rao@intel.com>
>> To: Jerin Jacob <jerin.jacob@caviumnetworks.com>
>> CC: hemant.agrawal@nxp.com, dev@dpdk.org, narender.vangati@intel.com,
>>   abhinandan.gujjar@intel.com, gage.eads@intel.com, nikhil.rao@intel.com
>> Subject: Re: [RFC] eventdev: event tx adapter APIs
>> User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101
>>   Thunderbird/52.8.0
>>
>>
>> Hi Jerin,
> 
>
>> The workers invoke rte_event_enqueue_burst() to their local port not to the
>> extra port as you described. The queue ID specified when
>> enqueuing is linked to the the adapter's port, the adapter reads these
>> events and transmits mbufs on the
>> ethernet port and queue specified in these mbufs. The diagram below
>> illustrates what I just described.
>>
>> +------+
>> |      |   +----+
>> |Worker+-->+port+--+
>> |      |   +----+  |                                         +----+
>> +------+           |                                     +-->+eth0|
>>                     |  +---------+            +-------+   |   +----+
>>                     +--+         |   +----+   |       +---+   +----+
>>                        |  Queue  +-->+port+-->+Adapter|------>+eth1|
>>                     +--+         |   +----+   |       +---+   +----+
>> +------+           |  +---------+            +-------+   |   +----+
>> |      |   +----+  |                                     +-->+eth2|
>> |Worker+-->+port+--+                                         +----+
>> |      |   +----+
>> +------+
> 
> 
> Makes sense. One suggestion here, Since we have ALL type queue and
> normal queues, Can we move the queue change or sched_type change code
> from the application and move that down to function pointer abstraction(any
> way adapter knows which queues to enqueue for), that way we can have same
> final stage code for ALL type queues and Normal queues.
> 
Yes, I see the queue/sched type change approach followed in 
pipeline_worker_tx.c, a queue id can be provided in 
rte_event_eth_tx_adapter_conf

+struct rte_event_eth_tx_adapter_conf {
+	uint8_t event_port_id;
+	/**< Event port identifier, the adapter dequeues mbuf events from this
+	 * port.
+	 */
+	uint16_t tx_metadata_off;
+	/**<  Offset of struct rte_event_eth_tx_adapter_meta in the private
+	 * area of the mbuf
+	 */
+	uint32_t max_nb_tx;
+	/**< The adapter can return early if it has processed at least
+	 * max_nb_tx mbufs. This isn't treated as a requirement; batching may
+	 * cause the adapter to process more than max_nb_tx mbufs.
+	 */
+};

</sniped>

>> The worker core will receive events pointing to mbufs that need to be
>> transmitted to different
>> ports/queues, as described above. The port and the queue will be populated
>> in the mbuf and the
>> API can be as below
>>
>> uint16_t rte_event_eth_tx_adapter_enqueue(uint8_t instance_id, uint8_t event_port_id, const struct rte_event ev[], uint16_t nb_events);
>>
>> Let me know if that works for you.
> 
> Yes. That API works for me. I think, we can leverage "struct
> rte_eventdev" area for adding new function pointer. Just like
> enqueue_new_burst, enqueue_forward_burst variants, we can add one more
> there, so that we can reuse that hot cacheline for all fastpath function pointer case.
> That would translate to adding "uint8_t dev_id" on the above API.
The dev_id can be derived from the instance_id, does that work ?

I need some clarification on the configuration API/flow. The 
eventdev_pipeline sample app checks if DEV_TX_OFFLOAD_MT_LOCKFREE flag 
is set on all ethernet devices and if so, uses the pipeline_worker_tx 
path as opposed to the "consumer" function, if we were to use the 
adapter to replace some of the sample code then it seems like the 
RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT is hardware assist for the 
pipeline worker tx mode, the adapter would support 2 modes (consumer and 
worker_tx, borrowing terminology from the sample), worker_tx would only 
be supported if the eventdev supports 
RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT (at least in the first version)

Thanks,
Nikhil

Re: [dpdk-dev] [RFC] eventdev: event tx adapter APIs

[Jerin Jacob]

-----Original Message-----
> Date: Tue, 5 Jun 2018 14:54:58 +0530
> From: "Rao, Nikhil" <nikhil.rao@intel.com>
> To: Jerin Jacob <jerin.jacob@caviumnetworks.com>
> CC: hemant.agrawal@nxp.com, dev@dpdk.org, narender.vangati@intel.com,
>  abhinandan.gujjar@intel.com, gage.eads@intel.com, nikhil.rao@intel.com
> Subject: Re: [RFC] eventdev: event tx adapter APIs
> User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101
>  Thunderbird/52.8.0
> 
> On 6/4/2018 10:41 AM, Jerin Jacob wrote:
> > -----Original Message-----
> > > Date: Fri, 1 Jun 2018 23:47:00 +0530
> > > From: "Rao, Nikhil" <nikhil.rao@intel.com>
> > > To: Jerin Jacob <jerin.jacob@caviumnetworks.com>
> > > CC: hemant.agrawal@nxp.com, dev@dpdk.org, narender.vangati@intel.com,
> > >   abhinandan.gujjar@intel.com, gage.eads@intel.com, nikhil.rao@intel.com
> > > Subject: Re: [RFC] eventdev: event tx adapter APIs
> > > User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101
> > >   Thunderbird/52.8.0
> > > 
> > > 
> > > Hi Jerin,
> > 
> > 
> > > The workers invoke rte_event_enqueue_burst() to their local port not to the
> > > extra port as you described. The queue ID specified when
> > > enqueuing is linked to the the adapter's port, the adapter reads these
> > > events and transmits mbufs on the
> > > ethernet port and queue specified in these mbufs. The diagram below
> > > illustrates what I just described.
> > > 
> > > +------+
> > > |      |   +----+
> > > |Worker+-->+port+--+
> > > |      |   +----+  |                                         +----+
> > > +------+           |                                     +-->+eth0|
> > >                     |  +---------+            +-------+   |   +----+
> > >                     +--+         |   +----+   |       +---+   +----+
> > >                        |  Queue  +-->+port+-->+Adapter|------>+eth1|
> > >                     +--+         |   +----+   |       +---+   +----+
> > > +------+           |  +---------+            +-------+   |   +----+
> > > |      |   +----+  |                                     +-->+eth2|
> > > |Worker+-->+port+--+                                         +----+
> > > |      |   +----+
> > > +------+
> > 
> > 
> > Makes sense. One suggestion here, Since we have ALL type queue and
> > normal queues, Can we move the queue change or sched_type change code
> > from the application and move that down to function pointer abstraction(any
> > way adapter knows which queues to enqueue for), that way we can have same
> > final stage code for ALL type queues and Normal queues.
> > 
> Yes, I see the queue/sched type change approach followed in
> pipeline_worker_tx.c, a queue id can be provided in
> rte_event_eth_tx_adapter_conf
> 
> +struct rte_event_eth_tx_adapter_conf {
> +	uint8_t event_port_id;
> +	/**< Event port identifier, the adapter dequeues mbuf events from this
> +	 * port.
> +	 */
> +	uint16_t tx_metadata_off;
> +	/**<  Offset of struct rte_event_eth_tx_adapter_meta in the private
> +	 * area of the mbuf
> +	 */
> +	uint32_t max_nb_tx;
> +	/**< The adapter can return early if it has processed at least
> +	 * max_nb_tx mbufs. This isn't treated as a requirement; batching may
> +	 * cause the adapter to process more than max_nb_tx mbufs.
> +	 */
> +};
> 
> </sniped>
> 
> > > The worker core will receive events pointing to mbufs that need to be
> > > transmitted to different
> > > ports/queues, as described above. The port and the queue will be populated
> > > in the mbuf and the
> > > API can be as below
> > > 
> > > uint16_t rte_event_eth_tx_adapter_enqueue(uint8_t instance_id, uint8_t event_port_id, const struct rte_event ev[], uint16_t nb_events);
> > > 
> > > Let me know if that works for you.
> > 
> > Yes. That API works for me. I think, we can leverage "struct
> > rte_eventdev" area for adding new function pointer. Just like
> > enqueue_new_burst, enqueue_forward_burst variants, we can add one more
> > there, so that we can reuse that hot cacheline for all fastpath function pointer case.
> > That would translate to adding "uint8_t dev_id" on the above API.

> The dev_id can be derived from the instance_id, does that work ?

Do we need to that in fastpath?, IMO, if you can do that in slow path then it is fine.

> 
> I need some clarification on the configuration API/flow. The
> eventdev_pipeline sample app checks if DEV_TX_OFFLOAD_MT_LOCKFREE flag is
> set on all ethernet devices and if so, uses the pipeline_worker_tx path as
> opposed to the "consumer" function,

Yes.

> if we were to use the adapter to replace
> some of the sample code then it seems like the
> RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT is hardware assist for the
> pipeline worker tx mode, the adapter would support 2 modes (consumer and
> worker_tx, borrowing terminology from the sample), worker_tx would only be
> supported if the eventdev supports
> RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT (at least in the first version)

I think the flow can be

1) rte_event_eth_tx_adapter_enqueue() function should simply call,

struct rte_eventdev *dev = &rte_eventdev[dev_id];
return (*dev->eth_tx_adapter_enqueue)(...);

2) You can expose generic version of "eth_tx_adapter_enqueue" in Tx
adapter. If drivers does not set the "eth_tx_adapter_enqueue" function
pointer or DEV_TX_OFFLOAD_MT_LOCKFREE flag is NOT set on all ethernet devices
_then_ in common code we can assign "eth_tx_adapter_enqueue" function
pointer as your generic Tx adapter function pointer.

3) I think, you can focus only on generic "consumer" case as you can not
test "worker_tx" case. We are planning to add more optimized raw
"worker_tx" case in driver(Point 2 will allow that by having driver
specific "eth_tx_adapter_enqueue" function pointer).

/Jerin

> 
> Thanks,
> Nikhil
> 

Re: [dpdk-dev] [RFC] eventdev: event tx adapter APIs

[Jerin Jacob]

-----Original Message-----
> Date: Tue, 5 Jun 2018 14:54:58 +0530
> From: "Rao, Nikhil" <nikhil.rao@intel.com>
> To: Jerin Jacob <jerin.jacob@caviumnetworks.com>
> CC: hemant.agrawal@nxp.com, dev@dpdk.org, narender.vangati@intel.com,
>  abhinandan.gujjar@intel.com, gage.eads@intel.com, nikhil.rao@intel.com
> Subject: Re: [RFC] eventdev: event tx adapter APIs
> User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101
>  Thunderbird/52.8.0
> 
> On 6/4/2018 10:41 AM, Jerin Jacob wrote:
> > -----Original Message-----
> > > Date: Fri, 1 Jun 2018 23:47:00 +0530
> > > From: "Rao, Nikhil" <nikhil.rao@intel.com>
> > > To: Jerin Jacob <jerin.jacob@caviumnetworks.com>
> > > CC: hemant.agrawal@nxp.com, dev@dpdk.org, narender.vangati@intel.com,
> > >   abhinandan.gujjar@intel.com, gage.eads@intel.com, nikhil.rao@intel.com
> > > Subject: Re: [RFC] eventdev: event tx adapter APIs
> > > User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101
> > >   Thunderbird/52.8.0
> > > 
> > > 
> > > Hi Jerin,
> > 
> > 
> > > The workers invoke rte_event_enqueue_burst() to their local port not to the
> > > extra port as you described. The queue ID specified when
> > > enqueuing is linked to the the adapter's port, the adapter reads these
> > > events and transmits mbufs on the
> > > ethernet port and queue specified in these mbufs. The diagram below
> > > illustrates what I just described.
> > > 
> > > +------+
> > > |      |   +----+
> > > |Worker+-->+port+--+
> > > |      |   +----+  |                                         +----+
> > > +------+           |                                     +-->+eth0|
> > >                     |  +---------+            +-------+   |   +----+
> > >                     +--+         |   +----+   |       +---+   +----+
> > >                        |  Queue  +-->+port+-->+Adapter|------>+eth1|
> > >                     +--+         |   +----+   |       +---+   +----+
> > > +------+           |  +---------+            +-------+   |   +----+
> > > |      |   +----+  |                                     +-->+eth2|
> > > |Worker+-->+port+--+                                         +----+
> > > |      |   +----+
> > > +------+
> > 
> > 
> > Makes sense. One suggestion here, Since we have ALL type queue and
> > normal queues, Can we move the queue change or sched_type change code
> > from the application and move that down to function pointer abstraction(any
> > way adapter knows which queues to enqueue for), that way we can have same
> > final stage code for ALL type queues and Normal queues.
> > 
> Yes, I see the queue/sched type change approach followed in
> pipeline_worker_tx.c, a queue id can be provided in
> rte_event_eth_tx_adapter_conf
> 
> +struct rte_event_eth_tx_adapter_conf {
> +	uint8_t event_port_id;
> +	/**< Event port identifier, the adapter dequeues mbuf events from this
> +	 * port.
> +	 */
> +	uint16_t tx_metadata_off;
> +	/**<  Offset of struct rte_event_eth_tx_adapter_meta in the private
> +	 * area of the mbuf
> +	 */
> +	uint32_t max_nb_tx;
> +	/**< The adapter can return early if it has processed at least
> +	 * max_nb_tx mbufs. This isn't treated as a requirement; batching may
> +	 * cause the adapter to process more than max_nb_tx mbufs.
> +	 */
> +};
> 
> </sniped>
> 
> > > The worker core will receive events pointing to mbufs that need to be
> > > transmitted to different
> > > ports/queues, as described above. The port and the queue will be populated
> > > in the mbuf and the
> > > API can be as below
> > > 
> > > uint16_t rte_event_eth_tx_adapter_enqueue(uint8_t instance_id, uint8_t event_port_id, const struct rte_event ev[], uint16_t nb_events);
> > > 
> > > Let me know if that works for you.
> > 
> > Yes. That API works for me. I think, we can leverage "struct
> > rte_eventdev" area for adding new function pointer. Just like
> > enqueue_new_burst, enqueue_forward_burst variants, we can add one more
> > there, so that we can reuse that hot cacheline for all fastpath function pointer case.
> > That would translate to adding "uint8_t dev_id" on the above API.

> The dev_id can be derived from the instance_id, does that work ?

Do we need to that in fastpath?, IMO, if you can do that in slow path then it is fine.

> 
> I need some clarification on the configuration API/flow. The
> eventdev_pipeline sample app checks if DEV_TX_OFFLOAD_MT_LOCKFREE flag is
> set on all ethernet devices and if so, uses the pipeline_worker_tx path as
> opposed to the "consumer" function,

Yes

> if we were to use the adapter to replace
> some of the sample code then it seems like the
> RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT is hardware assist for the
> pipeline worker tx mode, the adapter would support 2 modes (consumer and
> worker_tx, borrowing terminology from the sample), worker_tx would only be
> supported if the eventdev supports
> RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT (at least in the first version)

Yes.

1) I think, rte_event_eth_tx_adapter_enqueue() function can simply call,

struct rte_eventdev *dev = &rte_eventdevs[dev_id];
return (*dev->eth_tx_adapter_enqueue)(...);

2) You can expose generic version of "eth_tx_adapter_enqueue" in Tx
adapter. If drivers does not set the "eth_tx_adapter_enqueue" function
pointer or DEV_TX_OFFLOAD_MT_LOCKFREE flag is NOT set on all ethernet devices
_then_ in common code we can assign eth_tx_adapter_enqueue function
pointer as your generic Tx adapter function pointer.

3) I think, you can focus only on generic "consumer" case as you can not
test "worker_tx" case. We are planning to add more optimized raw
"worker_tx" case in driver(Point 2 will allow that by having driver
specific "eth_tx_adapter_enqueue" function pointer).

/Jerin

> 
> Thanks,
> Nikhil
> 

[dpdk-dev] [RFC v2] eventdev: event tx adapter APIs

[Nikhil Rao]

Add common APIs for the transmit stage of an event driven
DPDK application. Also add a transmit queue field to the mbuf
that is used by the adapter to transmit mbufs.

Signed-off-by: Nikhil Rao <nikhil.rao@intel.com>
---

Changelog
=========

v1->v2:
 * Add the tx_adapter_enqueue function to struct rte_eventdev.
   It is set to the common Tx adapter function when creating the adapter
   if the eventdev PMD does not support it or if the
   DEV_TX_OFFLOAD_MT_LOCKFREE flag is NOT set on all ethernet devices.
 * Add the rte_event_eth_tx_adapter_enqueue() API.
 * Add the txq_id field to struct rte_mbuf.

 lib/librte_eventdev/rte_event_eth_tx_adapter.h | 380 +++++++++++++++++++++++++
 lib/librte_eventdev/rte_eventdev.h             |   7 +-
 lib/librte_mbuf/rte_mbuf.h                     |  20 +-
 3 files changed, 405 insertions(+), 2 deletions(-)
 create mode 100644 lib/librte_eventdev/rte_event_eth_tx_adapter.h

diff --git a/lib/librte_eventdev/rte_event_eth_tx_adapter.h b/lib/librte_eventdev/rte_event_eth_tx_adapter.h
new file mode 100644
index 0000000..a0e8505
--- /dev/null
+++ b/lib/librte_eventdev/rte_event_eth_tx_adapter.h
@@ -0,0 +1,380 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation.
+ */
+
+#ifndef _RTE_EVENT_ETH_TX_ADAPTER_
+#define _RTE_EVENT_ETH_TX_ADAPTER_
+
+/**
+ * @file
+ *
+ * RTE Event Ethernet Tx Adapter
+ *
+ * The event ethernet Tx adapter provides configuration and data path APIs
+ * for the transmit stage of an event driven packet processing application.
+ * These APIs abstract the implementation of the transmit stage and allow the
+ * the application to use eventdev PMD support or a common implementation.
+ *
+ * In the common implementation, the application uses the adapter API to
+ * enqueue mbufs to the adapter which runs as a rte_service function. The
+ * service function deqeueues events from its event port and transmits the
+ * mbufs referenced by these events.
+ *
+ * The ethernet Tx event adapter APIs are:
+ *
+ *  - rte_event_eth_tx_adapter_create()
+ *  - rte_event_eth_tx_adapter_create_ext()
+ *  - rte_event_eth_tx_adapter_free()
+ *  - rte_event_eth_tx_adapter_start()
+ *  - rte_event_eth_tx_adapter_stop()
+ *  - rte_event_eth_tx_adapter_queue_start()
+ *  - rte_event_eth_tx_adapter_queue_stop()
+ *  - rte_event_eth_tx_adapter_enqueue()
+ *  - rte_event_eth_tx_adapter_stats_get()
+ *  - rte_event_eth_tx_adapter_stats_reset()
+ *
+ * The application creates the adapter using
+ * rte_event_eth_tx_adapter_create(). The adapter may internally create an event
+ * port using the port configuration parameter.
+ * The adapter is responsible for linking the queue as per its implementation,
+ * for example in the case of the service function, the adapter links this queue
+ * to the event port it will dequeue events from.
+ *
+ * The application uses rte_event_eth_tx_adapter_enqueue() to send mbufs to the
+ * adaptervia this event queue. The ethernet port and transmit queue index to
+ * transmit the mbuf on are specified in the mbuf.
+ *
+ * The application can start and stop the adapter using the
+ * rte_event_eth_tx_adapter_start/stop() calls.
+ *
+ * To support dynamic reconfiguration of Tx queues, the application can
+ * call rte_event_eth_tx_adapter_queue_start()/stop() to synchronize
+ * access to the Tx queue with the adapter. For example, if the application
+ * wants to reconfigure a Tx queue that could be concurrently
+ * being accessed by the adapter, it calls rte_event_eth_tx_adapter_queue_stop()
+ * first, reconfigures the queue and then calls
+ * rte_event_eth_tx_adapter_queue_start() which signals to the adapter
+ * that it is safe to resume access to the Tx queue.
+ *
+ * The common adapter implementation uses an EAL service function as described
+ * before and its execution is controlled using the rte_service APIs. The
+ * rte_event_eth_tx_adapter_service_id_get()
+ * function can be used to retrieve the adapter's service function ID.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+#include "rte_eventdev.h"
+
+#define RTE_EVENT_ETH_TX_ADAPTER_MAX_INSTANCE 32
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Adapter configuration structure
+ */
+struct rte_event_eth_tx_adapter_conf {
+	uint8_t event_port_id;
+	/**< Event port identifier, the adapter service function dequeues mbuf
+	 * events from this port.
+	 */
+	uint32_t max_nb_tx;
+	/**< The adapter can return early if it has processed at least
+	 * max_nb_tx mbufs. This isn't treated as a requirement; batching may
+	 * cause the adapter to process more than max_nb_tx mbufs.
+	 */
+};
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Function type used for adapter configuration callback. The callback is
+ * used to fill in members of the struct rte_event_eth_tx_adapter_conf, this
+ * callback is invoked when creating a SW service to transmit packets.
+ *
+ * @param id
+ *  Adapter identifier.
+ * @param dev_id
+ *  Event device identifier.
+ * @param [out] conf
+ *  Structure that needs to be populated by this callback.
+ * @param arg
+ *  Argument to the callback. This is the same as the conf_arg passed to the
+ *  rte_event_eth_tx_adapter_create_ext().
+ *
+ * @return
+ *   - 0: Success
+ *   - <0: Error code on failure
+ */
+typedef int (*rte_event_eth_tx_adapter_conf_cb) (uint8_t id, uint8_t dev_id,
+			struct rte_event_eth_tx_adapter_conf *conf,
+			void *arg);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * A structure used to retrieve statistics for an eth tx adapter instance.
+ */
+struct rte_event_eth_tx_adapter_stats {
+	uint64_t event_poll_count;
+	/*< Event port poll count */
+	uint64_t tx_packets;
+	/*< Number of packets transmitted */
+	uint64_t tx_dropped;
+	/*< Number of packets dropped */
+};
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Create a new event ethernet Tx adapter with the specified identifier.
+ *
+ * @param id
+ *  The identifier of the event ethernet Tx adapter.
+ * @param dev_id
+ *  The event device identifier.
+ * @param queue_id
+ *  The event queue identifier.
+ * @param port_config
+ *  Event port configuration, the adapter uses this configuration to
+ *  create an event port if needed. It uses this port to dequeue
+ *  events that are sent to it by rte_event_eth_tx_adapter_enqueue()
+ *
+ * @return
+ *   - 0: Success
+ *   - <0: Error code on failure
+ */
+int rte_event_eth_tx_adapter_create(uint8_t id, uint8_t dev_id,
+				uint8_t queue_id,
+				struct rte_event_port_conf *port_config);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Free an event adapter
+ *
+ * @param id
+ *  Adapter identifier.
+ * @return
+ *   - 0: Success
+ *   - <0: Error code on failure, If the adapter still has Tx queues
+ *      added to it, the function returns -EBUSY.
+ */
+int rte_event_eth_tx_adapter_free(uint8_t id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Start ethernet Tx event adapter
+ *
+ * @param id
+ *  Adapter identifier.
+ * @return
+ *  - 0: Success, Adapter started correctly.
+ *  - <0: Error code on failure.
+ */
+int rte_event_eth_tx_adapter_start(uint8_t id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Stop ethernet Tx event adapter
+ *
+ * @param id
+ *  Adapter identifier.
+ * @return
+ *  - 0: Success, Adapter started correctly.
+ *  - <0: Error code on failure.
+ */
+int rte_event_eth_tx_adapter_stop(uint8_t id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Signal the Tx adapter to start processing mbufs for a
+ * Tx queue. A queue value of -1 is used to indicate all
+ * queues within the device.
+ *
+ * @param id
+ *  Adapter identifier.
+ * @param eth_dev_id
+ *  Ethernet Port Identifier.
+ * @param queue
+ *  Tx queue index.
+ *
+ * @return
+ *  - 0: Success, Adapter started correctly.
+ *  - <0: Error code on failure.
+ */
+int rte_event_eth_tx_adapter_queue_start(uint8_t id,
+					uint16_t eth_dev_id,
+					int32_t queue);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Signal the Tx adapter to stop processing mbufs for a
+ * Tx queue. A queue value of -1 is used to indicate all
+ * queues within the device.
+ *
+ * @param id
+ *  Adapter identifier.
+ * @param eth_dev_id
+ *  Ethernet Port Identifier.
+ * @param queue
+ *  Tx queue index.
+ *
+ * @return
+ *  - 0: Success, Adapter started correctly.
+ *  - <0: Error code on failure.
+ */
+int rte_event_eth_tx_adapter_queue_stop(uint8_t id,
+					uint16_t eth_dev_id,
+					int32_t queue);
+
+static __rte_always_inline uint16_t
+__rte_event_eth_tx_adapter_enqueue(uint8_t id, uint8_t dev_id, uint8_t port_id,
+				const struct rte_event ev[],
+				uint16_t nb_events,
+				const event_tx_adapter_enqueue fn)
+{
+	const struct rte_eventdev *dev = &rte_eventdevs[dev_id];
+
+#ifdef RTE_LIBRTE_EVENTDEV_DEBUG
+	if (id >= RTE_EVENT_ETH_TX_ADAPTER_MAX_INSTANCE || 
+		dev_id >= RTE_EVENT_MAX_DEVS ||
+		!rte_eventdevs[dev_id].attached) {
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (port_id >= dev->data->nb_ports) {
+		rte_errno = -EINVAL;
+		return 0;
+	}
+#endif
+	/*
+	 * TODO: Do we need a special case for nb_events == 1
+	 */
+	return fn(id, dev, dev->data->ports[port_id], ev, nb_events);
+}
+
+/**
+ * Enqueue a burst of events objects or an event object supplied in *rte_event*
+ * structure on an  event device designated by its *dev_id* through the event
+ * port specified by *port_id*. The the event queue on which it will be enqueued
+ * id derived from the adapter id parameter.
+ *
+ * The *nb_events* parameter is the number of event objects to enqueue which are
+ * supplied in the *ev* array of *rte_event* structure.
+ *
+ * The rte_event_eth_tx_adapter_enqueue() function returns the number of
+ * events objects it actually enqueued. A return value equal to *nb_events*
+ * means that all event objects have been enqueued.
+ *
+ * @param id
+ *  The identifier of the tx adapter.
+ * @param dev_id
+ *  The identifier of the device.
+ * @param port_id
+ *  The identifier of the event port.
+ * @param ev
+ *  Points to an array of *nb_events* objects of type *rte_event* structure
+ *  which contain the event object enqueue operations to be processed.
+ * @param nb_events
+ *  The number of event objects to enqueue, typically number of
+ *  rte_event_port_enqueue_depth() available for this port.
+ *
+ * @return
+ *   The number of event objects actually enqueued on the event device. The
+ *   return value can be less than the value of the *nb_events* parameter when
+ *   the event devices queue is full or if invalid parameters are specified in a
+ *   *rte_event*. If the return value is less than *nb_events*, the remaining
+ *   events at the end of ev[] are not consumed and the caller has to take care
+ *   of them, and rte_errno is set accordingly. Possible errno values include:
+ *   - -EINVAL  The port ID is invalid, device ID is invalid, an event's queue
+ *              ID is invalid, or an event's sched type doesn't match the
+ *              capabilities of the destination queue.
+ *   - -ENOSPC  The event port was backpressured and unable to enqueue
+ *              one or more events. This error code is only applicable to
+ *              closed systems.
+ */
+static inline uint16_t
+rte_event_eth_tx_adapter_enqueue(uint8_t id, uint8_t dev_id,
+				uint8_t port_id,
+				const struct rte_event ev[],
+				uint16_t nb_events)
+{
+	const struct rte_eventdev *dev = &rte_eventdevs[dev_id];
+
+	return __rte_event_eth_tx_adapter_enqueue(id, dev_id, port_id, ev,
+						nb_events,
+						dev->tx_adapter_enqueue);
+}
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Retrieve statistics for an adapter
+ *
+ * @param id
+ *  Adapter identifier.
+ * @param [out] stats
+ *  A pointer to structure used to retrieve statistics for an adapter.
+ * @return
+ *  - 0: Success, retrieved successfully.
+ *  - <0: Error code on failure.
+ */
+int rte_event_eth_tx_adapter_stats_get(uint8_t id,
+				struct rte_event_eth_tx_adapter_stats *stats);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Reset statistics for an adapter.
+ *
+ * @param id
+ *  Adapter identifier.
+ * @return
+ *  - 0: Success, statistics reset successfully.
+ *  - <0: Error code on failure.
+ */
+int rte_event_eth_tx_adapter_stats_reset(uint8_t id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Retrieve the service ID of an adapter. If the adapter doesn't use
+ * a rte_service function, this function returns -ESRCH.
+ *
+ * @param id
+ *  Adapter identifier.
+ * @param [out] service_id
+ *  A pointer to a uint32_t, to be filled in with the service id.
+ * @return
+ *  - 0: Success
+ *  - <0: Error code on failure, if the adapter doesn't use a rte_service
+ * function, this function returns -ESRCH.
+ */
+int rte_event_eth_tx_adapter_service_id_get(uint8_t id, uint32_t *service_id);
+
+#ifdef __cplusplus
+}
+#endif
+#endif	/* _RTE_EVENT_ETH_TX_ADAPTER_ */
diff --git a/lib/librte_eventdev/rte_eventdev.h b/lib/librte_eventdev/rte_eventdev.h
index b6fd6ee..1bf28a0 100644
--- a/lib/librte_eventdev/rte_eventdev.h
+++ b/lib/librte_eventdev/rte_eventdev.h
@@ -1203,6 +1203,10 @@ typedef uint16_t (*event_dequeue_t)(void *port, struct rte_event *ev,
 typedef uint16_t (*event_dequeue_burst_t)(void *port, struct rte_event ev[],
 		uint16_t nb_events, uint64_t timeout_ticks);
 /**< @internal Dequeue burst of events from port of a device */
+typedef uint16_t (*event_tx_adapter_enqueue)(uint8_t id,
+			const struct rte_eventdev *dev, void *port,
+			const struct rte_event ev[], uint16_t nb_events);
+/**< @internal Enqueue burst of events on port of a device */
 
 #define RTE_EVENTDEV_NAME_MAX_LEN	(64)
 /**< @internal Max length of name of event PMD */
@@ -1266,7 +1270,8 @@ struct rte_eventdev {
 	/**< Pointer to PMD dequeue function. */
 	event_dequeue_burst_t dequeue_burst;
 	/**< Pointer to PMD dequeue burst function. */
-
+	event_tx_adapter_enqueue tx_adapter_enqueue;
+	/**< Pointer to PMD tx adapter enqueue function. */
 	struct rte_eventdev_data *data;
 	/**< Pointer to device data */
 	struct rte_eventdev_ops *dev_ops;
diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h
index 4fd9a0d..c59389c 100644
--- a/lib/librte_mbuf/rte_mbuf.h
+++ b/lib/librte_mbuf/rte_mbuf.h
@@ -464,7 +464,9 @@ struct rte_mbuf {
 	};
 	uint16_t nb_segs;         /**< Number of segments. */
 
-	/** Input port (16 bits to support more than 256 virtual ports). */
+	/** Input port (16 bits to support more than 256 virtual ports).
+	 * The event eth Tx adapter uses this field to specify the output port.
+	 */
 	uint16_t port;
 
 	uint64_t ol_flags;        /**< Offload features. */
@@ -511,6 +513,7 @@ struct rte_mbuf {
 	/** VLAN TCI (CPU order), valid if PKT_RX_VLAN is set. */
 	uint16_t vlan_tci;
 
+	RTE_STD_C11
 	union {
 		uint32_t rss;     /**< RSS hash result if RSS enabled */
 		struct {
@@ -531,6 +534,11 @@ struct rte_mbuf {
 			uint32_t lo;
 			uint32_t hi;
 		} sched;          /**< Hierarchical scheduler */
+		struct {
+			uint32_t resvd1; /* overlaps with rte_sched_port_hierarchy::color */
+			uint16_t resvd2;
+			uint16_t txq_id;
+		};
 		uint32_t usr;	  /**< User defined tags. See rte_distributor_process() */
 	} hash;                   /**< hash information */
 
@@ -1880,6 +1888,16 @@ static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
 #define rte_pktmbuf_data_len(m) ((m)->data_len)
 
 /**
+ * A macro that returns the txq field of the mbuf
+ *
+ * The value can be read or assigned.
+ *
+ * @param m
+ *   The packet mbuf.
+ */
+#define rte_pktmbuf_tx_queue(m) ((m)->txq_id)
+
+/**
  * Prepend len bytes to an mbuf data area.
  *
  * Returns a pointer to the new
-- 
1.8.3.1

Re: [dpdk-dev] [RFC v2] eventdev: event tx adapter APIs

[Rao, Nikhil]

Hi Folks,

Did you get a chance to review v2 ? Please let me know your feedback so 
I can proceed towards implementation.

Thanks,
Nikhil

Re: [dpdk-dev] [RFC v2] eventdev: event tx adapter APIs

[Jerin Jacob]

-----Original Message-----
> Date: Wed, 13 Jun 2018 03:02:43 +0530
> From: Nikhil Rao <nikhil.rao@intel.com>
> To: jerin.jacob@caviumnetworks.com, olivier.matz@6wind.com
> CC: hemant.agrawal@nxp.com, dev@dpdk.org, narender.vangati@intel.com,
>  abhinandan.gujjar@intel.com, gage.eads@intel.com, jia.guo@intel.com,
>  cristian.dumitrescu@intel.com, Nikhil Rao <nikhil.rao@intel.com>
> Subject: [RFC v2] eventdev: event tx adapter APIs
> X-Mailer: git-send-email 1.8.3.1
> 
> Add common APIs for the transmit stage of an event driven
> DPDK application. Also add a transmit queue field to the mbuf
> that is used by the adapter to transmit mbufs.
> 
> Signed-off-by: Nikhil Rao <nikhil.rao@intel.com>
> ---
> 
> Changelog
> =========
> 
> v1->v2:
>  * Add the tx_adapter_enqueue function to struct rte_eventdev.
>    It is set to the common Tx adapter function when creating the adapter
>    if the eventdev PMD does not support it or if the
>    DEV_TX_OFFLOAD_MT_LOCKFREE flag is NOT set on all ethernet devices.
>  * Add the rte_event_eth_tx_adapter_enqueue() API.
>  * Add the txq_id field to struct rte_mbuf.
> 

Overall it looks good. Some comments inline. I think you can change it
from RFC and send the implementation.

>  lib/librte_eventdev/rte_event_eth_tx_adapter.h | 380 +++++++++++++++++++++++++
>  lib/librte_eventdev/rte_eventdev.h             |   7 +-
>  lib/librte_mbuf/rte_mbuf.h                     |  20 +-

Please split mbuf changes in a separate patch.


>  3 files changed, 405 insertions(+), 2 deletions(-)
>  create mode 100644 lib/librte_eventdev/rte_event_eth_tx_adapter.h
> 
> diff --git a/lib/librte_eventdev/rte_event_eth_tx_adapter.h b/lib/librte_eventdev/rte_event_eth_tx_adapter.h
> new file mode 100644
> index 0000000..a0e8505
> --- /dev/null
> +++ b/lib/librte_eventdev/rte_event_eth_tx_adapter.h
> @@ -0,0 +1,380 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation.

2018

> + */
> +
> +#ifndef _RTE_EVENT_ETH_TX_ADAPTER_
> +#define _RTE_EVENT_ETH_TX_ADAPTER_
> +
> +/**
> + * @file
> + *
> + * RTE Event Ethernet Tx Adapter
> + *
> + * The event ethernet Tx adapter provides configuration and data path APIs
> + * for the transmit stage of an event driven packet processing application.
> + * These APIs abstract the implementation of the transmit stage and allow the

s/transmit stage/ethernet transmit stage/

> + * the application to use eventdev PMD support or a common implementation.
> + *
> + * In the common implementation, the application uses the adapter API to
> + * enqueue mbufs to the adapter which runs as a rte_service function. The
> + * service function deqeueues events from its event port and transmits the

s/deqeueues/dequeues

> + * mbufs referenced by these events.
> + *
> + * The ethernet Tx event adapter APIs are:
> + *
> + *  - rte_event_eth_tx_adapter_create()
> + *  - rte_event_eth_tx_adapter_create_ext()
> + *  - rte_event_eth_tx_adapter_free()
> + *  - rte_event_eth_tx_adapter_start()
> + *  - rte_event_eth_tx_adapter_stop()
> + *  - rte_event_eth_tx_adapter_queue_start()
> + *  - rte_event_eth_tx_adapter_queue_stop()
> + *  - rte_event_eth_tx_adapter_enqueue()
> + *  - rte_event_eth_tx_adapter_stats_get()
> + *  - rte_event_eth_tx_adapter_stats_reset()
> + *
> + * The application creates the adapter using
> + * rte_event_eth_tx_adapter_create(). The adapter may internally create an event
> + * port using the port configuration parameter.
> + * The adapter is responsible for linking the queue as per its implementation,
> + * for example in the case of the service function, the adapter links this queue
> + * to the event port it will dequeue events from.
> + *
> + * The application uses rte_event_eth_tx_adapter_enqueue() to send mbufs to the
> + * adaptervia this event queue. The ethernet port and transmit queue index to

s/adaptervia/adapter via

> + * transmit the mbuf on are specified in the mbuf.
> + *
> + * The application can start and stop the adapter using the
> + * rte_event_eth_tx_adapter_start/stop() calls.
> + *
> + * To support dynamic reconfiguration of Tx queues, the application can
> + * call rte_event_eth_tx_adapter_queue_start()/stop() to synchronize
> + * access to the Tx queue with the adapter. For example, if the application
> + * wants to reconfigure a Tx queue that could be concurrently
> + * being accessed by the adapter, it calls rte_event_eth_tx_adapter_queue_stop()
> + * first, reconfigures the queue and then calls
> + * rte_event_eth_tx_adapter_queue_start() which signals to the adapter
> + * that it is safe to resume access to the Tx queue.
> + *
> + * The common adapter implementation uses an EAL service function as described
> + * before and its execution is controlled using the rte_service APIs. The
> + * rte_event_eth_tx_adapter_service_id_get()
> + * function can be used to retrieve the adapter's service function ID.
> + */
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include <stdint.h>
> +
> +#include "rte_eventdev.h"
> +
> +#define RTE_EVENT_ETH_TX_ADAPTER_MAX_INSTANCE 32
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Adapter configuration structure
> + */
> +struct rte_event_eth_tx_adapter_conf {
> +	uint8_t event_port_id;
> +	/**< Event port identifier, the adapter service function dequeues mbuf
> +	 * events from this port.
> +	 */
> +	uint32_t max_nb_tx;
> +	/**< The adapter can return early if it has processed at least
> +	 * max_nb_tx mbufs. This isn't treated as a requirement; batching may
> +	 * cause the adapter to process more than max_nb_tx mbufs.
> +	 */
> +};
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Function type used for adapter configuration callback. The callback is
> + * used to fill in members of the struct rte_event_eth_tx_adapter_conf, this
> + * callback is invoked when creating a SW service to transmit packets.
> + *
> + * @param id
> + *  Adapter identifier.
> + * @param dev_id
> + *  Event device identifier.
> + * @param [out] conf
> + *  Structure that needs to be populated by this callback.
> + * @param arg
> + *  Argument to the callback. This is the same as the conf_arg passed to the
> + *  rte_event_eth_tx_adapter_create_ext().
> + *
> + * @return
> + *   - 0: Success
> + *   - <0: Error code on failure
> + */
> +typedef int (*rte_event_eth_tx_adapter_conf_cb) (uint8_t id, uint8_t dev_id,
> +			struct rte_event_eth_tx_adapter_conf *conf,
> +			void *arg);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * A structure used to retrieve statistics for an eth tx adapter instance.
> + */
> +struct rte_event_eth_tx_adapter_stats {
> +	uint64_t event_poll_count;
> +	/*< Event port poll count */
> +	uint64_t tx_packets;
> +	/*< Number of packets transmitted */
> +	uint64_t tx_dropped;
> +	/*< Number of packets dropped */
> +};
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Create a new event ethernet Tx adapter with the specified identifier.
> + *
> + * @param id
> + *  The identifier of the event ethernet Tx adapter.
> + * @param dev_id
> + *  The event device identifier.
> + * @param queue_id
> + *  The event queue identifier.
> + * @param port_config
> + *  Event port configuration, the adapter uses this configuration to
> + *  create an event port if needed. It uses this port to dequeue
> + *  events that are sent to it by rte_event_eth_tx_adapter_enqueue()
> + *
> + * @return
> + *   - 0: Success
> + *   - <0: Error code on failure
> + */

__rte_experimental missing in all the functions.

> +int rte_event_eth_tx_adapter_create(uint8_t id, uint8_t dev_id,
> +				uint8_t queue_id,
> +				struct rte_event_port_conf *port_config);
> +


# Missing the rte_event_eth_tx_adapter_create_ext() prototype?

#I assume you will have rte_event_eth_tx_adapter_caps_get() with the
RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT capability

> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Free an event adapter
> + *
> + * @param id
> + *  Adapter identifier.
> + * @return
> + *   - 0: Success
> + *   - <0: Error code on failure, If the adapter still has Tx queues
> + *      added to it, the function returns -EBUSY.
> + */
> +int rte_event_eth_tx_adapter_free(uint8_t id);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Start ethernet Tx event adapter
> + *
> + * @param id
> + *  Adapter identifier.
> + * @return
> + *  - 0: Success, Adapter started correctly.
> + *  - <0: Error code on failure.
> + */
> +int rte_event_eth_tx_adapter_start(uint8_t id);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Stop ethernet Tx event adapter
> + *
> + * @param id
> + *  Adapter identifier.
> + * @return
> + *  - 0: Success, Adapter started correctly.
> + *  - <0: Error code on failure.
> + */
> +int rte_event_eth_tx_adapter_stop(uint8_t id);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Signal the Tx adapter to start processing mbufs for a
> + * Tx queue. A queue value of -1 is used to indicate all
> + * queues within the device.
> + *
> + * @param id
> + *  Adapter identifier.
> + * @param eth_dev_id
> + *  Ethernet Port Identifier.
> + * @param queue
> + *  Tx queue index.
> + *
> + * @return
> + *  - 0: Success, Adapter started correctly.
> + *  - <0: Error code on failure.
> + */
> +int rte_event_eth_tx_adapter_queue_start(uint8_t id,
> +					uint16_t eth_dev_id,
> +					int32_t queue);
> +

How about changing to to rte_event_eth_tx_adapter_queue_add to inline
with Rx adapter?

> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Signal the Tx adapter to stop processing mbufs for a
> + * Tx queue. A queue value of -1 is used to indicate all
> + * queues within the device.
> + *
> + * @param id
> + *  Adapter identifier.
> + * @param eth_dev_id
> + *  Ethernet Port Identifier.
> + * @param queue
> + *  Tx queue index.
> + *
> + * @return
> + *  - 0: Success, Adapter started correctly.
> + *  - <0: Error code on failure.
> + */
> +int rte_event_eth_tx_adapter_queue_stop(uint8_t id,
> +					uint16_t eth_dev_id,
> +					int32_t queue);
> +
> +static __rte_always_inline uint16_t
> +__rte_event_eth_tx_adapter_enqueue(uint8_t id, uint8_t dev_id, uint8_t port_id,
> +				const struct rte_event ev[],
> +				uint16_t nb_events,
> +				const event_tx_adapter_enqueue fn)
> +{
> +	const struct rte_eventdev *dev = &rte_eventdevs[dev_id];
> +
> +#ifdef RTE_LIBRTE_EVENTDEV_DEBUG
> +	if (id >= RTE_EVENT_ETH_TX_ADAPTER_MAX_INSTANCE || 
> +		dev_id >= RTE_EVENT_MAX_DEVS ||
> +		!rte_eventdevs[dev_id].attached) {
> +		rte_errno = -EINVAL;
> +		return 0;
> +	}
> +
> +	if (port_id >= dev->data->nb_ports) {
> +		rte_errno = -EINVAL;
> +		return 0;
> +	}
> +#endif
> +	/*
> +	 * TODO: Do we need a special case for nb_events == 1
> +	 */

I think, we don't need the special case for nb_events == 1

> +	return fn(id, dev, dev->data->ports[port_id], ev, nb_events);
> +}
> +
> +/**
> + * Enqueue a burst of events objects or an event object supplied in *rte_event*
> + * structure on an  event device designated by its *dev_id* through the event
> + * port specified by *port_id*. The the event queue on which it will be enqueued
> + * id derived from the adapter id parameter.
> + *
> + * The *nb_events* parameter is the number of event objects to enqueue which are
> + * supplied in the *ev* array of *rte_event* structure.
> + *
> + * The rte_event_eth_tx_adapter_enqueue() function returns the number of
> + * events objects it actually enqueued. A return value equal to *nb_events*
> + * means that all event objects have been enqueued.
> + *
> + * @param id
> + *  The identifier of the tx adapter.
> + * @param dev_id
> + *  The identifier of the device.
> + * @param port_id
> + *  The identifier of the event port.
> + * @param ev
> + *  Points to an array of *nb_events* objects of type *rte_event* structure
> + *  which contain the event object enqueue operations to be processed.
> + * @param nb_events
> + *  The number of event objects to enqueue, typically number of
> + *  rte_event_port_enqueue_depth() available for this port.
> + *
> + * @return
> + *   The number of event objects actually enqueued on the event device. The
> + *   return value can be less than the value of the *nb_events* parameter when
> + *   the event devices queue is full or if invalid parameters are specified in a
> + *   *rte_event*. If the return value is less than *nb_events*, the remaining
> + *   events at the end of ev[] are not consumed and the caller has to take care
> + *   of them, and rte_errno is set accordingly. Possible errno values include:
> + *   - -EINVAL  The port ID is invalid, device ID is invalid, an event's queue
> + *              ID is invalid, or an event's sched type doesn't match the
> + *              capabilities of the destination queue.
> + *   - -ENOSPC  The event port was backpressured and unable to enqueue
> + *              one or more events. This error code is only applicable to
> + *              closed systems.
> + */
> +static inline uint16_t
> +rte_event_eth_tx_adapter_enqueue(uint8_t id, uint8_t dev_id,
> +				uint8_t port_id,
> +				const struct rte_event ev[],
> +				uint16_t nb_events)
> +{
> +	const struct rte_eventdev *dev = &rte_eventdevs[dev_id];
> +
> +	return __rte_event_eth_tx_adapter_enqueue(id, dev_id, port_id, ev,
> +						nb_events,
> +						dev->tx_adapter_enqueue);
> +}
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Retrieve statistics for an adapter
> + *
> + * @param id
> + *  Adapter identifier.
> + * @param [out] stats
> + *  A pointer to structure used to retrieve statistics for an adapter.
> + * @return
> + *  - 0: Success, retrieved successfully.
> + *  - <0: Error code on failure.
> + */
> +int rte_event_eth_tx_adapter_stats_get(uint8_t id,
> +				struct rte_event_eth_tx_adapter_stats *stats);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Reset statistics for an adapter.
> + *
> + * @param id
> + *  Adapter identifier.
> + * @return
> + *  - 0: Success, statistics reset successfully.
> + *  - <0: Error code on failure.
> + */
> +int rte_event_eth_tx_adapter_stats_reset(uint8_t id);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Retrieve the service ID of an adapter. If the adapter doesn't use
> + * a rte_service function, this function returns -ESRCH.
> + *
> + * @param id
> + *  Adapter identifier.
> + * @param [out] service_id
> + *  A pointer to a uint32_t, to be filled in with the service id.
> + * @return
> + *  - 0: Success
> + *  - <0: Error code on failure, if the adapter doesn't use a rte_service
> + * function, this function returns -ESRCH.
> + */
> +int rte_event_eth_tx_adapter_service_id_get(uint8_t id, uint32_t *service_id);
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +#endif	/* _RTE_EVENT_ETH_TX_ADAPTER_ */
> diff --git a/lib/librte_eventdev/rte_eventdev.h b/lib/librte_eventdev/rte_eventdev.h
> index b6fd6ee..1bf28a0 100644
> --- a/lib/librte_eventdev/rte_eventdev.h
> +++ b/lib/librte_eventdev/rte_eventdev.h
> @@ -1203,6 +1203,10 @@ typedef uint16_t (*event_dequeue_t)(void *port, struct rte_event *ev,
>  typedef uint16_t (*event_dequeue_burst_t)(void *port, struct rte_event ev[],
>  		uint16_t nb_events, uint64_t timeout_ticks);
>  /**< @internal Dequeue burst of events from port of a device */
> +typedef uint16_t (*event_tx_adapter_enqueue)(uint8_t id,
> +			const struct rte_eventdev *dev, void *port,
> +			const struct rte_event ev[], uint16_t nb_events);
> +/**< @internal Enqueue burst of events on port of a device */
>  
>  #define RTE_EVENTDEV_NAME_MAX_LEN	(64)
>  /**< @internal Max length of name of event PMD */
> @@ -1266,7 +1270,8 @@ struct rte_eventdev {
>  	/**< Pointer to PMD dequeue function. */
>  	event_dequeue_burst_t dequeue_burst;
>  	/**< Pointer to PMD dequeue burst function. */
> -
> +	event_tx_adapter_enqueue tx_adapter_enqueue;
> +	/**< Pointer to PMD tx adapter enqueue function. */
>  	struct rte_eventdev_data *data;
>  	/**< Pointer to device data */
>  	struct rte_eventdev_ops *dev_ops;
> diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h
> index 4fd9a0d..c59389c 100644
> --- a/lib/librte_mbuf/rte_mbuf.h
> +++ b/lib/librte_mbuf/rte_mbuf.h
> @@ -464,7 +464,9 @@ struct rte_mbuf {
>  	};
>  	uint16_t nb_segs;         /**< Number of segments. */
>  
> -	/** Input port (16 bits to support more than 256 virtual ports). */
> +	/** Input port (16 bits to support more than 256 virtual ports).
> +	 * The event eth Tx adapter uses this field to specify the output port.

See rte_event_eth_tx_adapter_enqueue()

> +	 */
>  	uint16_t port;
>  
>  	uint64_t ol_flags;        /**< Offload features. */
> @@ -511,6 +513,7 @@ struct rte_mbuf {
>  	/** VLAN TCI (CPU order), valid if PKT_RX_VLAN is set. */
>  	uint16_t vlan_tci;
>  
> +	RTE_STD_C11
>  	union {
>  		uint32_t rss;     /**< RSS hash result if RSS enabled */
>  		struct {
> @@ -531,6 +534,11 @@ struct rte_mbuf {
>  			uint32_t lo;
>  			uint32_t hi;
>  		} sched;          /**< Hierarchical scheduler */
> +		struct {
> +			uint32_t resvd1; /* overlaps with rte_sched_port_hierarchy::color */

s/overlaps/Overlaps/

> +			uint16_t resvd2;
> +			uint16_t txq_id;

/* The event eth Tx adapter transmit queue. See rte_event_eth_tx_adapter_enqueue() */

> +		};
>  		uint32_t usr;	  /**< User defined tags. See rte_distributor_process() */
>  	} hash;                   /**< hash information */
>  
> @@ -1880,6 +1888,16 @@ static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
>  #define rte_pktmbuf_data_len(m) ((m)->data_len)
>  
>  /**
> + * A macro that returns the txq field of the mbuf
> + *
> + * The value can be read or assigned.
> + *
> + * @param m
> + *   The packet mbuf.
> + */
> +#define rte_pktmbuf_tx_queue(m) ((m)->txq_id)

Should we change to rte_pktmbuf_tx_adapter_txq to make it explicit that
it is related to Tx adapter?

> +
> +/**
>   * Prepend len bytes to an mbuf data area.
>   *
>   * Returns a pointer to the new
> -- 
> 1.8.3.1
> 

Re: [dpdk-dev] [RFC v2] eventdev: event tx adapter APIs

[Rao, Nikhil]

On 6/17/2018 4:39 PM, Jerin Jacob wrote:

> -----Original Message-----
>> Date: Wed, 13 Jun 2018 03:02:43 +0530
>> From: Nikhil Rao <nikhil.rao@intel.com>
>> To: jerin.jacob@caviumnetworks.com, olivier.matz@6wind.com
>> CC: hemant.agrawal@nxp.com, dev@dpdk.org, narender.vangati@intel.com,
>>   abhinandan.gujjar@intel.com, gage.eads@intel.com, jia.guo@intel.com,
>>   cristian.dumitrescu@intel.com, Nikhil Rao <nikhil.rao@intel.com>
>> Subject: [RFC v2] eventdev: event tx adapter APIs
>> X-Mailer: git-send-email 1.8.3.1
>>
>> Add common APIs for the transmit stage of an event driven
>> DPDK application. Also add a transmit queue field to the mbuf
>> that is used by the adapter to transmit mbufs.
>>
>> Signed-off-by: Nikhil Rao <nikhil.rao@intel.com>
>> ---
>>
>> Changelog
>> =========
>>
>> v1->v2:
>>   * Add the tx_adapter_enqueue function to struct rte_eventdev.
>>     It is set to the common Tx adapter function when creating the adapter
>>     if the eventdev PMD does not support it or if the
>>     DEV_TX_OFFLOAD_MT_LOCKFREE flag is NOT set on all ethernet devices.
>>   * Add the rte_event_eth_tx_adapter_enqueue() API.
>>   * Add the txq_id field to struct rte_mbuf.
>>
> Overall it looks good. Some comments inline. I think you can change it
> from RFC and send the implementation.
>
>>   lib/librte_eventdev/rte_event_eth_tx_adapter.h | 380 +++++++++++++++++++++++++
>>   lib/librte_eventdev/rte_eventdev.h             |   7 +-
>>   lib/librte_mbuf/rte_mbuf.h                     |  20 +-
> Please split mbuf changes in a separate patch.
>
>
>>   3 files changed, 405 insertions(+), 2 deletions(-)
>>   create mode 100644 lib/librte_eventdev/rte_event_eth_tx_adapter.h
>>
>> diff --git a/lib/librte_eventdev/rte_event_eth_tx_adapter.h b/lib/librte_eventdev/rte_event_eth_tx_adapter.h
>> new file mode 100644
>> index 0000000..a0e8505
>> --- /dev/null
>> +++ b/lib/librte_eventdev/rte_event_eth_tx_adapter.h
>> @@ -0,0 +1,380 @@
>> +/* SPDX-License-Identifier: BSD-3-Clause
>> + * Copyright(c) 2017 Intel Corporation.
> 2018
>
>> + */
>> +
>> +#ifndef _RTE_EVENT_ETH_TX_ADAPTER_
>> +#define _RTE_EVENT_ETH_TX_ADAPTER_
>> +
>> +/**
>> + * @file
>> + *
>> + * RTE Event Ethernet Tx Adapter
>> + *
>> + * The event ethernet Tx adapter provides configuration and data path APIs
>> + * for the transmit stage of an event driven packet processing application.
>> + * These APIs abstract the implementation of the transmit stage and allow the
> s/transmit stage/ethernet transmit stage/
>
>> + * the application to use eventdev PMD support or a common implementation.
>> + *
>> + * In the common implementation, the application uses the adapter API to
>> + * enqueue mbufs to the adapter which runs as a rte_service function. The
>> + * service function deqeueues events from its event port and transmits the
> s/deqeueues/dequeues
>
>> + * mbufs referenced by these events.
>> + *
>> + * The ethernet Tx event adapter APIs are:
>> + *
>> + *  - rte_event_eth_tx_adapter_create()
>> + *  - rte_event_eth_tx_adapter_create_ext()
>> + *  - rte_event_eth_tx_adapter_free()
>> + *  - rte_event_eth_tx_adapter_start()
>> + *  - rte_event_eth_tx_adapter_stop()
>> + *  - rte_event_eth_tx_adapter_queue_start()
>> + *  - rte_event_eth_tx_adapter_queue_stop()
>> + *  - rte_event_eth_tx_adapter_enqueue()
>> + *  - rte_event_eth_tx_adapter_stats_get()
>> + *  - rte_event_eth_tx_adapter_stats_reset()
>> + *
>> + * The application creates the adapter using
>> + * rte_event_eth_tx_adapter_create(). The adapter may internally create an event
>> + * port using the port configuration parameter.
>> + * The adapter is responsible for linking the queue as per its implementation,
>> + * for example in the case of the service function, the adapter links this queue
>> + * to the event port it will dequeue events from.
>> + *
>> + * The application uses rte_event_eth_tx_adapter_enqueue() to send mbufs to the
>> + * adaptervia this event queue. The ethernet port and transmit queue index to
> s/adaptervia/adapter via
>
>> + * transmit the mbuf on are specified in the mbuf.
>> + *
>> + * The application can start and stop the adapter using the
>> + * rte_event_eth_tx_adapter_start/stop() calls.
>> + *
>> + * To support dynamic reconfiguration of Tx queues, the application can
>> + * call rte_event_eth_tx_adapter_queue_start()/stop() to synchronize
>> + * access to the Tx queue with the adapter. For example, if the application
>> + * wants to reconfigure a Tx queue that could be concurrently
>> + * being accessed by the adapter, it calls rte_event_eth_tx_adapter_queue_stop()
>> + * first, reconfigures the queue and then calls
>> + * rte_event_eth_tx_adapter_queue_start() which signals to the adapter
>> + * that it is safe to resume access to the Tx queue.
>> + *
>> + * The common adapter implementation uses an EAL service function as described
>> + * before and its execution is controlled using the rte_service APIs. The
>> + * rte_event_eth_tx_adapter_service_id_get()
>> + * function can be used to retrieve the adapter's service function ID.
>> + */
>> +
>> +#ifdef __cplusplus
>> +extern "C" {
>> +#endif
>> +
>> +#include <stdint.h>
>> +
>> +#include "rte_eventdev.h"
>> +
>> +#define RTE_EVENT_ETH_TX_ADAPTER_MAX_INSTANCE 32
>> +
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice
>> + *
>> + * Adapter configuration structure
>> + */
>> +struct rte_event_eth_tx_adapter_conf {
>> +	uint8_t event_port_id;
>> +	/**< Event port identifier, the adapter service function dequeues mbuf
>> +	 * events from this port.
>> +	 */
>> +	uint32_t max_nb_tx;
>> +	/**< The adapter can return early if it has processed at least
>> +	 * max_nb_tx mbufs. This isn't treated as a requirement; batching may
>> +	 * cause the adapter to process more than max_nb_tx mbufs.
>> +	 */
>> +};
>> +
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice
>> + *
>> + * Function type used for adapter configuration callback. The callback is
>> + * used to fill in members of the struct rte_event_eth_tx_adapter_conf, this
>> + * callback is invoked when creating a SW service to transmit packets.
>> + *
>> + * @param id
>> + *  Adapter identifier.
>> + * @param dev_id
>> + *  Event device identifier.
>> + * @param [out] conf
>> + *  Structure that needs to be populated by this callback.
>> + * @param arg
>> + *  Argument to the callback. This is the same as the conf_arg passed to the
>> + *  rte_event_eth_tx_adapter_create_ext().
>> + *
>> + * @return
>> + *   - 0: Success
>> + *   - <0: Error code on failure
>> + */
>> +typedef int (*rte_event_eth_tx_adapter_conf_cb) (uint8_t id, uint8_t dev_id,
>> +			struct rte_event_eth_tx_adapter_conf *conf,
>> +			void *arg);
>> +
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice
>> + *
>> + * A structure used to retrieve statistics for an eth tx adapter instance.
>> + */
>> +struct rte_event_eth_tx_adapter_stats {
>> +	uint64_t event_poll_count;
>> +	/*< Event port poll count */
>> +	uint64_t tx_packets;
>> +	/*< Number of packets transmitted */
>> +	uint64_t tx_dropped;
>> +	/*< Number of packets dropped */
>> +};
>> +
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice
>> + *
>> + * Create a new event ethernet Tx adapter with the specified identifier.
>> + *
>> + * @param id
>> + *  The identifier of the event ethernet Tx adapter.
>> + * @param dev_id
>> + *  The event device identifier.
>> + * @param queue_id
>> + *  The event queue identifier.
>> + * @param port_config
>> + *  Event port configuration, the adapter uses this configuration to
>> + *  create an event port if needed. It uses this port to dequeue
>> + *  events that are sent to it by rte_event_eth_tx_adapter_enqueue()
>> + *
>> + * @return
>> + *   - 0: Success
>> + *   - <0: Error code on failure
>> + */
> __rte_experimental missing in all the functions.
>
>> +int rte_event_eth_tx_adapter_create(uint8_t id, uint8_t dev_id,
>> +				uint8_t queue_id,
>> +				struct rte_event_port_conf *port_config);
>> +
>
> # Missing the rte_event_eth_tx_adapter_create_ext() prototype?
>
> #I assume you will have rte_event_eth_tx_adapter_caps_get() with the
> RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT capability

Yes.
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice
>> + *
>> + * Free an event adapter
>> + *
>> + * @param id
>> + *  Adapter identifier.
>> + * @return
>> + *   - 0: Success
>> + *   - <0: Error code on failure, If the adapter still has Tx queues
>> + *      added to it, the function returns -EBUSY.
>> + */
>> +int rte_event_eth_tx_adapter_free(uint8_t id);
>> +
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice
>> + *
>> + * Start ethernet Tx event adapter
>> + *
>> + * @param id
>> + *  Adapter identifier.
>> + * @return
>> + *  - 0: Success, Adapter started correctly.
>> + *  - <0: Error code on failure.
>> + */
>> +int rte_event_eth_tx_adapter_start(uint8_t id);
>> +
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice
>> + *
>> + * Stop ethernet Tx event adapter
>> + *
>> + * @param id
>> + *  Adapter identifier.
>> + * @return
>> + *  - 0: Success, Adapter started correctly.
>> + *  - <0: Error code on failure.
>> + */
>> +int rte_event_eth_tx_adapter_stop(uint8_t id);
>> +
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice
>> + *
>> + * Signal the Tx adapter to start processing mbufs for a
>> + * Tx queue. A queue value of -1 is used to indicate all
>> + * queues within the device.
>> + *
>> + * @param id
>> + *  Adapter identifier.
>> + * @param eth_dev_id
>> + *  Ethernet Port Identifier.
>> + * @param queue
>> + *  Tx queue index.
>> + *
>> + * @return
>> + *  - 0: Success, Adapter started correctly.
>> + *  - <0: Error code on failure.
>> + */
>> +int rte_event_eth_tx_adapter_queue_start(uint8_t id,
>> +					uint16_t eth_dev_id,
>> +					int32_t queue);
>> +
> How about changing to to rte_event_eth_tx_adapter_queue_add to inline
> with Rx adapter?

OK.
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice
>> + *
>> + * Signal the Tx adapter to stop processing mbufs for a
>> + * Tx queue. A queue value of -1 is used to indicate all
>> + * queues within the device.
>> + *
>> + * @param id
>> + *  Adapter identifier.
>> + * @param eth_dev_id
>> + *  Ethernet Port Identifier.
>> + * @param queue
>> + *  Tx queue index.
>> + *
>> + * @return
>> + *  - 0: Success, Adapter started correctly.
>> + *  - <0: Error code on failure.
>> + */
>> +int rte_event_eth_tx_adapter_queue_stop(uint8_t id,
>> +					uint16_t eth_dev_id,
>> +					int32_t queue);
>> +
I think the queue_del() function  should have to take id.
>> +static __rte_always_inline uint16_t
>> +__rte_event_eth_tx_adapter_enqueue(uint8_t id, uint8_t dev_id, uint8_t port_id,
>> +				const struct rte_event ev[],
>> +				uint16_t nb_events,
>> +				const event_tx_adapter_enqueue fn)
>> +{
>> +	const struct rte_eventdev *dev = &rte_eventdevs[dev_id];
>> +
>> +#ifdef RTE_LIBRTE_EVENTDEV_DEBUG
>> +	if (id >= RTE_EVENT_ETH_TX_ADAPTER_MAX_INSTANCE ||
>> +		dev_id >= RTE_EVENT_MAX_DEVS ||
>> +		!rte_eventdevs[dev_id].attached) {
>> +		rte_errno = -EINVAL;
>> +		return 0;
>> +	}
>> +
>> +	if (port_id >= dev->data->nb_ports) {
>> +		rte_errno = -EINVAL;
>> +		return 0;
>> +	}
>> +#endif
>> +	/*
>> +	 * TODO: Do we need a special case for nb_events == 1
>> +	 */
> I think, we don't need the special case for nb_events == 1
>
>> +	return fn(id, dev, dev->data->ports[port_id], ev, nb_events);
>> +}
>> +
>> +/**
>> + * Enqueue a burst of events objects or an event object supplied in *rte_event*
>> + * structure on an  event device designated by its *dev_id* through the event
>> + * port specified by *port_id*. The the event queue on which it will be enqueued
>> + * id derived from the adapter id parameter.
>> + *
>> + * The *nb_events* parameter is the number of event objects to enqueue which are
>> + * supplied in the *ev* array of *rte_event* structure.
>> + *
>> + * The rte_event_eth_tx_adapter_enqueue() function returns the number of
>> + * events objects it actually enqueued. A return value equal to *nb_events*
>> + * means that all event objects have been enqueued.
>> + *
>> + * @param id
>> + *  The identifier of the tx adapter.
>> + * @param dev_id
>> + *  The identifier of the device.
>> + * @param port_id
>> + *  The identifier of the event port.
>> + * @param ev
>> + *  Points to an array of *nb_events* objects of type *rte_event* structure
>> + *  which contain the event object enqueue operations to be processed.
>> + * @param nb_events
>> + *  The number of event objects to enqueue, typically number of
>> + *  rte_event_port_enqueue_depth() available for this port.
>> + *
>> + * @return
>> + *   The number of event objects actually enqueued on the event device. The
>> + *   return value can be less than the value of the *nb_events* parameter when
>> + *   the event devices queue is full or if invalid parameters are specified in a
>> + *   *rte_event*. If the return value is less than *nb_events*, the remaining
>> + *   events at the end of ev[] are not consumed and the caller has to take care
>> + *   of them, and rte_errno is set accordingly. Possible errno values include:
>> + *   - -EINVAL  The port ID is invalid, device ID is invalid, an event's queue
>> + *              ID is invalid, or an event's sched type doesn't match the
>> + *              capabilities of the destination queue.
>> + *   - -ENOSPC  The event port was backpressured and unable to enqueue
>> + *              one or more events. This error code is only applicable to
>> + *              closed systems.
>> + */
>> +static inline uint16_t
>> +rte_event_eth_tx_adapter_enqueue(uint8_t id, uint8_t dev_id,
>> +				uint8_t port_id,
>> +				const struct rte_event ev[],
>> +				uint16_t nb_events)
>> +{
>> +	const struct rte_eventdev *dev = &rte_eventdevs[dev_id];
>> +
>> +	return __rte_event_eth_tx_adapter_enqueue(id, dev_id, port_id, ev,
>> +						nb_events,
>> +						dev->tx_adapter_enqueue);
>> +}
>> +
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice
>> + *
>> + * Retrieve statistics for an adapter
>> + *
>> + * @param id
>> + *  Adapter identifier.
>> + * @param [out] stats
>> + *  A pointer to structure used to retrieve statistics for an adapter.
>> + * @return
>> + *  - 0: Success, retrieved successfully.
>> + *  - <0: Error code on failure.
>> + */
>> +int rte_event_eth_tx_adapter_stats_get(uint8_t id,
>> +				struct rte_event_eth_tx_adapter_stats *stats);
>> +
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice
>> + *
>> + * Reset statistics for an adapter.
>> + *
>> + * @param id
>> + *  Adapter identifier.
>> + * @return
>> + *  - 0: Success, statistics reset successfully.
>> + *  - <0: Error code on failure.
>> + */
>> +int rte_event_eth_tx_adapter_stats_reset(uint8_t id);
>> +
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice
>> + *
>> + * Retrieve the service ID of an adapter. If the adapter doesn't use
>> + * a rte_service function, this function returns -ESRCH.
>> + *
>> + * @param id
>> + *  Adapter identifier.
>> + * @param [out] service_id
>> + *  A pointer to a uint32_t, to be filled in with the service id.
>> + * @return
>> + *  - 0: Success
>> + *  - <0: Error code on failure, if the adapter doesn't use a rte_service
>> + * function, this function returns -ESRCH.
>> + */
>> +int rte_event_eth_tx_adapter_service_id_get(uint8_t id, uint32_t *service_id);
>> +
>> +#ifdef __cplusplus
>> +}
>> +#endif
>> +#endif	/* _RTE_EVENT_ETH_TX_ADAPTER_ */
>> diff --git a/lib/librte_eventdev/rte_eventdev.h b/lib/librte_eventdev/rte_eventdev.h
>> index b6fd6ee..1bf28a0 100644
>> --- a/lib/librte_eventdev/rte_eventdev.h
>> +++ b/lib/librte_eventdev/rte_eventdev.h
>> @@ -1203,6 +1203,10 @@ typedef uint16_t (*event_dequeue_t)(void *port, struct rte_event *ev,
>>   typedef uint16_t (*event_dequeue_burst_t)(void *port, struct rte_event ev[],
>>   		uint16_t nb_events, uint64_t timeout_ticks);
>>   /**< @internal Dequeue burst of events from port of a device */
>> +typedef uint16_t (*event_tx_adapter_enqueue)(uint8_t id,
>> +			const struct rte_eventdev *dev, void *port,
>> +			const struct rte_event ev[], uint16_t nb_events);
>> +/**< @internal Enqueue burst of events on port of a device */
>>   
>>   #define RTE_EVENTDEV_NAME_MAX_LEN	(64)
>>   /**< @internal Max length of name of event PMD */
>> @@ -1266,7 +1270,8 @@ struct rte_eventdev {
>>   	/**< Pointer to PMD dequeue function. */
>>   	event_dequeue_burst_t dequeue_burst;
>>   	/**< Pointer to PMD dequeue burst function. */
>> -
>> +	event_tx_adapter_enqueue tx_adapter_enqueue;
>> +	/**< Pointer to PMD tx adapter enqueue function. */
>>   	struct rte_eventdev_data *data;
>>   	/**< Pointer to device data */
>>   	struct rte_eventdev_ops *dev_ops;
>> diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h
>> index 4fd9a0d..c59389c 100644
>> --- a/lib/librte_mbuf/rte_mbuf.h
>> +++ b/lib/librte_mbuf/rte_mbuf.h
>> @@ -464,7 +464,9 @@ struct rte_mbuf {
>>   	};
>>   	uint16_t nb_segs;         /**< Number of segments. */
>>   
>> -	/** Input port (16 bits to support more than 256 virtual ports). */
>> +	/** Input port (16 bits to support more than 256 virtual ports).
>> +	 * The event eth Tx adapter uses this field to specify the output port.
> See rte_event_eth_tx_adapter_enqueue()
>
>> +	 */
>>   	uint16_t port;
>>   
>>   	uint64_t ol_flags;        /**< Offload features. */
>> @@ -511,6 +513,7 @@ struct rte_mbuf {
>>   	/** VLAN TCI (CPU order), valid if PKT_RX_VLAN is set. */
>>   	uint16_t vlan_tci;
>>   
>> +	RTE_STD_C11
>>   	union {
>>   		uint32_t rss;     /**< RSS hash result if RSS enabled */
>>   		struct {
>> @@ -531,6 +534,11 @@ struct rte_mbuf {
>>   			uint32_t lo;
>>   			uint32_t hi;
>>   		} sched;          /**< Hierarchical scheduler */
>> +		struct {
>> +			uint32_t resvd1; /* overlaps with rte_sched_port_hierarchy::color */
> s/overlaps/Overlaps/
>
>> +			uint16_t resvd2;
>> +			uint16_t txq_id;
> /* The event eth Tx adapter transmit queue. See rte_event_eth_tx_adapter_enqueue() */
>
>> +		};
>>   		uint32_t usr;	  /**< User defined tags. See rte_distributor_process() */
>>   	} hash;                   /**< hash information */
>>   
>> @@ -1880,6 +1888,16 @@ static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
>>   #define rte_pktmbuf_data_len(m) ((m)->data_len)
>>   
>>   /**
>> + * A macro that returns the txq field of the mbuf
>> + *
>> + * The value can be read or assigned.
>> + *
>> + * @param m
>> + *   The packet mbuf.
>> + */
>> +#define rte_pktmbuf_tx_queue(m) ((m)->txq_id)
> Should we change to rte_pktmbuf_tx_adapter_txq to make it explicit that
> it is related to Tx adapter?
I am fine with the suggestion.

Thanks for the review,
Nikhil