[dpdk-dev] [PATCH v6 1/3] net/tap: add queue and port ids in Rx/Tx queues structures

[dpdk-dev] [PATCH v6 1/3] net/tap: add queue and port ids in Rx/Tx queues structures

[Raslan Darawsheh]

Port and queue ids are added to easily map the file
descriptors stored in each process private.
---
changes in v6:
 - reword the commit log
----
Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
---
 drivers/net/tap/rte_eth_tap.c | 3 +++
 drivers/net/tap/rte_eth_tap.h | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index ad5ae98..edfb7da 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -1293,6 +1293,7 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->mp = mp;
 	rxq->trigger_seen = 1; /* force initial burst */
 	rxq->in_port = dev->data->port_id;
+	rxq->queue_id = rx_queue_id;
 	rxq->nb_rx_desc = nb_desc;
 	iovecs = rte_zmalloc_socket(dev->device->name, sizeof(*iovecs), 0,
 				    socket_id);
@@ -1359,6 +1360,8 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		return -1;
 	dev->data->tx_queues[tx_queue_id] = &internals->txq[tx_queue_id];
 	txq = dev->data->tx_queues[tx_queue_id];
+	txq->out_port = dev->data->port_id;
+	txq->queue_id = tx_queue_id;
 
 	offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
 	txq->csum = !!(offloads &
diff --git a/drivers/net/tap/rte_eth_tap.h b/drivers/net/tap/rte_eth_tap.h
index 44e2773..4502e24 100644
--- a/drivers/net/tap/rte_eth_tap.h
+++ b/drivers/net/tap/rte_eth_tap.h
@@ -46,6 +46,7 @@ struct rx_queue {
 	struct rte_mempool *mp;         /* Mempool for RX packets */
 	uint32_t trigger_seen;          /* Last seen Rx trigger value */
 	uint16_t in_port;               /* Port ID */
+	uint16_t queue_id;		/* queue ID*/
 	int fd;
 	struct pkt_stats stats;         /* Stats for this RX queue */
 	uint16_t nb_rx_desc;            /* max number of mbufs available */
@@ -62,6 +63,8 @@ struct tx_queue {
 	uint16_t csum:1;                /* Enable checksum offloading */
 	struct pkt_stats stats;         /* Stats for this TX queue */
 	struct rte_gso_ctx gso_ctx;     /* GSO context */
+	uint16_t out_port;              /* Port ID */
+	uint16_t queue_id;		/* queue ID*/
 };
 
 struct pmd_internals {
-- 
2.7.4

[dpdk-dev] [PATCH v6 2/3] net/tap: move fds of Rx/Tx queues to be in process private

[Raslan Darawsheh]

fd's cannot be shared between processes, and each process need to have
it's own fd's pointer.

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
---
 drivers/net/tap/rte_eth_tap.c | 93 +++++++++++++++++++++++++++++--------------
 drivers/net/tap/rte_eth_tap.h |  7 +++-
 drivers/net/tap/tap_flow.c    |  3 +-
 drivers/net/tap/tap_intr.c    |  5 ++-
 4 files changed, 73 insertions(+), 35 deletions(-)

diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index edfb7da..3372d54 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -315,6 +315,7 @@ static uint16_t
 pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 {
 	struct rx_queue *rxq = queue;
+	struct pmd_process_private *process_private;
 	uint16_t num_rx;
 	unsigned long num_rx_bytes = 0;
 	uint32_t trigger = tap_trigger;
@@ -323,6 +324,7 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		return 0;
 	if (trigger)
 		rxq->trigger_seen = trigger;
+	process_private = rte_eth_devices[rxq->in_port].process_private;
 	rte_compiler_barrier();
 	for (num_rx = 0; num_rx < nb_pkts; ) {
 		struct rte_mbuf *mbuf = rxq->pool;
@@ -331,9 +333,9 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		uint16_t data_off = rte_pktmbuf_headroom(mbuf);
 		int len;
 
-		len = readv(rxq->fd, *rxq->iovecs,
-			    1 +
-			    (rxq->rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ?
+		len = readv(process_private->rxq_fds[rxq->queue_id],
+			*rxq->iovecs,
+			1 + (rxq->rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ?
 			     rxq->nb_rx_desc : 1));
 		if (len < (int)sizeof(struct tun_pi))
 			break;
@@ -494,6 +496,9 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
 {
 	int i;
 	uint16_t l234_hlen;
+	struct pmd_process_private *process_private;
+
+	process_private = rte_eth_devices[txq->out_port].process_private;
 
 	for (i = 0; i < num_mbufs; i++) {
 		struct rte_mbuf *mbuf = pmbufs[i];
@@ -595,7 +600,7 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
 			tap_tx_l4_cksum(l4_cksum, l4_phdr_cksum, l4_raw_cksum);
 
 		/* copy the tx frame data */
-		n = writev(txq->fd, iovecs, j);
+		n = writev(process_private->txq_fds[txq->queue_id], iovecs, j);
 		if (n <= 0)
 			break;
 		(*num_packets)++;
@@ -970,19 +975,20 @@ tap_dev_close(struct rte_eth_dev *dev)
 {
 	int i;
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 
 	tap_link_set_down(dev);
 	tap_flow_flush(dev, NULL);
 	tap_flow_implicit_flush(internals, NULL);
 
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		if (internals->rxq[i].fd != -1) {
-			close(internals->rxq[i].fd);
-			internals->rxq[i].fd = -1;
+		if (process_private->rxq_fds[i] != -1) {
+			close(process_private->rxq_fds[i]);
+			process_private->rxq_fds[i] = -1;
 		}
-		if (internals->txq[i].fd != -1) {
-			close(internals->txq[i].fd);
-			internals->txq[i].fd = -1;
+		if (process_private->txq_fds[i] != -1) {
+			close(process_private->txq_fds[i]);
+			process_private->txq_fds[i] = -1;
 		}
 	}
 
@@ -1006,10 +1012,14 @@ static void
 tap_rx_queue_release(void *queue)
 {
 	struct rx_queue *rxq = queue;
+	struct pmd_process_private *process_private;
 
-	if (rxq && (rxq->fd > 0)) {
-		close(rxq->fd);
-		rxq->fd = -1;
+	if (!rxq)
+		return;
+	process_private = rte_eth_devices[rxq->in_port].process_private;
+	if (process_private->rxq_fds[rxq->queue_id] > 0) {
+		close(process_private->rxq_fds[rxq->queue_id]);
+		process_private->rxq_fds[rxq->queue_id] = -1;
 		rte_pktmbuf_free(rxq->pool);
 		rte_free(rxq->iovecs);
 		rxq->pool = NULL;
@@ -1021,10 +1031,15 @@ static void
 tap_tx_queue_release(void *queue)
 {
 	struct tx_queue *txq = queue;
+	struct pmd_process_private *process_private;
+
+	if (!txq)
+		return;
+	process_private = rte_eth_devices[txq->out_port].process_private;
 
-	if (txq && (txq->fd > 0)) {
-		close(txq->fd);
-		txq->fd = -1;
+	if (process_private->txq_fds[txq->queue_id] > 0) {
+		close(process_private->txq_fds[txq->queue_id]);
+		process_private->txq_fds[txq->queue_id] = -1;
 	}
 }
 
@@ -1209,18 +1224,19 @@ tap_setup_queue(struct rte_eth_dev *dev,
 	int *other_fd;
 	const char *dir;
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct rx_queue *rx = &internals->rxq[qid];
 	struct tx_queue *tx = &internals->txq[qid];
 	struct rte_gso_ctx *gso_ctx;
 
 	if (is_rx) {
-		fd = &rx->fd;
-		other_fd = &tx->fd;
+		fd = &process_private->rxq_fds[qid];
+		other_fd = &process_private->txq_fds[qid];
 		dir = "rx";
 		gso_ctx = NULL;
 	} else {
-		fd = &tx->fd;
-		other_fd = &rx->fd;
+		fd = &process_private->txq_fds[qid];
+		other_fd = &process_private->rxq_fds[qid];
 		dir = "tx";
 		gso_ctx = &tx->gso_ctx;
 	}
@@ -1273,6 +1289,7 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 		   struct rte_mempool *mp)
 {
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct rx_queue *rxq = &internals->rxq[rx_queue_id];
 	struct rte_mbuf **tmp = &rxq->pool;
 	long iov_max = sysconf(_SC_IOV_MAX);
@@ -1332,7 +1349,8 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 	}
 
 	TAP_LOG(DEBUG, "  RX TUNTAP device name %s, qid %d on fd %d",
-		internals->name, rx_queue_id, internals->rxq[rx_queue_id].fd);
+		internals->name, rx_queue_id,
+		process_private->rxq_fds[rx_queue_id]);
 
 	return 0;
 
@@ -1352,6 +1370,7 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		   const struct rte_eth_txconf *tx_conf)
 {
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct tx_queue *txq;
 	int ret;
 	uint64_t offloads;
@@ -1374,7 +1393,8 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		return -1;
 	TAP_LOG(DEBUG,
 		"  TX TUNTAP device name %s, qid %d on fd %d csum %s",
-		internals->name, tx_queue_id, internals->txq[tx_queue_id].fd,
+		internals->name, tx_queue_id,
+		process_private->txq_fds[tx_queue_id],
 		txq->csum ? "on" : "off");
 
 	return 0;
@@ -1622,6 +1642,7 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 	int numa_node = rte_socket_id();
 	struct rte_eth_dev *dev;
 	struct pmd_internals *pmd;
+	struct pmd_process_private *process_private;
 	struct rte_eth_dev_data *data;
 	struct ifreq ifr;
 	int i;
@@ -1636,7 +1657,16 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 		goto error_exit_nodev;
 	}
 
+	process_private = (struct pmd_process_private *)
+		rte_zmalloc_socket(tap_name, sizeof(struct pmd_process_private),
+			RTE_CACHE_LINE_SIZE, dev->device->numa_node);
+
+	if (process_private == NULL) {
+		TAP_LOG(ERR, "Failed to alloc memory for process private");
+		return -1;
+	}
 	pmd = dev->data->dev_private;
+	dev->process_private = process_private;
 	pmd->dev = dev;
 	snprintf(pmd->name, sizeof(pmd->name), "%s", tap_name);
 	pmd->type = type;
@@ -1672,8 +1702,8 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 	/* Presetup the fds to -1 as being not valid */
 	pmd->ka_fd = -1;
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		pmd->rxq[i].fd = -1;
-		pmd->txq[i].fd = -1;
+		process_private->rxq_fds[i] = -1;
+		process_private->txq_fds[i] = -1;
 	}
 
 	if (pmd->type == ETH_TUNTAP_TYPE_TAP) {
@@ -2073,6 +2103,7 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 {
 	struct rte_eth_dev *eth_dev = NULL;
 	struct pmd_internals *internals;
+	struct pmd_process_private *process_private;
 	int i;
 
 	/* find the ethdev entry */
@@ -2081,6 +2112,7 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 		return 0;
 
 	internals = eth_dev->data->dev_private;
+	process_private = eth_dev->process_private;
 
 	TAP_LOG(DEBUG, "Closing %s Ethernet device on numa %u",
 		(internals->type == ETH_TUNTAP_TYPE_TAP) ? "TAP" : "TUN",
@@ -2092,18 +2124,19 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 		tap_nl_final(internals->nlsk_fd);
 	}
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		if (internals->rxq[i].fd != -1) {
-			close(internals->rxq[i].fd);
-			internals->rxq[i].fd = -1;
+		if (process_private->rxq_fds[i] != -1) {
+			close(process_private->rxq_fds[i]);
+			process_private->rxq_fds[i] = -1;
 		}
-		if (internals->txq[i].fd != -1) {
-			close(internals->txq[i].fd);
-			internals->txq[i].fd = -1;
+		if (process_private->txq_fds[i] != -1) {
+			close(process_private->txq_fds[i]);
+			process_private->txq_fds[i] = -1;
 		}
 	}
 
 	close(internals->ioctl_sock);
 	rte_free(eth_dev->data->dev_private);
+	rte_free(eth_dev->process_private);
 	rte_eth_dev_release_port(eth_dev);
 
 	if (internals->ka_fd != -1) {
diff --git a/drivers/net/tap/rte_eth_tap.h b/drivers/net/tap/rte_eth_tap.h
index 4502e24..dc3579a 100644
--- a/drivers/net/tap/rte_eth_tap.h
+++ b/drivers/net/tap/rte_eth_tap.h
@@ -47,7 +47,6 @@ struct rx_queue {
 	uint32_t trigger_seen;          /* Last seen Rx trigger value */
 	uint16_t in_port;               /* Port ID */
 	uint16_t queue_id;		/* queue ID*/
-	int fd;
 	struct pkt_stats stats;         /* Stats for this RX queue */
 	uint16_t nb_rx_desc;            /* max number of mbufs available */
 	struct rte_eth_rxmode *rxmode;  /* RX features */
@@ -57,7 +56,6 @@ struct rx_queue {
 };
 
 struct tx_queue {
-	int fd;
 	int type;                       /* Type field - TUN|TAP */
 	uint16_t *mtu;                  /* Pointer to MTU from dev_data */
 	uint16_t csum:1;                /* Enable checksum offloading */
@@ -95,6 +93,11 @@ struct pmd_internals {
 	int ka_fd;                        /* keep-alive file descriptor */
 };
 
+struct pmd_process_private {
+	int rxq_fds[RTE_PMD_TAP_MAX_QUEUES];
+	int txq_fds[RTE_PMD_TAP_MAX_QUEUES];
+};
+
 /* tap_intr.c */
 
 int tap_rx_intr_vec_set(struct rte_eth_dev *dev, int set);
diff --git a/drivers/net/tap/tap_flow.c b/drivers/net/tap/tap_flow.c
index 0e01af6..3ba6945 100644
--- a/drivers/net/tap/tap_flow.c
+++ b/drivers/net/tap/tap_flow.c
@@ -1567,6 +1567,7 @@ tap_flow_isolate(struct rte_eth_dev *dev,
 		 struct rte_flow_error *error __rte_unused)
 {
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 
 	/* normalize 'set' variable to contain 0 or 1 values */
 	if (set)
@@ -1580,7 +1581,7 @@ tap_flow_isolate(struct rte_eth_dev *dev,
 	 * If netdevice is there, setup appropriate flow rules immediately.
 	 * Otherwise it will be set when bringing up the netdevice (tun_alloc).
 	 */
-	if (!pmd->rxq[0].fd)
+	if (!process_private->rxq_fds[0])
 		return 0;
 	if (set) {
 		struct rte_flow *remote_flow;
diff --git a/drivers/net/tap/tap_intr.c b/drivers/net/tap/tap_intr.c
index fc59018..7af0010 100644
--- a/drivers/net/tap/tap_intr.c
+++ b/drivers/net/tap/tap_intr.c
@@ -51,6 +51,7 @@ static int
 tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 {
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	unsigned int rxqs_n = pmd->dev->data->nb_rx_queues;
 	struct rte_intr_handle *intr_handle = &pmd->intr_handle;
 	unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
@@ -71,7 +72,7 @@ tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 		struct rx_queue *rxq = pmd->dev->data->rx_queues[i];
 
 		/* Skip queues that cannot request interrupts. */
-		if (!rxq || rxq->fd <= 0) {
+		if (!rxq || process_private->rxq_fds[i] <= 0) {
 			/* Use invalid intr_vec[] index to disable entry. */
 			intr_handle->intr_vec[i] =
 				RTE_INTR_VEC_RXTX_OFFSET +
@@ -79,7 +80,7 @@ tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 			continue;
 		}
 		intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
-		intr_handle->efds[count] = rxq->fd;
+		intr_handle->efds[count] = process_private->rxq_fds[i];
 		count++;
 	}
 	if (!count)
-- 
2.7.4

Re: [dpdk-dev] [PATCH v6 2/3] net/tap: move fds of Rx/Tx queues to be in process private

[Ferruh Yigit]

On 10/10/2018 3:39 PM, Raslan Darawsheh wrote:
> fd's cannot be shared between processes, and each process need to have
> it's own fd's pointer.
> 
> Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
> ---
>  drivers/net/tap/rte_eth_tap.c | 93 +++++++++++++++++++++++++++++--------------
>  drivers/net/tap/rte_eth_tap.h |  7 +++-
>  drivers/net/tap/tap_flow.c    |  3 +-
>  drivers/net/tap/tap_intr.c    |  5 ++-
>  4 files changed, 73 insertions(+), 35 deletions(-)
> 
> diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
> index edfb7da..3372d54 100644
> --- a/drivers/net/tap/rte_eth_tap.c
> +++ b/drivers/net/tap/rte_eth_tap.c
> @@ -315,6 +315,7 @@ static uint16_t
>  pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
>  {
>  	struct rx_queue *rxq = queue;
> +	struct pmd_process_private *process_private;
>  	uint16_t num_rx;
>  	unsigned long num_rx_bytes = 0;
>  	uint32_t trigger = tap_trigger;
> @@ -323,6 +324,7 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
>  		return 0;
>  	if (trigger)
>  		rxq->trigger_seen = trigger;
> +	process_private = rte_eth_devices[rxq->in_port].process_private;

This patch has a dependency to [1] which is waiting for new version, right?
Please note these kind of dependencies after "---" part of the commit log.

[1]
https://patches.dpdk.org/patch/46185/

Re: [dpdk-dev] [PATCH v6 2/3] net/tap: move fds of Rx/Tx queues to be in process private

[Raslan Darawsheh]

Hi Ferruh,

Yes you are right about that, I missed adding it in the commit log
Should I resend a new version with this noted?

Kindest regards,
Raslan Darawsheh

> -----Original Message-----
> From: Ferruh Yigit <ferruh.yigit@intel.com>
> Sent: Thursday, October 11, 2018 7:20 PM
> To: Raslan Darawsheh <rasland@mellanox.com>; keith.wiles@intel.com
> Cc: Thomas Monjalon <thomas@monjalon.net>; dev@dpdk.org; Shahaf
> Shuler <shahafs@mellanox.com>; Ori Kam <orika@mellanox.com>;
> Alejandro Lucero <alejandro.lucero@netronome.com>
> Subject: Re: [PATCH v6 2/3] net/tap: move fds of Rx/Tx queues to be in
> process private
> 
> On 10/10/2018 3:39 PM, Raslan Darawsheh wrote:
> > fd's cannot be shared between processes, and each process need to have
> > it's own fd's pointer.
> >
> > Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
> > ---
> >  drivers/net/tap/rte_eth_tap.c | 93
> > +++++++++++++++++++++++++++++--------------
> >  drivers/net/tap/rte_eth_tap.h |  7 +++-
> >  drivers/net/tap/tap_flow.c    |  3 +-
> >  drivers/net/tap/tap_intr.c    |  5 ++-
> >  4 files changed, 73 insertions(+), 35 deletions(-)
> >
> > diff --git a/drivers/net/tap/rte_eth_tap.c
> > b/drivers/net/tap/rte_eth_tap.c index edfb7da..3372d54 100644
> > --- a/drivers/net/tap/rte_eth_tap.c
> > +++ b/drivers/net/tap/rte_eth_tap.c
> > @@ -315,6 +315,7 @@ static uint16_t
> >  pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
> > {
> >  	struct rx_queue *rxq = queue;
> > +	struct pmd_process_private *process_private;
> >  	uint16_t num_rx;
> >  	unsigned long num_rx_bytes = 0;
> >  	uint32_t trigger = tap_trigger;
> > @@ -323,6 +324,7 @@ pmd_rx_burst(void *queue, struct rte_mbuf
> **bufs, uint16_t nb_pkts)
> >  		return 0;
> >  	if (trigger)
> >  		rxq->trigger_seen = trigger;
> > +	process_private = rte_eth_devices[rxq->in_port].process_private;
> 
> This patch has a dependency to [1] which is waiting for new version, right?
> Please note these kind of dependencies after "---" part of the commit log.
> 
> [1]
> https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpat
> ches.dpdk.org%2Fpatch%2F46185%2F&amp;data=02%7C01%7Crasland%40m
> ellanox.com%7C5299b99a73e648ba870508d62f955870%7Ca652971c7d2e4d9b
> a6a4d149256f461b%7C0%7C0%7C636748715807291110&amp;sdata=lEbny%2F
> gaGhBvfFlc8RzlB3YwvZ3uD43LCtq6w%2B07RFM%3D&amp;reserved=0

Re: [dpdk-dev] [PATCH v6 2/3] net/tap: move fds of Rx/Tx queues to be in process private

[Ferruh Yigit]

On 10/16/2018 11:07 AM, Raslan Darawsheh wrote:
> Hi Ferruh,
> 
> Yes you are right about that, I missed adding it in the commit log
> Should I resend a new version with this noted?

No, not required, this is just a reminder for further.

<...>

>>
>> This patch has a dependency to [1] which is waiting for new version, right?
>> Please note these kind of dependencies after "---" part of the commit log.
>>
>> [1]
>> https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpat
>> ches.dpdk.org%2Fpatch%2F46185%2F&amp;data=02%7C01%7Crasland%40m
>> ellanox.com%7C5299b99a73e648ba870508d62f955870%7Ca652971c7d2e4d9b
>> a6a4d149256f461b%7C0%7C0%7C636748715807291110&amp;sdata=lEbny%2F
>> gaGhBvfFlc8RzlB3YwvZ3uD43LCtq6w%2B07RFM%3D&amp;reserved=0

[dpdk-dev] [PATCH v7 1/3] net/tap: add queue and port ids in Rx/Tx queues structures

[Raslan Darawsheh]

Port and queue ids are added to easily map the file
descriptors stored in each process private.

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
---
 drivers/net/tap/rte_eth_tap.c | 3 +++
 drivers/net/tap/rte_eth_tap.h | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index ad5ae98..edfb7da 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -1293,6 +1293,7 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->mp = mp;
 	rxq->trigger_seen = 1; /* force initial burst */
 	rxq->in_port = dev->data->port_id;
+	rxq->queue_id = rx_queue_id;
 	rxq->nb_rx_desc = nb_desc;
 	iovecs = rte_zmalloc_socket(dev->device->name, sizeof(*iovecs), 0,
 				    socket_id);
@@ -1359,6 +1360,8 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		return -1;
 	dev->data->tx_queues[tx_queue_id] = &internals->txq[tx_queue_id];
 	txq = dev->data->tx_queues[tx_queue_id];
+	txq->out_port = dev->data->port_id;
+	txq->queue_id = tx_queue_id;
 
 	offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
 	txq->csum = !!(offloads &
diff --git a/drivers/net/tap/rte_eth_tap.h b/drivers/net/tap/rte_eth_tap.h
index 44e2773..4502e24 100644
--- a/drivers/net/tap/rte_eth_tap.h
+++ b/drivers/net/tap/rte_eth_tap.h
@@ -46,6 +46,7 @@ struct rx_queue {
 	struct rte_mempool *mp;         /* Mempool for RX packets */
 	uint32_t trigger_seen;          /* Last seen Rx trigger value */
 	uint16_t in_port;               /* Port ID */
+	uint16_t queue_id;		/* queue ID*/
 	int fd;
 	struct pkt_stats stats;         /* Stats for this RX queue */
 	uint16_t nb_rx_desc;            /* max number of mbufs available */
@@ -62,6 +63,8 @@ struct tx_queue {
 	uint16_t csum:1;                /* Enable checksum offloading */
 	struct pkt_stats stats;         /* Stats for this TX queue */
 	struct rte_gso_ctx gso_ctx;     /* GSO context */
+	uint16_t out_port;              /* Port ID */
+	uint16_t queue_id;		/* queue ID*/
 };
 
 struct pmd_internals {
-- 
2.7.4

[dpdk-dev] [PATCH v7 2/3] net/tap: move fds of Rx/Tx queues to be in process private

[Raslan Darawsheh]

fd's cannot be shared between processes, and each process need to have
it's own fd's pointer.

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>

---
   This patch has dependancy on the following patch
   http://patches.dpdk.org/patch/46185/
---
---
 drivers/net/tap/rte_eth_tap.c | 93 +++++++++++++++++++++++++++++--------------
 drivers/net/tap/rte_eth_tap.h |  7 +++-
 drivers/net/tap/tap_flow.c    |  3 +-
 drivers/net/tap/tap_intr.c    |  5 ++-
 4 files changed, 73 insertions(+), 35 deletions(-)

diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index edfb7da..3372d54 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -315,6 +315,7 @@ static uint16_t
 pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 {
 	struct rx_queue *rxq = queue;
+	struct pmd_process_private *process_private;
 	uint16_t num_rx;
 	unsigned long num_rx_bytes = 0;
 	uint32_t trigger = tap_trigger;
@@ -323,6 +324,7 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		return 0;
 	if (trigger)
 		rxq->trigger_seen = trigger;
+	process_private = rte_eth_devices[rxq->in_port].process_private;
 	rte_compiler_barrier();
 	for (num_rx = 0; num_rx < nb_pkts; ) {
 		struct rte_mbuf *mbuf = rxq->pool;
@@ -331,9 +333,9 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		uint16_t data_off = rte_pktmbuf_headroom(mbuf);
 		int len;
 
-		len = readv(rxq->fd, *rxq->iovecs,
-			    1 +
-			    (rxq->rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ?
+		len = readv(process_private->rxq_fds[rxq->queue_id],
+			*rxq->iovecs,
+			1 + (rxq->rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ?
 			     rxq->nb_rx_desc : 1));
 		if (len < (int)sizeof(struct tun_pi))
 			break;
@@ -494,6 +496,9 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
 {
 	int i;
 	uint16_t l234_hlen;
+	struct pmd_process_private *process_private;
+
+	process_private = rte_eth_devices[txq->out_port].process_private;
 
 	for (i = 0; i < num_mbufs; i++) {
 		struct rte_mbuf *mbuf = pmbufs[i];
@@ -595,7 +600,7 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
 			tap_tx_l4_cksum(l4_cksum, l4_phdr_cksum, l4_raw_cksum);
 
 		/* copy the tx frame data */
-		n = writev(txq->fd, iovecs, j);
+		n = writev(process_private->txq_fds[txq->queue_id], iovecs, j);
 		if (n <= 0)
 			break;
 		(*num_packets)++;
@@ -970,19 +975,20 @@ tap_dev_close(struct rte_eth_dev *dev)
 {
 	int i;
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 
 	tap_link_set_down(dev);
 	tap_flow_flush(dev, NULL);
 	tap_flow_implicit_flush(internals, NULL);
 
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		if (internals->rxq[i].fd != -1) {
-			close(internals->rxq[i].fd);
-			internals->rxq[i].fd = -1;
+		if (process_private->rxq_fds[i] != -1) {
+			close(process_private->rxq_fds[i]);
+			process_private->rxq_fds[i] = -1;
 		}
-		if (internals->txq[i].fd != -1) {
-			close(internals->txq[i].fd);
-			internals->txq[i].fd = -1;
+		if (process_private->txq_fds[i] != -1) {
+			close(process_private->txq_fds[i]);
+			process_private->txq_fds[i] = -1;
 		}
 	}
 
@@ -1006,10 +1012,14 @@ static void
 tap_rx_queue_release(void *queue)
 {
 	struct rx_queue *rxq = queue;
+	struct pmd_process_private *process_private;
 
-	if (rxq && (rxq->fd > 0)) {
-		close(rxq->fd);
-		rxq->fd = -1;
+	if (!rxq)
+		return;
+	process_private = rte_eth_devices[rxq->in_port].process_private;
+	if (process_private->rxq_fds[rxq->queue_id] > 0) {
+		close(process_private->rxq_fds[rxq->queue_id]);
+		process_private->rxq_fds[rxq->queue_id] = -1;
 		rte_pktmbuf_free(rxq->pool);
 		rte_free(rxq->iovecs);
 		rxq->pool = NULL;
@@ -1021,10 +1031,15 @@ static void
 tap_tx_queue_release(void *queue)
 {
 	struct tx_queue *txq = queue;
+	struct pmd_process_private *process_private;
+
+	if (!txq)
+		return;
+	process_private = rte_eth_devices[txq->out_port].process_private;
 
-	if (txq && (txq->fd > 0)) {
-		close(txq->fd);
-		txq->fd = -1;
+	if (process_private->txq_fds[txq->queue_id] > 0) {
+		close(process_private->txq_fds[txq->queue_id]);
+		process_private->txq_fds[txq->queue_id] = -1;
 	}
 }
 
@@ -1209,18 +1224,19 @@ tap_setup_queue(struct rte_eth_dev *dev,
 	int *other_fd;
 	const char *dir;
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct rx_queue *rx = &internals->rxq[qid];
 	struct tx_queue *tx = &internals->txq[qid];
 	struct rte_gso_ctx *gso_ctx;
 
 	if (is_rx) {
-		fd = &rx->fd;
-		other_fd = &tx->fd;
+		fd = &process_private->rxq_fds[qid];
+		other_fd = &process_private->txq_fds[qid];
 		dir = "rx";
 		gso_ctx = NULL;
 	} else {
-		fd = &tx->fd;
-		other_fd = &rx->fd;
+		fd = &process_private->txq_fds[qid];
+		other_fd = &process_private->rxq_fds[qid];
 		dir = "tx";
 		gso_ctx = &tx->gso_ctx;
 	}
@@ -1273,6 +1289,7 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 		   struct rte_mempool *mp)
 {
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct rx_queue *rxq = &internals->rxq[rx_queue_id];
 	struct rte_mbuf **tmp = &rxq->pool;
 	long iov_max = sysconf(_SC_IOV_MAX);
@@ -1332,7 +1349,8 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 	}
 
 	TAP_LOG(DEBUG, "  RX TUNTAP device name %s, qid %d on fd %d",
-		internals->name, rx_queue_id, internals->rxq[rx_queue_id].fd);
+		internals->name, rx_queue_id,
+		process_private->rxq_fds[rx_queue_id]);
 
 	return 0;
 
@@ -1352,6 +1370,7 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		   const struct rte_eth_txconf *tx_conf)
 {
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct tx_queue *txq;
 	int ret;
 	uint64_t offloads;
@@ -1374,7 +1393,8 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		return -1;
 	TAP_LOG(DEBUG,
 		"  TX TUNTAP device name %s, qid %d on fd %d csum %s",
-		internals->name, tx_queue_id, internals->txq[tx_queue_id].fd,
+		internals->name, tx_queue_id,
+		process_private->txq_fds[tx_queue_id],
 		txq->csum ? "on" : "off");
 
 	return 0;
@@ -1622,6 +1642,7 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 	int numa_node = rte_socket_id();
 	struct rte_eth_dev *dev;
 	struct pmd_internals *pmd;
+	struct pmd_process_private *process_private;
 	struct rte_eth_dev_data *data;
 	struct ifreq ifr;
 	int i;
@@ -1636,7 +1657,16 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 		goto error_exit_nodev;
 	}
 
+	process_private = (struct pmd_process_private *)
+		rte_zmalloc_socket(tap_name, sizeof(struct pmd_process_private),
+			RTE_CACHE_LINE_SIZE, dev->device->numa_node);
+
+	if (process_private == NULL) {
+		TAP_LOG(ERR, "Failed to alloc memory for process private");
+		return -1;
+	}
 	pmd = dev->data->dev_private;
+	dev->process_private = process_private;
 	pmd->dev = dev;
 	snprintf(pmd->name, sizeof(pmd->name), "%s", tap_name);
 	pmd->type = type;
@@ -1672,8 +1702,8 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 	/* Presetup the fds to -1 as being not valid */
 	pmd->ka_fd = -1;
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		pmd->rxq[i].fd = -1;
-		pmd->txq[i].fd = -1;
+		process_private->rxq_fds[i] = -1;
+		process_private->txq_fds[i] = -1;
 	}
 
 	if (pmd->type == ETH_TUNTAP_TYPE_TAP) {
@@ -2073,6 +2103,7 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 {
 	struct rte_eth_dev *eth_dev = NULL;
 	struct pmd_internals *internals;
+	struct pmd_process_private *process_private;
 	int i;
 
 	/* find the ethdev entry */
@@ -2081,6 +2112,7 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 		return 0;
 
 	internals = eth_dev->data->dev_private;
+	process_private = eth_dev->process_private;
 
 	TAP_LOG(DEBUG, "Closing %s Ethernet device on numa %u",
 		(internals->type == ETH_TUNTAP_TYPE_TAP) ? "TAP" : "TUN",
@@ -2092,18 +2124,19 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 		tap_nl_final(internals->nlsk_fd);
 	}
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		if (internals->rxq[i].fd != -1) {
-			close(internals->rxq[i].fd);
-			internals->rxq[i].fd = -1;
+		if (process_private->rxq_fds[i] != -1) {
+			close(process_private->rxq_fds[i]);
+			process_private->rxq_fds[i] = -1;
 		}
-		if (internals->txq[i].fd != -1) {
-			close(internals->txq[i].fd);
-			internals->txq[i].fd = -1;
+		if (process_private->txq_fds[i] != -1) {
+			close(process_private->txq_fds[i]);
+			process_private->txq_fds[i] = -1;
 		}
 	}
 
 	close(internals->ioctl_sock);
 	rte_free(eth_dev->data->dev_private);
+	rte_free(eth_dev->process_private);
 	rte_eth_dev_release_port(eth_dev);
 
 	if (internals->ka_fd != -1) {
diff --git a/drivers/net/tap/rte_eth_tap.h b/drivers/net/tap/rte_eth_tap.h
index 4502e24..dc3579a 100644
--- a/drivers/net/tap/rte_eth_tap.h
+++ b/drivers/net/tap/rte_eth_tap.h
@@ -47,7 +47,6 @@ struct rx_queue {
 	uint32_t trigger_seen;          /* Last seen Rx trigger value */
 	uint16_t in_port;               /* Port ID */
 	uint16_t queue_id;		/* queue ID*/
-	int fd;
 	struct pkt_stats stats;         /* Stats for this RX queue */
 	uint16_t nb_rx_desc;            /* max number of mbufs available */
 	struct rte_eth_rxmode *rxmode;  /* RX features */
@@ -57,7 +56,6 @@ struct rx_queue {
 };
 
 struct tx_queue {
-	int fd;
 	int type;                       /* Type field - TUN|TAP */
 	uint16_t *mtu;                  /* Pointer to MTU from dev_data */
 	uint16_t csum:1;                /* Enable checksum offloading */
@@ -95,6 +93,11 @@ struct pmd_internals {
 	int ka_fd;                        /* keep-alive file descriptor */
 };
 
+struct pmd_process_private {
+	int rxq_fds[RTE_PMD_TAP_MAX_QUEUES];
+	int txq_fds[RTE_PMD_TAP_MAX_QUEUES];
+};
+
 /* tap_intr.c */
 
 int tap_rx_intr_vec_set(struct rte_eth_dev *dev, int set);
diff --git a/drivers/net/tap/tap_flow.c b/drivers/net/tap/tap_flow.c
index 0e01af6..3ba6945 100644
--- a/drivers/net/tap/tap_flow.c
+++ b/drivers/net/tap/tap_flow.c
@@ -1567,6 +1567,7 @@ tap_flow_isolate(struct rte_eth_dev *dev,
 		 struct rte_flow_error *error __rte_unused)
 {
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 
 	/* normalize 'set' variable to contain 0 or 1 values */
 	if (set)
@@ -1580,7 +1581,7 @@ tap_flow_isolate(struct rte_eth_dev *dev,
 	 * If netdevice is there, setup appropriate flow rules immediately.
 	 * Otherwise it will be set when bringing up the netdevice (tun_alloc).
 	 */
-	if (!pmd->rxq[0].fd)
+	if (!process_private->rxq_fds[0])
 		return 0;
 	if (set) {
 		struct rte_flow *remote_flow;
diff --git a/drivers/net/tap/tap_intr.c b/drivers/net/tap/tap_intr.c
index fc59018..7af0010 100644
--- a/drivers/net/tap/tap_intr.c
+++ b/drivers/net/tap/tap_intr.c
@@ -51,6 +51,7 @@ static int
 tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 {
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	unsigned int rxqs_n = pmd->dev->data->nb_rx_queues;
 	struct rte_intr_handle *intr_handle = &pmd->intr_handle;
 	unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
@@ -71,7 +72,7 @@ tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 		struct rx_queue *rxq = pmd->dev->data->rx_queues[i];
 
 		/* Skip queues that cannot request interrupts. */
-		if (!rxq || rxq->fd <= 0) {
+		if (!rxq || process_private->rxq_fds[i] <= 0) {
 			/* Use invalid intr_vec[] index to disable entry. */
 			intr_handle->intr_vec[i] =
 				RTE_INTR_VEC_RXTX_OFFSET +
@@ -79,7 +80,7 @@ tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 			continue;
 		}
 		intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
-		intr_handle->efds[count] = rxq->fd;
+		intr_handle->efds[count] = process_private->rxq_fds[i];
 		count++;
 	}
 	if (!count)
-- 
2.7.4

[dpdk-dev] [PATCH v7 3/3] net/tap: allow secondary process to access primary device queues

[Raslan Darawsheh]

In the case the device is created by the primary process,
the secondary must request some file descriptors to attach the queues.
The file descriptors are shared via IPC Unix socket.

Thanks to the IPC synchronization, the secondary process
is now able to do Rx/Tx on a TAP created by the primary process.

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>

---
    v2:
       - translate file descriptors via IPC API
       - add documentation
    v3:
       - rabse the commit
       - use private static array for fd's to be local for each process

    v4:
       - removed TODO and FIXME tags
       - used strlcpy instead of strcpy

    v5: rebase the commit on top of Alejandro Lucero patch for secondary
        process private pointer.
        http://patches.dpdk.org/patch/46185/

    v6: reword the commit log

    v7: rely on tap_device_count for registration
---
---
 doc/guides/nics/tap.rst                |  16 ++++
 doc/guides/rel_notes/release_18_11.rst |   5 ++
 drivers/net/tap/Makefile               |   1 +
 drivers/net/tap/meson.build            |   1 +
 drivers/net/tap/rte_eth_tap.c          | 150 ++++++++++++++++++++++++++++++++-
 5 files changed, 172 insertions(+), 1 deletion(-)

diff --git a/doc/guides/nics/tap.rst b/doc/guides/nics/tap.rst
index 2714868..9a3d7b3 100644
--- a/doc/guides/nics/tap.rst
+++ b/doc/guides/nics/tap.rst
@@ -152,6 +152,22 @@ Distribute IPv4 TCP packets using RSS to a given MAC address over queues 0-3::
    testpmd> flow create 0 priority 4 ingress pattern eth dst is 0a:0b:0c:0d:0e:0f \
             / ipv4 / tcp / end actions rss queues 0 1 2 3 end / end
 
+Multi-process sharing
+---------------------
+
+It is possible to attach an existing TAP device in a secondary process,
+by declaring it as a vdev with the same name as in the primary process,
+and without any parameter.
+
+The port attached in a secondary process will give access to the
+statistics and the queues.
+Therefore it can be used for monitoring or Rx/Tx processing.
+
+The IPC synchronization of Rx/Tx queues is currently limited:
+
+  - Maximum 8 queues shared
+  - Synchronized on probing, but not on later port update
+
 Example
 -------
 
diff --git a/doc/guides/rel_notes/release_18_11.rst b/doc/guides/rel_notes/release_18_11.rst
index 2133a5b..3240b52 100644
--- a/doc/guides/rel_notes/release_18_11.rst
+++ b/doc/guides/rel_notes/release_18_11.rst
@@ -104,6 +104,11 @@ New Features
   the specified port. The port must be stopped before the command call in order
   to reconfigure queues.
 
+* **Added TAP Rx/Tx queues sharing with a secondary process.**
+
+  A secondary process can attach a TAP device created in the primary process,
+  probe the queues, and process Rx/Tx in a secondary process.
+
 
 API Changes
 -----------
diff --git a/drivers/net/tap/Makefile b/drivers/net/tap/Makefile
index 3243365..7748283 100644
--- a/drivers/net/tap/Makefile
+++ b/drivers/net/tap/Makefile
@@ -22,6 +22,7 @@ CFLAGS += -O3
 CFLAGS += -I$(SRCDIR)
 CFLAGS += -I.
 CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash
 LDLIBS += -lrte_bus_vdev -lrte_gso
diff --git a/drivers/net/tap/meson.build b/drivers/net/tap/meson.build
index 37f65b7..f7e8852 100644
--- a/drivers/net/tap/meson.build
+++ b/drivers/net/tap/meson.build
@@ -35,6 +35,7 @@ args = [
 	  'TCA_ACT_BPF_FD' ],
 ]
 config = configuration_data()
+allow_experimental_apis = true
 foreach arg:args
 	config.set(arg[0], cc.has_header_symbol(arg[1], arg[2]))
 endforeach
diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index 3372d54..cfb2648 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -16,6 +16,8 @@
 #include <rte_debug.h>
 #include <rte_ip.h>
 #include <rte_string_fns.h>
+#include <rte_ethdev.h>
+#include <rte_errno.h>
 
 #include <assert.h>
 #include <sys/types.h>
@@ -62,6 +64,10 @@
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
+/* IPC key for queue fds sync */
+#define TAP_MP_KEY "tap_mp_sync_queues"
+
+static int tap_devices_count;
 static struct rte_vdev_driver pmd_tap_drv;
 static struct rte_vdev_driver pmd_tun_drv;
 
@@ -100,6 +106,17 @@ enum ioctl_mode {
 	REMOTE_ONLY,
 };
 
+/* Message header to synchronize queues via IPC */
+struct ipc_queues {
+	char port_name[RTE_DEV_NAME_MAX_LEN];
+	int rxq_count;
+	int txq_count;
+	/*
+	 * The file descriptors are in the dedicated part
+	 * of the Unix message to be translated by the kernel.
+	 */
+};
+
 static int tap_intr_handle_set(struct rte_eth_dev *dev, int set);
 
 /**
@@ -2006,6 +2023,102 @@ rte_pmd_tun_probe(struct rte_vdev_device *dev)
 	return ret;
 }
 
+/* Request queue file descriptors from secondary to primary. */
+static int
+tap_mp_attach_queues(const char *port_name, struct rte_eth_dev *dev)
+{
+	int ret;
+	struct timespec timeout = {.tv_sec = 1, .tv_nsec = 0};
+	struct rte_mp_msg request, *reply;
+	struct rte_mp_reply replies;
+	struct ipc_queues *request_param = (struct ipc_queues *)request.param;
+	struct ipc_queues *reply_param;
+	struct pmd_process_private *process_private = dev->process_private;
+	int queue, fd_iterator;
+
+	/* Prepare the request */
+	strlcpy(request.name, TAP_MP_KEY, sizeof(request.name));
+	strlcpy(request_param->port_name, port_name,
+		sizeof(request_param->port_name));
+	request.len_param = sizeof(*request_param);
+	/* Send request and receive reply */
+	ret = rte_mp_request_sync(&request, &replies, &timeout);
+	if (ret < 0) {
+		TAP_LOG(ERR, "Failed to request queues from primary: %d",
+			rte_errno);
+		return -1;
+	}
+	reply = &replies.msgs[0];
+	reply_param = (struct ipc_queues *)reply->param;
+	TAP_LOG(DEBUG, "Received IPC reply for %s", reply_param->port_name);
+
+	/* Attach the queues from received file descriptors */
+	dev->data->nb_rx_queues = reply_param->rxq_count;
+	dev->data->nb_tx_queues = reply_param->txq_count;
+	fd_iterator = 0;
+	for (queue = 0; queue < reply_param->rxq_count; queue++)
+		process_private->rxq_fds[queue] = reply->fds[fd_iterator++];
+	for (queue = 0; queue < reply_param->txq_count; queue++)
+		process_private->txq_fds[queue] = reply->fds[fd_iterator++];
+
+	return 0;
+}
+
+/* Send the queue file descriptors from the primary process to secondary. */
+static int
+tap_mp_sync_queues(const struct rte_mp_msg *request, const void *peer)
+{
+	struct rte_eth_dev *dev;
+	struct pmd_process_private *process_private;
+	struct rte_mp_msg reply;
+	const struct ipc_queues *request_param =
+		(const struct ipc_queues *)request->param;
+	struct ipc_queues *reply_param =
+		(struct ipc_queues *)reply.param;
+	uint16_t port_id;
+	int queue;
+	int ret;
+
+	/* Get requested port */
+	TAP_LOG(DEBUG, "Received IPC request for %s", request_param->port_name);
+	ret = rte_eth_dev_get_port_by_name(request_param->port_name, &port_id);
+	if (ret) {
+		TAP_LOG(ERR, "Failed to get port id for %s",
+			request_param->port_name);
+		return -1;
+	}
+	dev = &rte_eth_devices[port_id];
+	process_private = dev->process_private;
+
+	/* Fill file descriptors for all queues */
+	reply.num_fds = 0;
+	reply_param->rxq_count = 0;
+	for (queue = 0; queue < dev->data->nb_rx_queues; queue++) {
+		reply.fds[reply.num_fds++] = process_private->rxq_fds[queue];
+		reply_param->rxq_count++;
+	}
+	RTE_ASSERT(reply_param->rxq_count == dev->data->nb_rx_queues);
+	RTE_ASSERT(reply_param->txq_count == dev->data->nb_tx_queues);
+	RTE_ASSERT(reply.num_fds <= RTE_MP_MAX_FD_NUM);
+
+	reply_param->txq_count = 0;
+	for (queue = 0; queue < dev->data->nb_tx_queues; queue++) {
+		reply.fds[reply.num_fds++] = process_private->txq_fds[queue];
+		reply_param->txq_count++;
+	}
+
+	/* Send reply */
+	strlcpy(reply.name, request->name, sizeof(reply.name));
+	strlcpy(reply_param->port_name, request_param->port_name,
+		sizeof(reply_param->port_name));
+	reply.len_param = sizeof(*reply_param);
+	if (rte_mp_reply(&reply, peer) < 0) {
+		TAP_LOG(ERR, "Failed to reply an IPC request to sync queues");
+		return -1;
+	}
+	return 0;
+}
+
 /* Open a TAP interface device.
  */
 static int
@@ -2032,9 +2145,28 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 			TAP_LOG(ERR, "Failed to probe %s", name);
 			return -1;
 		}
-		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &ops;
 		eth_dev->device = &dev->device;
+		eth_dev->rx_pkt_burst = pmd_rx_burst;
+		eth_dev->tx_pkt_burst = pmd_tx_burst;
+		if (!rte_eal_primary_proc_alive(NULL)) {
+			TAP_LOG(ERR, "Primary process is missing");
+			return -1;
+		}
+		eth_dev->process_private = (struct pmd_process_private *)
+			rte_zmalloc_socket(name,
+				sizeof(struct pmd_process_private),
+				RTE_CACHE_LINE_SIZE,
+				eth_dev->device->numa_node);
+		if (eth_dev->process_private == NULL) {
+			TAP_LOG(ERR,
+				"Failed to alloc memory for process private");
+			return -1;
+		}
+
+		ret = tap_mp_attach_queues(name, eth_dev);
+		if (ret != 0)
+			return -1;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
@@ -2082,6 +2214,16 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 	TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s",
 		name, tap_name);
 
+	/* Register IPC feed callback */
+	if (!tap_devices_count) {
+		ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
+		if (ret < 0) {
+			TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
+				tuntap_name, strerror(rte_errno));
+			goto leave;
+		}
+	}
+	tap_devices_count++;
 	ret = eth_dev_tap_create(dev, tap_name, remote_iface, &user_mac,
 		ETH_TUNTAP_TYPE_TAP);
 
@@ -2089,6 +2231,9 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 	if (ret == -1) {
 		TAP_LOG(ERR, "Failed to create pmd for %s as %s",
 			name, tap_name);
+		if (!tap_devices_count)
+			rte_mp_action_unregister(TAP_MP_KEY);
+		tap_devices_count--;
 		tap_unit--;		/* Restore the unit number */
 	}
 	rte_kvargs_free(kvlist);
@@ -2137,6 +2282,9 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 	close(internals->ioctl_sock);
 	rte_free(eth_dev->data->dev_private);
 	rte_free(eth_dev->process_private);
+	if (tap_devices_count == 1)
+		rte_mp_action_unregister(TAP_MP_KEY);
+	tap_devices_count--;
 	rte_eth_dev_release_port(eth_dev);
 
 	if (internals->ka_fd != -1) {
-- 
2.7.4

Re: [dpdk-dev] [PATCH v7 3/3] net/tap: allow secondary process to access primary device queues

[Ferruh Yigit]

On 10/17/2018 9:56 AM, Raslan Darawsheh wrote:
> @@ -2082,6 +2214,16 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
>  	TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s",
>  		name, tap_name);
>  
> +	/* Register IPC feed callback */
> +	if (!tap_devices_count) {
> +		ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
> +		if (ret < 0) {
> +			TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
> +				tuntap_name, strerror(rte_errno));
> +			goto leave;
> +		}
> +	}
> +	tap_devices_count++;
>  	ret = eth_dev_tap_create(dev, tap_name, remote_iface, &user_mac,
>  		ETH_TUNTAP_TYPE_TAP);
>  
> @@ -2089,6 +2231,9 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
>  	if (ret == -1) {
>  		TAP_LOG(ERR, "Failed to create pmd for %s as %s",
>  			name, tap_name);
> +		if (!tap_devices_count)
> +			rte_mp_action_unregister(TAP_MP_KEY);
> +		tap_devices_count--;
>  		tap_unit--;		/* Restore the unit number */
>  	}
>  	rte_kvargs_free(kvlist);
Fail recovery part seems broken, it can be like [1] or [2], but both requires a
new variable.
I double checked the logic in prev version of the patch that uses EEXIST return
values, that is also broken. Overall the challenge is in error recovery part we
don't know if we enter there before or after increasing dev_count, that is why a
local variable required.

If you can fix the error recovery path using EEXIST without needing a new
variable, I think that is better, but if not I suggest following [2] since the
logic of increase the dev_count after device successfully created makes sense to
me, but both works.

Thanks,
ferruh


[1]
         /* Register IPC feed callback */
         if (!tap_devices_count) {
                 ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
                 if (ret < 0) {
                         TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
                                 tuntap_name, strerror(rte_errno));
                         goto leave;
                 }
         }
         tap_devices_count++;
         tap_devices_count_increased = 1;
         ret = eth_dev_tap_create(dev, tap_name, remote_iface, &user_mac,
                 ETH_TUNTAP_TYPE_TAP);

 leave:
         if (ret == -1) {
                 TAP_LOG(ERR, "Failed to create pmd for %s as %s",
                         name, tap_name);
                 if (tap_devices_count_increased == 1) {
                         if (tap_devices_count == 1)
                                 rte_mp_action_unregister(TAP_MP_KEY);
                         tap_devices_count--;
                 }
                 tap_unit--;             /* Restore the unit number */
         }
         rte_kvargs_free(kvlist);



[2]

         /* Register IPC feed callback */
         if (!tap_devices_count) {
                 ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
                 if (ret < 0) {
                         TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
                                 tuntap_name, strerror(rte_errno));
                         goto leave;
                 }
                 mp_action_registered = 1;
         }
         ret = eth_dev_tap_create(dev, tap_name, remote_iface, &user_mac,
                 ETH_TUNTAP_TYPE_TAP);


 leave:
         if (ret == -1) {
                 TAP_LOG(ERR, "Failed to create pmd for %s as %s",
                         name, tap_name);
                 if (mp_action_registered == 1)
                         rte_mp_action_unregister(TAP_MP_KEY);
                 tap_unit--;             /* Restore the unit number */
         } else {
                 tap_devices_count++;
         }
         rte_kvargs_free(kvlist);

Re: [dpdk-dev] [PATCH v7 3/3] net/tap: allow secondary process to access primary device queues

[Raslan Darawsheh]

You right about that fixed in the new version

Kindest regards,
Raslan Darawsheh

> -----Original Message-----
> From: Ferruh Yigit <ferruh.yigit@intel.com>
> Sent: Wednesday, October 17, 2018 3:07 PM
> To: Raslan Darawsheh <rasland@mellanox.com>; keith.wiles@intel.com
> Cc: Thomas Monjalon <thomas@monjalon.net>; dev@dpdk.org; Shahaf
> Shuler <shahafs@mellanox.com>; Ori Kam <orika@mellanox.com>
> Subject: Re: [PATCH v7 3/3] net/tap: allow secondary process to access
> primary device queues
> 
> On 10/17/2018 9:56 AM, Raslan Darawsheh wrote:
> > @@ -2082,6 +2214,16 @@ rte_pmd_tap_probe(struct rte_vdev_device
> *dev)
> >  	TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s",
> >  		name, tap_name);
> >
> > +	/* Register IPC feed callback */
> > +	if (!tap_devices_count) {
> > +		ret = rte_mp_action_register(TAP_MP_KEY,
> tap_mp_sync_queues);
> > +		if (ret < 0) {
> > +			TAP_LOG(ERR, "%s: Failed to register IPC callback:
> %s",
> > +				tuntap_name, strerror(rte_errno));
> > +			goto leave;
> > +		}
> > +	}
> > +	tap_devices_count++;
> >  	ret = eth_dev_tap_create(dev, tap_name, remote_iface,
> &user_mac,
> >  		ETH_TUNTAP_TYPE_TAP);
> >
> > @@ -2089,6 +2231,9 @@ rte_pmd_tap_probe(struct rte_vdev_device
> *dev)
> >  	if (ret == -1) {
> >  		TAP_LOG(ERR, "Failed to create pmd for %s as %s",
> >  			name, tap_name);
> > +		if (!tap_devices_count)
> > +			rte_mp_action_unregister(TAP_MP_KEY);
> > +		tap_devices_count--;
> >  		tap_unit--;		/* Restore the unit number */
> >  	}
> >  	rte_kvargs_free(kvlist);
> Fail recovery part seems broken, it can be like [1] or [2], but both requires a
> new variable.
> I double checked the logic in prev version of the patch that uses EEXIST
> return values, that is also broken. Overall the challenge is in error recovery
> part we don't know if we enter there before or after increasing dev_count,
> that is why a local variable required.
> 
> If you can fix the error recovery path using EEXIST without needing a new
> variable, I think that is better, but if not I suggest following [2] since the logic
> of increase the dev_count after device successfully created makes sense to
> me, but both works.
> 
> Thanks,
> ferruh
> 
> 
> [1]
>          /* Register IPC feed callback */
>          if (!tap_devices_count) {
>                  ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
>                  if (ret < 0) {
>                          TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
>                                  tuntap_name, strerror(rte_errno));
>                          goto leave;
>                  }
>          }
>          tap_devices_count++;
>          tap_devices_count_increased = 1;
>          ret = eth_dev_tap_create(dev, tap_name, remote_iface, &user_mac,
>                  ETH_TUNTAP_TYPE_TAP);
> 
>  leave:
>          if (ret == -1) {
>                  TAP_LOG(ERR, "Failed to create pmd for %s as %s",
>                          name, tap_name);
>                  if (tap_devices_count_increased == 1) {
>                          if (tap_devices_count == 1)
>                                  rte_mp_action_unregister(TAP_MP_KEY);
>                          tap_devices_count--;
>                  }
>                  tap_unit--;             /* Restore the unit number */
>          }
>          rte_kvargs_free(kvlist);
> 
> 
> 
> [2]
> 
>          /* Register IPC feed callback */
>          if (!tap_devices_count) {
>                  ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
>                  if (ret < 0) {
>                          TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
>                                  tuntap_name, strerror(rte_errno));
>                          goto leave;
>                  }
>                  mp_action_registered = 1;
>          }
>          ret = eth_dev_tap_create(dev, tap_name, remote_iface, &user_mac,
>                  ETH_TUNTAP_TYPE_TAP);
> 
> 
>  leave:
>          if (ret == -1) {
>                  TAP_LOG(ERR, "Failed to create pmd for %s as %s",
>                          name, tap_name);
>                  if (mp_action_registered == 1)
>                          rte_mp_action_unregister(TAP_MP_KEY);
>                  tap_unit--;             /* Restore the unit number */
>          } else {
>                  tap_devices_count++;
>          }
>          rte_kvargs_free(kvlist);

[dpdk-dev] [PATCH v8 1/3] net/tap: add queue and port ids in Rx/Tx queues structures

[Raslan Darawsheh]

Port and queue ids are added to easily map the file
descriptors stored in each process private.

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
---
 drivers/net/tap/rte_eth_tap.c | 3 +++
 drivers/net/tap/rte_eth_tap.h | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index ad5ae98..edfb7da 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -1293,6 +1293,7 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->mp = mp;
 	rxq->trigger_seen = 1; /* force initial burst */
 	rxq->in_port = dev->data->port_id;
+	rxq->queue_id = rx_queue_id;
 	rxq->nb_rx_desc = nb_desc;
 	iovecs = rte_zmalloc_socket(dev->device->name, sizeof(*iovecs), 0,
 				    socket_id);
@@ -1359,6 +1360,8 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		return -1;
 	dev->data->tx_queues[tx_queue_id] = &internals->txq[tx_queue_id];
 	txq = dev->data->tx_queues[tx_queue_id];
+	txq->out_port = dev->data->port_id;
+	txq->queue_id = tx_queue_id;
 
 	offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
 	txq->csum = !!(offloads &
diff --git a/drivers/net/tap/rte_eth_tap.h b/drivers/net/tap/rte_eth_tap.h
index 44e2773..4502e24 100644
--- a/drivers/net/tap/rte_eth_tap.h
+++ b/drivers/net/tap/rte_eth_tap.h
@@ -46,6 +46,7 @@ struct rx_queue {
 	struct rte_mempool *mp;         /* Mempool for RX packets */
 	uint32_t trigger_seen;          /* Last seen Rx trigger value */
 	uint16_t in_port;               /* Port ID */
+	uint16_t queue_id;		/* queue ID*/
 	int fd;
 	struct pkt_stats stats;         /* Stats for this RX queue */
 	uint16_t nb_rx_desc;            /* max number of mbufs available */
@@ -62,6 +63,8 @@ struct tx_queue {
 	uint16_t csum:1;                /* Enable checksum offloading */
 	struct pkt_stats stats;         /* Stats for this TX queue */
 	struct rte_gso_ctx gso_ctx;     /* GSO context */
+	uint16_t out_port;              /* Port ID */
+	uint16_t queue_id;		/* queue ID*/
 };
 
 struct pmd_internals {
-- 
2.7.4

[dpdk-dev] [PATCH v8 2/3] net/tap: move fds of Rx/Tx queues to be in process private

[Raslan Darawsheh]

fd's cannot be shared between processes, and each process need to have
it's own fd's pointer.

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>

---
   This patch has dependancy on the following patch
   http://patches.dpdk.org/patch/46185/
---
---
 drivers/net/tap/rte_eth_tap.c | 93 +++++++++++++++++++++++++++++--------------
 drivers/net/tap/rte_eth_tap.h |  7 +++-
 drivers/net/tap/tap_flow.c    |  3 +-
 drivers/net/tap/tap_intr.c    |  5 ++-
 4 files changed, 73 insertions(+), 35 deletions(-)

diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index edfb7da..3372d54 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -315,6 +315,7 @@ static uint16_t
 pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 {
 	struct rx_queue *rxq = queue;
+	struct pmd_process_private *process_private;
 	uint16_t num_rx;
 	unsigned long num_rx_bytes = 0;
 	uint32_t trigger = tap_trigger;
@@ -323,6 +324,7 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		return 0;
 	if (trigger)
 		rxq->trigger_seen = trigger;
+	process_private = rte_eth_devices[rxq->in_port].process_private;
 	rte_compiler_barrier();
 	for (num_rx = 0; num_rx < nb_pkts; ) {
 		struct rte_mbuf *mbuf = rxq->pool;
@@ -331,9 +333,9 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		uint16_t data_off = rte_pktmbuf_headroom(mbuf);
 		int len;
 
-		len = readv(rxq->fd, *rxq->iovecs,
-			    1 +
-			    (rxq->rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ?
+		len = readv(process_private->rxq_fds[rxq->queue_id],
+			*rxq->iovecs,
+			1 + (rxq->rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ?
 			     rxq->nb_rx_desc : 1));
 		if (len < (int)sizeof(struct tun_pi))
 			break;
@@ -494,6 +496,9 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
 {
 	int i;
 	uint16_t l234_hlen;
+	struct pmd_process_private *process_private;
+
+	process_private = rte_eth_devices[txq->out_port].process_private;
 
 	for (i = 0; i < num_mbufs; i++) {
 		struct rte_mbuf *mbuf = pmbufs[i];
@@ -595,7 +600,7 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
 			tap_tx_l4_cksum(l4_cksum, l4_phdr_cksum, l4_raw_cksum);
 
 		/* copy the tx frame data */
-		n = writev(txq->fd, iovecs, j);
+		n = writev(process_private->txq_fds[txq->queue_id], iovecs, j);
 		if (n <= 0)
 			break;
 		(*num_packets)++;
@@ -970,19 +975,20 @@ tap_dev_close(struct rte_eth_dev *dev)
 {
 	int i;
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 
 	tap_link_set_down(dev);
 	tap_flow_flush(dev, NULL);
 	tap_flow_implicit_flush(internals, NULL);
 
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		if (internals->rxq[i].fd != -1) {
-			close(internals->rxq[i].fd);
-			internals->rxq[i].fd = -1;
+		if (process_private->rxq_fds[i] != -1) {
+			close(process_private->rxq_fds[i]);
+			process_private->rxq_fds[i] = -1;
 		}
-		if (internals->txq[i].fd != -1) {
-			close(internals->txq[i].fd);
-			internals->txq[i].fd = -1;
+		if (process_private->txq_fds[i] != -1) {
+			close(process_private->txq_fds[i]);
+			process_private->txq_fds[i] = -1;
 		}
 	}
 
@@ -1006,10 +1012,14 @@ static void
 tap_rx_queue_release(void *queue)
 {
 	struct rx_queue *rxq = queue;
+	struct pmd_process_private *process_private;
 
-	if (rxq && (rxq->fd > 0)) {
-		close(rxq->fd);
-		rxq->fd = -1;
+	if (!rxq)
+		return;
+	process_private = rte_eth_devices[rxq->in_port].process_private;
+	if (process_private->rxq_fds[rxq->queue_id] > 0) {
+		close(process_private->rxq_fds[rxq->queue_id]);
+		process_private->rxq_fds[rxq->queue_id] = -1;
 		rte_pktmbuf_free(rxq->pool);
 		rte_free(rxq->iovecs);
 		rxq->pool = NULL;
@@ -1021,10 +1031,15 @@ static void
 tap_tx_queue_release(void *queue)
 {
 	struct tx_queue *txq = queue;
+	struct pmd_process_private *process_private;
+
+	if (!txq)
+		return;
+	process_private = rte_eth_devices[txq->out_port].process_private;
 
-	if (txq && (txq->fd > 0)) {
-		close(txq->fd);
-		txq->fd = -1;
+	if (process_private->txq_fds[txq->queue_id] > 0) {
+		close(process_private->txq_fds[txq->queue_id]);
+		process_private->txq_fds[txq->queue_id] = -1;
 	}
 }
 
@@ -1209,18 +1224,19 @@ tap_setup_queue(struct rte_eth_dev *dev,
 	int *other_fd;
 	const char *dir;
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct rx_queue *rx = &internals->rxq[qid];
 	struct tx_queue *tx = &internals->txq[qid];
 	struct rte_gso_ctx *gso_ctx;
 
 	if (is_rx) {
-		fd = &rx->fd;
-		other_fd = &tx->fd;
+		fd = &process_private->rxq_fds[qid];
+		other_fd = &process_private->txq_fds[qid];
 		dir = "rx";
 		gso_ctx = NULL;
 	} else {
-		fd = &tx->fd;
-		other_fd = &rx->fd;
+		fd = &process_private->txq_fds[qid];
+		other_fd = &process_private->rxq_fds[qid];
 		dir = "tx";
 		gso_ctx = &tx->gso_ctx;
 	}
@@ -1273,6 +1289,7 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 		   struct rte_mempool *mp)
 {
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct rx_queue *rxq = &internals->rxq[rx_queue_id];
 	struct rte_mbuf **tmp = &rxq->pool;
 	long iov_max = sysconf(_SC_IOV_MAX);
@@ -1332,7 +1349,8 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 	}
 
 	TAP_LOG(DEBUG, "  RX TUNTAP device name %s, qid %d on fd %d",
-		internals->name, rx_queue_id, internals->rxq[rx_queue_id].fd);
+		internals->name, rx_queue_id,
+		process_private->rxq_fds[rx_queue_id]);
 
 	return 0;
 
@@ -1352,6 +1370,7 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		   const struct rte_eth_txconf *tx_conf)
 {
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct tx_queue *txq;
 	int ret;
 	uint64_t offloads;
@@ -1374,7 +1393,8 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		return -1;
 	TAP_LOG(DEBUG,
 		"  TX TUNTAP device name %s, qid %d on fd %d csum %s",
-		internals->name, tx_queue_id, internals->txq[tx_queue_id].fd,
+		internals->name, tx_queue_id,
+		process_private->txq_fds[tx_queue_id],
 		txq->csum ? "on" : "off");
 
 	return 0;
@@ -1622,6 +1642,7 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 	int numa_node = rte_socket_id();
 	struct rte_eth_dev *dev;
 	struct pmd_internals *pmd;
+	struct pmd_process_private *process_private;
 	struct rte_eth_dev_data *data;
 	struct ifreq ifr;
 	int i;
@@ -1636,7 +1657,16 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 		goto error_exit_nodev;
 	}
 
+	process_private = (struct pmd_process_private *)
+		rte_zmalloc_socket(tap_name, sizeof(struct pmd_process_private),
+			RTE_CACHE_LINE_SIZE, dev->device->numa_node);
+
+	if (process_private == NULL) {
+		TAP_LOG(ERR, "Failed to alloc memory for process private");
+		return -1;
+	}
 	pmd = dev->data->dev_private;
+	dev->process_private = process_private;
 	pmd->dev = dev;
 	snprintf(pmd->name, sizeof(pmd->name), "%s", tap_name);
 	pmd->type = type;
@@ -1672,8 +1702,8 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 	/* Presetup the fds to -1 as being not valid */
 	pmd->ka_fd = -1;
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		pmd->rxq[i].fd = -1;
-		pmd->txq[i].fd = -1;
+		process_private->rxq_fds[i] = -1;
+		process_private->txq_fds[i] = -1;
 	}
 
 	if (pmd->type == ETH_TUNTAP_TYPE_TAP) {
@@ -2073,6 +2103,7 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 {
 	struct rte_eth_dev *eth_dev = NULL;
 	struct pmd_internals *internals;
+	struct pmd_process_private *process_private;
 	int i;
 
 	/* find the ethdev entry */
@@ -2081,6 +2112,7 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 		return 0;
 
 	internals = eth_dev->data->dev_private;
+	process_private = eth_dev->process_private;
 
 	TAP_LOG(DEBUG, "Closing %s Ethernet device on numa %u",
 		(internals->type == ETH_TUNTAP_TYPE_TAP) ? "TAP" : "TUN",
@@ -2092,18 +2124,19 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 		tap_nl_final(internals->nlsk_fd);
 	}
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		if (internals->rxq[i].fd != -1) {
-			close(internals->rxq[i].fd);
-			internals->rxq[i].fd = -1;
+		if (process_private->rxq_fds[i] != -1) {
+			close(process_private->rxq_fds[i]);
+			process_private->rxq_fds[i] = -1;
 		}
-		if (internals->txq[i].fd != -1) {
-			close(internals->txq[i].fd);
-			internals->txq[i].fd = -1;
+		if (process_private->txq_fds[i] != -1) {
+			close(process_private->txq_fds[i]);
+			process_private->txq_fds[i] = -1;
 		}
 	}
 
 	close(internals->ioctl_sock);
 	rte_free(eth_dev->data->dev_private);
+	rte_free(eth_dev->process_private);
 	rte_eth_dev_release_port(eth_dev);
 
 	if (internals->ka_fd != -1) {
diff --git a/drivers/net/tap/rte_eth_tap.h b/drivers/net/tap/rte_eth_tap.h
index 4502e24..dc3579a 100644
--- a/drivers/net/tap/rte_eth_tap.h
+++ b/drivers/net/tap/rte_eth_tap.h
@@ -47,7 +47,6 @@ struct rx_queue {
 	uint32_t trigger_seen;          /* Last seen Rx trigger value */
 	uint16_t in_port;               /* Port ID */
 	uint16_t queue_id;		/* queue ID*/
-	int fd;
 	struct pkt_stats stats;         /* Stats for this RX queue */
 	uint16_t nb_rx_desc;            /* max number of mbufs available */
 	struct rte_eth_rxmode *rxmode;  /* RX features */
@@ -57,7 +56,6 @@ struct rx_queue {
 };
 
 struct tx_queue {
-	int fd;
 	int type;                       /* Type field - TUN|TAP */
 	uint16_t *mtu;                  /* Pointer to MTU from dev_data */
 	uint16_t csum:1;                /* Enable checksum offloading */
@@ -95,6 +93,11 @@ struct pmd_internals {
 	int ka_fd;                        /* keep-alive file descriptor */
 };
 
+struct pmd_process_private {
+	int rxq_fds[RTE_PMD_TAP_MAX_QUEUES];
+	int txq_fds[RTE_PMD_TAP_MAX_QUEUES];
+};
+
 /* tap_intr.c */
 
 int tap_rx_intr_vec_set(struct rte_eth_dev *dev, int set);
diff --git a/drivers/net/tap/tap_flow.c b/drivers/net/tap/tap_flow.c
index 0e01af6..3ba6945 100644
--- a/drivers/net/tap/tap_flow.c
+++ b/drivers/net/tap/tap_flow.c
@@ -1567,6 +1567,7 @@ tap_flow_isolate(struct rte_eth_dev *dev,
 		 struct rte_flow_error *error __rte_unused)
 {
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 
 	/* normalize 'set' variable to contain 0 or 1 values */
 	if (set)
@@ -1580,7 +1581,7 @@ tap_flow_isolate(struct rte_eth_dev *dev,
 	 * If netdevice is there, setup appropriate flow rules immediately.
 	 * Otherwise it will be set when bringing up the netdevice (tun_alloc).
 	 */
-	if (!pmd->rxq[0].fd)
+	if (!process_private->rxq_fds[0])
 		return 0;
 	if (set) {
 		struct rte_flow *remote_flow;
diff --git a/drivers/net/tap/tap_intr.c b/drivers/net/tap/tap_intr.c
index fc59018..7af0010 100644
--- a/drivers/net/tap/tap_intr.c
+++ b/drivers/net/tap/tap_intr.c
@@ -51,6 +51,7 @@ static int
 tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 {
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	unsigned int rxqs_n = pmd->dev->data->nb_rx_queues;
 	struct rte_intr_handle *intr_handle = &pmd->intr_handle;
 	unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
@@ -71,7 +72,7 @@ tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 		struct rx_queue *rxq = pmd->dev->data->rx_queues[i];
 
 		/* Skip queues that cannot request interrupts. */
-		if (!rxq || rxq->fd <= 0) {
+		if (!rxq || process_private->rxq_fds[i] <= 0) {
 			/* Use invalid intr_vec[] index to disable entry. */
 			intr_handle->intr_vec[i] =
 				RTE_INTR_VEC_RXTX_OFFSET +
@@ -79,7 +80,7 @@ tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 			continue;
 		}
 		intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
-		intr_handle->efds[count] = rxq->fd;
+		intr_handle->efds[count] = process_private->rxq_fds[i];
 		count++;
 	}
 	if (!count)
-- 
2.7.4

[dpdk-dev] [PATCH v8 3/3] net/tap: allow secondary process to access primary device queues

[Raslan Darawsheh]

In the case the device is created by the primary process,
the secondary must request some file descriptors to attach the queues.
The file descriptors are shared via IPC Unix socket.

Thanks to the IPC synchronization, the secondary process
is now able to do Rx/Tx on a TAP created by the primary process.

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>

---
    v2:
       - translate file descriptors via IPC API
       - add documentation
    v3:
       - rabse the commit
       - use private static array for fd's to be local for each process

    v4:
       - removed TODO and FIXME tags
       - used strlcpy instead of strcpy

    v5: rebase the commit on top of Alejandro Lucero patch for secondary
        process private pointer.
        http://patches.dpdk.org/patch/46185/

    v6: reword the commit log

    v7: rely on tap_device_count for registration
    v8: fix the fail recovery
---

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
---
 doc/guides/nics/tap.rst                |  16 ++++
 doc/guides/rel_notes/release_18_11.rst |   5 ++
 drivers/net/tap/Makefile               |   1 +
 drivers/net/tap/meson.build            |   1 +
 drivers/net/tap/rte_eth_tap.c          | 150 ++++++++++++++++++++++++++++++++-
 5 files changed, 172 insertions(+), 1 deletion(-)

diff --git a/doc/guides/nics/tap.rst b/doc/guides/nics/tap.rst
index 2714868..9a3d7b3 100644
--- a/doc/guides/nics/tap.rst
+++ b/doc/guides/nics/tap.rst
@@ -152,6 +152,22 @@ Distribute IPv4 TCP packets using RSS to a given MAC address over queues 0-3::
    testpmd> flow create 0 priority 4 ingress pattern eth dst is 0a:0b:0c:0d:0e:0f \
             / ipv4 / tcp / end actions rss queues 0 1 2 3 end / end
 
+Multi-process sharing
+---------------------
+
+It is possible to attach an existing TAP device in a secondary process,
+by declaring it as a vdev with the same name as in the primary process,
+and without any parameter.
+
+The port attached in a secondary process will give access to the
+statistics and the queues.
+Therefore it can be used for monitoring or Rx/Tx processing.
+
+The IPC synchronization of Rx/Tx queues is currently limited:
+
+  - Maximum 8 queues shared
+  - Synchronized on probing, but not on later port update
+
 Example
 -------
 
diff --git a/doc/guides/rel_notes/release_18_11.rst b/doc/guides/rel_notes/release_18_11.rst
index 2133a5b..3240b52 100644
--- a/doc/guides/rel_notes/release_18_11.rst
+++ b/doc/guides/rel_notes/release_18_11.rst
@@ -104,6 +104,11 @@ New Features
   the specified port. The port must be stopped before the command call in order
   to reconfigure queues.
 
+* **Added TAP Rx/Tx queues sharing with a secondary process.**
+
+  A secondary process can attach a TAP device created in the primary process,
+  probe the queues, and process Rx/Tx in a secondary process.
+
 
 API Changes
 -----------
diff --git a/drivers/net/tap/Makefile b/drivers/net/tap/Makefile
index 3243365..7748283 100644
--- a/drivers/net/tap/Makefile
+++ b/drivers/net/tap/Makefile
@@ -22,6 +22,7 @@ CFLAGS += -O3
 CFLAGS += -I$(SRCDIR)
 CFLAGS += -I.
 CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash
 LDLIBS += -lrte_bus_vdev -lrte_gso
diff --git a/drivers/net/tap/meson.build b/drivers/net/tap/meson.build
index 37f65b7..f7e8852 100644
--- a/drivers/net/tap/meson.build
+++ b/drivers/net/tap/meson.build
@@ -35,6 +35,7 @@ args = [
 	  'TCA_ACT_BPF_FD' ],
 ]
 config = configuration_data()
+allow_experimental_apis = true
 foreach arg:args
 	config.set(arg[0], cc.has_header_symbol(arg[1], arg[2]))
 endforeach
diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index 3372d54..cfb2648 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -16,6 +16,8 @@
 #include <rte_debug.h>
 #include <rte_ip.h>
 #include <rte_string_fns.h>
+#include <rte_ethdev.h>
+#include <rte_errno.h>
 
 #include <assert.h>
 #include <sys/types.h>
@@ -62,6 +64,10 @@
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
+/* IPC key for queue fds sync */
+#define TAP_MP_KEY "tap_mp_sync_queues"
+
+static int tap_devices_count;
 static struct rte_vdev_driver pmd_tap_drv;
 static struct rte_vdev_driver pmd_tun_drv;
 
@@ -100,6 +106,17 @@ enum ioctl_mode {
 	REMOTE_ONLY,
 };
 
+/* Message header to synchronize queues via IPC */
+struct ipc_queues {
+	char port_name[RTE_DEV_NAME_MAX_LEN];
+	int rxq_count;
+	int txq_count;
+	/*
+	 * The file descriptors are in the dedicated part
+	 * of the Unix message to be translated by the kernel.
+	 */
+};
+
 static int tap_intr_handle_set(struct rte_eth_dev *dev, int set);
 
 /**
@@ -2006,6 +2023,102 @@ rte_pmd_tun_probe(struct rte_vdev_device *dev)
 	return ret;
 }
 
+/* Request queue file descriptors from secondary to primary. */
+static int
+tap_mp_attach_queues(const char *port_name, struct rte_eth_dev *dev)
+{
+	int ret;
+	struct timespec timeout = {.tv_sec = 1, .tv_nsec = 0};
+	struct rte_mp_msg request, *reply;
+	struct rte_mp_reply replies;
+	struct ipc_queues *request_param = (struct ipc_queues *)request.param;
+	struct ipc_queues *reply_param;
+	struct pmd_process_private *process_private = dev->process_private;
+	int queue, fd_iterator;
+
+	/* Prepare the request */
+	strlcpy(request.name, TAP_MP_KEY, sizeof(request.name));
+	strlcpy(request_param->port_name, port_name,
+		sizeof(request_param->port_name));
+	request.len_param = sizeof(*request_param);
+	/* Send request and receive reply */
+	ret = rte_mp_request_sync(&request, &replies, &timeout);
+	if (ret < 0) {
+		TAP_LOG(ERR, "Failed to request queues from primary: %d",
+			rte_errno);
+		return -1;
+	}
+	reply = &replies.msgs[0];
+	reply_param = (struct ipc_queues *)reply->param;
+	TAP_LOG(DEBUG, "Received IPC reply for %s", reply_param->port_name);
+
+	/* Attach the queues from received file descriptors */
+	dev->data->nb_rx_queues = reply_param->rxq_count;
+	dev->data->nb_tx_queues = reply_param->txq_count;
+	fd_iterator = 0;
+	for (queue = 0; queue < reply_param->rxq_count; queue++)
+		process_private->rxq_fds[queue] = reply->fds[fd_iterator++];
+	for (queue = 0; queue < reply_param->txq_count; queue++)
+		process_private->txq_fds[queue] = reply->fds[fd_iterator++];
+
+	return 0;
+}
+
+/* Send the queue file descriptors from the primary process to secondary. */
+static int
+tap_mp_sync_queues(const struct rte_mp_msg *request, const void *peer)
+{
+	struct rte_eth_dev *dev;
+	struct pmd_process_private *process_private;
+	struct rte_mp_msg reply;
+	const struct ipc_queues *request_param =
+		(const struct ipc_queues *)request->param;
+	struct ipc_queues *reply_param =
+		(struct ipc_queues *)reply.param;
+	uint16_t port_id;
+	int queue;
+	int ret;
+
+	/* Get requested port */
+	TAP_LOG(DEBUG, "Received IPC request for %s", request_param->port_name);
+	ret = rte_eth_dev_get_port_by_name(request_param->port_name, &port_id);
+	if (ret) {
+		TAP_LOG(ERR, "Failed to get port id for %s",
+			request_param->port_name);
+		return -1;
+	}
+	dev = &rte_eth_devices[port_id];
+	process_private = dev->process_private;
+
+	/* Fill file descriptors for all queues */
+	reply.num_fds = 0;
+	reply_param->rxq_count = 0;
+	for (queue = 0; queue < dev->data->nb_rx_queues; queue++) {
+		reply.fds[reply.num_fds++] = process_private->rxq_fds[queue];
+		reply_param->rxq_count++;
+	}
+	RTE_ASSERT(reply_param->rxq_count == dev->data->nb_rx_queues);
+	RTE_ASSERT(reply_param->txq_count == dev->data->nb_tx_queues);
+	RTE_ASSERT(reply.num_fds <= RTE_MP_MAX_FD_NUM);
+
+	reply_param->txq_count = 0;
+	for (queue = 0; queue < dev->data->nb_tx_queues; queue++) {
+		reply.fds[reply.num_fds++] = process_private->txq_fds[queue];
+		reply_param->txq_count++;
+	}
+
+	/* Send reply */
+	strlcpy(reply.name, request->name, sizeof(reply.name));
+	strlcpy(reply_param->port_name, request_param->port_name,
+		sizeof(reply_param->port_name));
+	reply.len_param = sizeof(*reply_param);
+	if (rte_mp_reply(&reply, peer) < 0) {
+		TAP_LOG(ERR, "Failed to reply an IPC request to sync queues");
+		return -1;
+	}
+	return 0;
+}
+
 /* Open a TAP interface device.
  */
 static int
@@ -2032,9 +2145,28 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 			TAP_LOG(ERR, "Failed to probe %s", name);
 			return -1;
 		}
-		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &ops;
 		eth_dev->device = &dev->device;
+		eth_dev->rx_pkt_burst = pmd_rx_burst;
+		eth_dev->tx_pkt_burst = pmd_tx_burst;
+		if (!rte_eal_primary_proc_alive(NULL)) {
+			TAP_LOG(ERR, "Primary process is missing");
+			return -1;
+		}
+		eth_dev->process_private = (struct pmd_process_private *)
+			rte_zmalloc_socket(name,
+				sizeof(struct pmd_process_private),
+				RTE_CACHE_LINE_SIZE,
+				eth_dev->device->numa_node);
+		if (eth_dev->process_private == NULL) {
+			TAP_LOG(ERR,
+				"Failed to alloc memory for process private");
+			return -1;
+		}
+
+		ret = tap_mp_attach_queues(name, eth_dev);
+		if (ret != 0)
+			return -1;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
@@ -2082,6 +2214,16 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 	TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s",
 		name, tap_name);
 
+	/* Register IPC feed callback */
+	if (!tap_devices_count) {
+		ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
+		if (ret < 0) {
+			TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
+				tuntap_name, strerror(rte_errno));
+			goto leave;
+		}
+	}
+	tap_devices_count++;
 	ret = eth_dev_tap_create(dev, tap_name, remote_iface, &user_mac,
 		ETH_TUNTAP_TYPE_TAP);
 
@@ -2089,6 +2231,9 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 	if (ret == -1) {
 		TAP_LOG(ERR, "Failed to create pmd for %s as %s",
 			name, tap_name);
+		if (!tap_devices_count)
+			rte_mp_action_unregister(TAP_MP_KEY);
+		tap_devices_count--;
 		tap_unit--;		/* Restore the unit number */
 	}
 	rte_kvargs_free(kvlist);
@@ -2137,6 +2282,9 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 	close(internals->ioctl_sock);
 	rte_free(eth_dev->data->dev_private);
 	rte_free(eth_dev->process_private);
+	if (tap_devices_count == 1)
+		rte_mp_action_unregister(TAP_MP_KEY);
+	tap_devices_count--;
 	rte_eth_dev_release_port(eth_dev);
 
 	if (internals->ka_fd != -1) {
-- 
2.7.4

Re: [dpdk-dev] [PATCH v8 3/3] net/tap: allow secondary process to access primary device queues

[Ferruh Yigit]

On 10/17/2018 3:45 PM, Raslan Darawsheh wrote:
> In the case the device is created by the primary process,
> the secondary must request some file descriptors to attach the queues.
> The file descriptors are shared via IPC Unix socket.
> 
> Thanks to the IPC synchronization, the secondary process
> is now able to do Rx/Tx on a TAP created by the primary process.
> 
> Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
> Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
> 
> ---
>     v2:
>        - translate file descriptors via IPC API
>        - add documentation
>     v3:
>        - rabse the commit
>        - use private static array for fd's to be local for each process
> 
>     v4:
>        - removed TODO and FIXME tags
>        - used strlcpy instead of strcpy
> 
>     v5: rebase the commit on top of Alejandro Lucero patch for secondary
>         process private pointer.
>         http://patches.dpdk.org/patch/46185/
> 
>     v6: reword the commit log
> 
>     v7: rely on tap_device_count for registration
>     v8: fix the fail recovery

The code is exact same with v7, perhaps send wrong branch/code ? (or testing
your luck :)

Re: [dpdk-dev] [PATCH v8 3/3] net/tap: allow secondary process to access primary device queues

[Raslan Darawsheh]

Hi Ferruh,

Yes seems I sent the wrong code I'm sending a V9 with proper code.

Kindest regards,
Raslan Darawsheh

> -----Original Message-----
> From: Ferruh Yigit <ferruh.yigit@intel.com>
> Sent: Wednesday, October 17, 2018 7:02 PM
> To: Raslan Darawsheh <rasland@mellanox.com>; keith.wiles@intel.com
> Cc: Thomas Monjalon <thomas@monjalon.net>; dev@dpdk.org; Shahaf
> Shuler <shahafs@mellanox.com>; Ori Kam <orika@mellanox.com>
> Subject: Re: [PATCH v8 3/3] net/tap: allow secondary process to access
> primary device queues
> 
> On 10/17/2018 3:45 PM, Raslan Darawsheh wrote:
> > In the case the device is created by the primary process, the
> > secondary must request some file descriptors to attach the queues.
> > The file descriptors are shared via IPC Unix socket.
> >
> > Thanks to the IPC synchronization, the secondary process is now able
> > to do Rx/Tx on a TAP created by the primary process.
> >
> > Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
> > Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
> >
> > ---
> >     v2:
> >        - translate file descriptors via IPC API
> >        - add documentation
> >     v3:
> >        - rabse the commit
> >        - use private static array for fd's to be local for each
> > process
> >
> >     v4:
> >        - removed TODO and FIXME tags
> >        - used strlcpy instead of strcpy
> >
> >     v5: rebase the commit on top of Alejandro Lucero patch for secondary
> >         process private pointer.
> >
> >
> https://emea01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fpatc
> >
> hes.dpdk.org%2Fpatch%2F46185%2F&amp;data=02%7C01%7Crasland%40me
> llanox.
> >
> com%7C402ffdbc4dba4780cf1508d63449e8dc%7Ca652971c7d2e4d9ba6a4d14
> 9256f4
> >
> 61b%7C0%7C0%7C636753889385487993&amp;sdata=b3BlYqRNeU7uN0qvbXh
> 5%2BXP0u
> > 0ssna%2BvW5ARKjnJwAM%3D&amp;reserved=0
> >
> >     v6: reword the commit log
> >
> >     v7: rely on tap_device_count for registration
> >     v8: fix the fail recovery
> 
> The code is exact same with v7, perhaps send wrong branch/code ? (or
> testing your luck :)

[dpdk-dev] [PATCH v9 1/3] net/tap: add queue and port ids in Rx/Tx queues structures

[Raslan Darawsheh]

Port and queue ids are added to easily map the file
descriptors stored in each process private.

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
---
 drivers/net/tap/rte_eth_tap.c | 3 +++
 drivers/net/tap/rte_eth_tap.h | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index ad5ae98..edfb7da 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -1293,6 +1293,7 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->mp = mp;
 	rxq->trigger_seen = 1; /* force initial burst */
 	rxq->in_port = dev->data->port_id;
+	rxq->queue_id = rx_queue_id;
 	rxq->nb_rx_desc = nb_desc;
 	iovecs = rte_zmalloc_socket(dev->device->name, sizeof(*iovecs), 0,
 				    socket_id);
@@ -1359,6 +1360,8 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		return -1;
 	dev->data->tx_queues[tx_queue_id] = &internals->txq[tx_queue_id];
 	txq = dev->data->tx_queues[tx_queue_id];
+	txq->out_port = dev->data->port_id;
+	txq->queue_id = tx_queue_id;
 
 	offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
 	txq->csum = !!(offloads &
diff --git a/drivers/net/tap/rte_eth_tap.h b/drivers/net/tap/rte_eth_tap.h
index 44e2773..4502e24 100644
--- a/drivers/net/tap/rte_eth_tap.h
+++ b/drivers/net/tap/rte_eth_tap.h
@@ -46,6 +46,7 @@ struct rx_queue {
 	struct rte_mempool *mp;         /* Mempool for RX packets */
 	uint32_t trigger_seen;          /* Last seen Rx trigger value */
 	uint16_t in_port;               /* Port ID */
+	uint16_t queue_id;		/* queue ID*/
 	int fd;
 	struct pkt_stats stats;         /* Stats for this RX queue */
 	uint16_t nb_rx_desc;            /* max number of mbufs available */
@@ -62,6 +63,8 @@ struct tx_queue {
 	uint16_t csum:1;                /* Enable checksum offloading */
 	struct pkt_stats stats;         /* Stats for this TX queue */
 	struct rte_gso_ctx gso_ctx;     /* GSO context */
+	uint16_t out_port;              /* Port ID */
+	uint16_t queue_id;		/* queue ID*/
 };
 
 struct pmd_internals {
-- 
2.7.4

[dpdk-dev] [PATCH v9 2/3] net/tap: move fds of Rx/Tx queues to be in process private

[Raslan Darawsheh]

fd's cannot be shared between processes, and each process need to have
it's own fd's pointer.

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>

---
   This patch has dependancy on the following patch
   http://patches.dpdk.org/patch/46185/
---
---
 drivers/net/tap/rte_eth_tap.c | 93 +++++++++++++++++++++++++++++--------------
 drivers/net/tap/rte_eth_tap.h |  7 +++-
 drivers/net/tap/tap_flow.c    |  3 +-
 drivers/net/tap/tap_intr.c    |  5 ++-
 4 files changed, 73 insertions(+), 35 deletions(-)

diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index edfb7da..3372d54 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -315,6 +315,7 @@ static uint16_t
 pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 {
 	struct rx_queue *rxq = queue;
+	struct pmd_process_private *process_private;
 	uint16_t num_rx;
 	unsigned long num_rx_bytes = 0;
 	uint32_t trigger = tap_trigger;
@@ -323,6 +324,7 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		return 0;
 	if (trigger)
 		rxq->trigger_seen = trigger;
+	process_private = rte_eth_devices[rxq->in_port].process_private;
 	rte_compiler_barrier();
 	for (num_rx = 0; num_rx < nb_pkts; ) {
 		struct rte_mbuf *mbuf = rxq->pool;
@@ -331,9 +333,9 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		uint16_t data_off = rte_pktmbuf_headroom(mbuf);
 		int len;
 
-		len = readv(rxq->fd, *rxq->iovecs,
-			    1 +
-			    (rxq->rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ?
+		len = readv(process_private->rxq_fds[rxq->queue_id],
+			*rxq->iovecs,
+			1 + (rxq->rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ?
 			     rxq->nb_rx_desc : 1));
 		if (len < (int)sizeof(struct tun_pi))
 			break;
@@ -494,6 +496,9 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
 {
 	int i;
 	uint16_t l234_hlen;
+	struct pmd_process_private *process_private;
+
+	process_private = rte_eth_devices[txq->out_port].process_private;
 
 	for (i = 0; i < num_mbufs; i++) {
 		struct rte_mbuf *mbuf = pmbufs[i];
@@ -595,7 +600,7 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
 			tap_tx_l4_cksum(l4_cksum, l4_phdr_cksum, l4_raw_cksum);
 
 		/* copy the tx frame data */
-		n = writev(txq->fd, iovecs, j);
+		n = writev(process_private->txq_fds[txq->queue_id], iovecs, j);
 		if (n <= 0)
 			break;
 		(*num_packets)++;
@@ -970,19 +975,20 @@ tap_dev_close(struct rte_eth_dev *dev)
 {
 	int i;
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 
 	tap_link_set_down(dev);
 	tap_flow_flush(dev, NULL);
 	tap_flow_implicit_flush(internals, NULL);
 
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		if (internals->rxq[i].fd != -1) {
-			close(internals->rxq[i].fd);
-			internals->rxq[i].fd = -1;
+		if (process_private->rxq_fds[i] != -1) {
+			close(process_private->rxq_fds[i]);
+			process_private->rxq_fds[i] = -1;
 		}
-		if (internals->txq[i].fd != -1) {
-			close(internals->txq[i].fd);
-			internals->txq[i].fd = -1;
+		if (process_private->txq_fds[i] != -1) {
+			close(process_private->txq_fds[i]);
+			process_private->txq_fds[i] = -1;
 		}
 	}
 
@@ -1006,10 +1012,14 @@ static void
 tap_rx_queue_release(void *queue)
 {
 	struct rx_queue *rxq = queue;
+	struct pmd_process_private *process_private;
 
-	if (rxq && (rxq->fd > 0)) {
-		close(rxq->fd);
-		rxq->fd = -1;
+	if (!rxq)
+		return;
+	process_private = rte_eth_devices[rxq->in_port].process_private;
+	if (process_private->rxq_fds[rxq->queue_id] > 0) {
+		close(process_private->rxq_fds[rxq->queue_id]);
+		process_private->rxq_fds[rxq->queue_id] = -1;
 		rte_pktmbuf_free(rxq->pool);
 		rte_free(rxq->iovecs);
 		rxq->pool = NULL;
@@ -1021,10 +1031,15 @@ static void
 tap_tx_queue_release(void *queue)
 {
 	struct tx_queue *txq = queue;
+	struct pmd_process_private *process_private;
+
+	if (!txq)
+		return;
+	process_private = rte_eth_devices[txq->out_port].process_private;
 
-	if (txq && (txq->fd > 0)) {
-		close(txq->fd);
-		txq->fd = -1;
+	if (process_private->txq_fds[txq->queue_id] > 0) {
+		close(process_private->txq_fds[txq->queue_id]);
+		process_private->txq_fds[txq->queue_id] = -1;
 	}
 }
 
@@ -1209,18 +1224,19 @@ tap_setup_queue(struct rte_eth_dev *dev,
 	int *other_fd;
 	const char *dir;
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct rx_queue *rx = &internals->rxq[qid];
 	struct tx_queue *tx = &internals->txq[qid];
 	struct rte_gso_ctx *gso_ctx;
 
 	if (is_rx) {
-		fd = &rx->fd;
-		other_fd = &tx->fd;
+		fd = &process_private->rxq_fds[qid];
+		other_fd = &process_private->txq_fds[qid];
 		dir = "rx";
 		gso_ctx = NULL;
 	} else {
-		fd = &tx->fd;
-		other_fd = &rx->fd;
+		fd = &process_private->txq_fds[qid];
+		other_fd = &process_private->rxq_fds[qid];
 		dir = "tx";
 		gso_ctx = &tx->gso_ctx;
 	}
@@ -1273,6 +1289,7 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 		   struct rte_mempool *mp)
 {
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct rx_queue *rxq = &internals->rxq[rx_queue_id];
 	struct rte_mbuf **tmp = &rxq->pool;
 	long iov_max = sysconf(_SC_IOV_MAX);
@@ -1332,7 +1349,8 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 	}
 
 	TAP_LOG(DEBUG, "  RX TUNTAP device name %s, qid %d on fd %d",
-		internals->name, rx_queue_id, internals->rxq[rx_queue_id].fd);
+		internals->name, rx_queue_id,
+		process_private->rxq_fds[rx_queue_id]);
 
 	return 0;
 
@@ -1352,6 +1370,7 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		   const struct rte_eth_txconf *tx_conf)
 {
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct tx_queue *txq;
 	int ret;
 	uint64_t offloads;
@@ -1374,7 +1393,8 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		return -1;
 	TAP_LOG(DEBUG,
 		"  TX TUNTAP device name %s, qid %d on fd %d csum %s",
-		internals->name, tx_queue_id, internals->txq[tx_queue_id].fd,
+		internals->name, tx_queue_id,
+		process_private->txq_fds[tx_queue_id],
 		txq->csum ? "on" : "off");
 
 	return 0;
@@ -1622,6 +1642,7 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 	int numa_node = rte_socket_id();
 	struct rte_eth_dev *dev;
 	struct pmd_internals *pmd;
+	struct pmd_process_private *process_private;
 	struct rte_eth_dev_data *data;
 	struct ifreq ifr;
 	int i;
@@ -1636,7 +1657,16 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 		goto error_exit_nodev;
 	}
 
+	process_private = (struct pmd_process_private *)
+		rte_zmalloc_socket(tap_name, sizeof(struct pmd_process_private),
+			RTE_CACHE_LINE_SIZE, dev->device->numa_node);
+
+	if (process_private == NULL) {
+		TAP_LOG(ERR, "Failed to alloc memory for process private");
+		return -1;
+	}
 	pmd = dev->data->dev_private;
+	dev->process_private = process_private;
 	pmd->dev = dev;
 	snprintf(pmd->name, sizeof(pmd->name), "%s", tap_name);
 	pmd->type = type;
@@ -1672,8 +1702,8 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 	/* Presetup the fds to -1 as being not valid */
 	pmd->ka_fd = -1;
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		pmd->rxq[i].fd = -1;
-		pmd->txq[i].fd = -1;
+		process_private->rxq_fds[i] = -1;
+		process_private->txq_fds[i] = -1;
 	}
 
 	if (pmd->type == ETH_TUNTAP_TYPE_TAP) {
@@ -2073,6 +2103,7 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 {
 	struct rte_eth_dev *eth_dev = NULL;
 	struct pmd_internals *internals;
+	struct pmd_process_private *process_private;
 	int i;
 
 	/* find the ethdev entry */
@@ -2081,6 +2112,7 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 		return 0;
 
 	internals = eth_dev->data->dev_private;
+	process_private = eth_dev->process_private;
 
 	TAP_LOG(DEBUG, "Closing %s Ethernet device on numa %u",
 		(internals->type == ETH_TUNTAP_TYPE_TAP) ? "TAP" : "TUN",
@@ -2092,18 +2124,19 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 		tap_nl_final(internals->nlsk_fd);
 	}
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		if (internals->rxq[i].fd != -1) {
-			close(internals->rxq[i].fd);
-			internals->rxq[i].fd = -1;
+		if (process_private->rxq_fds[i] != -1) {
+			close(process_private->rxq_fds[i]);
+			process_private->rxq_fds[i] = -1;
 		}
-		if (internals->txq[i].fd != -1) {
-			close(internals->txq[i].fd);
-			internals->txq[i].fd = -1;
+		if (process_private->txq_fds[i] != -1) {
+			close(process_private->txq_fds[i]);
+			process_private->txq_fds[i] = -1;
 		}
 	}
 
 	close(internals->ioctl_sock);
 	rte_free(eth_dev->data->dev_private);
+	rte_free(eth_dev->process_private);
 	rte_eth_dev_release_port(eth_dev);
 
 	if (internals->ka_fd != -1) {
diff --git a/drivers/net/tap/rte_eth_tap.h b/drivers/net/tap/rte_eth_tap.h
index 4502e24..dc3579a 100644
--- a/drivers/net/tap/rte_eth_tap.h
+++ b/drivers/net/tap/rte_eth_tap.h
@@ -47,7 +47,6 @@ struct rx_queue {
 	uint32_t trigger_seen;          /* Last seen Rx trigger value */
 	uint16_t in_port;               /* Port ID */
 	uint16_t queue_id;		/* queue ID*/
-	int fd;
 	struct pkt_stats stats;         /* Stats for this RX queue */
 	uint16_t nb_rx_desc;            /* max number of mbufs available */
 	struct rte_eth_rxmode *rxmode;  /* RX features */
@@ -57,7 +56,6 @@ struct rx_queue {
 };
 
 struct tx_queue {
-	int fd;
 	int type;                       /* Type field - TUN|TAP */
 	uint16_t *mtu;                  /* Pointer to MTU from dev_data */
 	uint16_t csum:1;                /* Enable checksum offloading */
@@ -95,6 +93,11 @@ struct pmd_internals {
 	int ka_fd;                        /* keep-alive file descriptor */
 };
 
+struct pmd_process_private {
+	int rxq_fds[RTE_PMD_TAP_MAX_QUEUES];
+	int txq_fds[RTE_PMD_TAP_MAX_QUEUES];
+};
+
 /* tap_intr.c */
 
 int tap_rx_intr_vec_set(struct rte_eth_dev *dev, int set);
diff --git a/drivers/net/tap/tap_flow.c b/drivers/net/tap/tap_flow.c
index 0e01af6..3ba6945 100644
--- a/drivers/net/tap/tap_flow.c
+++ b/drivers/net/tap/tap_flow.c
@@ -1567,6 +1567,7 @@ tap_flow_isolate(struct rte_eth_dev *dev,
 		 struct rte_flow_error *error __rte_unused)
 {
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 
 	/* normalize 'set' variable to contain 0 or 1 values */
 	if (set)
@@ -1580,7 +1581,7 @@ tap_flow_isolate(struct rte_eth_dev *dev,
 	 * If netdevice is there, setup appropriate flow rules immediately.
 	 * Otherwise it will be set when bringing up the netdevice (tun_alloc).
 	 */
-	if (!pmd->rxq[0].fd)
+	if (!process_private->rxq_fds[0])
 		return 0;
 	if (set) {
 		struct rte_flow *remote_flow;
diff --git a/drivers/net/tap/tap_intr.c b/drivers/net/tap/tap_intr.c
index fc59018..7af0010 100644
--- a/drivers/net/tap/tap_intr.c
+++ b/drivers/net/tap/tap_intr.c
@@ -51,6 +51,7 @@ static int
 tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 {
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	unsigned int rxqs_n = pmd->dev->data->nb_rx_queues;
 	struct rte_intr_handle *intr_handle = &pmd->intr_handle;
 	unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
@@ -71,7 +72,7 @@ tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 		struct rx_queue *rxq = pmd->dev->data->rx_queues[i];
 
 		/* Skip queues that cannot request interrupts. */
-		if (!rxq || rxq->fd <= 0) {
+		if (!rxq || process_private->rxq_fds[i] <= 0) {
 			/* Use invalid intr_vec[] index to disable entry. */
 			intr_handle->intr_vec[i] =
 				RTE_INTR_VEC_RXTX_OFFSET +
@@ -79,7 +80,7 @@ tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 			continue;
 		}
 		intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
-		intr_handle->efds[count] = rxq->fd;
+		intr_handle->efds[count] = process_private->rxq_fds[i];
 		count++;
 	}
 	if (!count)
-- 
2.7.4

[dpdk-dev] [PATCH v9 3/3] net/tap: allow secondary process to access primary device queues

[Raslan Darawsheh]

In the case the device is created by the primary process,
the secondary must request some file descriptors to attach the queues.
The file descriptors are shared via IPC Unix socket.

Thanks to the IPC synchronization, the secondary process
is now able to do Rx/Tx on a TAP created by the primary process.

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>

---
    v2:
       - translate file descriptors via IPC API
       - add documentation
    v3:
       - rabse the commit
       - use private static array for fd's to be local for each process

    v4:
       - removed TODO and FIXME tags
       - used strlcpy instead of strcpy

    v5: rebase the commit on top of Alejandro Lucero patch for secondary
        process private pointer.
        http://patches.dpdk.org/patch/46185/

    v6: reword the commit log

    v7: rely on tap_device_count for registration
    v8: fix the fail recovery
    v9: sent the good fix for the fail recovery
---
---
 doc/guides/nics/tap.rst                |  16 ++++
 doc/guides/rel_notes/release_18_11.rst |   5 ++
 drivers/net/tap/Makefile               |   1 +
 drivers/net/tap/meson.build            |   1 +
 drivers/net/tap/rte_eth_tap.c          | 154 ++++++++++++++++++++++++++++++++-
 5 files changed, 176 insertions(+), 1 deletion(-)

diff --git a/doc/guides/nics/tap.rst b/doc/guides/nics/tap.rst
index 2714868..9a3d7b3 100644
--- a/doc/guides/nics/tap.rst
+++ b/doc/guides/nics/tap.rst
@@ -152,6 +152,22 @@ Distribute IPv4 TCP packets using RSS to a given MAC address over queues 0-3::
    testpmd> flow create 0 priority 4 ingress pattern eth dst is 0a:0b:0c:0d:0e:0f \
             / ipv4 / tcp / end actions rss queues 0 1 2 3 end / end
 
+Multi-process sharing
+---------------------
+
+It is possible to attach an existing TAP device in a secondary process,
+by declaring it as a vdev with the same name as in the primary process,
+and without any parameter.
+
+The port attached in a secondary process will give access to the
+statistics and the queues.
+Therefore it can be used for monitoring or Rx/Tx processing.
+
+The IPC synchronization of Rx/Tx queues is currently limited:
+
+  - Maximum 8 queues shared
+  - Synchronized on probing, but not on later port update
+
 Example
 -------
 
diff --git a/doc/guides/rel_notes/release_18_11.rst b/doc/guides/rel_notes/release_18_11.rst
index 2133a5b..3240b52 100644
--- a/doc/guides/rel_notes/release_18_11.rst
+++ b/doc/guides/rel_notes/release_18_11.rst
@@ -104,6 +104,11 @@ New Features
   the specified port. The port must be stopped before the command call in order
   to reconfigure queues.
 
+* **Added TAP Rx/Tx queues sharing with a secondary process.**
+
+  A secondary process can attach a TAP device created in the primary process,
+  probe the queues, and process Rx/Tx in a secondary process.
+
 
 API Changes
 -----------
diff --git a/drivers/net/tap/Makefile b/drivers/net/tap/Makefile
index 3243365..7748283 100644
--- a/drivers/net/tap/Makefile
+++ b/drivers/net/tap/Makefile
@@ -22,6 +22,7 @@ CFLAGS += -O3
 CFLAGS += -I$(SRCDIR)
 CFLAGS += -I.
 CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash
 LDLIBS += -lrte_bus_vdev -lrte_gso
diff --git a/drivers/net/tap/meson.build b/drivers/net/tap/meson.build
index 37f65b7..f7e8852 100644
--- a/drivers/net/tap/meson.build
+++ b/drivers/net/tap/meson.build
@@ -35,6 +35,7 @@ args = [
 	  'TCA_ACT_BPF_FD' ],
 ]
 config = configuration_data()
+allow_experimental_apis = true
 foreach arg:args
 	config.set(arg[0], cc.has_header_symbol(arg[1], arg[2]))
 endforeach
diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index 3372d54..09171b3 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -16,6 +16,8 @@
 #include <rte_debug.h>
 #include <rte_ip.h>
 #include <rte_string_fns.h>
+#include <rte_ethdev.h>
+#include <rte_errno.h>
 
 #include <assert.h>
 #include <sys/types.h>
@@ -62,6 +64,10 @@
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
+/* IPC key for queue fds sync */
+#define TAP_MP_KEY "tap_mp_sync_queues"
+
+static int tap_devices_count;
 static struct rte_vdev_driver pmd_tap_drv;
 static struct rte_vdev_driver pmd_tun_drv;
 
@@ -100,6 +106,17 @@ enum ioctl_mode {
 	REMOTE_ONLY,
 };
 
+/* Message header to synchronize queues via IPC */
+struct ipc_queues {
+	char port_name[RTE_DEV_NAME_MAX_LEN];
+	int rxq_count;
+	int txq_count;
+	/*
+	 * The file descriptors are in the dedicated part
+	 * of the Unix message to be translated by the kernel.
+	 */
+};
+
 static int tap_intr_handle_set(struct rte_eth_dev *dev, int set);
 
 /**
@@ -2006,6 +2023,102 @@ rte_pmd_tun_probe(struct rte_vdev_device *dev)
 	return ret;
 }
 
+/* Request queue file descriptors from secondary to primary. */
+static int
+tap_mp_attach_queues(const char *port_name, struct rte_eth_dev *dev)
+{
+	int ret;
+	struct timespec timeout = {.tv_sec = 1, .tv_nsec = 0};
+	struct rte_mp_msg request, *reply;
+	struct rte_mp_reply replies;
+	struct ipc_queues *request_param = (struct ipc_queues *)request.param;
+	struct ipc_queues *reply_param;
+	struct pmd_process_private *process_private = dev->process_private;
+	int queue, fd_iterator;
+
+	/* Prepare the request */
+	strlcpy(request.name, TAP_MP_KEY, sizeof(request.name));
+	strlcpy(request_param->port_name, port_name,
+		sizeof(request_param->port_name));
+	request.len_param = sizeof(*request_param);
+	/* Send request and receive reply */
+	ret = rte_mp_request_sync(&request, &replies, &timeout);
+	if (ret < 0) {
+		TAP_LOG(ERR, "Failed to request queues from primary: %d",
+			rte_errno);
+		return -1;
+	}
+	reply = &replies.msgs[0];
+	reply_param = (struct ipc_queues *)reply->param;
+	TAP_LOG(DEBUG, "Received IPC reply for %s", reply_param->port_name);
+
+	/* Attach the queues from received file descriptors */
+	dev->data->nb_rx_queues = reply_param->rxq_count;
+	dev->data->nb_tx_queues = reply_param->txq_count;
+	fd_iterator = 0;
+	for (queue = 0; queue < reply_param->rxq_count; queue++)
+		process_private->rxq_fds[queue] = reply->fds[fd_iterator++];
+	for (queue = 0; queue < reply_param->txq_count; queue++)
+		process_private->txq_fds[queue] = reply->fds[fd_iterator++];
+
+	return 0;
+}
+
+/* Send the queue file descriptors from the primary process to secondary. */
+static int
+tap_mp_sync_queues(const struct rte_mp_msg *request, const void *peer)
+{
+	struct rte_eth_dev *dev;
+	struct pmd_process_private *process_private;
+	struct rte_mp_msg reply;
+	const struct ipc_queues *request_param =
+		(const struct ipc_queues *)request->param;
+	struct ipc_queues *reply_param =
+		(struct ipc_queues *)reply.param;
+	uint16_t port_id;
+	int queue;
+	int ret;
+
+	/* Get requested port */
+	TAP_LOG(DEBUG, "Received IPC request for %s", request_param->port_name);
+	ret = rte_eth_dev_get_port_by_name(request_param->port_name, &port_id);
+	if (ret) {
+		TAP_LOG(ERR, "Failed to get port id for %s",
+			request_param->port_name);
+		return -1;
+	}
+	dev = &rte_eth_devices[port_id];
+	process_private = dev->process_private;
+
+	/* Fill file descriptors for all queues */
+	reply.num_fds = 0;
+	reply_param->rxq_count = 0;
+	for (queue = 0; queue < dev->data->nb_rx_queues; queue++) {
+		reply.fds[reply.num_fds++] = process_private->rxq_fds[queue];
+		reply_param->rxq_count++;
+	}
+	RTE_ASSERT(reply_param->rxq_count == dev->data->nb_rx_queues);
+	RTE_ASSERT(reply_param->txq_count == dev->data->nb_tx_queues);
+	RTE_ASSERT(reply.num_fds <= RTE_MP_MAX_FD_NUM);
+
+	reply_param->txq_count = 0;
+	for (queue = 0; queue < dev->data->nb_tx_queues; queue++) {
+		reply.fds[reply.num_fds++] = process_private->txq_fds[queue];
+		reply_param->txq_count++;
+	}
+
+	/* Send reply */
+	strlcpy(reply.name, request->name, sizeof(reply.name));
+	strlcpy(reply_param->port_name, request_param->port_name,
+		sizeof(reply_param->port_name));
+	reply.len_param = sizeof(*reply_param);
+	if (rte_mp_reply(&reply, peer) < 0) {
+		TAP_LOG(ERR, "Failed to reply an IPC request to sync queues");
+		return -1;
+	}
+	return 0;
+}
+
 /* Open a TAP interface device.
  */
 static int
@@ -2019,6 +2132,7 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 	char remote_iface[RTE_ETH_NAME_MAX_LEN];
 	struct ether_addr user_mac = { .addr_bytes = {0} };
 	struct rte_eth_dev *eth_dev;
+	int tap_devices_count_increased = 0;
 
 	strcpy(tuntap_name, "TAP");
 
@@ -2032,9 +2146,28 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 			TAP_LOG(ERR, "Failed to probe %s", name);
 			return -1;
 		}
-		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &ops;
 		eth_dev->device = &dev->device;
+		eth_dev->rx_pkt_burst = pmd_rx_burst;
+		eth_dev->tx_pkt_burst = pmd_tx_burst;
+		if (!rte_eal_primary_proc_alive(NULL)) {
+			TAP_LOG(ERR, "Primary process is missing");
+			return -1;
+		}
+		eth_dev->process_private = (struct pmd_process_private *)
+			rte_zmalloc_socket(name,
+				sizeof(struct pmd_process_private),
+				RTE_CACHE_LINE_SIZE,
+				eth_dev->device->numa_node);
+		if (eth_dev->process_private == NULL) {
+			TAP_LOG(ERR,
+				"Failed to alloc memory for process private");
+			return -1;
+		}
+
+		ret = tap_mp_attach_queues(name, eth_dev);
+		if (ret != 0)
+			return -1;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
@@ -2082,6 +2215,17 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 	TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s",
 		name, tap_name);
 
+	/* Register IPC feed callback */
+	if (!tap_devices_count) {
+		ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
+		if (ret < 0) {
+			TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
+				tuntap_name, strerror(rte_errno));
+			goto leave;
+		}
+	}
+	tap_devices_count++;
+	tap_devices_count_increased = 1;
 	ret = eth_dev_tap_create(dev, tap_name, remote_iface, &user_mac,
 		ETH_TUNTAP_TYPE_TAP);
 
@@ -2089,6 +2233,11 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 	if (ret == -1) {
 		TAP_LOG(ERR, "Failed to create pmd for %s as %s",
 			name, tap_name);
+		if (tap_devices_count_increased == 1) {
+			if (!tap_devices_count)
+				rte_mp_action_unregister(TAP_MP_KEY);
+			tap_devices_count--;
+		}
 		tap_unit--;		/* Restore the unit number */
 	}
 	rte_kvargs_free(kvlist);
@@ -2137,6 +2286,9 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 	close(internals->ioctl_sock);
 	rte_free(eth_dev->data->dev_private);
 	rte_free(eth_dev->process_private);
+	if (tap_devices_count == 1)
+		rte_mp_action_unregister(TAP_MP_KEY);
+	tap_devices_count--;
 	rte_eth_dev_release_port(eth_dev);
 
 	if (internals->ka_fd != -1) {
-- 
2.7.4

Re: [dpdk-dev] [PATCH v9 3/3] net/tap: allow secondary process to access primary device queues

[Ferruh Yigit]

On 10/18/2018 9:15 AM, Raslan Darawsheh wrote:
> @@ -2082,6 +2215,17 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
>  	TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s",
>  		name, tap_name);
>  
> +	/* Register IPC feed callback */
> +	if (!tap_devices_count) {
> +		ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
> +		if (ret < 0) {
> +			TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
> +				tuntap_name, strerror(rte_errno));
> +			goto leave;
> +		}
> +	}
> +	tap_devices_count++;
> +	tap_devices_count_increased = 1;
>  	ret = eth_dev_tap_create(dev, tap_name, remote_iface, &user_mac,
>  		ETH_TUNTAP_TYPE_TAP);
>  
> @@ -2089,6 +2233,11 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
>  	if (ret == -1) {
>  		TAP_LOG(ERR, "Failed to create pmd for %s as %s",
>  			name, tap_name);
> +		if (tap_devices_count_increased == 1) {
> +			if (!tap_devices_count)
> +				rte_mp_action_unregister(TAP_MP_KEY);
> +			tap_devices_count--;
> +		}
>  		tap_unit--;		/* Restore the unit number */
>  	}
>  	rte_kvargs_free(kvlist);

Hi Raslan,

This is still wrong, please check the code snippet I have shared.

For first probe, if rte_mp_action_register() run successfully:
  tap_devices_count++;
  tap_devices_count_increased = 1;

later if eth_dev_tap_create() fails, following won't unregister:
  if (!tap_devices_count)
    rte_mp_action_unregister(TAP_MP_KEY);

[dpdk-dev] [PATCH v10 1/3] net/tap: add queue and port ids in Rx/Tx queues structures

[Raslan Darawsheh]

Port and queue ids are added to easily map the file
descriptors stored in each process private.

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
---
 drivers/net/tap/rte_eth_tap.c | 3 +++
 drivers/net/tap/rte_eth_tap.h | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index ad5ae98..edfb7da 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -1293,6 +1293,7 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 	rxq->mp = mp;
 	rxq->trigger_seen = 1; /* force initial burst */
 	rxq->in_port = dev->data->port_id;
+	rxq->queue_id = rx_queue_id;
 	rxq->nb_rx_desc = nb_desc;
 	iovecs = rte_zmalloc_socket(dev->device->name, sizeof(*iovecs), 0,
 				    socket_id);
@@ -1359,6 +1360,8 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		return -1;
 	dev->data->tx_queues[tx_queue_id] = &internals->txq[tx_queue_id];
 	txq = dev->data->tx_queues[tx_queue_id];
+	txq->out_port = dev->data->port_id;
+	txq->queue_id = tx_queue_id;
 
 	offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
 	txq->csum = !!(offloads &
diff --git a/drivers/net/tap/rte_eth_tap.h b/drivers/net/tap/rte_eth_tap.h
index 44e2773..4502e24 100644
--- a/drivers/net/tap/rte_eth_tap.h
+++ b/drivers/net/tap/rte_eth_tap.h
@@ -46,6 +46,7 @@ struct rx_queue {
 	struct rte_mempool *mp;         /* Mempool for RX packets */
 	uint32_t trigger_seen;          /* Last seen Rx trigger value */
 	uint16_t in_port;               /* Port ID */
+	uint16_t queue_id;		/* queue ID*/
 	int fd;
 	struct pkt_stats stats;         /* Stats for this RX queue */
 	uint16_t nb_rx_desc;            /* max number of mbufs available */
@@ -62,6 +63,8 @@ struct tx_queue {
 	uint16_t csum:1;                /* Enable checksum offloading */
 	struct pkt_stats stats;         /* Stats for this TX queue */
 	struct rte_gso_ctx gso_ctx;     /* GSO context */
+	uint16_t out_port;              /* Port ID */
+	uint16_t queue_id;		/* queue ID*/
 };
 
 struct pmd_internals {
-- 
2.7.4

[dpdk-dev] [PATCH v10 2/3] net/tap: move fds of Rx/Tx queues to be in process private

[Raslan Darawsheh]

fd's cannot be shared between processes, and each process need to have
it's own fd's pointer.

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>

---
   This patch has dependancy on the following patch
   http://patches.dpdk.org/patch/46185/
---
---
 drivers/net/tap/rte_eth_tap.c | 93 +++++++++++++++++++++++++++++--------------
 drivers/net/tap/rte_eth_tap.h |  7 +++-
 drivers/net/tap/tap_flow.c    |  3 +-
 drivers/net/tap/tap_intr.c    |  5 ++-
 4 files changed, 73 insertions(+), 35 deletions(-)

diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index edfb7da..3372d54 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -315,6 +315,7 @@ static uint16_t
 pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 {
 	struct rx_queue *rxq = queue;
+	struct pmd_process_private *process_private;
 	uint16_t num_rx;
 	unsigned long num_rx_bytes = 0;
 	uint32_t trigger = tap_trigger;
@@ -323,6 +324,7 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		return 0;
 	if (trigger)
 		rxq->trigger_seen = trigger;
+	process_private = rte_eth_devices[rxq->in_port].process_private;
 	rte_compiler_barrier();
 	for (num_rx = 0; num_rx < nb_pkts; ) {
 		struct rte_mbuf *mbuf = rxq->pool;
@@ -331,9 +333,9 @@ pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		uint16_t data_off = rte_pktmbuf_headroom(mbuf);
 		int len;
 
-		len = readv(rxq->fd, *rxq->iovecs,
-			    1 +
-			    (rxq->rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ?
+		len = readv(process_private->rxq_fds[rxq->queue_id],
+			*rxq->iovecs,
+			1 + (rxq->rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ?
 			     rxq->nb_rx_desc : 1));
 		if (len < (int)sizeof(struct tun_pi))
 			break;
@@ -494,6 +496,9 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
 {
 	int i;
 	uint16_t l234_hlen;
+	struct pmd_process_private *process_private;
+
+	process_private = rte_eth_devices[txq->out_port].process_private;
 
 	for (i = 0; i < num_mbufs; i++) {
 		struct rte_mbuf *mbuf = pmbufs[i];
@@ -595,7 +600,7 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
 			tap_tx_l4_cksum(l4_cksum, l4_phdr_cksum, l4_raw_cksum);
 
 		/* copy the tx frame data */
-		n = writev(txq->fd, iovecs, j);
+		n = writev(process_private->txq_fds[txq->queue_id], iovecs, j);
 		if (n <= 0)
 			break;
 		(*num_packets)++;
@@ -970,19 +975,20 @@ tap_dev_close(struct rte_eth_dev *dev)
 {
 	int i;
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 
 	tap_link_set_down(dev);
 	tap_flow_flush(dev, NULL);
 	tap_flow_implicit_flush(internals, NULL);
 
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		if (internals->rxq[i].fd != -1) {
-			close(internals->rxq[i].fd);
-			internals->rxq[i].fd = -1;
+		if (process_private->rxq_fds[i] != -1) {
+			close(process_private->rxq_fds[i]);
+			process_private->rxq_fds[i] = -1;
 		}
-		if (internals->txq[i].fd != -1) {
-			close(internals->txq[i].fd);
-			internals->txq[i].fd = -1;
+		if (process_private->txq_fds[i] != -1) {
+			close(process_private->txq_fds[i]);
+			process_private->txq_fds[i] = -1;
 		}
 	}
 
@@ -1006,10 +1012,14 @@ static void
 tap_rx_queue_release(void *queue)
 {
 	struct rx_queue *rxq = queue;
+	struct pmd_process_private *process_private;
 
-	if (rxq && (rxq->fd > 0)) {
-		close(rxq->fd);
-		rxq->fd = -1;
+	if (!rxq)
+		return;
+	process_private = rte_eth_devices[rxq->in_port].process_private;
+	if (process_private->rxq_fds[rxq->queue_id] > 0) {
+		close(process_private->rxq_fds[rxq->queue_id]);
+		process_private->rxq_fds[rxq->queue_id] = -1;
 		rte_pktmbuf_free(rxq->pool);
 		rte_free(rxq->iovecs);
 		rxq->pool = NULL;
@@ -1021,10 +1031,15 @@ static void
 tap_tx_queue_release(void *queue)
 {
 	struct tx_queue *txq = queue;
+	struct pmd_process_private *process_private;
+
+	if (!txq)
+		return;
+	process_private = rte_eth_devices[txq->out_port].process_private;
 
-	if (txq && (txq->fd > 0)) {
-		close(txq->fd);
-		txq->fd = -1;
+	if (process_private->txq_fds[txq->queue_id] > 0) {
+		close(process_private->txq_fds[txq->queue_id]);
+		process_private->txq_fds[txq->queue_id] = -1;
 	}
 }
 
@@ -1209,18 +1224,19 @@ tap_setup_queue(struct rte_eth_dev *dev,
 	int *other_fd;
 	const char *dir;
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct rx_queue *rx = &internals->rxq[qid];
 	struct tx_queue *tx = &internals->txq[qid];
 	struct rte_gso_ctx *gso_ctx;
 
 	if (is_rx) {
-		fd = &rx->fd;
-		other_fd = &tx->fd;
+		fd = &process_private->rxq_fds[qid];
+		other_fd = &process_private->txq_fds[qid];
 		dir = "rx";
 		gso_ctx = NULL;
 	} else {
-		fd = &tx->fd;
-		other_fd = &rx->fd;
+		fd = &process_private->txq_fds[qid];
+		other_fd = &process_private->rxq_fds[qid];
 		dir = "tx";
 		gso_ctx = &tx->gso_ctx;
 	}
@@ -1273,6 +1289,7 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 		   struct rte_mempool *mp)
 {
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct rx_queue *rxq = &internals->rxq[rx_queue_id];
 	struct rte_mbuf **tmp = &rxq->pool;
 	long iov_max = sysconf(_SC_IOV_MAX);
@@ -1332,7 +1349,8 @@ tap_rx_queue_setup(struct rte_eth_dev *dev,
 	}
 
 	TAP_LOG(DEBUG, "  RX TUNTAP device name %s, qid %d on fd %d",
-		internals->name, rx_queue_id, internals->rxq[rx_queue_id].fd);
+		internals->name, rx_queue_id,
+		process_private->rxq_fds[rx_queue_id]);
 
 	return 0;
 
@@ -1352,6 +1370,7 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		   const struct rte_eth_txconf *tx_conf)
 {
 	struct pmd_internals *internals = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	struct tx_queue *txq;
 	int ret;
 	uint64_t offloads;
@@ -1374,7 +1393,8 @@ tap_tx_queue_setup(struct rte_eth_dev *dev,
 		return -1;
 	TAP_LOG(DEBUG,
 		"  TX TUNTAP device name %s, qid %d on fd %d csum %s",
-		internals->name, tx_queue_id, internals->txq[tx_queue_id].fd,
+		internals->name, tx_queue_id,
+		process_private->txq_fds[tx_queue_id],
 		txq->csum ? "on" : "off");
 
 	return 0;
@@ -1622,6 +1642,7 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 	int numa_node = rte_socket_id();
 	struct rte_eth_dev *dev;
 	struct pmd_internals *pmd;
+	struct pmd_process_private *process_private;
 	struct rte_eth_dev_data *data;
 	struct ifreq ifr;
 	int i;
@@ -1636,7 +1657,16 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 		goto error_exit_nodev;
 	}
 
+	process_private = (struct pmd_process_private *)
+		rte_zmalloc_socket(tap_name, sizeof(struct pmd_process_private),
+			RTE_CACHE_LINE_SIZE, dev->device->numa_node);
+
+	if (process_private == NULL) {
+		TAP_LOG(ERR, "Failed to alloc memory for process private");
+		return -1;
+	}
 	pmd = dev->data->dev_private;
+	dev->process_private = process_private;
 	pmd->dev = dev;
 	snprintf(pmd->name, sizeof(pmd->name), "%s", tap_name);
 	pmd->type = type;
@@ -1672,8 +1702,8 @@ eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name,
 	/* Presetup the fds to -1 as being not valid */
 	pmd->ka_fd = -1;
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		pmd->rxq[i].fd = -1;
-		pmd->txq[i].fd = -1;
+		process_private->rxq_fds[i] = -1;
+		process_private->txq_fds[i] = -1;
 	}
 
 	if (pmd->type == ETH_TUNTAP_TYPE_TAP) {
@@ -2073,6 +2103,7 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 {
 	struct rte_eth_dev *eth_dev = NULL;
 	struct pmd_internals *internals;
+	struct pmd_process_private *process_private;
 	int i;
 
 	/* find the ethdev entry */
@@ -2081,6 +2112,7 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 		return 0;
 
 	internals = eth_dev->data->dev_private;
+	process_private = eth_dev->process_private;
 
 	TAP_LOG(DEBUG, "Closing %s Ethernet device on numa %u",
 		(internals->type == ETH_TUNTAP_TYPE_TAP) ? "TAP" : "TUN",
@@ -2092,18 +2124,19 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 		tap_nl_final(internals->nlsk_fd);
 	}
 	for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) {
-		if (internals->rxq[i].fd != -1) {
-			close(internals->rxq[i].fd);
-			internals->rxq[i].fd = -1;
+		if (process_private->rxq_fds[i] != -1) {
+			close(process_private->rxq_fds[i]);
+			process_private->rxq_fds[i] = -1;
 		}
-		if (internals->txq[i].fd != -1) {
-			close(internals->txq[i].fd);
-			internals->txq[i].fd = -1;
+		if (process_private->txq_fds[i] != -1) {
+			close(process_private->txq_fds[i]);
+			process_private->txq_fds[i] = -1;
 		}
 	}
 
 	close(internals->ioctl_sock);
 	rte_free(eth_dev->data->dev_private);
+	rte_free(eth_dev->process_private);
 	rte_eth_dev_release_port(eth_dev);
 
 	if (internals->ka_fd != -1) {
diff --git a/drivers/net/tap/rte_eth_tap.h b/drivers/net/tap/rte_eth_tap.h
index 4502e24..dc3579a 100644
--- a/drivers/net/tap/rte_eth_tap.h
+++ b/drivers/net/tap/rte_eth_tap.h
@@ -47,7 +47,6 @@ struct rx_queue {
 	uint32_t trigger_seen;          /* Last seen Rx trigger value */
 	uint16_t in_port;               /* Port ID */
 	uint16_t queue_id;		/* queue ID*/
-	int fd;
 	struct pkt_stats stats;         /* Stats for this RX queue */
 	uint16_t nb_rx_desc;            /* max number of mbufs available */
 	struct rte_eth_rxmode *rxmode;  /* RX features */
@@ -57,7 +56,6 @@ struct rx_queue {
 };
 
 struct tx_queue {
-	int fd;
 	int type;                       /* Type field - TUN|TAP */
 	uint16_t *mtu;                  /* Pointer to MTU from dev_data */
 	uint16_t csum:1;                /* Enable checksum offloading */
@@ -95,6 +93,11 @@ struct pmd_internals {
 	int ka_fd;                        /* keep-alive file descriptor */
 };
 
+struct pmd_process_private {
+	int rxq_fds[RTE_PMD_TAP_MAX_QUEUES];
+	int txq_fds[RTE_PMD_TAP_MAX_QUEUES];
+};
+
 /* tap_intr.c */
 
 int tap_rx_intr_vec_set(struct rte_eth_dev *dev, int set);
diff --git a/drivers/net/tap/tap_flow.c b/drivers/net/tap/tap_flow.c
index 0e01af6..3ba6945 100644
--- a/drivers/net/tap/tap_flow.c
+++ b/drivers/net/tap/tap_flow.c
@@ -1567,6 +1567,7 @@ tap_flow_isolate(struct rte_eth_dev *dev,
 		 struct rte_flow_error *error __rte_unused)
 {
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 
 	/* normalize 'set' variable to contain 0 or 1 values */
 	if (set)
@@ -1580,7 +1581,7 @@ tap_flow_isolate(struct rte_eth_dev *dev,
 	 * If netdevice is there, setup appropriate flow rules immediately.
 	 * Otherwise it will be set when bringing up the netdevice (tun_alloc).
 	 */
-	if (!pmd->rxq[0].fd)
+	if (!process_private->rxq_fds[0])
 		return 0;
 	if (set) {
 		struct rte_flow *remote_flow;
diff --git a/drivers/net/tap/tap_intr.c b/drivers/net/tap/tap_intr.c
index fc59018..7af0010 100644
--- a/drivers/net/tap/tap_intr.c
+++ b/drivers/net/tap/tap_intr.c
@@ -51,6 +51,7 @@ static int
 tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 {
 	struct pmd_internals *pmd = dev->data->dev_private;
+	struct pmd_process_private *process_private = dev->process_private;
 	unsigned int rxqs_n = pmd->dev->data->nb_rx_queues;
 	struct rte_intr_handle *intr_handle = &pmd->intr_handle;
 	unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
@@ -71,7 +72,7 @@ tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 		struct rx_queue *rxq = pmd->dev->data->rx_queues[i];
 
 		/* Skip queues that cannot request interrupts. */
-		if (!rxq || rxq->fd <= 0) {
+		if (!rxq || process_private->rxq_fds[i] <= 0) {
 			/* Use invalid intr_vec[] index to disable entry. */
 			intr_handle->intr_vec[i] =
 				RTE_INTR_VEC_RXTX_OFFSET +
@@ -79,7 +80,7 @@ tap_rx_intr_vec_install(struct rte_eth_dev *dev)
 			continue;
 		}
 		intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
-		intr_handle->efds[count] = rxq->fd;
+		intr_handle->efds[count] = process_private->rxq_fds[i];
 		count++;
 	}
 	if (!count)
-- 
2.7.4

[dpdk-dev] [PATCH v10 3/3] net/tap: allow secondary process to access primary device queues

[Raslan Darawsheh]

In the case the device is created by the primary process,
the secondary must request some file descriptors to attach the queues.
The file descriptors are shared via IPC Unix socket.

Thanks to the IPC synchronization, the secondary process
is now able to do Rx/Tx on a TAP created by the primary process.

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>

---
    v2:
       - translate file descriptors via IPC API
       - add documentation
    v3:
       - rabse the commit
       - use private static array for fd's to be local for each process

    v4:
       - removed TODO and FIXME tags
       - used strlcpy instead of strcpy

    v5: rebase the commit on top of Alejandro Lucero patch for secondary
        process private pointer.
        http://patches.dpdk.org/patch/46185/

    v6: reword the commit log

    v7: rely on tap_device_count for registration
    v8: fix the fail recovery
    v9: sent the good fix for the fail recovery
    v10: fix unrigester in case a failiure happend in the dev_tap_create()

---
---
 doc/guides/nics/tap.rst                |  16 ++++
 doc/guides/rel_notes/release_18_11.rst |   5 ++
 drivers/net/tap/Makefile               |   1 +
 drivers/net/tap/meson.build            |   1 +
 drivers/net/tap/rte_eth_tap.c          | 154 ++++++++++++++++++++++++++++++++-
 5 files changed, 176 insertions(+), 1 deletion(-)

diff --git a/doc/guides/nics/tap.rst b/doc/guides/nics/tap.rst
index 2714868..9a3d7b3 100644
--- a/doc/guides/nics/tap.rst
+++ b/doc/guides/nics/tap.rst
@@ -152,6 +152,22 @@ Distribute IPv4 TCP packets using RSS to a given MAC address over queues 0-3::
    testpmd> flow create 0 priority 4 ingress pattern eth dst is 0a:0b:0c:0d:0e:0f \
             / ipv4 / tcp / end actions rss queues 0 1 2 3 end / end
 
+Multi-process sharing
+---------------------
+
+It is possible to attach an existing TAP device in a secondary process,
+by declaring it as a vdev with the same name as in the primary process,
+and without any parameter.
+
+The port attached in a secondary process will give access to the
+statistics and the queues.
+Therefore it can be used for monitoring or Rx/Tx processing.
+
+The IPC synchronization of Rx/Tx queues is currently limited:
+
+  - Maximum 8 queues shared
+  - Synchronized on probing, but not on later port update
+
 Example
 -------
 
diff --git a/doc/guides/rel_notes/release_18_11.rst b/doc/guides/rel_notes/release_18_11.rst
index 2133a5b..3240b52 100644
--- a/doc/guides/rel_notes/release_18_11.rst
+++ b/doc/guides/rel_notes/release_18_11.rst
@@ -104,6 +104,11 @@ New Features
   the specified port. The port must be stopped before the command call in order
   to reconfigure queues.
 
+* **Added TAP Rx/Tx queues sharing with a secondary process.**
+
+  A secondary process can attach a TAP device created in the primary process,
+  probe the queues, and process Rx/Tx in a secondary process.
+
 
 API Changes
 -----------
diff --git a/drivers/net/tap/Makefile b/drivers/net/tap/Makefile
index 3243365..7748283 100644
--- a/drivers/net/tap/Makefile
+++ b/drivers/net/tap/Makefile
@@ -22,6 +22,7 @@ CFLAGS += -O3
 CFLAGS += -I$(SRCDIR)
 CFLAGS += -I.
 CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash
 LDLIBS += -lrte_bus_vdev -lrte_gso
diff --git a/drivers/net/tap/meson.build b/drivers/net/tap/meson.build
index 37f65b7..f7e8852 100644
--- a/drivers/net/tap/meson.build
+++ b/drivers/net/tap/meson.build
@@ -35,6 +35,7 @@ args = [
 	  'TCA_ACT_BPF_FD' ],
 ]
 config = configuration_data()
+allow_experimental_apis = true
 foreach arg:args
 	config.set(arg[0], cc.has_header_symbol(arg[1], arg[2]))
 endforeach
diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index 3372d54..3cef5f6 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -16,6 +16,8 @@
 #include <rte_debug.h>
 #include <rte_ip.h>
 #include <rte_string_fns.h>
+#include <rte_ethdev.h>
+#include <rte_errno.h>
 
 #include <assert.h>
 #include <sys/types.h>
@@ -62,6 +64,10 @@
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
+/* IPC key for queue fds sync */
+#define TAP_MP_KEY "tap_mp_sync_queues"
+
+static int tap_devices_count;
 static struct rte_vdev_driver pmd_tap_drv;
 static struct rte_vdev_driver pmd_tun_drv;
 
@@ -100,6 +106,17 @@ enum ioctl_mode {
 	REMOTE_ONLY,
 };
 
+/* Message header to synchronize queues via IPC */
+struct ipc_queues {
+	char port_name[RTE_DEV_NAME_MAX_LEN];
+	int rxq_count;
+	int txq_count;
+	/*
+	 * The file descriptors are in the dedicated part
+	 * of the Unix message to be translated by the kernel.
+	 */
+};
+
 static int tap_intr_handle_set(struct rte_eth_dev *dev, int set);
 
 /**
@@ -2006,6 +2023,102 @@ rte_pmd_tun_probe(struct rte_vdev_device *dev)
 	return ret;
 }
 
+/* Request queue file descriptors from secondary to primary. */
+static int
+tap_mp_attach_queues(const char *port_name, struct rte_eth_dev *dev)
+{
+	int ret;
+	struct timespec timeout = {.tv_sec = 1, .tv_nsec = 0};
+	struct rte_mp_msg request, *reply;
+	struct rte_mp_reply replies;
+	struct ipc_queues *request_param = (struct ipc_queues *)request.param;
+	struct ipc_queues *reply_param;
+	struct pmd_process_private *process_private = dev->process_private;
+	int queue, fd_iterator;
+
+	/* Prepare the request */
+	strlcpy(request.name, TAP_MP_KEY, sizeof(request.name));
+	strlcpy(request_param->port_name, port_name,
+		sizeof(request_param->port_name));
+	request.len_param = sizeof(*request_param);
+	/* Send request and receive reply */
+	ret = rte_mp_request_sync(&request, &replies, &timeout);
+	if (ret < 0) {
+		TAP_LOG(ERR, "Failed to request queues from primary: %d",
+			rte_errno);
+		return -1;
+	}
+	reply = &replies.msgs[0];
+	reply_param = (struct ipc_queues *)reply->param;
+	TAP_LOG(DEBUG, "Received IPC reply for %s", reply_param->port_name);
+
+	/* Attach the queues from received file descriptors */
+	dev->data->nb_rx_queues = reply_param->rxq_count;
+	dev->data->nb_tx_queues = reply_param->txq_count;
+	fd_iterator = 0;
+	for (queue = 0; queue < reply_param->rxq_count; queue++)
+		process_private->rxq_fds[queue] = reply->fds[fd_iterator++];
+	for (queue = 0; queue < reply_param->txq_count; queue++)
+		process_private->txq_fds[queue] = reply->fds[fd_iterator++];
+
+	return 0;
+}
+
+/* Send the queue file descriptors from the primary process to secondary. */
+static int
+tap_mp_sync_queues(const struct rte_mp_msg *request, const void *peer)
+{
+	struct rte_eth_dev *dev;
+	struct pmd_process_private *process_private;
+	struct rte_mp_msg reply;
+	const struct ipc_queues *request_param =
+		(const struct ipc_queues *)request->param;
+	struct ipc_queues *reply_param =
+		(struct ipc_queues *)reply.param;
+	uint16_t port_id;
+	int queue;
+	int ret;
+
+	/* Get requested port */
+	TAP_LOG(DEBUG, "Received IPC request for %s", request_param->port_name);
+	ret = rte_eth_dev_get_port_by_name(request_param->port_name, &port_id);
+	if (ret) {
+		TAP_LOG(ERR, "Failed to get port id for %s",
+			request_param->port_name);
+		return -1;
+	}
+	dev = &rte_eth_devices[port_id];
+	process_private = dev->process_private;
+
+	/* Fill file descriptors for all queues */
+	reply.num_fds = 0;
+	reply_param->rxq_count = 0;
+	for (queue = 0; queue < dev->data->nb_rx_queues; queue++) {
+		reply.fds[reply.num_fds++] = process_private->rxq_fds[queue];
+		reply_param->rxq_count++;
+	}
+	RTE_ASSERT(reply_param->rxq_count == dev->data->nb_rx_queues);
+	RTE_ASSERT(reply_param->txq_count == dev->data->nb_tx_queues);
+	RTE_ASSERT(reply.num_fds <= RTE_MP_MAX_FD_NUM);
+
+	reply_param->txq_count = 0;
+	for (queue = 0; queue < dev->data->nb_tx_queues; queue++) {
+		reply.fds[reply.num_fds++] = process_private->txq_fds[queue];
+		reply_param->txq_count++;
+	}
+
+	/* Send reply */
+	strlcpy(reply.name, request->name, sizeof(reply.name));
+	strlcpy(reply_param->port_name, request_param->port_name,
+		sizeof(reply_param->port_name));
+	reply.len_param = sizeof(*reply_param);
+	if (rte_mp_reply(&reply, peer) < 0) {
+		TAP_LOG(ERR, "Failed to reply an IPC request to sync queues");
+		return -1;
+	}
+	return 0;
+}
+
 /* Open a TAP interface device.
  */
 static int
@@ -2019,6 +2132,7 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 	char remote_iface[RTE_ETH_NAME_MAX_LEN];
 	struct ether_addr user_mac = { .addr_bytes = {0} };
 	struct rte_eth_dev *eth_dev;
+	int tap_devices_count_increased = 0;
 
 	strcpy(tuntap_name, "TAP");
 
@@ -2032,9 +2146,28 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 			TAP_LOG(ERR, "Failed to probe %s", name);
 			return -1;
 		}
-		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &ops;
 		eth_dev->device = &dev->device;
+		eth_dev->rx_pkt_burst = pmd_rx_burst;
+		eth_dev->tx_pkt_burst = pmd_tx_burst;
+		if (!rte_eal_primary_proc_alive(NULL)) {
+			TAP_LOG(ERR, "Primary process is missing");
+			return -1;
+		}
+		eth_dev->process_private = (struct pmd_process_private *)
+			rte_zmalloc_socket(name,
+				sizeof(struct pmd_process_private),
+				RTE_CACHE_LINE_SIZE,
+				eth_dev->device->numa_node);
+		if (eth_dev->process_private == NULL) {
+			TAP_LOG(ERR,
+				"Failed to alloc memory for process private");
+			return -1;
+		}
+
+		ret = tap_mp_attach_queues(name, eth_dev);
+		if (ret != 0)
+			return -1;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
@@ -2082,6 +2215,17 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 	TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s",
 		name, tap_name);
 
+	/* Register IPC feed callback */
+	if (!tap_devices_count) {
+		ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
+		if (ret < 0) {
+			TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
+				tuntap_name, strerror(rte_errno));
+			goto leave;
+		}
+	}
+	tap_devices_count++;
+	tap_devices_count_increased = 1;
 	ret = eth_dev_tap_create(dev, tap_name, remote_iface, &user_mac,
 		ETH_TUNTAP_TYPE_TAP);
 
@@ -2089,6 +2233,11 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 	if (ret == -1) {
 		TAP_LOG(ERR, "Failed to create pmd for %s as %s",
 			name, tap_name);
+		if (tap_devices_count_increased == 1) {
+			if (tap_devices_count == 1)
+				rte_mp_action_unregister(TAP_MP_KEY);
+			tap_devices_count--;
+		}
 		tap_unit--;		/* Restore the unit number */
 	}
 	rte_kvargs_free(kvlist);
@@ -2137,6 +2286,9 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 	close(internals->ioctl_sock);
 	rte_free(eth_dev->data->dev_private);
 	rte_free(eth_dev->process_private);
+	if (tap_devices_count == 1)
+		rte_mp_action_unregister(TAP_MP_KEY);
+	tap_devices_count--;
 	rte_eth_dev_release_port(eth_dev);
 
 	if (internals->ka_fd != -1) {
-- 
2.7.4

Re: [dpdk-dev] [PATCH v10 1/3] net/tap: add queue and port ids in Rx/Tx queues structures

[Ferruh Yigit]

On 10/18/2018 11:17 AM, Raslan Darawsheh wrote:
> Port and queue ids are added to easily map the file
> descriptors stored in each process private.
> 
> Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>

Series applied to dpdk-next-net/master, thanks.

Re: [dpdk-dev] [PATCH v10 1/3] net/tap: add queue and port ids in Rx/Tx queues structures

[Ferruh Yigit]

On 10/18/2018 11:17 AM, Raslan Darawsheh wrote:
> Port and queue ids are added to easily map the file
> descriptors stored in each process private.
> 
> Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>

For series,
Reviewed-by: Ferruh Yigit <ferruh.yigit@intel.com>

[dpdk-dev] [PATCH v6 3/3] net/tap: allow secondary process to access primary device queues

[Raslan Darawsheh]

In the case the device is created by the primary process,
the secondary must request some file descriptors to attach the queues.
The file descriptors are shared via IPC Unix socket.

Thanks to the IPC synchronization, the secondary process
is now able to do Rx/Tx on a TAP created by the primary process.

Signed-off-by: Raslan Darawsheh <rasland@mellanox.com>
Signed-off-by: Thomas Monjalon <thomas@monjalon.net>

---
    v2:
       - translate file descriptors via IPC API
       - add documentation
    v3:
       - rabse the commit
       - use private static array for fd's to be local for each process

    v4:
       - removed TODO and FIXME tags
       - used strlcpy instead of strcpy

    v5: rebase the commit on top of Alejandro Lucero patch for secondary
        process private pointer.
        http://patches.dpdk.org/patch/46185/

    v6: reword the commit log
---
---
 doc/guides/nics/tap.rst                |  16 ++++
 doc/guides/rel_notes/release_18_11.rst |   5 ++
 drivers/net/tap/Makefile               |   1 +
 drivers/net/tap/meson.build            |   1 +
 drivers/net/tap/rte_eth_tap.c          | 147 ++++++++++++++++++++++++++++++++-
 5 files changed, 169 insertions(+), 1 deletion(-)

diff --git a/doc/guides/nics/tap.rst b/doc/guides/nics/tap.rst
index 2714868..9a3d7b3 100644
--- a/doc/guides/nics/tap.rst
+++ b/doc/guides/nics/tap.rst
@@ -152,6 +152,22 @@ Distribute IPv4 TCP packets using RSS to a given MAC address over queues 0-3::
    testpmd> flow create 0 priority 4 ingress pattern eth dst is 0a:0b:0c:0d:0e:0f \
             / ipv4 / tcp / end actions rss queues 0 1 2 3 end / end
 
+Multi-process sharing
+---------------------
+
+It is possible to attach an existing TAP device in a secondary process,
+by declaring it as a vdev with the same name as in the primary process,
+and without any parameter.
+
+The port attached in a secondary process will give access to the
+statistics and the queues.
+Therefore it can be used for monitoring or Rx/Tx processing.
+
+The IPC synchronization of Rx/Tx queues is currently limited:
+
+  - Maximum 8 queues shared
+  - Synchronized on probing, but not on later port update
+
 Example
 -------
 
diff --git a/doc/guides/rel_notes/release_18_11.rst b/doc/guides/rel_notes/release_18_11.rst
index 2133a5b..3240b52 100644
--- a/doc/guides/rel_notes/release_18_11.rst
+++ b/doc/guides/rel_notes/release_18_11.rst
@@ -104,6 +104,11 @@ New Features
   the specified port. The port must be stopped before the command call in order
   to reconfigure queues.
 
+* **Added TAP Rx/Tx queues sharing with a secondary process.**
+
+  A secondary process can attach a TAP device created in the primary process,
+  probe the queues, and process Rx/Tx in a secondary process.
+
 
 API Changes
 -----------
diff --git a/drivers/net/tap/Makefile b/drivers/net/tap/Makefile
index 3243365..7748283 100644
--- a/drivers/net/tap/Makefile
+++ b/drivers/net/tap/Makefile
@@ -22,6 +22,7 @@ CFLAGS += -O3
 CFLAGS += -I$(SRCDIR)
 CFLAGS += -I.
 CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash
 LDLIBS += -lrte_bus_vdev -lrte_gso
diff --git a/drivers/net/tap/meson.build b/drivers/net/tap/meson.build
index 37f65b7..f7e8852 100644
--- a/drivers/net/tap/meson.build
+++ b/drivers/net/tap/meson.build
@@ -35,6 +35,7 @@ args = [
 	  'TCA_ACT_BPF_FD' ],
 ]
 config = configuration_data()
+allow_experimental_apis = true
 foreach arg:args
 	config.set(arg[0], cc.has_header_symbol(arg[1], arg[2]))
 endforeach
diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index 3372d54..83c9288 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -16,6 +16,8 @@
 #include <rte_debug.h>
 #include <rte_ip.h>
 #include <rte_string_fns.h>
+#include <rte_ethdev.h>
+#include <rte_errno.h>
 
 #include <assert.h>
 #include <sys/types.h>
@@ -62,6 +64,10 @@
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
+/* IPC key for queue fds sync */
+#define TAP_MP_KEY "tap_mp_sync_queues"
+
+static int tap_devices_count;
 static struct rte_vdev_driver pmd_tap_drv;
 static struct rte_vdev_driver pmd_tun_drv;
 
@@ -100,6 +106,17 @@ enum ioctl_mode {
 	REMOTE_ONLY,
 };
 
+/* Message header to synchronize queues via IPC */
+struct ipc_queues {
+	char port_name[RTE_DEV_NAME_MAX_LEN];
+	int rxq_count;
+	int txq_count;
+	/*
+	 * The file descriptors are in the dedicated part
+	 * of the Unix message to be translated by the kernel.
+	 */
+};
+
 static int tap_intr_handle_set(struct rte_eth_dev *dev, int set);
 
 /**
@@ -2006,6 +2023,102 @@ rte_pmd_tun_probe(struct rte_vdev_device *dev)
 	return ret;
 }
 
+/* Request queue file descriptors from secondary to primary. */
+static int
+tap_mp_attach_queues(const char *port_name, struct rte_eth_dev *dev)
+{
+	int ret;
+	struct timespec timeout = {.tv_sec = 1, .tv_nsec = 0};
+	struct rte_mp_msg request, *reply;
+	struct rte_mp_reply replies;
+	struct ipc_queues *request_param = (struct ipc_queues *)request.param;
+	struct ipc_queues *reply_param;
+	struct pmd_process_private *process_private = dev->process_private;
+	int queue, fd_iterator;
+
+	/* Prepare the request */
+	strlcpy(request.name, TAP_MP_KEY, sizeof(request.name));
+	strlcpy(request_param->port_name, port_name,
+		sizeof(request_param->port_name));
+	request.len_param = sizeof(*request_param);
+	/* Send request and receive reply */
+	ret = rte_mp_request_sync(&request, &replies, &timeout);
+	if (ret < 0) {
+		TAP_LOG(ERR, "Failed to request queues from primary: %d",
+			rte_errno);
+		return -1;
+	}
+	reply = &replies.msgs[0];
+	reply_param = (struct ipc_queues *)reply->param;
+	TAP_LOG(DEBUG, "Received IPC reply for %s", reply_param->port_name);
+
+	/* Attach the queues from received file descriptors */
+	dev->data->nb_rx_queues = reply_param->rxq_count;
+	dev->data->nb_tx_queues = reply_param->txq_count;
+	fd_iterator = 0;
+	for (queue = 0; queue < reply_param->rxq_count; queue++)
+		process_private->rxq_fds[queue] = reply->fds[fd_iterator++];
+	for (queue = 0; queue < reply_param->txq_count; queue++)
+		process_private->txq_fds[queue] = reply->fds[fd_iterator++];
+
+	return 0;
+}
+
+/* Send the queue file descriptors from the primary process to secondary. */
+static int
+tap_mp_sync_queues(const struct rte_mp_msg *request, const void *peer)
+{
+	struct rte_eth_dev *dev;
+	struct pmd_process_private *process_private;
+	struct rte_mp_msg reply;
+	const struct ipc_queues *request_param =
+		(const struct ipc_queues *)request->param;
+	struct ipc_queues *reply_param =
+		(struct ipc_queues *)reply.param;
+	uint16_t port_id;
+	int queue;
+	int ret;
+
+	/* Get requested port */
+	TAP_LOG(DEBUG, "Received IPC request for %s", request_param->port_name);
+	ret = rte_eth_dev_get_port_by_name(request_param->port_name, &port_id);
+	if (ret) {
+		TAP_LOG(ERR, "Failed to get port id for %s",
+			request_param->port_name);
+		return -1;
+	}
+	dev = &rte_eth_devices[port_id];
+	process_private = dev->process_private;
+
+	/* Fill file descriptors for all queues */
+	reply.num_fds = 0;
+	reply_param->rxq_count = 0;
+	for (queue = 0; queue < dev->data->nb_rx_queues; queue++) {
+		reply.fds[reply.num_fds++] = process_private->rxq_fds[queue];
+		reply_param->rxq_count++;
+	}
+	RTE_ASSERT(reply_param->rxq_count == dev->data->nb_rx_queues);
+	RTE_ASSERT(reply_param->txq_count == dev->data->nb_tx_queues);
+	RTE_ASSERT(reply.num_fds <= RTE_MP_MAX_FD_NUM);
+
+	reply_param->txq_count = 0;
+	for (queue = 0; queue < dev->data->nb_tx_queues; queue++) {
+		reply.fds[reply.num_fds++] = process_private->txq_fds[queue];
+		reply_param->txq_count++;
+	}
+
+	/* Send reply */
+	strlcpy(reply.name, request->name, sizeof(reply.name));
+	strlcpy(reply_param->port_name, request_param->port_name,
+		sizeof(reply_param->port_name));
+	reply.len_param = sizeof(*reply_param);
+	if (rte_mp_reply(&reply, peer) < 0) {
+		TAP_LOG(ERR, "Failed to reply an IPC request to sync queues");
+		return -1;
+	}
+	return 0;
+}
+
 /* Open a TAP interface device.
  */
 static int
@@ -2032,9 +2145,28 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 			TAP_LOG(ERR, "Failed to probe %s", name);
 			return -1;
 		}
-		/* TODO: request info from primary to set up Rx and Tx */
 		eth_dev->dev_ops = &ops;
 		eth_dev->device = &dev->device;
+		eth_dev->rx_pkt_burst = pmd_rx_burst;
+		eth_dev->tx_pkt_burst = pmd_tx_burst;
+		if (!rte_eal_primary_proc_alive(NULL)) {
+			TAP_LOG(ERR, "Primary process is missing");
+			return -1;
+		}
+		eth_dev->process_private = (struct pmd_process_private *)
+			rte_zmalloc_socket(name,
+				sizeof(struct pmd_process_private),
+				RTE_CACHE_LINE_SIZE,
+				eth_dev->device->numa_node);
+		if (eth_dev->process_private == NULL) {
+			TAP_LOG(ERR,
+				"Failed to alloc memory for process private");
+			return -1;
+		}
+
+		ret = tap_mp_attach_queues(name, eth_dev);
+		if (ret != 0)
+			return -1;
 		rte_eth_dev_probing_finish(eth_dev);
 		return 0;
 	}
@@ -2082,6 +2214,14 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 	TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s",
 		name, tap_name);
 
+	/* Register IPC feed callback */
+	ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
+	if (ret < 0 && rte_errno != EEXIST && tap_devices_count) {
+		TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
+			tuntap_name, strerror(rte_errno));
+		goto leave;
+	}
+	tap_devices_count++;
 	ret = eth_dev_tap_create(dev, tap_name, remote_iface, &user_mac,
 		ETH_TUNTAP_TYPE_TAP);
 
@@ -2089,6 +2229,8 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
 	if (ret == -1) {
 		TAP_LOG(ERR, "Failed to create pmd for %s as %s",
 			name, tap_name);
+		if (!tap_devices_count)
+			rte_mp_action_unregister(TAP_MP_KEY);
 		tap_unit--;		/* Restore the unit number */
 	}
 	rte_kvargs_free(kvlist);
@@ -2137,6 +2279,9 @@ rte_pmd_tap_remove(struct rte_vdev_device *dev)
 	close(internals->ioctl_sock);
 	rte_free(eth_dev->data->dev_private);
 	rte_free(eth_dev->process_private);
+	if (tap_devices_count == 1)
+		rte_mp_action_unregister(TAP_MP_KEY);
+	tap_devices_count--;
 	rte_eth_dev_release_port(eth_dev);
 
 	if (internals->ka_fd != -1) {
-- 
2.7.4

Re: [dpdk-dev] [PATCH v6 3/3] net/tap: allow secondary process to access primary device queues

[Ferruh Yigit]

On 10/10/2018 3:39 PM, Raslan Darawsheh wrote:
> @@ -2082,6 +2214,14 @@ rte_pmd_tap_probe(struct rte_vdev_device *dev)
>  	TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s",
>  		name, tap_name);
>  
> +	/* Register IPC feed callback */
> +	ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
> +	if (ret < 0 && rte_errno != EEXIST && tap_devices_count) {
> +		TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
> +			tuntap_name, strerror(rte_errno));
> +		goto leave;
> +	}
> +	tap_devices_count++;
>  	ret = eth_dev_tap_create(dev, tap_name, remote_iface, &user_mac,
>  		ETH_TUNTAP_TYPE_TAP);

Why not rely on "tap_devices_count" but call rte_mp_action_register() every time
and try to figure out status from ret?

if (tap_devices_count == 0) {
  ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
  if (ret < 0) {
    TAP_LOG();
    goto leave;
  }
}
tap_devices_count++;

Re: [dpdk-dev] [PATCH v6 3/3] net/tap: allow secondary process to access primary device queues

[Raslan Darawsheh]

Hi Ferruh,

I didn't do it this way since I don't want it to unregister in case it was the second device for example and it failed to prob.
Which means only the first probed tap and the last removed tap will handle this registration and deregistration. 

Kindest regards,
Raslan Darawsheh

> -----Original Message-----
> From: Ferruh Yigit <ferruh.yigit@intel.com>
> Sent: Thursday, October 11, 2018 7:33 PM
> To: Raslan Darawsheh <rasland@mellanox.com>; keith.wiles@intel.com
> Cc: Thomas Monjalon <thomas@monjalon.net>; dev@dpdk.org; Shahaf
> Shuler <shahafs@mellanox.com>; Ori Kam <orika@mellanox.com>
> Subject: Re: [PATCH v6 3/3] net/tap: allow secondary process to access
> primary device queues
> 
> On 10/10/2018 3:39 PM, Raslan Darawsheh wrote:
> > @@ -2082,6 +2214,14 @@ rte_pmd_tap_probe(struct rte_vdev_device
> *dev)
> >  	TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s",
> >  		name, tap_name);
> >
> > +	/* Register IPC feed callback */
> > +	ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
> > +	if (ret < 0 && rte_errno != EEXIST && tap_devices_count) {
> > +		TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
> > +			tuntap_name, strerror(rte_errno));
> > +		goto leave;
> > +	}
> > +	tap_devices_count++;
> >  	ret = eth_dev_tap_create(dev, tap_name, remote_iface,
> &user_mac,
> >  		ETH_TUNTAP_TYPE_TAP);
> 
> Why not rely on "tap_devices_count" but call rte_mp_action_register()
> every time and try to figure out status from ret?
> 
> if (tap_devices_count == 0) {
>   ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
>   if (ret < 0) {
>     TAP_LOG();
>     goto leave;
>   }
> }
> tap_devices_count++;

Re: [dpdk-dev] [PATCH v6 3/3] net/tap: allow secondary process to access primary device queues

[Ferruh Yigit]

On 10/16/2018 11:06 AM, Raslan Darawsheh wrote:
> Hi Ferruh,
> 
> I didn't do it this way since I don't want it to unregister in case it was the second device for example and it failed to prob.
> Which means only the first probed tap and the last removed tap will handle this registration and deregistration. 

What I am saying will do the same, only register in the first device probed and
unregister in the last device.

`tap_devices_count` is already there, why you don't rely on it, but call
rte_mp_action_register() anyway and rely on it will return EEXIST in second and
later calls.

Both will work, just I think using `tap_devices_count` is simpler.

> 
> Kindest regards,
> Raslan Darawsheh
> 
>> -----Original Message-----
>> From: Ferruh Yigit <ferruh.yigit@intel.com>
>> Sent: Thursday, October 11, 2018 7:33 PM
>> To: Raslan Darawsheh <rasland@mellanox.com>; keith.wiles@intel.com
>> Cc: Thomas Monjalon <thomas@monjalon.net>; dev@dpdk.org; Shahaf
>> Shuler <shahafs@mellanox.com>; Ori Kam <orika@mellanox.com>
>> Subject: Re: [PATCH v6 3/3] net/tap: allow secondary process to access
>> primary device queues
>>
>> On 10/10/2018 3:39 PM, Raslan Darawsheh wrote:
>>> @@ -2082,6 +2214,14 @@ rte_pmd_tap_probe(struct rte_vdev_device
>> *dev)
>>>  	TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s",
>>>  		name, tap_name);
>>>
>>> +	/* Register IPC feed callback */
>>> +	ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
>>> +	if (ret < 0 && rte_errno != EEXIST && tap_devices_count) {
>>> +		TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
>>> +			tuntap_name, strerror(rte_errno));
>>> +		goto leave;
>>> +	}
>>> +	tap_devices_count++;
>>>  	ret = eth_dev_tap_create(dev, tap_name, remote_iface,
>> &user_mac,
>>>  		ETH_TUNTAP_TYPE_TAP);
>>
>> Why not rely on "tap_devices_count" but call rte_mp_action_register()
>> every time and try to figure out status from ret?
>>
>> if (tap_devices_count == 0) {
>>   ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
>>   if (ret < 0) {
>>     TAP_LOG();
>>     goto leave;
>>   }
>> }
>> tap_devices_count++;
> 

Re: [dpdk-dev] [PATCH v6 3/3] net/tap: allow secondary process to access primary device queues

[Raslan Darawsheh]

Hi,

> -----Original Message-----
> From: Ferruh Yigit <ferruh.yigit@intel.com>
> Sent: Tuesday, October 16, 2018 2:28 PM
> To: Raslan Darawsheh <rasland@mellanox.com>; keith.wiles@intel.com
> Cc: Thomas Monjalon <thomas@monjalon.net>; dev@dpdk.org; Shahaf
> Shuler <shahafs@mellanox.com>; Ori Kam <orika@mellanox.com>
> Subject: Re: [PATCH v6 3/3] net/tap: allow secondary process to access
> primary device queues
> 
> On 10/16/2018 11:06 AM, Raslan Darawsheh wrote:
> > Hi Ferruh,
> >
> > I didn't do it this way since I don't want it to unregister in case it was the
> second device for example and it failed to prob.
> > Which means only the first probed tap and the last removed tap will handle
> this registration and deregistration.
> 
> What I am saying will do the same, only register in the first device probed and
> unregister in the last device.
> 
> `tap_devices_count` is already there, why you don't rely on it, but call
> rte_mp_action_register() anyway and rely on it will return EEXIST in second
> and later calls.
> 
> Both will work, just I think using `tap_devices_count` is simpler.
> 
You are right about probing.
but, not in the closing of the ports, 
since any port will try to unregister the first one which will do the unregister will succeed.
meanwhile, I need only the last one to unregister.

> >
> > Kindest regards,
> > Raslan Darawsheh
> >
> >> -----Original Message-----
> >> From: Ferruh Yigit <ferruh.yigit@intel.com>
> >> Sent: Thursday, October 11, 2018 7:33 PM
> >> To: Raslan Darawsheh <rasland@mellanox.com>; keith.wiles@intel.com
> >> Cc: Thomas Monjalon <thomas@monjalon.net>; dev@dpdk.org; Shahaf
> >> Shuler <shahafs@mellanox.com>; Ori Kam <orika@mellanox.com>
> >> Subject: Re: [PATCH v6 3/3] net/tap: allow secondary process to
> >> access primary device queues
> >>
> >> On 10/10/2018 3:39 PM, Raslan Darawsheh wrote:
> >>> @@ -2082,6 +2214,14 @@ rte_pmd_tap_probe(struct rte_vdev_device
> >> *dev)
> >>>  	TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s",
> >>>  		name, tap_name);
> >>>
> >>> +	/* Register IPC feed callback */
> >>> +	ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
> >>> +	if (ret < 0 && rte_errno != EEXIST && tap_devices_count) {
> >>> +		TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
> >>> +			tuntap_name, strerror(rte_errno));
> >>> +		goto leave;
> >>> +	}
> >>> +	tap_devices_count++;
> >>>  	ret = eth_dev_tap_create(dev, tap_name, remote_iface,
> >> &user_mac,
> >>>  		ETH_TUNTAP_TYPE_TAP);
> >>
> >> Why not rely on "tap_devices_count" but call rte_mp_action_register()
> >> every time and try to figure out status from ret?
> >>
> >> if (tap_devices_count == 0) {
> >>   ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
> >>   if (ret < 0) {
> >>     TAP_LOG();
> >>     goto leave;
> >>   }
> >> }
> >> tap_devices_count++;
> >


Kindest regards,
Raslan Darawsheh

Re: [dpdk-dev] [PATCH v6 3/3] net/tap: allow secondary process to access primary device queues

[Ferruh Yigit]

On 10/17/2018 7:54 AM, Raslan Darawsheh wrote:
> Hi,
> 
>> -----Original Message-----
>> From: Ferruh Yigit <ferruh.yigit@intel.com>
>> Sent: Tuesday, October 16, 2018 2:28 PM
>> To: Raslan Darawsheh <rasland@mellanox.com>; keith.wiles@intel.com
>> Cc: Thomas Monjalon <thomas@monjalon.net>; dev@dpdk.org; Shahaf
>> Shuler <shahafs@mellanox.com>; Ori Kam <orika@mellanox.com>
>> Subject: Re: [PATCH v6 3/3] net/tap: allow secondary process to access
>> primary device queues
>>
>> On 10/16/2018 11:06 AM, Raslan Darawsheh wrote:
>>> Hi Ferruh,
>>>
>>> I didn't do it this way since I don't want it to unregister in case it was the
>> second device for example and it failed to prob.
>>> Which means only the first probed tap and the last removed tap will handle
>> this registration and deregistration.
>>
>> What I am saying will do the same, only register in the first device probed and
>> unregister in the last device.
>>
>> `tap_devices_count` is already there, why you don't rely on it, but call
>> rte_mp_action_register() anyway and rely on it will return EEXIST in second
>> and later calls.
>>
>> Both will work, just I think using `tap_devices_count` is simpler.
>>
> You are right about probing.
> but, not in the closing of the ports, 
> since any port will try to unregister the first one which will do the unregister will succeed.
> meanwhile, I need only the last one to unregister.

The unregister done already with `tap_devices_count` checks, all I am suggesting
is doing same thing for register part.

> 
>>>
>>> Kindest regards,
>>> Raslan Darawsheh
>>>
>>>> -----Original Message-----
>>>> From: Ferruh Yigit <ferruh.yigit@intel.com>
>>>> Sent: Thursday, October 11, 2018 7:33 PM
>>>> To: Raslan Darawsheh <rasland@mellanox.com>; keith.wiles@intel.com
>>>> Cc: Thomas Monjalon <thomas@monjalon.net>; dev@dpdk.org; Shahaf
>>>> Shuler <shahafs@mellanox.com>; Ori Kam <orika@mellanox.com>
>>>> Subject: Re: [PATCH v6 3/3] net/tap: allow secondary process to
>>>> access primary device queues
>>>>
>>>> On 10/10/2018 3:39 PM, Raslan Darawsheh wrote:
>>>>> @@ -2082,6 +2214,14 @@ rte_pmd_tap_probe(struct rte_vdev_device
>>>> *dev)
>>>>>  	TAP_LOG(NOTICE, "Initializing pmd_tap for %s as %s",
>>>>>  		name, tap_name);
>>>>>
>>>>> +	/* Register IPC feed callback */
>>>>> +	ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
>>>>> +	if (ret < 0 && rte_errno != EEXIST && tap_devices_count) {
>>>>> +		TAP_LOG(ERR, "%s: Failed to register IPC callback: %s",
>>>>> +			tuntap_name, strerror(rte_errno));
>>>>> +		goto leave;
>>>>> +	}
>>>>> +	tap_devices_count++;
>>>>>  	ret = eth_dev_tap_create(dev, tap_name, remote_iface,
>>>> &user_mac,
>>>>>  		ETH_TUNTAP_TYPE_TAP);
>>>>
>>>> Why not rely on "tap_devices_count" but call rte_mp_action_register()
>>>> every time and try to figure out status from ret?
>>>>
>>>> if (tap_devices_count == 0) {
>>>>   ret = rte_mp_action_register(TAP_MP_KEY, tap_mp_sync_queues);
>>>>   if (ret < 0) {
>>>>     TAP_LOG();
>>>>     goto leave;
>>>>   }
>>>> }
>>>> tap_devices_count++;
>>>
> 
> 
> Kindest regards,
> Raslan Darawsheh
>