[dpdk-dev] [PATCH 0/2] netvsc pmd fixes for 19.05

[dpdk-dev] [PATCH 0/2] netvsc pmd fixes for 19.05

[Stephen Hemminger]

Netvsc needed to reset incoming port as was recently done
for failsafe. Implement the new close semantics.

Stephen Hemminger (2):
  net/netvsc: reset mbuf port on VF receive
  net/netvsc: free all queues on close

 drivers/net/netvsc/hn_ethdev.c |  8 +++-
 drivers/net/netvsc/hn_rxtx.c   | 86 ++++++++++++++++++++++++----------
 drivers/net/netvsc/hn_var.h    |  1 +
 3 files changed, 68 insertions(+), 27 deletions(-)

-- 
2.20.1

[dpdk-dev] [PATCH 0/2] netvsc pmd fixes for 19.05

[Stephen Hemminger]

Netvsc needed to reset incoming port as was recently done
for failsafe. Implement the new close semantics.

Stephen Hemminger (2):
  net/netvsc: reset mbuf port on VF receive
  net/netvsc: free all queues on close

 drivers/net/netvsc/hn_ethdev.c |  8 +++-
 drivers/net/netvsc/hn_rxtx.c   | 86 ++++++++++++++++++++++++----------
 drivers/net/netvsc/hn_var.h    |  1 +
 3 files changed, 68 insertions(+), 27 deletions(-)

-- 
2.20.1

[dpdk-dev] [PATCH 1/2] net/netvsc: reset mbuf port on VF receive

[Stephen Hemminger]

Redo the receive logic to set m->port on packets received on VF.

When using VF, still need to check for packets and completions
arriving on the VMBus path even if application is not doing bursting
(ie n_rx == 0).

Also, fix comment.

Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>
---
 drivers/net/netvsc/hn_rxtx.c | 47 +++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 19 deletions(-)

diff --git a/drivers/net/netvsc/hn_rxtx.c b/drivers/net/netvsc/hn_rxtx.c
index c67e9ae25376..7856f7e6ec48 100644
--- a/drivers/net/netvsc/hn_rxtx.c
+++ b/drivers/net/netvsc/hn_rxtx.c
@@ -1393,6 +1393,24 @@ hn_xmit_pkts(void *ptxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	return nb_tx;
 }
 
+static uint16_t
+hn_recv_vf(uint16_t vf_port, const struct hn_rx_queue *rxq,
+	   struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+	uint16_t i, n;
+
+	if (unlikely(nb_pkts == 0))
+		return 0;
+
+	n = rte_eth_rx_burst(vf_port, rxq->queue_id, rx_pkts, nb_pkts);
+
+	/* relabel the received mbufs */
+	for (i = 0; i < n; i++)
+		rx_pkts[i]->port = rxq->port_id;
+
+	return n;
+}
+
 uint16_t
 hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 {
@@ -1404,30 +1422,21 @@ hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	if (unlikely(hv->closed))
 		return 0;
 
-	/* Transmit over VF if present and up */
+	/* Receive from VF if present and up */
 	vf_dev = hn_get_vf_dev(hv);
 
-	if (vf_dev && vf_dev->data->dev_started) {
-		/* Normally, with SR-IOV the ring buffer will be empty */
+	/* Check for new completions */
+	if (likely(rte_ring_count(rxq->rx_ring) < nb_pkts))
 		hn_process_events(hv, rxq->queue_id, 0);
 
-		/* Get mbufs some bufs off of staging ring */
-		nb_rcv = rte_ring_sc_dequeue_burst(rxq->rx_ring,
-						   (void **)rx_pkts,
-						   nb_pkts / 2, NULL);
-		/* And rest off of VF */
-		nb_rcv += rte_eth_rx_burst(vf_dev->data->port_id,
-					   rxq->queue_id,
-					   rx_pkts + nb_rcv, nb_pkts - nb_rcv);
-	} else {
-		/* If receive ring is not full then get more */
-		if (rte_ring_count(rxq->rx_ring) < nb_pkts)
-			hn_process_events(hv, rxq->queue_id, 0);
+	/* Always check the vmbus path for multicast and new flows */
+	nb_rcv = rte_ring_sc_dequeue_burst(rxq->rx_ring,
+					   (void **)rx_pkts, nb_pkts, NULL);
 
-		nb_rcv = rte_ring_sc_dequeue_burst(rxq->rx_ring,
-						   (void **)rx_pkts,
-						   nb_pkts, NULL);
-	}
+	/* If VF is available, check that as well */
+	if (vf_dev && vf_dev->data->dev_started)
+		nb_rcv += hn_recv_vf(vf_dev->data->port_id, rxq,
+				     rx_pkts + nb_rcv, nb_pkts - nb_rcv);
 
 	return nb_rcv;
 }
-- 
2.20.1

[dpdk-dev] [PATCH 1/2] net/netvsc: reset mbuf port on VF receive

[Stephen Hemminger]

Redo the receive logic to set m->port on packets received on VF.

When using VF, still need to check for packets and completions
arriving on the VMBus path even if application is not doing bursting
(ie n_rx == 0).

Also, fix comment.

Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>
---
 drivers/net/netvsc/hn_rxtx.c | 47 +++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 19 deletions(-)

diff --git a/drivers/net/netvsc/hn_rxtx.c b/drivers/net/netvsc/hn_rxtx.c
index c67e9ae25376..7856f7e6ec48 100644
--- a/drivers/net/netvsc/hn_rxtx.c
+++ b/drivers/net/netvsc/hn_rxtx.c
@@ -1393,6 +1393,24 @@ hn_xmit_pkts(void *ptxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	return nb_tx;
 }
 
+static uint16_t
+hn_recv_vf(uint16_t vf_port, const struct hn_rx_queue *rxq,
+	   struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+	uint16_t i, n;
+
+	if (unlikely(nb_pkts == 0))
+		return 0;
+
+	n = rte_eth_rx_burst(vf_port, rxq->queue_id, rx_pkts, nb_pkts);
+
+	/* relabel the received mbufs */
+	for (i = 0; i < n; i++)
+		rx_pkts[i]->port = rxq->port_id;
+
+	return n;
+}
+
 uint16_t
 hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 {
@@ -1404,30 +1422,21 @@ hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	if (unlikely(hv->closed))
 		return 0;
 
-	/* Transmit over VF if present and up */
+	/* Receive from VF if present and up */
 	vf_dev = hn_get_vf_dev(hv);
 
-	if (vf_dev && vf_dev->data->dev_started) {
-		/* Normally, with SR-IOV the ring buffer will be empty */
+	/* Check for new completions */
+	if (likely(rte_ring_count(rxq->rx_ring) < nb_pkts))
 		hn_process_events(hv, rxq->queue_id, 0);
 
-		/* Get mbufs some bufs off of staging ring */
-		nb_rcv = rte_ring_sc_dequeue_burst(rxq->rx_ring,
-						   (void **)rx_pkts,
-						   nb_pkts / 2, NULL);
-		/* And rest off of VF */
-		nb_rcv += rte_eth_rx_burst(vf_dev->data->port_id,
-					   rxq->queue_id,
-					   rx_pkts + nb_rcv, nb_pkts - nb_rcv);
-	} else {
-		/* If receive ring is not full then get more */
-		if (rte_ring_count(rxq->rx_ring) < nb_pkts)
-			hn_process_events(hv, rxq->queue_id, 0);
+	/* Always check the vmbus path for multicast and new flows */
+	nb_rcv = rte_ring_sc_dequeue_burst(rxq->rx_ring,
+					   (void **)rx_pkts, nb_pkts, NULL);
 
-		nb_rcv = rte_ring_sc_dequeue_burst(rxq->rx_ring,
-						   (void **)rx_pkts,
-						   nb_pkts, NULL);
-	}
+	/* If VF is available, check that as well */
+	if (vf_dev && vf_dev->data->dev_started)
+		nb_rcv += hn_recv_vf(vf_dev->data->port_id, rxq,
+				     rx_pkts + nb_rcv, nb_pkts - nb_rcv);
 
 	return nb_rcv;
 }
-- 
2.20.1

Re: [dpdk-dev] [PATCH 1/2] net/netvsc: reset mbuf port on VF receive

[Ferruh Yigit]

On 4/29/2019 9:33 PM, Stephen Hemminger wrote:
> Redo the receive logic to set m->port on packets received on VF.
> 
> When using VF, still need to check for packets and completions
> arriving on the VMBus path even if application is not doing bursting
> (ie n_rx == 0).
> 
> Also, fix comment.
> 
> Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>

If this is a fix, can you please describe what is fixed and put a fixes line,
that will help for backporting the fix.

Re: [dpdk-dev] [PATCH 1/2] net/netvsc: reset mbuf port on VF receive

[Ferruh Yigit]

On 4/29/2019 9:33 PM, Stephen Hemminger wrote:
> Redo the receive logic to set m->port on packets received on VF.
> 
> When using VF, still need to check for packets and completions
> arriving on the VMBus path even if application is not doing bursting
> (ie n_rx == 0).
> 
> Also, fix comment.
> 
> Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>

If this is a fix, can you please describe what is fixed and put a fixes line,
that will help for backporting the fix.

Re: [dpdk-dev] [PATCH 1/2] net/netvsc: reset mbuf port on VF receive

[Stephen Hemminger]

On Mon, 29 Apr 2019 13:33:24 -0700
Stephen Hemminger <stephen@networkplumber.org> wrote:

> Redo the receive logic to set m->port on packets received on VF.
> 
> When using VF, still need to check for packets and completions
> arriving on the VMBus path even if application is not doing bursting
> (ie n_rx == 0).
> 
> Also, fix comment.
> 
> Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>


Fixes: dc7680e8597c ("net/netvsc: support integrated VF")

Re: [dpdk-dev] [PATCH 1/2] net/netvsc: reset mbuf port on VF receive

[Stephen Hemminger]

On Mon, 29 Apr 2019 13:33:24 -0700
Stephen Hemminger <stephen@networkplumber.org> wrote:

> Redo the receive logic to set m->port on packets received on VF.
> 
> When using VF, still need to check for packets and completions
> arriving on the VMBus path even if application is not doing bursting
> (ie n_rx == 0).
> 
> Also, fix comment.
> 
> Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>


Fixes: dc7680e8597c ("net/netvsc: support integrated VF")

[dpdk-dev] [PATCH 2/2] net/netvsc: free all queues on close

[Stephen Hemminger]

When dev_close is called, the netvsc driver will clean up all
queues including the primary ring buffer.

Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>
---
 drivers/net/netvsc/hn_ethdev.c |  8 +++++--
 drivers/net/netvsc/hn_rxtx.c   | 39 ++++++++++++++++++++++++++++------
 drivers/net/netvsc/hn_var.h    |  1 +
 3 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c
index 407ee484935a..553cb06f6e33 100644
--- a/drivers/net/netvsc/hn_ethdev.c
+++ b/drivers/net/netvsc/hn_ethdev.c
@@ -112,6 +112,9 @@ eth_dev_vmbus_allocate(struct rte_vmbus_device *dev, size_t private_data_size)
 	eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
 	eth_dev->intr_handle = &dev->intr_handle;
 
+	/* allow ethdev to remove on close */
+	eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
+
 	return eth_dev;
 }
 
@@ -632,11 +635,12 @@ hn_dev_stop(struct rte_eth_dev *dev)
 }
 
 static void
-hn_dev_close(struct rte_eth_dev *dev __rte_unused)
+hn_dev_close(struct rte_eth_dev *dev)
 {
-	PMD_INIT_LOG(DEBUG, "close");
+	PMD_INIT_FUNC_TRACE();
 
 	hn_vf_close(dev);
+	hn_dev_free_queues(dev);
 }
 
 static const struct eth_dev_ops hn_eth_dev_ops = {
diff --git a/drivers/net/netvsc/hn_rxtx.c b/drivers/net/netvsc/hn_rxtx.c
index 7856f7e6ec48..1bebae7cddd4 100644
--- a/drivers/net/netvsc/hn_rxtx.c
+++ b/drivers/net/netvsc/hn_rxtx.c
@@ -840,12 +840,9 @@ hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	return error;
 }
 
-void
-hn_dev_rx_queue_release(void *arg)
+static void
+hn_rx_queue_free(struct hn_rx_queue *rxq, bool keep_primary)
 {
-	struct hn_rx_queue *rxq = arg;
-
-	PMD_INIT_FUNC_TRACE();
 
 	if (!rxq)
 		return;
@@ -857,12 +854,22 @@ hn_dev_rx_queue_release(void *arg)
 	hn_vf_rx_queue_release(rxq->hv, rxq->queue_id);
 
 	/* Keep primary queue to allow for control operations */
-	if (rxq != rxq->hv->primary) {
+	if (keep_primary && rxq != rxq->hv->primary) {
 		rte_free(rxq->event_buf);
 		rte_free(rxq);
 	}
 }
 
+void
+hn_dev_rx_queue_release(void *arg)
+{
+	struct hn_rx_queue *rxq = arg;
+
+	PMD_INIT_FUNC_TRACE();
+
+	hn_rx_queue_free(rxq, true);
+}
+
 int
 hn_dev_tx_done_cleanup(void *arg, uint32_t free_cnt)
 {
@@ -1440,3 +1447,23 @@ hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 	return nb_rcv;
 }
+
+void
+hn_dev_free_queues(struct rte_eth_dev *dev)
+{
+	unsigned int i;
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		struct hn_rx_queue *rxq = dev->data->rx_queues[i];
+
+		hn_rx_queue_free(rxq, false);
+		dev->data->rx_queues[i] = NULL;
+	}
+	dev->data->nb_rx_queues = 0;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		hn_dev_tx_queue_release(dev->data->tx_queues[i]);
+		dev->data->tx_queues[i] = NULL;
+	}
+	dev->data->nb_tx_queues = 0;
+}
diff --git a/drivers/net/netvsc/hn_var.h b/drivers/net/netvsc/hn_var.h
index 8383f3246ca4..de885d898e6a 100644
--- a/drivers/net/netvsc/hn_var.h
+++ b/drivers/net/netvsc/hn_var.h
@@ -173,6 +173,7 @@ int	hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
 			      const struct rte_eth_rxconf *rx_conf,
 			      struct rte_mempool *mp);
 void	hn_dev_rx_queue_release(void *arg);
+void	hn_dev_free_queues(struct rte_eth_dev *dev);
 
 /* Check if VF is attached */
 static inline bool
-- 
2.20.1

[dpdk-dev] [PATCH 2/2] net/netvsc: free all queues on close

[Stephen Hemminger]

When dev_close is called, the netvsc driver will clean up all
queues including the primary ring buffer.

Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>
---
 drivers/net/netvsc/hn_ethdev.c |  8 +++++--
 drivers/net/netvsc/hn_rxtx.c   | 39 ++++++++++++++++++++++++++++------
 drivers/net/netvsc/hn_var.h    |  1 +
 3 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c
index 407ee484935a..553cb06f6e33 100644
--- a/drivers/net/netvsc/hn_ethdev.c
+++ b/drivers/net/netvsc/hn_ethdev.c
@@ -112,6 +112,9 @@ eth_dev_vmbus_allocate(struct rte_vmbus_device *dev, size_t private_data_size)
 	eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
 	eth_dev->intr_handle = &dev->intr_handle;
 
+	/* allow ethdev to remove on close */
+	eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
+
 	return eth_dev;
 }
 
@@ -632,11 +635,12 @@ hn_dev_stop(struct rte_eth_dev *dev)
 }
 
 static void
-hn_dev_close(struct rte_eth_dev *dev __rte_unused)
+hn_dev_close(struct rte_eth_dev *dev)
 {
-	PMD_INIT_LOG(DEBUG, "close");
+	PMD_INIT_FUNC_TRACE();
 
 	hn_vf_close(dev);
+	hn_dev_free_queues(dev);
 }
 
 static const struct eth_dev_ops hn_eth_dev_ops = {
diff --git a/drivers/net/netvsc/hn_rxtx.c b/drivers/net/netvsc/hn_rxtx.c
index 7856f7e6ec48..1bebae7cddd4 100644
--- a/drivers/net/netvsc/hn_rxtx.c
+++ b/drivers/net/netvsc/hn_rxtx.c
@@ -840,12 +840,9 @@ hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	return error;
 }
 
-void
-hn_dev_rx_queue_release(void *arg)
+static void
+hn_rx_queue_free(struct hn_rx_queue *rxq, bool keep_primary)
 {
-	struct hn_rx_queue *rxq = arg;
-
-	PMD_INIT_FUNC_TRACE();
 
 	if (!rxq)
 		return;
@@ -857,12 +854,22 @@ hn_dev_rx_queue_release(void *arg)
 	hn_vf_rx_queue_release(rxq->hv, rxq->queue_id);
 
 	/* Keep primary queue to allow for control operations */
-	if (rxq != rxq->hv->primary) {
+	if (keep_primary && rxq != rxq->hv->primary) {
 		rte_free(rxq->event_buf);
 		rte_free(rxq);
 	}
 }
 
+void
+hn_dev_rx_queue_release(void *arg)
+{
+	struct hn_rx_queue *rxq = arg;
+
+	PMD_INIT_FUNC_TRACE();
+
+	hn_rx_queue_free(rxq, true);
+}
+
 int
 hn_dev_tx_done_cleanup(void *arg, uint32_t free_cnt)
 {
@@ -1440,3 +1447,23 @@ hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 	return nb_rcv;
 }
+
+void
+hn_dev_free_queues(struct rte_eth_dev *dev)
+{
+	unsigned int i;
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		struct hn_rx_queue *rxq = dev->data->rx_queues[i];
+
+		hn_rx_queue_free(rxq, false);
+		dev->data->rx_queues[i] = NULL;
+	}
+	dev->data->nb_rx_queues = 0;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		hn_dev_tx_queue_release(dev->data->tx_queues[i]);
+		dev->data->tx_queues[i] = NULL;
+	}
+	dev->data->nb_tx_queues = 0;
+}
diff --git a/drivers/net/netvsc/hn_var.h b/drivers/net/netvsc/hn_var.h
index 8383f3246ca4..de885d898e6a 100644
--- a/drivers/net/netvsc/hn_var.h
+++ b/drivers/net/netvsc/hn_var.h
@@ -173,6 +173,7 @@ int	hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
 			      const struct rte_eth_rxconf *rx_conf,
 			      struct rte_mempool *mp);
 void	hn_dev_rx_queue_release(void *arg);
+void	hn_dev_free_queues(struct rte_eth_dev *dev);
 
 /* Check if VF is attached */
 static inline bool
-- 
2.20.1

Re: [dpdk-dev] [PATCH 2/2] net/netvsc: free all queues on close

[Ferruh Yigit]

On 4/29/2019 9:33 PM, Stephen Hemminger wrote:
> When dev_close is called, the netvsc driver will clean up all
> queues including the primary ring buffer.
> 
> Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>
> ---
>  drivers/net/netvsc/hn_ethdev.c |  8 +++++--
>  drivers/net/netvsc/hn_rxtx.c   | 39 ++++++++++++++++++++++++++++------
>  drivers/net/netvsc/hn_var.h    |  1 +
>  3 files changed, 40 insertions(+), 8 deletions(-)
> 
> diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c
> index 407ee484935a..553cb06f6e33 100644
> --- a/drivers/net/netvsc/hn_ethdev.c
> +++ b/drivers/net/netvsc/hn_ethdev.c
> @@ -112,6 +112,9 @@ eth_dev_vmbus_allocate(struct rte_vmbus_device *dev, size_t private_data_size)
>  	eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
>  	eth_dev->intr_handle = &dev->intr_handle;
>  
> +	/* allow ethdev to remove on close */
> +	eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
> +
>  	return eth_dev;
>  }
>  
> @@ -632,11 +635,12 @@ hn_dev_stop(struct rte_eth_dev *dev)
>  }
>  
>  static void
> -hn_dev_close(struct rte_eth_dev *dev __rte_unused)
> +hn_dev_close(struct rte_eth_dev *dev)
>  {
> -	PMD_INIT_LOG(DEBUG, "close");
> +	PMD_INIT_FUNC_TRACE();
>  
>  	hn_vf_close(dev);
> +	hn_dev_free_queues(dev);
>  }
>  
>  static const struct eth_dev_ops hn_eth_dev_ops = {
> diff --git a/drivers/net/netvsc/hn_rxtx.c b/drivers/net/netvsc/hn_rxtx.c
> index 7856f7e6ec48..1bebae7cddd4 100644
> --- a/drivers/net/netvsc/hn_rxtx.c
> +++ b/drivers/net/netvsc/hn_rxtx.c
> @@ -840,12 +840,9 @@ hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
>  	return error;
>  }
>  
> -void
> -hn_dev_rx_queue_release(void *arg)
> +static void
> +hn_rx_queue_free(struct hn_rx_queue *rxq, bool keep_primary)
>  {
> -	struct hn_rx_queue *rxq = arg;
> -
> -	PMD_INIT_FUNC_TRACE();
>  
>  	if (!rxq)
>  		return;
> @@ -857,12 +854,22 @@ hn_dev_rx_queue_release(void *arg)
>  	hn_vf_rx_queue_release(rxq->hv, rxq->queue_id);
>  
>  	/* Keep primary queue to allow for control operations */
> -	if (rxq != rxq->hv->primary) {
> +	if (keep_primary && rxq != rxq->hv->primary) {

Isn't something wrong in this logic, when 'keep_primary' is 'false', it will
prevent taking this branch and it will prevent freeing _any_ 'rxq->event_buf' &
'rxq'. There is already a primary queue check after '&&', this flag looks like
working more like enable/disable freeing queues more than being related to primary.

And int he '.dev_close()' path [1] this function is called with 'false', so
queue resources are not cleaned properly.

[1]
hn_dev_close
  hn_dev_free_queues
    foreach rx_queue
      hn_rx_queue_free(rxq, false);

>  		rte_free(rxq->event_buf);
>  		rte_free(rxq);
>  	}
>  }
>  
> +void
> +hn_dev_rx_queue_release(void *arg)
> +{
> +	struct hn_rx_queue *rxq = arg;
> +
> +	PMD_INIT_FUNC_TRACE();
> +
> +	hn_rx_queue_free(rxq, true);
> +}
> +
>  int
>  hn_dev_tx_done_cleanup(void *arg, uint32_t free_cnt)
>  {
> @@ -1440,3 +1447,23 @@ hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
>  
>  	return nb_rcv;
>  }
> +
> +void
> +hn_dev_free_queues(struct rte_eth_dev *dev)
> +{
> +	unsigned int i;
> +
> +	for (i = 0; i < dev->data->nb_rx_queues; i++) {
> +		struct hn_rx_queue *rxq = dev->data->rx_queues[i];
> +
> +		hn_rx_queue_free(rxq, false);
> +		dev->data->rx_queues[i] = NULL;
> +	}
> +	dev->data->nb_rx_queues = 0;
> +
> +	for (i = 0; i < dev->data->nb_tx_queues; i++) {
> +		hn_dev_tx_queue_release(dev->data->tx_queues[i]);
> +		dev->data->tx_queues[i] = NULL;
> +	}
> +	dev->data->nb_tx_queues = 0;
> +}
> diff --git a/drivers/net/netvsc/hn_var.h b/drivers/net/netvsc/hn_var.h
> index 8383f3246ca4..de885d898e6a 100644
> --- a/drivers/net/netvsc/hn_var.h
> +++ b/drivers/net/netvsc/hn_var.h
> @@ -173,6 +173,7 @@ int	hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
>  			      const struct rte_eth_rxconf *rx_conf,
>  			      struct rte_mempool *mp);
>  void	hn_dev_rx_queue_release(void *arg);
> +void	hn_dev_free_queues(struct rte_eth_dev *dev);
>  
>  /* Check if VF is attached */
>  static inline bool
> 

Re: [dpdk-dev] [PATCH 2/2] net/netvsc: free all queues on close

[Ferruh Yigit]

On 4/29/2019 9:33 PM, Stephen Hemminger wrote:
> When dev_close is called, the netvsc driver will clean up all
> queues including the primary ring buffer.
> 
> Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>
> ---
>  drivers/net/netvsc/hn_ethdev.c |  8 +++++--
>  drivers/net/netvsc/hn_rxtx.c   | 39 ++++++++++++++++++++++++++++------
>  drivers/net/netvsc/hn_var.h    |  1 +
>  3 files changed, 40 insertions(+), 8 deletions(-)
> 
> diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c
> index 407ee484935a..553cb06f6e33 100644
> --- a/drivers/net/netvsc/hn_ethdev.c
> +++ b/drivers/net/netvsc/hn_ethdev.c
> @@ -112,6 +112,9 @@ eth_dev_vmbus_allocate(struct rte_vmbus_device *dev, size_t private_data_size)
>  	eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
>  	eth_dev->intr_handle = &dev->intr_handle;
>  
> +	/* allow ethdev to remove on close */
> +	eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
> +
>  	return eth_dev;
>  }
>  
> @@ -632,11 +635,12 @@ hn_dev_stop(struct rte_eth_dev *dev)
>  }
>  
>  static void
> -hn_dev_close(struct rte_eth_dev *dev __rte_unused)
> +hn_dev_close(struct rte_eth_dev *dev)
>  {
> -	PMD_INIT_LOG(DEBUG, "close");
> +	PMD_INIT_FUNC_TRACE();
>  
>  	hn_vf_close(dev);
> +	hn_dev_free_queues(dev);
>  }
>  
>  static const struct eth_dev_ops hn_eth_dev_ops = {
> diff --git a/drivers/net/netvsc/hn_rxtx.c b/drivers/net/netvsc/hn_rxtx.c
> index 7856f7e6ec48..1bebae7cddd4 100644
> --- a/drivers/net/netvsc/hn_rxtx.c
> +++ b/drivers/net/netvsc/hn_rxtx.c
> @@ -840,12 +840,9 @@ hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
>  	return error;
>  }
>  
> -void
> -hn_dev_rx_queue_release(void *arg)
> +static void
> +hn_rx_queue_free(struct hn_rx_queue *rxq, bool keep_primary)
>  {
> -	struct hn_rx_queue *rxq = arg;
> -
> -	PMD_INIT_FUNC_TRACE();
>  
>  	if (!rxq)
>  		return;
> @@ -857,12 +854,22 @@ hn_dev_rx_queue_release(void *arg)
>  	hn_vf_rx_queue_release(rxq->hv, rxq->queue_id);
>  
>  	/* Keep primary queue to allow for control operations */
> -	if (rxq != rxq->hv->primary) {
> +	if (keep_primary && rxq != rxq->hv->primary) {

Isn't something wrong in this logic, when 'keep_primary' is 'false', it will
prevent taking this branch and it will prevent freeing _any_ 'rxq->event_buf' &
'rxq'. There is already a primary queue check after '&&', this flag looks like
working more like enable/disable freeing queues more than being related to primary.

And int he '.dev_close()' path [1] this function is called with 'false', so
queue resources are not cleaned properly.

[1]
hn_dev_close
  hn_dev_free_queues
    foreach rx_queue
      hn_rx_queue_free(rxq, false);

>  		rte_free(rxq->event_buf);
>  		rte_free(rxq);
>  	}
>  }
>  
> +void
> +hn_dev_rx_queue_release(void *arg)
> +{
> +	struct hn_rx_queue *rxq = arg;
> +
> +	PMD_INIT_FUNC_TRACE();
> +
> +	hn_rx_queue_free(rxq, true);
> +}
> +
>  int
>  hn_dev_tx_done_cleanup(void *arg, uint32_t free_cnt)
>  {
> @@ -1440,3 +1447,23 @@ hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
>  
>  	return nb_rcv;
>  }
> +
> +void
> +hn_dev_free_queues(struct rte_eth_dev *dev)
> +{
> +	unsigned int i;
> +
> +	for (i = 0; i < dev->data->nb_rx_queues; i++) {
> +		struct hn_rx_queue *rxq = dev->data->rx_queues[i];
> +
> +		hn_rx_queue_free(rxq, false);
> +		dev->data->rx_queues[i] = NULL;
> +	}
> +	dev->data->nb_rx_queues = 0;
> +
> +	for (i = 0; i < dev->data->nb_tx_queues; i++) {
> +		hn_dev_tx_queue_release(dev->data->tx_queues[i]);
> +		dev->data->tx_queues[i] = NULL;
> +	}
> +	dev->data->nb_tx_queues = 0;
> +}
> diff --git a/drivers/net/netvsc/hn_var.h b/drivers/net/netvsc/hn_var.h
> index 8383f3246ca4..de885d898e6a 100644
> --- a/drivers/net/netvsc/hn_var.h
> +++ b/drivers/net/netvsc/hn_var.h
> @@ -173,6 +173,7 @@ int	hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
>  			      const struct rte_eth_rxconf *rx_conf,
>  			      struct rte_mempool *mp);
>  void	hn_dev_rx_queue_release(void *arg);
> +void	hn_dev_free_queues(struct rte_eth_dev *dev);
>  
>  /* Check if VF is attached */
>  static inline bool
> 

[dpdk-dev] [PATCH v2 0/2] netvsc PMD fixes for 19.05

[Stephen Hemminger]

Netvsc needed to reset incoming port as was recently done
for failsafe. Implement the new close semantics.

Stephen Hemminger (2):
  net/netvsc: reset mbuf port on VF receive
  net/netvsc: free all queues on close

 drivers/net/netvsc/hn_ethdev.c |  8 ++-
 drivers/net/netvsc/hn_rxtx.c   | 93 ++++++++++++++++++++++++----------
 drivers/net/netvsc/hn_var.h    |  1 +
 drivers/net/netvsc/hn_vf.c     | 11 +++-
 4 files changed, 82 insertions(+), 31 deletions(-)

-- 
2.20.1

[dpdk-dev] [PATCH v2 0/2] netvsc PMD fixes for 19.05

[Stephen Hemminger]

Netvsc needed to reset incoming port as was recently done
for failsafe. Implement the new close semantics.

Stephen Hemminger (2):
  net/netvsc: reset mbuf port on VF receive
  net/netvsc: free all queues on close

 drivers/net/netvsc/hn_ethdev.c |  8 ++-
 drivers/net/netvsc/hn_rxtx.c   | 93 ++++++++++++++++++++++++----------
 drivers/net/netvsc/hn_var.h    |  1 +
 drivers/net/netvsc/hn_vf.c     | 11 +++-
 4 files changed, 82 insertions(+), 31 deletions(-)

-- 
2.20.1

[dpdk-dev] [PATCH v2 1/2] net/netvsc: reset mbuf port on VF receive

[Stephen Hemminger]

From: Stephen Hemminger <sthemmin@microsoft.com>

Redo the receive logic to set m->port on packets received on
VF.

When using VF, still need to check for packets and completions
arriving on the VMBus path even if application is not doing bursting
(ie n_rx == 0).

Also, fix comment.

Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>
Fixes: dc7680e8597c ("net/netvsc: support integrated VF")
---
v2 - add fixes

 drivers/net/netvsc/hn_rxtx.c | 47 +++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 19 deletions(-)

diff --git a/drivers/net/netvsc/hn_rxtx.c b/drivers/net/netvsc/hn_rxtx.c
index c67e9ae25376..7856f7e6ec48 100644
--- a/drivers/net/netvsc/hn_rxtx.c
+++ b/drivers/net/netvsc/hn_rxtx.c
@@ -1393,6 +1393,24 @@ hn_xmit_pkts(void *ptxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	return nb_tx;
 }
 
+static uint16_t
+hn_recv_vf(uint16_t vf_port, const struct hn_rx_queue *rxq,
+	   struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+	uint16_t i, n;
+
+	if (unlikely(nb_pkts == 0))
+		return 0;
+
+	n = rte_eth_rx_burst(vf_port, rxq->queue_id, rx_pkts, nb_pkts);
+
+	/* relabel the received mbufs */
+	for (i = 0; i < n; i++)
+		rx_pkts[i]->port = rxq->port_id;
+
+	return n;
+}
+
 uint16_t
 hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 {
@@ -1404,30 +1422,21 @@ hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	if (unlikely(hv->closed))
 		return 0;
 
-	/* Transmit over VF if present and up */
+	/* Receive from VF if present and up */
 	vf_dev = hn_get_vf_dev(hv);
 
-	if (vf_dev && vf_dev->data->dev_started) {
-		/* Normally, with SR-IOV the ring buffer will be empty */
+	/* Check for new completions */
+	if (likely(rte_ring_count(rxq->rx_ring) < nb_pkts))
 		hn_process_events(hv, rxq->queue_id, 0);
 
-		/* Get mbufs some bufs off of staging ring */
-		nb_rcv = rte_ring_sc_dequeue_burst(rxq->rx_ring,
-						   (void **)rx_pkts,
-						   nb_pkts / 2, NULL);
-		/* And rest off of VF */
-		nb_rcv += rte_eth_rx_burst(vf_dev->data->port_id,
-					   rxq->queue_id,
-					   rx_pkts + nb_rcv, nb_pkts - nb_rcv);
-	} else {
-		/* If receive ring is not full then get more */
-		if (rte_ring_count(rxq->rx_ring) < nb_pkts)
-			hn_process_events(hv, rxq->queue_id, 0);
+	/* Always check the vmbus path for multicast and new flows */
+	nb_rcv = rte_ring_sc_dequeue_burst(rxq->rx_ring,
+					   (void **)rx_pkts, nb_pkts, NULL);
 
-		nb_rcv = rte_ring_sc_dequeue_burst(rxq->rx_ring,
-						   (void **)rx_pkts,
-						   nb_pkts, NULL);
-	}
+	/* If VF is available, check that as well */
+	if (vf_dev && vf_dev->data->dev_started)
+		nb_rcv += hn_recv_vf(vf_dev->data->port_id, rxq,
+				     rx_pkts + nb_rcv, nb_pkts - nb_rcv);
 
 	return nb_rcv;
 }
-- 
2.20.1

[dpdk-dev] [PATCH v2 1/2] net/netvsc: reset mbuf port on VF receive

[Stephen Hemminger]

From: Stephen Hemminger <sthemmin@microsoft.com>

Redo the receive logic to set m->port on packets received on
VF.

When using VF, still need to check for packets and completions
arriving on the VMBus path even if application is not doing bursting
(ie n_rx == 0).

Also, fix comment.

Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>
Fixes: dc7680e8597c ("net/netvsc: support integrated VF")
---
v2 - add fixes

 drivers/net/netvsc/hn_rxtx.c | 47 +++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 19 deletions(-)

diff --git a/drivers/net/netvsc/hn_rxtx.c b/drivers/net/netvsc/hn_rxtx.c
index c67e9ae25376..7856f7e6ec48 100644
--- a/drivers/net/netvsc/hn_rxtx.c
+++ b/drivers/net/netvsc/hn_rxtx.c
@@ -1393,6 +1393,24 @@ hn_xmit_pkts(void *ptxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	return nb_tx;
 }
 
+static uint16_t
+hn_recv_vf(uint16_t vf_port, const struct hn_rx_queue *rxq,
+	   struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+	uint16_t i, n;
+
+	if (unlikely(nb_pkts == 0))
+		return 0;
+
+	n = rte_eth_rx_burst(vf_port, rxq->queue_id, rx_pkts, nb_pkts);
+
+	/* relabel the received mbufs */
+	for (i = 0; i < n; i++)
+		rx_pkts[i]->port = rxq->port_id;
+
+	return n;
+}
+
 uint16_t
 hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 {
@@ -1404,30 +1422,21 @@ hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	if (unlikely(hv->closed))
 		return 0;
 
-	/* Transmit over VF if present and up */
+	/* Receive from VF if present and up */
 	vf_dev = hn_get_vf_dev(hv);
 
-	if (vf_dev && vf_dev->data->dev_started) {
-		/* Normally, with SR-IOV the ring buffer will be empty */
+	/* Check for new completions */
+	if (likely(rte_ring_count(rxq->rx_ring) < nb_pkts))
 		hn_process_events(hv, rxq->queue_id, 0);
 
-		/* Get mbufs some bufs off of staging ring */
-		nb_rcv = rte_ring_sc_dequeue_burst(rxq->rx_ring,
-						   (void **)rx_pkts,
-						   nb_pkts / 2, NULL);
-		/* And rest off of VF */
-		nb_rcv += rte_eth_rx_burst(vf_dev->data->port_id,
-					   rxq->queue_id,
-					   rx_pkts + nb_rcv, nb_pkts - nb_rcv);
-	} else {
-		/* If receive ring is not full then get more */
-		if (rte_ring_count(rxq->rx_ring) < nb_pkts)
-			hn_process_events(hv, rxq->queue_id, 0);
+	/* Always check the vmbus path for multicast and new flows */
+	nb_rcv = rte_ring_sc_dequeue_burst(rxq->rx_ring,
+					   (void **)rx_pkts, nb_pkts, NULL);
 
-		nb_rcv = rte_ring_sc_dequeue_burst(rxq->rx_ring,
-						   (void **)rx_pkts,
-						   nb_pkts, NULL);
-	}
+	/* If VF is available, check that as well */
+	if (vf_dev && vf_dev->data->dev_started)
+		nb_rcv += hn_recv_vf(vf_dev->data->port_id, rxq,
+				     rx_pkts + nb_rcv, nb_pkts - nb_rcv);
 
 	return nb_rcv;
 }
-- 
2.20.1

[dpdk-dev] [PATCH v2 2/2] net/netvsc: free all queues on close

[Stephen Hemminger]

From: Stephen Hemminger <sthemmin@microsoft.com>

When dev_close is called, the netvsc driver will clean up all
queues including the primary ring buffer.

Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>
---
v2 - fix logic on close of primary
     avoid double free if close is called twice

 drivers/net/netvsc/hn_ethdev.c |  8 ++++--
 drivers/net/netvsc/hn_rxtx.c   | 46 +++++++++++++++++++++++++++-------
 drivers/net/netvsc/hn_var.h    |  1 +
 drivers/net/netvsc/hn_vf.c     | 11 +++++++-
 4 files changed, 54 insertions(+), 12 deletions(-)

diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c
index 407ee484935a..553cb06f6e33 100644
--- a/drivers/net/netvsc/hn_ethdev.c
+++ b/drivers/net/netvsc/hn_ethdev.c
@@ -112,6 +112,9 @@ eth_dev_vmbus_allocate(struct rte_vmbus_device *dev, size_t private_data_size)
 	eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
 	eth_dev->intr_handle = &dev->intr_handle;
 
+	/* allow ethdev to remove on close */
+	eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
+
 	return eth_dev;
 }
 
@@ -632,11 +635,12 @@ hn_dev_stop(struct rte_eth_dev *dev)
 }
 
 static void
-hn_dev_close(struct rte_eth_dev *dev __rte_unused)
+hn_dev_close(struct rte_eth_dev *dev)
 {
-	PMD_INIT_LOG(DEBUG, "close");
+	PMD_INIT_FUNC_TRACE();
 
 	hn_vf_close(dev);
+	hn_dev_free_queues(dev);
 }
 
 static const struct eth_dev_ops hn_eth_dev_ops = {
diff --git a/drivers/net/netvsc/hn_rxtx.c b/drivers/net/netvsc/hn_rxtx.c
index 7856f7e6ec48..7d7b557785fa 100644
--- a/drivers/net/netvsc/hn_rxtx.c
+++ b/drivers/net/netvsc/hn_rxtx.c
@@ -840,12 +840,9 @@ hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	return error;
 }
 
-void
-hn_dev_rx_queue_release(void *arg)
+static void
+hn_rx_queue_free(struct hn_rx_queue *rxq, bool keep_primary)
 {
-	struct hn_rx_queue *rxq = arg;
-
-	PMD_INIT_FUNC_TRACE();
 
 	if (!rxq)
 		return;
@@ -857,10 +854,21 @@ hn_dev_rx_queue_release(void *arg)
 	hn_vf_rx_queue_release(rxq->hv, rxq->queue_id);
 
 	/* Keep primary queue to allow for control operations */
-	if (rxq != rxq->hv->primary) {
-		rte_free(rxq->event_buf);
-		rte_free(rxq);
-	}
+	if (keep_primary && rxq == rxq->hv->primary)
+		return;
+
+	rte_free(rxq->event_buf);
+	rte_free(rxq);
+}
+
+void
+hn_dev_rx_queue_release(void *arg)
+{
+	struct hn_rx_queue *rxq = arg;
+
+	PMD_INIT_FUNC_TRACE();
+
+	hn_rx_queue_free(rxq, true);
 }
 
 int
@@ -1440,3 +1448,23 @@ hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 	return nb_rcv;
 }
+
+void
+hn_dev_free_queues(struct rte_eth_dev *dev)
+{
+	unsigned int i;
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		struct hn_rx_queue *rxq = dev->data->rx_queues[i];
+
+		hn_rx_queue_free(rxq, false);
+		dev->data->rx_queues[i] = NULL;
+	}
+	dev->data->nb_rx_queues = 0;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		hn_dev_tx_queue_release(dev->data->tx_queues[i]);
+		dev->data->tx_queues[i] = NULL;
+	}
+	dev->data->nb_tx_queues = 0;
+}
diff --git a/drivers/net/netvsc/hn_var.h b/drivers/net/netvsc/hn_var.h
index 8383f3246ca4..de885d898e6a 100644
--- a/drivers/net/netvsc/hn_var.h
+++ b/drivers/net/netvsc/hn_var.h
@@ -173,6 +173,7 @@ int	hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
 			      const struct rte_eth_rxconf *rx_conf,
 			      struct rte_mempool *mp);
 void	hn_dev_rx_queue_release(void *arg);
+void	hn_dev_free_queues(struct rte_eth_dev *dev);
 
 /* Check if VF is attached */
 static inline bool
diff --git a/drivers/net/netvsc/hn_vf.c b/drivers/net/netvsc/hn_vf.c
index 883272ff498c..b980bb8a4d8a 100644
--- a/drivers/net/netvsc/hn_vf.c
+++ b/drivers/net/netvsc/hn_vf.c
@@ -362,7 +362,16 @@ void hn_vf_reset(struct rte_eth_dev *dev)
 
 void hn_vf_close(struct rte_eth_dev *dev)
 {
-	VF_ETHDEV_FUNC(dev, rte_eth_dev_close);
+	struct hn_data *hv = dev->data->dev_private;
+	uint16_t vf_port;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_port = hv->vf_port;
+	if (vf_port != HN_INVALID_PORT)
+		rte_eth_dev_close(vf_port);
+
+	hv->vf_port = HN_INVALID_PORT;
+	rte_spinlock_unlock(&hv->vf_lock);
 }
 
 void hn_vf_stats_reset(struct rte_eth_dev *dev)
-- 
2.20.1

[dpdk-dev] [PATCH v2 2/2] net/netvsc: free all queues on close

[Stephen Hemminger]

From: Stephen Hemminger <sthemmin@microsoft.com>

When dev_close is called, the netvsc driver will clean up all
queues including the primary ring buffer.

Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>
---
v2 - fix logic on close of primary
     avoid double free if close is called twice

 drivers/net/netvsc/hn_ethdev.c |  8 ++++--
 drivers/net/netvsc/hn_rxtx.c   | 46 +++++++++++++++++++++++++++-------
 drivers/net/netvsc/hn_var.h    |  1 +
 drivers/net/netvsc/hn_vf.c     | 11 +++++++-
 4 files changed, 54 insertions(+), 12 deletions(-)

diff --git a/drivers/net/netvsc/hn_ethdev.c b/drivers/net/netvsc/hn_ethdev.c
index 407ee484935a..553cb06f6e33 100644
--- a/drivers/net/netvsc/hn_ethdev.c
+++ b/drivers/net/netvsc/hn_ethdev.c
@@ -112,6 +112,9 @@ eth_dev_vmbus_allocate(struct rte_vmbus_device *dev, size_t private_data_size)
 	eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
 	eth_dev->intr_handle = &dev->intr_handle;
 
+	/* allow ethdev to remove on close */
+	eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
+
 	return eth_dev;
 }
 
@@ -632,11 +635,12 @@ hn_dev_stop(struct rte_eth_dev *dev)
 }
 
 static void
-hn_dev_close(struct rte_eth_dev *dev __rte_unused)
+hn_dev_close(struct rte_eth_dev *dev)
 {
-	PMD_INIT_LOG(DEBUG, "close");
+	PMD_INIT_FUNC_TRACE();
 
 	hn_vf_close(dev);
+	hn_dev_free_queues(dev);
 }
 
 static const struct eth_dev_ops hn_eth_dev_ops = {
diff --git a/drivers/net/netvsc/hn_rxtx.c b/drivers/net/netvsc/hn_rxtx.c
index 7856f7e6ec48..7d7b557785fa 100644
--- a/drivers/net/netvsc/hn_rxtx.c
+++ b/drivers/net/netvsc/hn_rxtx.c
@@ -840,12 +840,9 @@ hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	return error;
 }
 
-void
-hn_dev_rx_queue_release(void *arg)
+static void
+hn_rx_queue_free(struct hn_rx_queue *rxq, bool keep_primary)
 {
-	struct hn_rx_queue *rxq = arg;
-
-	PMD_INIT_FUNC_TRACE();
 
 	if (!rxq)
 		return;
@@ -857,10 +854,21 @@ hn_dev_rx_queue_release(void *arg)
 	hn_vf_rx_queue_release(rxq->hv, rxq->queue_id);
 
 	/* Keep primary queue to allow for control operations */
-	if (rxq != rxq->hv->primary) {
-		rte_free(rxq->event_buf);
-		rte_free(rxq);
-	}
+	if (keep_primary && rxq == rxq->hv->primary)
+		return;
+
+	rte_free(rxq->event_buf);
+	rte_free(rxq);
+}
+
+void
+hn_dev_rx_queue_release(void *arg)
+{
+	struct hn_rx_queue *rxq = arg;
+
+	PMD_INIT_FUNC_TRACE();
+
+	hn_rx_queue_free(rxq, true);
 }
 
 int
@@ -1440,3 +1448,23 @@ hn_recv_pkts(void *prxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 	return nb_rcv;
 }
+
+void
+hn_dev_free_queues(struct rte_eth_dev *dev)
+{
+	unsigned int i;
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		struct hn_rx_queue *rxq = dev->data->rx_queues[i];
+
+		hn_rx_queue_free(rxq, false);
+		dev->data->rx_queues[i] = NULL;
+	}
+	dev->data->nb_rx_queues = 0;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		hn_dev_tx_queue_release(dev->data->tx_queues[i]);
+		dev->data->tx_queues[i] = NULL;
+	}
+	dev->data->nb_tx_queues = 0;
+}
diff --git a/drivers/net/netvsc/hn_var.h b/drivers/net/netvsc/hn_var.h
index 8383f3246ca4..de885d898e6a 100644
--- a/drivers/net/netvsc/hn_var.h
+++ b/drivers/net/netvsc/hn_var.h
@@ -173,6 +173,7 @@ int	hn_dev_rx_queue_setup(struct rte_eth_dev *dev,
 			      const struct rte_eth_rxconf *rx_conf,
 			      struct rte_mempool *mp);
 void	hn_dev_rx_queue_release(void *arg);
+void	hn_dev_free_queues(struct rte_eth_dev *dev);
 
 /* Check if VF is attached */
 static inline bool
diff --git a/drivers/net/netvsc/hn_vf.c b/drivers/net/netvsc/hn_vf.c
index 883272ff498c..b980bb8a4d8a 100644
--- a/drivers/net/netvsc/hn_vf.c
+++ b/drivers/net/netvsc/hn_vf.c
@@ -362,7 +362,16 @@ void hn_vf_reset(struct rte_eth_dev *dev)
 
 void hn_vf_close(struct rte_eth_dev *dev)
 {
-	VF_ETHDEV_FUNC(dev, rte_eth_dev_close);
+	struct hn_data *hv = dev->data->dev_private;
+	uint16_t vf_port;
+
+	rte_spinlock_lock(&hv->vf_lock);
+	vf_port = hv->vf_port;
+	if (vf_port != HN_INVALID_PORT)
+		rte_eth_dev_close(vf_port);
+
+	hv->vf_port = HN_INVALID_PORT;
+	rte_spinlock_unlock(&hv->vf_lock);
 }
 
 void hn_vf_stats_reset(struct rte_eth_dev *dev)
-- 
2.20.1

Re: [dpdk-dev] [PATCH v2 0/2] netvsc PMD fixes for 19.05

[Ferruh Yigit]

On 4/30/2019 7:12 PM, Stephen Hemminger wrote:
> Netvsc needed to reset incoming port as was recently done
> for failsafe. Implement the new close semantics.
> 
> Stephen Hemminger (2):
>   net/netvsc: reset mbuf port on VF receive
>   net/netvsc: free all queues on close

Series applied to dpdk-next-net/master, thanks.

Re: [dpdk-dev] [PATCH v2 0/2] netvsc PMD fixes for 19.05

[Ferruh Yigit]

On 4/30/2019 7:12 PM, Stephen Hemminger wrote:
> Netvsc needed to reset incoming port as was recently done
> for failsafe. Implement the new close semantics.
> 
> Stephen Hemminger (2):
>   net/netvsc: reset mbuf port on VF receive
>   net/netvsc: free all queues on close

Series applied to dpdk-next-net/master, thanks.