[dpdk-dev] [PATCH] net/ixgbe: fix Rx queue interrupt mapping in VF

[dpdk-dev] [PATCH] net/ixgbe: fix Rx queue interrupt mapping in VF

[Wei Dai]

When a VF port is bound to VFIO-PCI, miscellaneous interrupt is
mapped to MSI-X vector 0 and Rx queues interrupt are mapped to
other vectors in vfio_enable_msix( ). To simplify implementation,
all VFIO-PCI bound ixgbe VF Rx queue interrupts can be mapped in
vector 1. And as current igb_uio only support only one vector,
ixgbe VF PMD should use vector 0 for igb_uio and vector 1 for
VFIO-PCI. Without this patch, VF Rx queue interrupt is mapped
to vector 0 in register settings and mapped to VFIO vector 1
in vfio_enable_msix( ), and then all Rx queue interrupts will
be missed.

Fixes: b13bfab4cdbe ("eal: reserve VFIO vector zero for misc interrupt")
Cc: stable@dpdk.org

Signed-off-by: Wei Dai <wei.dai@intel.com>
---
 drivers/net/ixgbe/ixgbe_ethdev.c | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/drivers/net/ixgbe/ixgbe_ethdev.c b/drivers/net/ixgbe/ixgbe_ethdev.c
index 9ca5cbc..39a3d8c 100644
--- a/drivers/net/ixgbe/ixgbe_ethdev.c
+++ b/drivers/net/ixgbe/ixgbe_ethdev.c
@@ -5029,7 +5029,10 @@ ixgbevf_dev_start(struct rte_eth_dev *dev)
 
 	/* check and configure queue intr-vector mapping */
 	if (dev->data->dev_conf.intr_conf.rxq != 0) {
-		intr_vector = dev->data->nb_rx_queues;
+		/* According to datasheet, only vector 0/1/2 can be used,
+		 * now only one vector is used for Rx queue
+		 */
+		intr_vector = 1;
 		if (rte_intr_efd_enable(intr_handle, intr_vector))
 			return -1;
 	}
@@ -5556,9 +5559,12 @@ ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
 	uint32_t mask;
 	struct ixgbe_hw *hw =
 		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	uint32_t vec = IXGBE_MISC_VEC_ID;
 
 	mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
-	mask |= (1 << IXGBE_MISC_VEC_ID);
+	if (rte_intr_allow_others(intr_handle))
+		vec = IXGBE_RX_VEC_START;
+	mask |= (1 << vec);
 	RTE_SET_USED(queue_id);
 	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
 
@@ -5573,9 +5579,14 @@ ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 	uint32_t mask;
 	struct ixgbe_hw *hw =
 		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
+	uint32_t vec = IXGBE_MISC_VEC_ID;
 
 	mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
-	mask &= ~(1 << IXGBE_MISC_VEC_ID);
+	if (rte_intr_allow_others(intr_handle))
+		vec = IXGBE_RX_VEC_START;
+	mask &= ~(1 << vec);
 	RTE_SET_USED(queue_id);
 	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
 
@@ -5717,6 +5728,7 @@ ixgbevf_configure_msix(struct rte_eth_dev *dev)
 		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	uint32_t q_idx;
 	uint32_t vector_idx = IXGBE_MISC_VEC_ID;
+	uint32_t base = IXGBE_MISC_VEC_ID;
 
 	/* Configure VF other cause ivar */
 	ixgbevf_set_ivar_map(hw, -1, 1, vector_idx);
@@ -5727,6 +5739,11 @@ ixgbevf_configure_msix(struct rte_eth_dev *dev)
 	if (!rte_intr_dp_is_en(intr_handle))
 		return;
 
+	if (rte_intr_allow_others(intr_handle)) {
+		base = IXGBE_RX_VEC_START;
+		vector_idx = IXGBE_RX_VEC_START;
+	}
+
 	/* Configure all RX queues of VF */
 	for (q_idx = 0; q_idx < dev->data->nb_rx_queues; q_idx++) {
 		/* Force all queue use vector 0,
@@ -5734,6 +5751,8 @@ ixgbevf_configure_msix(struct rte_eth_dev *dev)
 		 */
 		ixgbevf_set_ivar_map(hw, 0, q_idx, vector_idx);
 		intr_handle->intr_vec[q_idx] = vector_idx;
+		if (vector_idx < base + intr_handle->nb_efd - 1)
+			vector_idx++;
 	}
 }
 
-- 
2.7.5

Re: [dpdk-dev] [PATCH] net/ixgbe: fix Rx queue interrupt mapping in VF

[Dai, Wei]

Hi, Jianwei
How about your test result ?

Hi, Wenzhuo
Would you please review this patch ?

Thanks a lot !

> -----Original Message-----
> From: Dai, Wei
> Sent: Wednesday, September 20, 2017 6:18 PM
> To: Lu, Wenzhuo <wenzhuo.lu@intel.com>; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>
> Cc: dev@dpdk.org; Dai, Wei <wei.dai@intel.com>; stable@dpdk.org
> Subject: [PATCH] net/ixgbe: fix Rx queue interrupt mapping in VF
> 
> When a VF port is bound to VFIO-PCI, miscellaneous interrupt is mapped to
> MSI-X vector 0 and Rx queues interrupt are mapped to other vectors in
> vfio_enable_msix( ). To simplify implementation, all VFIO-PCI bound ixgbe VF
> Rx queue interrupts can be mapped in vector 1. And as current igb_uio only
> support only one vector, ixgbe VF PMD should use vector 0 for igb_uio and
> vector 1 for VFIO-PCI. Without this patch, VF Rx queue interrupt is mapped
> to vector 0 in register settings and mapped to VFIO vector 1 in
> vfio_enable_msix( ), and then all Rx queue interrupts will be missed.
> 
> Fixes: b13bfab4cdbe ("eal: reserve VFIO vector zero for misc interrupt")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Wei Dai <wei.dai@intel.com>
> ---
>  drivers/net/ixgbe/ixgbe_ethdev.c | 25 ++++++++++++++++++++++---
>  1 file changed, 22 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/net/ixgbe/ixgbe_ethdev.c
> b/drivers/net/ixgbe/ixgbe_ethdev.c
> index 9ca5cbc..39a3d8c 100644
> --- a/drivers/net/ixgbe/ixgbe_ethdev.c
> +++ b/drivers/net/ixgbe/ixgbe_ethdev.c
> @@ -5029,7 +5029,10 @@ ixgbevf_dev_start(struct rte_eth_dev *dev)
> 
>  	/* check and configure queue intr-vector mapping */
>  	if (dev->data->dev_conf.intr_conf.rxq != 0) {
> -		intr_vector = dev->data->nb_rx_queues;
> +		/* According to datasheet, only vector 0/1/2 can be used,
> +		 * now only one vector is used for Rx queue
> +		 */
> +		intr_vector = 1;
>  		if (rte_intr_efd_enable(intr_handle, intr_vector))
>  			return -1;
>  	}
> @@ -5556,9 +5559,12 @@ ixgbevf_dev_rx_queue_intr_enable(struct
> rte_eth_dev *dev, uint16_t queue_id)
>  	uint32_t mask;
>  	struct ixgbe_hw *hw =
>  		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
> +	uint32_t vec = IXGBE_MISC_VEC_ID;
> 
>  	mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
> -	mask |= (1 << IXGBE_MISC_VEC_ID);
> +	if (rte_intr_allow_others(intr_handle))
> +		vec = IXGBE_RX_VEC_START;
> +	mask |= (1 << vec);
>  	RTE_SET_USED(queue_id);
>  	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
> 
> @@ -5573,9 +5579,14 @@ ixgbevf_dev_rx_queue_intr_disable(struct
> rte_eth_dev *dev, uint16_t queue_id)
>  	uint32_t mask;
>  	struct ixgbe_hw *hw =
>  		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
> +	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
> +	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
> +	uint32_t vec = IXGBE_MISC_VEC_ID;
> 
>  	mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
> -	mask &= ~(1 << IXGBE_MISC_VEC_ID);
> +	if (rte_intr_allow_others(intr_handle))
> +		vec = IXGBE_RX_VEC_START;
> +	mask &= ~(1 << vec);
>  	RTE_SET_USED(queue_id);
>  	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
> 
> @@ -5717,6 +5728,7 @@ ixgbevf_configure_msix(struct rte_eth_dev *dev)
>  		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
>  	uint32_t q_idx;
>  	uint32_t vector_idx = IXGBE_MISC_VEC_ID;
> +	uint32_t base = IXGBE_MISC_VEC_ID;
> 
>  	/* Configure VF other cause ivar */
>  	ixgbevf_set_ivar_map(hw, -1, 1, vector_idx); @@ -5727,6 +5739,11
> @@ ixgbevf_configure_msix(struct rte_eth_dev *dev)
>  	if (!rte_intr_dp_is_en(intr_handle))
>  		return;
> 
> +	if (rte_intr_allow_others(intr_handle)) {
> +		base = IXGBE_RX_VEC_START;
> +		vector_idx = IXGBE_RX_VEC_START;
> +	}
> +
>  	/* Configure all RX queues of VF */
>  	for (q_idx = 0; q_idx < dev->data->nb_rx_queues; q_idx++) {
>  		/* Force all queue use vector 0,
> @@ -5734,6 +5751,8 @@ ixgbevf_configure_msix(struct rte_eth_dev *dev)
>  		 */
>  		ixgbevf_set_ivar_map(hw, 0, q_idx, vector_idx);
>  		intr_handle->intr_vec[q_idx] = vector_idx;
> +		if (vector_idx < base + intr_handle->nb_efd - 1)
> +			vector_idx++;
>  	}
>  }
> 
> --
> 2.7.5

Re: [dpdk-dev] [PATCH] net/ixgbe: fix Rx queue interrupt mapping in VF

[Lu, Wenzhuo]

Hi,

> -----Original Message-----
> From: Dai, Wei
> Sent: Wednesday, September 20, 2017 6:18 PM
> To: Lu, Wenzhuo <wenzhuo.lu@intel.com>; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>
> Cc: dev@dpdk.org; Dai, Wei <wei.dai@intel.com>; stable@dpdk.org
> Subject: [PATCH] net/ixgbe: fix Rx queue interrupt mapping in VF
> 
> When a VF port is bound to VFIO-PCI, miscellaneous interrupt is mapped to
> MSI-X vector 0 and Rx queues interrupt are mapped to other vectors in
> vfio_enable_msix( ). To simplify implementation, all VFIO-PCI bound ixgbe VF
> Rx queue interrupts can be mapped in vector 1. And as current igb_uio only
> support only one vector, ixgbe VF PMD should use vector 0 for igb_uio and
> vector 1 for VFIO-PCI. Without this patch, VF Rx queue interrupt is mapped
> to vector 0 in register settings and mapped to VFIO vector 1 in
> vfio_enable_msix( ), and then all Rx queue interrupts will be missed.
> 
> Fixes: b13bfab4cdbe ("eal: reserve VFIO vector zero for misc interrupt")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Wei Dai <wei.dai@intel.com>
Acked-by: Wenzhuo Lu <wenzhuo.lu@intel.com>

Re: [dpdk-dev] [PATCH] net/ixgbe: fix Rx queue interrupt mapping in VF

[Dai, Wei]

I look through from web browser and not find following mail from Jianwei Ma <Jianwei.ma@intel.com>
Hope this can add his test result in the mail list to community.

> -----Original Message-----
> From: Ma, Jianwei
> Sent: Tuesday, October 10, 2017 4:48 PM
> To: Dai, Wei <wei.dai@intel.com>; Lu, Wenzhuo <wenzhuo.lu@intel.com>;
> Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Cc: dev@dpdk.org; stable@dpdk.org
> Subject: RE: [PATCH] net/ixgbe: fix Rx queue interrupt mapping in VF
> 
> Verified with l3fwd-power. It worked well with this patch plus
> http://dpdk.org/dev/patchwork/patch/29221/  "[dpdk-dev,v2] net/ixgbe:
> fix VFIO interrupt mapping in VF"
> 
> -----Original Message-----
> From: Dai, Wei
> Sent: Tuesday, October 10, 2017 11:26
> To: Lu, Wenzhuo <wenzhuo.lu@intel.com>; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>; Ma, Jianwei <jianwei.ma@intel.com>
> Cc: dev@dpdk.org; stable@dpdk.org
> Subject: RE: [PATCH] net/ixgbe: fix Rx queue interrupt mapping in VF
> 
> Hi, Jianwei
> How about your test result ?
> 
> Hi, Wenzhuo
> Would you please review this patch ?
> 
> Thanks a lot !
> 
> > -----Original Message-----
> > From: Dai, Wei
> > Sent: Wednesday, September 20, 2017 6:18 PM
> > To: Lu, Wenzhuo <wenzhuo.lu@intel.com>; Ananyev, Konstantin
> > <konstantin.ananyev@intel.com>
> > Cc: dev@dpdk.org; Dai, Wei <wei.dai@intel.com>; stable@dpdk.org
> > Subject: [PATCH] net/ixgbe: fix Rx queue interrupt mapping in VF
> >
> > When a VF port is bound to VFIO-PCI, miscellaneous interrupt is mapped
> > to MSI-X vector 0 and Rx queues interrupt are mapped to other vectors
> > in vfio_enable_msix( ). To simplify implementation, all VFIO-PCI bound
> > ixgbe VF Rx queue interrupts can be mapped in vector 1. And as current
> > igb_uio only support only one vector, ixgbe VF PMD should use vector 0
> > for igb_uio and vector 1 for VFIO-PCI. Without this patch, VF Rx queue
> > interrupt is mapped to vector 0 in register settings and mapped to
> > VFIO vector 1 in vfio_enable_msix( ), and then all Rx queue interrupts will
> be missed.
> >
> > Fixes: b13bfab4cdbe ("eal: reserve VFIO vector zero for misc
> > interrupt")
> > Cc: stable@dpdk.org
> >
> > Signed-off-by: Wei Dai <wei.dai@intel.com>
Tested-by: Jianwei Ma <Jianwei.ma@intel.com>
> > ---
> >  drivers/net/ixgbe/ixgbe_ethdev.c | 25 ++++++++++++++++++++++---
> >  1 file changed, 22 insertions(+), 3 deletions(-)
> >
> > diff --git a/drivers/net/ixgbe/ixgbe_ethdev.c
> > b/drivers/net/ixgbe/ixgbe_ethdev.c
> > index 9ca5cbc..39a3d8c 100644
> > --- a/drivers/net/ixgbe/ixgbe_ethdev.c
> > +++ b/drivers/net/ixgbe/ixgbe_ethdev.c
> > @@ -5029,7 +5029,10 @@ ixgbevf_dev_start(struct rte_eth_dev *dev)
> >
> >  	/* check and configure queue intr-vector mapping */
> >  	if (dev->data->dev_conf.intr_conf.rxq != 0) {
> > -		intr_vector = dev->data->nb_rx_queues;
> > +		/* According to datasheet, only vector 0/1/2 can be used,
> > +		 * now only one vector is used for Rx queue
> > +		 */
> > +		intr_vector = 1;
> >  		if (rte_intr_efd_enable(intr_handle, intr_vector))
> >  			return -1;
> >  	}
> > @@ -5556,9 +5559,12 @@ ixgbevf_dev_rx_queue_intr_enable(struct
> > rte_eth_dev *dev, uint16_t queue_id)
> >  	uint32_t mask;
> >  	struct ixgbe_hw *hw =
> >  		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
> > +	uint32_t vec = IXGBE_MISC_VEC_ID;
> >
> >  	mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
> > -	mask |= (1 << IXGBE_MISC_VEC_ID);
> > +	if (rte_intr_allow_others(intr_handle))
> > +		vec = IXGBE_RX_VEC_START;
> > +	mask |= (1 << vec);
> >  	RTE_SET_USED(queue_id);
> >  	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
> >
> > @@ -5573,9 +5579,14 @@ ixgbevf_dev_rx_queue_intr_disable(struct
> > rte_eth_dev *dev, uint16_t queue_id)
> >  	uint32_t mask;
> >  	struct ixgbe_hw *hw =
> >  		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
> > +	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
> > +	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
> > +	uint32_t vec = IXGBE_MISC_VEC_ID;
> >
> >  	mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
> > -	mask &= ~(1 << IXGBE_MISC_VEC_ID);
> > +	if (rte_intr_allow_others(intr_handle))
> > +		vec = IXGBE_RX_VEC_START;
> > +	mask &= ~(1 << vec);
> >  	RTE_SET_USED(queue_id);
> >  	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
> >
> > @@ -5717,6 +5728,7 @@ ixgbevf_configure_msix(struct rte_eth_dev
> *dev)
> >  		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
> >  	uint32_t q_idx;
> >  	uint32_t vector_idx = IXGBE_MISC_VEC_ID;
> > +	uint32_t base = IXGBE_MISC_VEC_ID;
> >
> >  	/* Configure VF other cause ivar */
> >  	ixgbevf_set_ivar_map(hw, -1, 1, vector_idx); @@ -5727,6 +5739,11
> @@
> > ixgbevf_configure_msix(struct rte_eth_dev *dev)
> >  	if (!rte_intr_dp_is_en(intr_handle))
> >  		return;
> >
> > +	if (rte_intr_allow_others(intr_handle)) {
> > +		base = IXGBE_RX_VEC_START;
> > +		vector_idx = IXGBE_RX_VEC_START;
> > +	}
> > +
> >  	/* Configure all RX queues of VF */
> >  	for (q_idx = 0; q_idx < dev->data->nb_rx_queues; q_idx++) {
> >  		/* Force all queue use vector 0,
> > @@ -5734,6 +5751,8 @@ ixgbevf_configure_msix(struct rte_eth_dev
> *dev)
> >  		 */
> >  		ixgbevf_set_ivar_map(hw, 0, q_idx, vector_idx);
> >  		intr_handle->intr_vec[q_idx] = vector_idx;
> > +		if (vector_idx < base + intr_handle->nb_efd - 1)
> > +			vector_idx++;
> >  	}
> >  }
> >
> > --
> > 2.7.5

Re: [dpdk-dev] [PATCH] net/ixgbe: fix Rx queue interrupt mapping in VF

[Ferruh Yigit]

On 10/11/2017 4:13 AM, Lu, Wenzhuo wrote:
> Hi,
> 
>> -----Original Message-----
>> From: Dai, Wei
>> Sent: Wednesday, September 20, 2017 6:18 PM
>> To: Lu, Wenzhuo <wenzhuo.lu@intel.com>; Ananyev, Konstantin
>> <konstantin.ananyev@intel.com>
>> Cc: dev@dpdk.org; Dai, Wei <wei.dai@intel.com>; stable@dpdk.org
>> Subject: [PATCH] net/ixgbe: fix Rx queue interrupt mapping in VF
>>
>> When a VF port is bound to VFIO-PCI, miscellaneous interrupt is mapped to
>> MSI-X vector 0 and Rx queues interrupt are mapped to other vectors in
>> vfio_enable_msix( ). To simplify implementation, all VFIO-PCI bound ixgbe VF
>> Rx queue interrupts can be mapped in vector 1. And as current igb_uio only
>> support only one vector, ixgbe VF PMD should use vector 0 for igb_uio and
>> vector 1 for VFIO-PCI. Without this patch, VF Rx queue interrupt is mapped
>> to vector 0 in register settings and mapped to VFIO vector 1 in
>> vfio_enable_msix( ), and then all Rx queue interrupts will be missed.
>>
>> Fixes: b13bfab4cdbe ("eal: reserve VFIO vector zero for misc interrupt")
>> Cc: stable@dpdk.org
>>
>> Signed-off-by: Wei Dai <wei.dai@intel.com>
> Acked-by: Wenzhuo Lu <wenzhuo.lu@intel.com>

Applied to dpdk-next-net/master, thanks.