Re: [dpdk-dev] MSI-X vector #1 seems to be stalled sometimes after VF reset (ixgbe)

[Ruslan Nikolaev <ruslan@purestorage.com>]

interrupts_excerpts.patch:

drivers/net/ixgbe/ixgbe_ethdev.c

eth_ixgbevf_dev_init:

@@ -1462,8 +1467,9 @@
        rte_intr_callback_register(&pci_dev->intr_handle,
                                   ixgbevf_dev_interrupt_handler,
                                   (void *)eth_dev);
-       rte_intr_enable(&pci_dev->intr_handle);
-       ixgbevf_intr_enable(hw);
+
+       /* XXX: not enabling interrupts here since they are enabled in dev_start anyway,
+          and we do not have correct number of interrupt vectors here yet. */

        PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x mac.type=%s",
                     eth_dev->data->port_id, pci_dev->id.vendor_id,

ixgbevf_dev_start:
@@ -4168,7 +4174,7 @@

        /* check and configure queue intr-vector mapping */
        if (dev->data->dev_conf.intr_conf.rxq != 0) {
-               intr_vector = dev->data->nb_rx_queues;
+               intr_vector = 1;
                if (rte_intr_efd_enable(intr_handle, intr_vector))
                        return -1;
        }

@@ -4789,31 +4801,27 @@
 static int
 ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
 {
-       uint32_t mask;
+       /* use vector #0 for mailbox interrupts and vector #1 for all RX queues */
+       uint8_t idx = (queue_id < IXGBE_MAX_QUEUE_NUM_PER_VF);
+       uint32_t mask = 1U << idx;
        struct ixgbe_hw *hw =
                IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);

-       mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
-       mask |= (1 << IXGBE_MISC_VEC_ID);
-       RTE_SET_USED(queue_id);
        IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);

-       rte_intr_enable(&dev->pci_dev->intr_handle);
-
        return 0;
 }

 static int
 ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 {
-       uint32_t mask;
+       /* use vector #0 for mailbox interrupts and vector #1 for all RX queues */
+       uint8_t idx = (queue_id < IXGBE_MAX_QUEUE_NUM_PER_VF);
+       uint32_t mask = 1U << idx;
        struct ixgbe_hw *hw =
                IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);

-       mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
-       mask &= ~(1 << IXGBE_MISC_VEC_ID);
-       RTE_SET_USED(queue_id);
-       IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
+       IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask);

        return 0;
 }

ixgbevf_configure_msix:
@@ -4948,10 +4956,9 @@
        struct ixgbe_hw *hw =
                IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
        uint32_t q_idx;
-       uint32_t vector_idx = IXGBE_MISC_VEC_ID;

        /* Configure VF other cause ivar */
-       ixgbevf_set_ivar_map(hw, -1, 1, vector_idx);
+       ixgbevf_set_ivar_map(hw, -1, 1, IXGBE_MISC_VEC_ID);

        /* won't configure msix register if no mapping is done
         * between intr vector and event fd.
@@ -4961,11 +4968,9 @@

        /* Configure all RX queues of VF */
        for (q_idx = 0; q_idx < dev->data->nb_rx_queues; q_idx++) {
-               /* Force all queue use vector 0,
-                * as IXGBE_VF_MAXMSIVECOTR = 1
-                */
-               ixgbevf_set_ivar_map(hw, 0, q_idx, vector_idx);
-               intr_handle->intr_vec[q_idx] = vector_idx;
+               /* Force all queues to use vector 1 */
+               ixgbevf_set_ivar_map(hw, 0, q_idx, IXGBE_RX_VEC_START);
+               intr_handle->intr_vec[q_idx] = IXGBE_RX_VEC_START;
        }
 }