Re: [dpdk-stable] [dpdk-dev] [PATCH 6/9] net/virtio: fix queue setup consistency

[Olivier MATZ <olivier.matz@6wind.com>]

Platform description
--------------------

guest (dpdk)
+----------------+
|                |
|                |
|    port0       |
+----------------+
       |
       | virtio
       |
+----------------+
|     tap0       |
|                |
|                |
+----------------+
host (linux, vhost-net)

Host configuration
------------------

Start qemu with:

- a ne2k management interface to avoi any conflict with dpdk
- a virtio net device, connected to a tap interface through vhost-net
- mergeable buffers disabled

  /usr/bin/qemu-system-x86_64 -k fr -daemonize --enable-kvm -m 2G -cpu host \
    -smp 3 -serial telnet::40564,server,nowait -serial null \
    -qmp tcp::44340,server,nowait -monitor telnet::49229,server,nowait \
    -device ne2k_pci,mac=de:ad:de:01:02:03,netdev=user.0,addr=03 \
    -netdev user,id=user.0,hostfwd=tcp::34965-:22 \
    -netdev type=tap,id=vhostnet0,script=no,vhost=on,queues=8 \
    -device virtio-net-pci,mrg_rxbuf=off,netdev=vhostnet0,mq=on,vectors=17 \
    -hda "${VM_PATH}/ubuntu-16.04-template.qcow2" \
    -snapshot -vga none -display none

Guest configuration
-------------------

Apply a patch that reverts initialization of queues in testpmd
(initialize rx queue first), and displays some logs in virtio:

  --- a/app/test-pmd/testpmd.c
  +++ b/app/test-pmd/testpmd.c
  @@ -1461,34 +1461,10 @@ start_port(portid_t pid)
                  }
                  if (port->need_reconfig_queues > 0) {
                          port->need_reconfig_queues = 0;
  -                       /* setup tx queues */
  -                       for (qi = 0; qi < nb_txq; qi++) {
  -                               if ((numa_support) &&
  -                                       (txring_numa[pi] != NUMA_NO_CONFIG))
  -                                       diag = rte_eth_tx_queue_setup(pi, qi,
  -                                               nb_txd,txring_numa[pi],
  -                                               &(port->tx_conf));
  -                               else
  -                                       diag = rte_eth_tx_queue_setup(pi, qi,
  -                                               nb_txd,port->socket_id,
  -                                               &(port->tx_conf));
  -
  -                               if (diag == 0)
  -                                       continue;
  -
  -                               /* Fail to setup tx queue, return */
  -                               if (rte_atomic16_cmpset(&(port->port_status),
  -                                                       RTE_PORT_HANDLING,
  -                                                       RTE_PORT_STOPPED) == 0)
  -                                       printf("Port %d can not be set back "
  -                                                       "to stopped\n", pi);
  -                               printf("Fail to configure port %d tx queues\n", pi);
  -                               /* try to reconfigure queues next time */
  -                               port->need_reconfig_queues = 1;
  -                               return -1;
  -                       }
                          /* setup rx queues */
                          for (qi = 0; qi < nb_rxq; qi++) {
  +                               printf("rte_eth_rx_queue_setup %d %d\n",
  +                                       pi, qi);
                                  if ((numa_support) &&
                                          (rxring_numa[pi] != NUMA_NO_CONFIG)) {
                                          struct rte_mempool * mp =
  @@ -1500,7 +1476,6 @@ start_port(portid_t pid)
                                                          rxring_numa[pi]);
                                                  return -1;
                                          }
  -
                                          diag = rte_eth_rx_queue_setup(pi, qi,
                                               nb_rxd,rxring_numa[pi],
                                               &(port->rx_conf),mp);
  @@ -1532,6 +1507,34 @@ start_port(portid_t pid)
                                  port->need_reconfig_queues = 1;
                                  return -1;
                          }
  +                       /* setup tx queues */
  +                       for (qi = 0; qi < nb_txq; qi++) {
  +                               printf("rte_eth_tx_queue_setup %d %d\n",
  +                                       pi, qi);
  +                               if ((numa_support) &&
  +                                       (txring_numa[pi] != NUMA_NO_CONFIG))
  +                                       diag = rte_eth_tx_queue_setup(pi, qi,
  +                                               nb_txd,txring_numa[pi],
  +                                               &(port->tx_conf));
  +                               else
  +                                       diag = rte_eth_tx_queue_setup(pi, qi,
  +                                               nb_txd,port->socket_id,
  +                                               &(port->tx_conf));
  +
  +                               if (diag == 0)
  +                                       continue;
  +
  +                               /* Fail to setup tx queue, return */
  +                               if (rte_atomic16_cmpset(&(port->port_status),
  +                                                       RTE_PORT_HANDLING,
  +                                                       RTE_PORT_STOPPED) == 0)
  +                                       printf("Port %d can not be set back "
  +                                                       "to stopped\n", pi);
  +                               printf("Fail to configure port %d tx queues\n", pi);
  +                               /* try to reconfigure queues next time */
  +                               port->need_reconfig_queues = 1;
  +                               return -1;
  +                       }
                  }
   
                  for (event_type = RTE_ETH_EVENT_UNKNOWN;
  --- a/drivers/net/virtio/virtio_rxtx.c
  +++ b/drivers/net/virtio/virtio_rxtx.c
  @@ -445,6 +445,8 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
          nbufs = 0;
          error = ENOSPC;
   
  +        printf("rx_queue_setup() use_simple_rxtx=%d\n",
  +               hw->use_simple_rxtx);
          if (hw->use_simple_rxtx) {
                  for (desc_idx = 0; desc_idx < vq->vq_nentries;
                       desc_idx++) {
  @@ -563,6 +565,8 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
   
          vq->vq_free_thresh = tx_free_thresh;
   
  +        printf("tx_queue_setup() use_simple_rxtx=%d\n",
  +               hw->use_simple_rxtx);
          if (hw->use_simple_rxtx) {
                  uint16_t mid_idx  = vq->vq_nentries >> 1;
   

Compile dpdk:

  cd dpdk.org
  make config T=x86_64-native-linuxapp-gcc
  sed -i 's,CONFIG_RTE_LIBRTE_VIRTIO_DEBUG_INIT=n,CONFIG_RTE_LIBRTE_VIRTIO_DEBUG_INIT=y,' build/.config
  sed -i 's,CONFIG_RTE_LIBRTE_VIRTIO_DEBUG_DRIVER=n,CONFIG_RTE_LIBRTE_VIRTIO_DEBUG_DRIVER=y,' build/.config
  make -j4

Prepare environment:

  mkdir -p /mnt/huge
  mount -t hugetlbfs nodev /mnt/huge
  echo 256 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
  modprobe uio_pci_generic
  python usertools/dpdk-devbind.py -b uio_pci_generic 0000:00:02.0

  ./build/app/testpmd -l 0,1 --log-level 7 -- --total-num-mbufs=16384 \
    -i --port-topology=chained --disable-hw-vlan-filter \
    --disable-hw-vlan-strip --txqflags=0xf01

  ...
  Configuring Port 0 (socket 0)
  rte_eth_rx_queue_setup 0 0
  rx_queue_setup() use_simple_rxtx=0
  rte_eth_tx_queue_setup 0 0
  PMD: virtio_update_rxtx_handler(): Using simple rx/tx path
  tx_queue_setup() use_simple_rxtx=1
  ...

Configure testpmd:

  set fwd rxonly
  set verbose 1
  start

Without the fix, there is a segfault in virtio_recv_pkts_vec()

It works ok with the patch.