[DPDK/vhost/virtio Bug 1800] Virtio: Packed Queue desc_event_flags Causing Testpmd Crash

[DPDK/vhost/virtio Bug 1800] Virtio: Packed Queue desc_event_flags Causing Testpmd Crash

[bugzilla]

http://bugs.dpdk.org/show_bug.cgi?id=1800

            Bug ID: 1800
           Summary: Virtio: Packed Queue desc_event_flags Causing Testpmd
                    Crash
           Product: DPDK
           Version: 19.11
          Hardware: x86
                OS: Linux
            Status: UNCONFIRMED
          Severity: major
          Priority: Normal
         Component: vhost/virtio
          Assignee: dev@dpdk.org
          Reporter: 1104121601@qq.com
  Target Milestone: ---

1.  Environment
DPDK version: dpdk-19.11.1
Other software versions: N/A
OS: CentOS Linux release 7.4.1708 (Core)
Compiler: gcc version 4.8.5 20150623 (Red Hat 4.8.5-44) (GCC)
Hardware platform: Intel(R) Xeon(R) D-1736  CPU @ 2.30GHz


2.  Test Setup
export RTE_SDK=/root/dpdk-19.11.1
export RTE_TARGET=x86_64-native-linux-gcc 
./dpdk-setup.sh
instasll and bind pci jump..


3.  Steps to reproduce: 
Start testpmd with packed ring and test the ring -> desc_event_flags function
of the packed queue.

gdb --args testpmd -c 0xff -n 4 --huge-dir=/mnt/huge_dpdk --socket-mem=1024
--socket-limit=1024 -w 0000:15:00.1 --file-prefix=test3 -d
/usr/lib64/librte_pmd_virtio.so \ 
-- --total-num-mbufs=115200 --rxq=4 --txq=4 --forward-mode=txonly --nb-cores=4
--stats-period 1 --burst=512 --rxd=512 --txd=512 --eth-peer=0,10:70:fd:2a:60:39

testpmd> set fwd txonly
Set txonly packet forwarding mode
testpmd> start


3.1 Actual Result
testpmd> start
txonly packet forwarding - ports=1 - cores=4 - streams=4 - NUMA support
enabled, MP allocation mode: native
Logical Core 1 (socket 0) forwards packets on 1 streams:
  RX P=0/Q=0 (socket 0) -> TX P=0/Q=0 (socket 0) peer=10:70:FD:2A:60:39
Logical Core 2 (socket 0) forwards packets on 1 streams:
  RX P=0/Q=1 (socket 0) -> TX P=0/Q=1 (socket 0) peer=10:70:FD:2A:60:39
Logical Core 3 (socket 0) forwards packets on 1 streams:
  RX P=0/Q=2 (socket 0) -> TX P=0/Q=2 (socket 0) peer=10:70:FD:2A:60:39
Logical Core 4 (socket 0) forwards packets on 1 streams:
  RX P=0/Q=3 (socket 0) -> TX P=0/Q=3 (socket 0) peer=10:70:FD:2A:60:39

  txonly packet forwarding packets/burst=1
  packet len=64 - nb packet segments=1
  nb forwarding cores=4 - nb forwarding ports=1
  port 0: RX queue number: 4 Tx queue number: 4
    Rx offloads=0x0 Tx offloads=0x0
    RX queue: 0
      RX desc=512 - RX free threshold=0
      RX threshold registers: pthresh=0 hthresh=0  wthresh=0
      RX Offloads=0x0
    TX queue: 0
      TX desc=512 - TX free threshold=0
      TX threshold registers: pthresh=0 hthresh=0  wthresh=0
      TX offloads=0x0 - TX RS bit threshold=0
testpmd>
Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7fffe7f97700 (LWP 11521)]
virtio_xmit_cleanup_normal_packed (num=0, vq=0x17f99fac0) at
/usr/src/debug/dpdk-19.11.1/drivers/net/virtio/virtio_rxtx.c:327
327                             rte_pktmbuf_free(dxp->cookie);
(gdb) bt
#0  virtio_xmit_cleanup_normal_packed (num=0, vq=0x17f99fac0) at
/usr/src/debug/dpdk-19.11.1/drivers/net/virtio/virtio_rxtx.c:327
#1  virtio_xmit_cleanup_packed (in_order=0, num=1, vq=0x17f99fac0) at
/usr/src/debug/dpdk-19.11.1/drivers/net/virtio/virtio_rxtx.c:341
#2  virtio_xmit_pkts_packed (tx_queue=0x17f99fb40, tx_pkts=0x7fffe7f94550,
nb_pkts=<optimized out>) at
/usr/src/debug/dpdk-19.11.1/drivers/net/virtio/virtio_rxtx.c:2057
#3  0x0000000000471b55 in rte_eth_tx_burst (nb_pkts=<optimized out>,
tx_pkts=0x7fffe7f94550, queue_id=1, port_id=0)
    at
/usr/src/debug/dpdk-19.11.1/x86_64-default-linux-gcc/include/rte_ethdev.h:4666
#4  pkt_burst_transmit (fs=<optimized out>) at
/usr/src/debug/dpdk-19.11.1/app/test-pmd/txonly.c:305
#5  0x0000000000433d2a in run_pkt_fwd_on_lcore (fc=0x178f8fd00,
pkt_fwd=0x471270 <pkt_burst_transmit>) at
/usr/src/debug/dpdk-19.11.1/app/test-pmd/testpmd.c:1806
#6  0x0000000000433e57 in start_pkt_forward_on_core (fwd_arg=<optimized out>)
at /usr/src/debug/dpdk-19.11.1/app/test-pmd/testpmd.c:1832
#7  0x00007fffec96e7ef in eal_thread_loop (arg=<optimized out>) at
/usr/src/debug/dpdk-19.11.1/lib/librte_eal/linux/eal/eal_thread.c:153
#8  0x00007fffeaabdea5 in start_thread () from /lib64/libpthread.so.0
#9  0x00007fffea7e6b0d in clone () from /lib64/libc.so.6
(gdb) l
322                             vq->vq_used_cons_idx -= size;
323                             vq->vq_packed.used_wrap_counter ^= 1;
324                     }
325                     vq_ring_free_id_packed(vq, id);
326                     if (dxp->cookie != NULL) {
327                             rte_pktmbuf_free(dxp->cookie);
328                             dxp->cookie = NULL;
329                     }
330                     used_idx = vq->vq_used_cons_idx;
331             }
(gdb) p dxp->cookie
$14 = (void *) 0x168e90000
(gdb) p vq[0]
$15 = {hw = 0x17fab9dc0, {vq_split = {ring = {num = 4096, desc = 0x17f98d000,
avail = 0x17f99d000, used = 0x17f99e000}}, vq_packed = {ring = {num = 4096,
desc = 0x17f98d000,
        driver = 0x17f99d000, device = 0x17f99e000}, used_wrap_counter = true,
cached_flags = 128, event_flags_shadow = 1}}, vq_used_cons_idx = 51,
vq_nentries = 4096, vq_free_cnt = 1,
  vq_avail_idx = 562, vq_free_thresh = 32, vq_ring_virt_mem = 0x17f98d000,
vq_ring_size = 73728, {rxq = {vq = 0x17f99fac0, fake_mbuf = {cacheline0 =
0x17f99fb80, buf_addr = 0x0, {
          buf_iova = 0, buf_physaddr = 0}, rearm_data = 0x17f99fb90, data_off =
57906, {refcnt_atomic = {cnt = 5}, refcnt = 5}, nb_segs = 0, port = 0, ol_flags
= 0,
        rx_descriptor_fields1 = 0x17f99fba0, {packet_type = 0, {l2_type = 0,
l3_type = 0, l4_type = 0, tun_type = 0, {inner_esp_next_proto = 0 '\000',
{inner_l2_type = 0 '\000',
                inner_l3_type = 0 '\000'}}, inner_l4_type = 0}}, pkt_len = 0,
data_len = 0, vlan_tci = 0, {hash = {rss = 0, fdir = {{{hash = 0, id = 0}, lo =
0}, hi = 0}, sched = {
              queue_id = 0, traffic_class = 0 '\000', color = 0 '\000',
reserved = 0}, txadapter = {reserved1 = 0, reserved2 = 0, txq = 0}, usr = 0}},
vlan_tci_outer = 0, buf_len = 0,
        timestamp = 0, cacheline1 = 0x17f99fbc0, {userdata = 0x0, udata64 = 0},
pool = 0x100006168, next = 0x0, {tx_offload = 0, {l2_len = 0, l3_len = 0,
l4_len = 0, tso_segsz = 0,
            outer_l3_len = 0, outer_l2_len = 0}}, priv_size = 0, timesync = 0,
seqn = 0, shinfo = 0x0, jd_flags = 0, dynfield1 = {0}}, mbuf_initializer = 0,
mpool = 0x0, queue_id = 0,
      port_id = 0, stats = {packets = 0, bytes = 0, errors = 0, multicast = 0,
broadcast = 0, size_bins = {0, 0, 0, 0, 0, 0, 0, 0}}, mz = 0x0}, txq = {vq =
0x17f99fac0,
      virtio_net_hdr_mz = 0x1000061b0, virtio_net_hdr_mem = 6435098496,
queue_id = 1, port_id = 0, stats = {packets = 385586, bytes = 24677504, errors
= 0, multicast = 0, broadcast = 0,
        size_bins = {0, 385586, 0, 0, 0, 0, 0, 0}}, mz = 0x100006168}, cq = {vq
= 0x17f99fac0, virtio_net_hdr_mz = 0x1000061b0, virtio_net_hdr_mem =
6435098496, port_id = 1,
      mz = 0x5e232, lock = {locked = 24677504}}}, vq_ring_mem = 6435688448,
vq_desc_head_idx = 562, vq_desc_tail_idx = 50, vq_queue_index = 3, offset = 8,
notify_addr = 0x2200001000,
  sw_ring = 0x0, vq_descx = 0x17f99fce0}
(gdb) p vq->vq_descx[50]
$16 = {cookie = 0x168e90000, ndescs = 1, next = 32768} (bad mbuf addr)
(gdb) p vq->vq_descx[51]
$17 = {cookie = 0x168e95bc0, ndescs = 1, next = 52} (normal mbuf addr)
(gdb) p vq->vq_descx[52]
$18 = {cookie = 0x168e95280, ndescs = 1, next = 53}
(gdb) p vq->vq_descx[53]
$19 = {cookie = 0x168e94940, ndescs = 1, next = 54}
(gdb) p vq->vq_descx[54]
$20 = {cookie = 0x168e94000, ndescs = 1, next = 55}
(gdb)


3.2   Expected Result
Keep sending...

testpmd> show port stats all

  ######################## NIC statistics for port 0  ########################
  RX-packets: 8          RX-missed: 0          RX-bytes:  934
  RX-errors: 0
  RX-nombuf:  0
  TX-packets: 440662528  TX-errors: 0          TX-bytes:  28202432512

  Throughput (since last show)
  Rx-pps:            0          Rx-bps:            0
  Tx-pps:     31153304          Tx-bps:  15950370552
  ############################################################################
testpmd> show port stats all


4.  Detailed Analysis
  After Testpmd runs for several minutes, it crashes unexpectedly. 
  Debugging shows that the content of vq_descx (within struct virtqueue) is
being modified unexpectedly—likely due to address mismatches in the queue
memory layout.

  After debugging, it was found that the virtio-pci driver uses inconsistent
address calculation logic for two critical steps:
    When informing the hardware (via the modern_setup_queue interface) of the
physical addresses for the driver and device of the packed queue.
    When calculating the virtual addresses of the driver and device for the
driver own use (via vring_init_packed).
  This mismatch leads to the hardware and the driver referencing different
memory for the device queue, causing unintended overwrites of vq_descx.

  Below is a detailed process.


4.1   Address Calculation in modern_setup_queue (Hardware-facing)
  The modern_setup_queue function configures the queue addresses and passes
them to the hardware. 
  For packed queues, it calculates desc_addr, avail_addr (driver addr), and
used_addr (device addr) as follows:

  static int
  modern_setup_queue(struct virtio_hw *hw, struct virtqueue *vq)
  {
    uint64_t desc_addr, avail_addr, used_addr;
    uint16_t notify_off;

    if (!check_vq_phys_addr_ok(vq))
      return -1;

    desc_addr = vq->vq_ring_mem;
    avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc);
    used_addr = RTE_ALIGN_CEIL(avail_addr + offsetof(struct vring_avail,
                ring[vq->vq_nentries]),
            VIRTIO_PCI_VRING_ALIGN);

    rte_write16(vq->vq_queue_index, &hw->common_cfg->queue_select);

    io_write64_twopart(desc_addr, &hw->common_cfg->queue_desc_lo,
                &hw->common_cfg->queue_desc_hi);
    io_write64_twopart(avail_addr, &hw->common_cfg->queue_avail_lo,
                &hw->common_cfg->queue_avail_hi);
    io_write64_twopart(used_addr, &hw->common_cfg->queue_used_lo,
                &hw->common_cfg->queue_used_hi);

    notify_off = rte_read16(&hw->common_cfg->queue_notify_off);
    vq->notify_addr = (void *)((uint8_t *)hw->notify_base +
          notify_off * hw->notify_off_multiplier);

    rte_write16(1, &hw->common_cfg->queue_enable);

    PMD_INIT_LOG(DEBUG, "queue %u addresses:", vq->vq_queue_index);
    PMD_INIT_LOG(DEBUG, "\t desc_addr: %" PRIx64, desc_addr);
    PMD_INIT_LOG(DEBUG, "\t aval_addr: %" PRIx64, avail_addr);
    PMD_INIT_LOG(DEBUG, "\t used_addr: %" PRIx64, used_addr);
    PMD_INIT_LOG(DEBUG, "\t notify addr: %p (notify offset: %u)",
      vq->notify_addr, notify_off);

    return 0;
  }

  Key parameters for our test:
    vq_nentries = 0x1000 (4096 entries)
    sizeof(struct vring_desc) = 16 (0x10 in hex)
    offsetof(struct vring_avail, ring[vq->vq_nentries]) = 4 + (0x1000 * 2)
(base offset 4 + 2 bytes per ring pointer)
    Alignment requirement: VIRTIO_PCI_VRING_ALIGN = 4096 (0x1000 in hex)

  Example calculation (assuming desc_addr = 0x0):
    avail_addr = 0x0 + (0x1000 * 0x10) = 0x10000 (driver address passed to
hardware)
    used_addr = RTE_ALIGN_CEIL(0x10000 + 4 + (0x1000 * 2), 0x1000) =
RTE_ALIGN_CEIL(0x12004, 0x1000) = 0x13000 (device ddress passed to hardware)


4.2   Address Calculation in vring_init_packed (Driver-internal)
  The vring_init_packed function calculates the driver-internal virtual
addresses for the packed queue driver and device addr:

    static inline void
    vring_init_packed(struct vring_packed *vr, uint8_t *p, unsigned long align,
        unsigned int num)
    {
      vr->num = num;
      vr->desc = (struct vring_packed_desc *)p;
      vr->driver = (struct vring_packed_desc_event *)(p +
          vr->num * sizeof(struct vring_packed_desc));
      vr->device = (struct vring_packed_desc_event *)
        RTE_ALIGN_CEIL(((uintptr_t)vr->driver +
            sizeof(struct vring_packed_desc_event)), align);
    }

  Example calculation (same desc_addr = 0x0, p = 0x0):
    vr->driver = 0x0 + (0x1000 * 0x10) = 0x10000 (matches avail_addr from
modern_setup_queue)
    vr->device = RTE_ALIGN_CEIL(0x10000 + sizeof(struct
vring_packed_desc_event), 0x1000)
      sizeof(struct vring_packed_desc_event) = 4 (standard definition), this
becomes RTE_ALIGN_CEIL(0x10004, 0x1000) = 0x11000 (driver-internal device
address)


4.3   Critical Mismatch
  Hardware uses 0x13000 as the device address.
  Driver uses 0x11000 as the device address.


4.4   Modification Applied to Fix the Issue
  We modified modern_setup_queue to use the same address logic as
vring_init_packed for packed queues. After this change, Testpmd runs stably
without crashes:

    static int
    modern_setup_queue(struct virtio_hw *hw, struct virtqueue *vq)
    {
      uint64_t desc_addr, avail_addr, used_addr;
      uint16_t notify_off;

      if (!check_vq_phys_addr_ok(vq))
        return -1;

      desc_addr = vq->vq_ring_mem;
      avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc);
      if (vtpci_packed_queue(hw))
        used_addr = RTE_ALIGN_CEIL(((uintptr_t)avail_addr + 
                    sizeof(struct vring_packed_desc_event)), 
              VIRTIO_PCI_VRING_ALIGN);
      else
        used_addr = RTE_ALIGN_CEIL(avail_addr + offsetof(struct vring_avail,
                    ring[vq->vq_nentries]),
              VIRTIO_PCI_VRING_ALIGN);

      rte_write16(vq->vq_queue_index, &hw->common_cfg->queue_select);

      io_write64_twopart(desc_addr, &hw->common_cfg->queue_desc_lo,
                  &hw->common_cfg->queue_desc_hi);
      io_write64_twopart(avail_addr, &hw->common_cfg->queue_avail_lo,
                  &hw->common_cfg->queue_avail_hi);
      io_write64_twopart(used_addr, &hw->common_cfg->queue_used_lo,
                  &hw->common_cfg->queue_used_hi);

      notify_off = rte_read16(&hw->common_cfg->queue_notify_off);
      vq->notify_addr = (void *)((uint8_t *)hw->notify_base +
            notify_off * hw->notify_off_multiplier);

      rte_write16(1, &hw->common_cfg->queue_enable);

      PMD_INIT_LOG(DEBUG, "queue %u addresses:", vq->vq_queue_index);
      PMD_INIT_LOG(DEBUG, "\t desc_addr: %" PRIx64, desc_addr);
      PMD_INIT_LOG(DEBUG, "\t aval_addr: %" PRIx64, avail_addr);
      PMD_INIT_LOG(DEBUG, "\t used_addr: %" PRIx64, used_addr);
      PMD_INIT_LOG(DEBUG, "\t notify addr: %p (notify offset: %u)",
        vq->notify_addr, notify_off);

      return 0;
    }


4.5   Reference from Virtio-User/Vhost-User
  We noticed that the virtio-user and vhost-user drivers already use the same
logic as our modified modern_setup_queue for packed queues. For example, in
virtio_user_setup_queue_packed:
    static void
    virtio_user_setup_queue_packed(struct virtqueue *vq,
                struct virtio_user_dev *dev)
    {
      uint16_t queue_idx = vq->vq_queue_index;
      struct vring_packed *vring;
      uint64_t desc_addr;
      uint64_t avail_addr;
      uint64_t used_addr;
      uint16_t i;

      vring  = &dev->packed_vrings[queue_idx];
      desc_addr = (uintptr_t)vq->vq_ring_virt_mem;
      avail_addr = desc_addr + vq->vq_nentries *
        sizeof(struct vring_packed_desc);
      used_addr = RTE_ALIGN_CEIL(avail_addr +
            sizeof(struct vring_packed_desc_event),
            VIRTIO_PCI_VRING_ALIGN);
      vring->num = vq->vq_nentries;
      vring->desc = (void *)(uintptr_t)desc_addr;
      vring->driver = (void *)(uintptr_t)avail_addr;
      vring->device = (void *)(uintptr_t)used_addr;
      dev->packed_queues[queue_idx].avail_wrap_counter = true;
      dev->packed_queues[queue_idx].used_wrap_counter = true;

      for (i = 0; i < vring->num; i++)
        vring->desc[i].flags = 0;
    }



Question for Clarification
  Both the higher and lower dpdk versions exhibit this behavior.

  Therefore, for packed queues, why do modern_setup_queue and vring_init_packed
use different logic to calculate the device address?
  Is this inconsistency due to incorrect usage on my part, or are there special
considerations specific to packed queue mode (e.g., hardware compatibility,
protocol requirements)?
  We would greatly appreciate your help in clarifying this confusion. Thank
you!

-- 
You are receiving this mail because:
You are the assignee for the bug.