From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by dpdk.org (Postfix) with ESMTP id 1BFFBAFD7 for ; Tue, 27 May 2014 12:30:38 +0200 (CEST) Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga102.fm.intel.com with ESMTP; 27 May 2014 03:30:49 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="4.98,918,1392192000"; d="scan'208";a="545426448" Received: from fmsmsx104.amr.corp.intel.com ([10.19.9.35]) by fmsmga002.fm.intel.com with ESMTP; 27 May 2014 03:30:25 -0700 Received: from fmsmsx117.amr.corp.intel.com (10.18.116.17) by FMSMSX104.amr.corp.intel.com (10.19.9.35) with Microsoft SMTP Server (TLS) id 14.3.123.3; Tue, 27 May 2014 03:30:25 -0700 Received: from shsmsx104.ccr.corp.intel.com (10.239.4.70) by fmsmsx117.amr.corp.intel.com (10.18.116.17) with Microsoft SMTP Server (TLS) id 14.3.123.3; Tue, 27 May 2014 03:30:24 -0700 Received: from shsmsx101.ccr.corp.intel.com ([169.254.1.7]) by SHSMSX104.ccr.corp.intel.com ([169.254.5.192]) with mapi id 14.03.0123.003; Tue, 27 May 2014 18:30:22 +0800 From: "Fu, JingguoX" To: "dev@dpdk.org" , "Ouyang, Changchun" Thread-Topic: [dpdk-dev] [PATCH v3] virtio: Support multiple queues feature in DPDK based virtio-net frontend. Thread-Index: AQHPeOG32lDrXSThmUegy5nWP5MSYZtUI5aQ Date: Tue, 27 May 2014 10:30:21 +0000 Message-ID: <6BD6202160B55B409D4232931158226242A4C3@SHSMSX101.ccr.corp.intel.com> References: <1401108837-21582-1-git-send-email-changchun.ouyang@intel.com> In-Reply-To: <1401108837-21582-1-git-send-email-changchun.ouyang@intel.com> Accept-Language: en-US Content-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.239.127.40] Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Subject: Re: [dpdk-dev] [PATCH v3] virtio: Support multiple queues feature in DPDK based virtio-net frontend. X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 27 May 2014 10:30:41 -0000 This v3 patch continues fixing some errors and warnings reported by checkpa= tch.pl. This patch supports multiple queues feature in DPDK based virtio-net fronte= nd. It firstly gets max queue number of virtio-net from virtio PCI configuratio= n and then send command to negotiate the queue number with backend; When receivin= g and transmitting packets, it negotiates multiple virtio-net queues which serve = RX/TX; To utilize this feature, the backend also need support multiple queues feat= ure and enable it. Signed-off-by: Ouyang Changchun Test-by: Fu Jingguo Compile pass >> Server: Grizzly >> Server OS: 3.13.9-200.fc20.x86_64 >> Guest OS: 3.11.10-301 >> git commit 57f0ba5f8b8588dfa6ffcd001447ef6337afa6cd >> Gcc Version: 4.8.2 Function test pass --- lib/librte_pmd_virtio/virtio_ethdev.c | 377 ++++++++++++++++++++++++++++--= ---- lib/librte_pmd_virtio/virtio_ethdev.h | 40 ++-- lib/librte_pmd_virtio/virtio_pci.h | 4 +- lib/librte_pmd_virtio/virtio_rxtx.c | 92 +++++++-- lib/librte_pmd_virtio/virtqueue.h | 61 ++++-- 5 files changed, 458 insertions(+), 116 deletions(-) diff --git a/lib/librte_pmd_virtio/virtio_ethdev.c b/lib/librte_pmd_virtio/= virtio_ethdev.c index 49e236b..c2b4dfb 100644 --- a/lib/librte_pmd_virtio/virtio_ethdev.c +++ b/lib/librte_pmd_virtio/virtio_ethdev.c @@ -81,6 +81,12 @@ static void virtio_dev_stats_get(struct rte_eth_dev *dev= , struct rte_eth_stats * static void virtio_dev_stats_reset(struct rte_eth_dev *dev); static void virtio_dev_free_mbufs(struct rte_eth_dev *dev); =20 +static int virtio_dev_queue_stats_mapping_set( + __rte_unused struct rte_eth_dev *eth_dev, + __rte_unused uint16_t queue_id, + __rte_unused uint8_t stat_idx, + __rte_unused uint8_t is_rx); + /* * The set of PCI devices this driver supports */ @@ -92,6 +98,135 @@ static struct rte_pci_id pci_id_virtio_map[] =3D { { .vendor_id =3D 0, /* sentinel */ }, }; =20 +static int +virtio_send_command(struct virtqueue *vq, struct virtio_pmd_ctrl *ctrl, + int *dlen, int pkt_num) +{ + uint32_t head =3D vq->vq_desc_head_idx, i; + int k, sum =3D 0; + virtio_net_ctrl_ack status =3D ~0; + struct virtio_pmd_ctrl result; + + ctrl->status =3D status; + + if (!vq->hw->cvq) { + PMD_INIT_LOG(ERR, "%s(): Control queue is " + "not supported by this device.\n", __func__); + return -1; + } + + PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx =3D %d, status =3D %d, " + "vq->hw->cvq =3D %p vq =3D %p\n", + vq->vq_desc_head_idx, status, vq->hw->cvq, vq); + + if ((vq->vq_free_cnt < ((uint32_t)pkt_num + 2)) || (pkt_num < 1)) + return -1; + + memcpy(vq->virtio_net_hdr_mz->addr, ctrl, + sizeof(struct virtio_pmd_ctrl)); + + /* + * Format is enforced in qemu code: + * One TX packet for header; + * At least one TX packet per argument; + * One RX packet for ACK. + */ + vq->vq_ring.desc[head].flags =3D VRING_DESC_F_NEXT; + vq->vq_ring.desc[head].addr =3D vq->virtio_net_hdr_mz->phys_addr; + vq->vq_ring.desc[head].len =3D sizeof(struct virtio_net_ctrl_hdr); + vq->vq_free_cnt--; + i =3D vq->vq_ring.desc[head].next; + + for (k =3D 0; k < pkt_num; k++) { + vq->vq_ring.desc[i].flags =3D VRING_DESC_F_NEXT; + vq->vq_ring.desc[i].addr =3D vq->virtio_net_hdr_mz->phys_addr + + sizeof(struct virtio_net_ctrl_hdr) + + sizeof(ctrl->status) + sizeof(uint8_t)*sum; + vq->vq_ring.desc[i].len =3D dlen[k]; + sum +=3D dlen[k]; + vq->vq_free_cnt--; + i =3D vq->vq_ring.desc[i].next; + } + + vq->vq_ring.desc[i].flags =3D VRING_DESC_F_WRITE; + vq->vq_ring.desc[i].addr =3D vq->virtio_net_hdr_mz->phys_addr + + sizeof(struct virtio_net_ctrl_hdr); + vq->vq_ring.desc[i].len =3D sizeof(ctrl->status); + vq->vq_free_cnt--; + + vq->vq_desc_head_idx =3D vq->vq_ring.desc[i].next; + + vq_update_avail_ring(vq, head); + vq_update_avail_idx(vq); + + PMD_INIT_LOG(DEBUG, "vq->vq_queue_index =3D %d\n", vq->vq_queue_index); + + virtqueue_notify(vq); + + while (vq->vq_used_cons_idx =3D=3D vq->vq_ring.used->idx) + usleep(100); + + while (vq->vq_used_cons_idx !=3D vq->vq_ring.used->idx) { + uint32_t idx, desc_idx, used_idx; + struct vring_used_elem *uep; + + rmb(); + + used_idx =3D (uint32_t)(vq->vq_used_cons_idx + & (vq->vq_nentries - 1)); + uep =3D &vq->vq_ring.used->ring[used_idx]; + idx =3D (uint32_t) uep->id; + desc_idx =3D idx; + + while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) { + desc_idx =3D vq->vq_ring.desc[desc_idx].next; + vq->vq_free_cnt++; + } + + vq->vq_ring.desc[desc_idx].next =3D vq->vq_desc_head_idx; + vq->vq_desc_head_idx =3D idx; + + vq->vq_used_cons_idx++; + vq->vq_free_cnt++; + } + + PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=3D%d\nvq->vq_desc_head_idx=3D%d\n", + vq->vq_free_cnt, vq->vq_desc_head_idx); + + memcpy(&result, vq->virtio_net_hdr_mz->addr, + sizeof(struct virtio_pmd_ctrl)); + + return result.status; +} + +static int +virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues) +{ + struct virtio_hw *hw + =3D VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct virtio_pmd_ctrl ctrl; + int dlen[1]; + int ret; + + ctrl.hdr.class =3D VIRTIO_NET_CTRL_MQ; + ctrl.hdr.cmd =3D VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; + memcpy(ctrl.data, &nb_queues, sizeof(uint16_t)); + + PMD_INIT_LOG(DEBUG, "ctrl.data=3D%d\n", *(int *)ctrl.data); + + dlen[0] =3D sizeof(uint16_t); + + ret =3D virtio_send_command(hw->cvq, &ctrl, dlen, 1); + + if (ret) { + PMD_INIT_LOG(ERR, "Multiqueue configured but send command " + "failed, this is too late now...\n"); + return -EINVAL; + } + + return 0; +} + int virtio_dev_queue_setup(struct rte_eth_dev *dev, int queue_type, uint16_t queue_idx, @@ -134,7 +269,7 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, =20 if (queue_type =3D=3D VTNET_RQ) { rte_snprintf(vq_name, sizeof(vq_name), "port%d_rvq%d", - dev->data->port_id, queue_idx); + dev->data->port_id, queue_idx); vq =3D rte_zmalloc(vq_name, sizeof(struct virtqueue) + vq_size * sizeof(struct vq_desc_extra), CACHE_LINE_SIZE); memcpy(vq->vq_name, vq_name, sizeof(vq->vq_name)); @@ -146,8 +281,9 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, memcpy(vq->vq_name, vq_name, sizeof(vq->vq_name)); } else if(queue_type =3D=3D VTNET_CQ) { rte_snprintf(vq_name, sizeof(vq_name), "port%d_cvq", - dev->data->port_id); - vq =3D rte_zmalloc(vq_name, sizeof(struct virtqueue), + dev->data->port_id); + vq =3D rte_zmalloc(vq_name, sizeof(struct virtqueue) + + vq_size * sizeof(struct vq_desc_extra), CACHE_LINE_SIZE); memcpy(vq->vq_name, vq_name, sizeof(vq->vq_name)); } @@ -155,6 +291,7 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, PMD_INIT_LOG(ERR, "%s: Can not allocate virtqueue\n", __func__); return (-ENOMEM);=20 } + vq->hw =3D hw; vq->port_id =3D dev->data->port_id; vq->queue_id =3D queue_idx; @@ -171,11 +308,12 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d\n", size, vq-= >vq_ring_size); =20 mz =3D rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, - socket_id, 0, VIRTIO_PCI_VRING_ALIGN); + socket_id, 0, VIRTIO_PCI_VRING_ALIGN); if (mz =3D=3D NULL) { rte_free(vq); return (-ENOMEM); } + /* * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit, * and only accepts 32 bit page frame number.=20 @@ -186,6 +324,7 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, rte_free(vq); return (-ENOMEM); } + memset(mz->addr, 0, sizeof(mz->len)); vq->mz =3D mz; vq->vq_ring_mem =3D mz->phys_addr; @@ -197,8 +336,8 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, =20 if (queue_type =3D=3D VTNET_TQ) { /*=20 - * For each xmit packet, allocate a virtio_net_hdr - */ + * For each xmit packet, allocate a virtio_net_hdr + */ rte_snprintf(vq_name, sizeof(vq_name), "port%d_tvq%d_hdrzone", dev->data->port_id, queue_idx); vq->virtio_net_hdr_mz =3D rte_memzone_reserve_aligned(vq_name, @@ -206,10 +345,12 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, socket_id, 0, CACHE_LINE_SIZE); if (vq->virtio_net_hdr_mz =3D=3D NULL) { rte_free(vq); - return (-ENOMEM); + return -ENOMEM; } - vq->virtio_net_hdr_mem =3D (void *)(uintptr_t)vq->virtio_net_hdr_mz->phy= s_addr; - memset(vq->virtio_net_hdr_mz->addr, 0, vq_size * sizeof(struct virtio_ne= t_hdr)); + vq->virtio_net_hdr_mem =3D + (void *)(uintptr_t)vq->virtio_net_hdr_mz->phys_addr; + memset(vq->virtio_net_hdr_mz->addr, 0, + vq_size * sizeof(struct virtio_net_hdr)); } else if (queue_type =3D=3D VTNET_CQ) { /* Allocate a page for control vq command, data and status */ rte_snprintf(vq_name, sizeof(vq_name), "port%d_cvq_hdrzone", @@ -218,9 +359,10 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, PAGE_SIZE, socket_id, 0, CACHE_LINE_SIZE); if (vq->virtio_net_hdr_mz =3D=3D NULL) { rte_free(vq); - return (-ENOMEM); + return -ENOMEM; } - vq->virtio_net_hdr_mem =3D (void *)(uintptr_t)vq->virtio_net_hdr_mz->phy= s_addr; + vq->virtio_net_hdr_mem =3D + (void *)(uintptr_t)vq->virtio_net_hdr_mz->phys_addr; memset(vq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE); } =20 @@ -231,12 +373,12 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, VIRTIO_WRITE_REG_4(hw, VIRTIO_PCI_QUEUE_PFN, mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); *pvq =3D vq; - return (0); + return 0; } =20 static int -virtio_dev_cq_queue_setup(struct rte_eth_dev *dev, - unsigned int socket_id) +virtio_dev_cq_queue_setup(struct rte_eth_dev *dev, uint16_t vtpci_queue_id= x, + uint32_t socket_id) { struct virtqueue *vq; uint16_t nb_desc =3D 0; @@ -245,15 +387,16 @@ virtio_dev_cq_queue_setup(struct rte_eth_dev *dev, VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private); =20 PMD_INIT_FUNC_TRACE(); - ret =3D virtio_dev_queue_setup(dev, VTNET_CQ, 0, VTNET_SQ_CQ_QUEUE_IDX, - nb_desc, socket_id, &vq); + ret =3D virtio_dev_queue_setup(dev, VTNET_CQ, VTNET_SQ_CQ_QUEUE_IDX, + vtpci_queue_idx, nb_desc, socket_id, &vq); + if (ret < 0) { PMD_INIT_LOG(ERR, "control vq initialization failed\n"); return ret; } =20 hw->cvq =3D vq; - return (0); + return 0; } =20 static void @@ -264,26 +407,29 @@ virtio_dev_close(struct rte_eth_dev *dev) virtio_dev_stop(dev); } =20 - /* * dev_ops for virtio, bare necessities for basic operation */ static struct eth_dev_ops virtio_eth_dev_ops =3D { - .dev_configure =3D virtio_dev_configure, - .dev_start =3D virtio_dev_start, - .dev_stop =3D virtio_dev_stop, - .dev_close =3D virtio_dev_close, - - .dev_infos_get =3D virtio_dev_info_get, - .stats_get =3D virtio_dev_stats_get, - .stats_reset =3D virtio_dev_stats_reset, - .link_update =3D virtio_dev_link_update, - .mac_addr_add =3D NULL, - .mac_addr_remove =3D NULL, - .rx_queue_setup =3D virtio_dev_rx_queue_setup, - .rx_queue_release =3D virtio_dev_rx_queue_release, /* meaningfull o= nly to multiple queue */ - .tx_queue_setup =3D virtio_dev_tx_queue_setup, - .tx_queue_release =3D virtio_dev_tx_queue_release /* meaningfull onl= y to multiple queue */ + .dev_configure =3D virtio_dev_configure, + .dev_start =3D virtio_dev_start, + .dev_stop =3D virtio_dev_stop, + .dev_close =3D virtio_dev_close, + + .dev_infos_get =3D virtio_dev_info_get, + .stats_get =3D virtio_dev_stats_get, + .stats_reset =3D virtio_dev_stats_reset, + .link_update =3D virtio_dev_link_update, + .mac_addr_add =3D NULL, + .mac_addr_remove =3D NULL, + .rx_queue_setup =3D virtio_dev_rx_queue_setup, + /* meaningfull only to multiple queue */ + .rx_queue_release =3D virtio_dev_rx_queue_release, + .tx_queue_setup =3D virtio_dev_tx_queue_setup, + /* meaningfull only to multiple queue */ + .tx_queue_release =3D virtio_dev_tx_queue_release, + /* collect stats per queue */ + .queue_stats_mapping_set =3D virtio_dev_queue_stats_mapping_set }; =20 static inline int @@ -370,7 +516,7 @@ static void virtio_negotiate_features(struct virtio_hw *hw) { uint32_t guest_features, mask; - mask =3D VIRTIO_NET_F_CTRL_VQ | VIRTIO_NET_F_CTRL_RX | VIRTIO_NET_F_CTRL_= VLAN; + mask =3D VIRTIO_NET_F_CTRL_RX | VIRTIO_NET_F_CTRL_VLAN; mask |=3D VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM ; =20 /* TSO and LRO are only available when their corresponding @@ -388,12 +534,21 @@ virtio_negotiate_features(struct virtio_hw *hw) =20 /* Prepare guest_features: feature that driver wants to support */ guest_features =3D VTNET_FEATURES & ~mask; + PMD_INIT_LOG(DEBUG, "guest_features before negotiate =3D %x\n", + guest_features); =20 /* Read device(host) feature bits */ hw->host_features =3D VIRTIO_READ_REG_4(hw, VIRTIO_PCI_HOST_FEATURES); + PMD_INIT_LOG(DEBUG, "host_features before negotiate =3D %x\n", + hw->host_features); =20 - /* Negotiate features: Subset of device feature bits are written back (gu= est feature bits) */ + /* + * Negotiate features: Subset of device feature bits are written back + * guest feature bits. + */ hw->guest_features =3D vtpci_negotiate_features(hw, guest_features); + PMD_INIT_LOG(DEBUG, "features after negotiate =3D %x\n", + hw->guest_features); } =20 #ifdef RTE_EXEC_ENV_LINUXAPP @@ -501,9 +656,13 @@ static int eth_virtio_dev_init(__rte_unused struct eth_driver *eth_drv, struct rte_eth_dev *eth_dev) { + struct virtio_net_config *config; + struct virtio_net_config local_config; + uint32_t offset_conf =3D sizeof(config->mac); struct rte_pci_device *pci_dev; struct virtio_hw *hw =3D VIRTIO_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + if (RTE_PKTMBUF_HEADROOM < sizeof(struct virtio_net_hdr) ) { PMD_INIT_LOG(ERR,=20 "MBUF HEADROOM should be enough to hold virtio net hdr\n"); @@ -561,9 +720,6 @@ eth_virtio_dev_init(__rte_unused struct eth_driver *eth= _drv, #endif hw->io_base =3D (uint32_t)(uintptr_t)pci_dev->mem_resource[0].addr; =20 - hw->max_rx_queues =3D VIRTIO_MAX_RX_QUEUES; - hw->max_tx_queues =3D VIRTIO_MAX_TX_QUEUES; - /* Reset the device although not necessary at startup */ vtpci_reset(hw); =20 @@ -573,6 +729,7 @@ eth_virtio_dev_init(__rte_unused struct eth_driver *eth= _drv, /* Tell the host we've known how to drive the device. */ vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); virtio_negotiate_features(hw); + /* Setting up rx_header size for the device */ if(vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) hw->vtnet_hdr_size =3D sizeof(struct virtio_net_hdr_mrg_rxbuf); @@ -587,6 +744,7 @@ eth_virtio_dev_init(__rte_unused struct eth_driver *eth= _drv, ETHER_ADDR_LEN); return (-ENOMEM); } + /* Copy the permanent MAC address to: virtio_hw */ virtio_get_hwaddr(hw); ether_addr_copy((struct ether_addr *) hw->mac_addr, @@ -594,19 +752,67 @@ eth_virtio_dev_init(__rte_unused struct eth_driver *e= th_drv, PMD_INIT_LOG(DEBUG, "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X\n", hw->mac_= addr[0], hw->mac_addr[1],hw->mac_addr[2], hw->mac_addr[3], hw->mac_addr[4], hw->= mac_addr[5]); =20 - if(vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) - virtio_dev_cq_queue_setup(eth_dev, SOCKET_ID_ANY); + if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) { + config =3D &local_config; + + if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { + offset_conf +=3D sizeof(config->status); + } else { + PMD_INIT_LOG(DEBUG, "VIRTIO_NET_F_STATUS is " + "not supported\n"); + config->status =3D 0; + } + + if (vtpci_with_feature(hw, VIRTIO_NET_F_MQ)) { + offset_conf +=3D sizeof(config->max_virtqueue_pairs); + } else { + PMD_INIT_LOG(DEBUG, "VIRTIO_NET_F_MQ is " + "not supported\n"); + config->max_virtqueue_pairs =3D 1; + } + + vtpci_read_dev_config(hw, 0, (uint8_t *)config, offset_conf); + + hw->max_rx_queues =3D + (VIRTIO_MAX_RX_QUEUES < config->max_virtqueue_pairs) ? + VIRTIO_MAX_RX_QUEUES : config->max_virtqueue_pairs; + hw->max_tx_queues =3D + (VIRTIO_MAX_TX_QUEUES < config->max_virtqueue_pairs) ? + VIRTIO_MAX_TX_QUEUES : config->max_virtqueue_pairs; + + virtio_dev_cq_queue_setup(eth_dev, + config->max_virtqueue_pairs * 2, + SOCKET_ID_ANY); + + PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=3D%d\n", + config->max_virtqueue_pairs); + PMD_INIT_LOG(DEBUG, "config->status=3D%d\n", config->status); + PMD_INIT_LOG(DEBUG, + "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X\n", + config->mac[0], config->mac[1], + config->mac[2], config->mac[3], + config->mac[4], config->mac[5]); + } else { + hw->max_rx_queues =3D 1; + hw->max_tx_queues =3D 1; + } + + eth_dev->data->nb_rx_queues =3D hw->max_rx_queues; + eth_dev->data->nb_tx_queues =3D hw->max_tx_queues; =20 + PMD_INIT_LOG(DEBUG, "hw->max_rx_queues=3D%d hw->max_tx_queues=3D%d\n", + hw->max_rx_queues, hw->max_tx_queues); PMD_INIT_LOG(DEBUG, "port %d vendorID=3D0x%x deviceID=3D0x%x", eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id); - return (0); + return 0; } =20 static struct eth_driver rte_virtio_pmd =3D { { .name =3D "rte_virtio_pmd", .id_table =3D pci_id_virtio_map, + .drv_flags =3D RTE_PCI_DRV_NEED_IGB_UIO, }, .eth_dev_init =3D eth_virtio_dev_init, .dev_private_size =3D sizeof(struct virtio_adapter), @@ -652,6 +858,7 @@ virtio_dev_configure(__rte_unused struct rte_eth_dev *d= ev) static int virtio_dev_start(struct rte_eth_dev *dev) { + uint16_t nb_queues, i; uint16_t status; struct virtio_hw *hw =3D VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -664,12 +871,13 @@ virtio_dev_start(struct rte_eth_dev *dev) =20 hw->adapter_stopped =3D 0; =20 + virtio_dev_cq_start(dev); + /* Do final configuration before rx/tx engine starts */ virtio_dev_rxtx_start(dev); =20 /* Check VIRTIO_NET_F_STATUS for link status*/ if(vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { - vtpci_read_dev_config(hw, offsetof(struct virtio_net_config, status), &status, sizeof(status)); @@ -686,34 +894,66 @@ virtio_dev_start(struct rte_eth_dev *dev) *Otherwise the tap backend might already stop its queue due to fullness. *vhost backend will have no chance to be waked up */ - virtqueue_notify(dev->data->rx_queues[0]); + nb_queues =3D dev->data->nb_rx_queues; + if (nb_queues > 1) { + if (virtio_set_multiple_queues(dev, nb_queues) !=3D 0) + return -EINVAL; + } + + PMD_INIT_LOG(DEBUG, "nb_queues=3D%d\n", nb_queues); + + for (i =3D 0; i < nb_queues; i++) + virtqueue_notify(dev->data->rx_queues[i]); + PMD_INIT_LOG(DEBUG, "Notified backend at initialization\n"); - return (0); + + for (i =3D 0; i < dev->data->nb_rx_queues; i++) + VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]); + + for (i =3D 0; i < dev->data->nb_tx_queues; i++) + VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]); + + return 0; } =20 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev) { - struct rte_mbuf * buf; - int i =3D 0; - PMD_INIT_LOG(DEBUG, "Before freeing rxq used and unused buf \n"); - VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[0]); - while( (buf =3D(struct rte_mbuf *)virtqueue_detatch_unused(dev->data->rx_= queues[0])) !=3D NULL) { - rte_pktmbuf_free_seg(buf); - i++; + struct rte_mbuf *buf; + int i, mbuf_num =3D 0; + for (i =3D 0; i < dev->data->nb_rx_queues; i++) { + PMD_INIT_LOG(DEBUG, "Before freeing rxq[%d] used and " + "unused buf\n", i); + VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]); + + while ((buf =3D (struct rte_mbuf *)virtqueue_detatch_unused( + dev->data->rx_queues[i])) !=3D NULL) { + rte_pktmbuf_free_seg(buf); + mbuf_num++; + } + + PMD_INIT_LOG(DEBUG, "free %d mbufs\n", mbuf_num); + PMD_INIT_LOG(DEBUG, "After freeing rxq[%d] used and " + "unused buf\n", i); + VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]); } - PMD_INIT_LOG(DEBUG, "free %d mbufs\n", i); - PMD_INIT_LOG(DEBUG, "After freeing rxq used and unused buf\n"); - VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[0]); - PMD_INIT_LOG(DEBUG, "Before freeing txq used and unused bufs\n"); - VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[0]); - i =3D 0; - while( (buf =3D (struct rte_mbuf *)virtqueue_detatch_unused(dev->data->tx= _queues[0])) !=3D NULL) { - rte_pktmbuf_free_seg(buf); - i++; + + for (i =3D 0; i < dev->data->nb_tx_queues; i++) { + PMD_INIT_LOG(DEBUG, "Before freeing txq[%d] used and " + "unused bufs\n", i); + VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]); + + mbuf_num =3D 0; + while ((buf =3D (struct rte_mbuf *)virtqueue_detatch_unused( + dev->data->tx_queues[i])) !=3D NULL) { + rte_pktmbuf_free_seg(buf); + mbuf_num++; + } + + PMD_INIT_LOG(DEBUG, "free %d mbufs\n", mbuf_num); + PMD_INIT_LOG(DEBUG, "After freeing txq[%d] used and " + "unused buf\n", i); + VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]); } - PMD_INIT_LOG(DEBUG, "free %d mbufs\n", i); - PMD_INIT_LOG(DEBUG, "After freeing txq used and unused buf\n"); - VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[0]); } =20 /* @@ -776,6 +1016,17 @@ virtio_dev_info_get(struct rte_eth_dev *dev, struct r= te_eth_dev_info *dev_info) dev_info->max_mac_addrs =3D VIRTIO_MAX_MAC_ADDRS; } =20 +/* + * It enables testpmd to collect per queue stats. + */ +static int +virtio_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *eth_de= v, +__rte_unused uint16_t queue_id, __rte_unused uint8_t stat_idx, +__rte_unused uint8_t is_rx) +{ + return 0; +} + static struct rte_driver rte_virtio_driver =3D { .type =3D PMD_PDEV, .init =3D rte_virtio_pmd_init, diff --git a/lib/librte_pmd_virtio/virtio_ethdev.h b/lib/librte_pmd_virtio/= virtio_ethdev.h index 80c9d63..da26213 100644 --- a/lib/librte_pmd_virtio/virtio_ethdev.h +++ b/lib/librte_pmd_virtio/virtio_ethdev.h @@ -49,29 +49,35 @@ #define PAGE_SIZE 4096 #endif =20 -#define VIRTIO_MAX_RX_QUEUES 1 -#define VIRTIO_MAX_TX_QUEUES 1 +#define VIRTIO_MAX_RX_QUEUES 128 +#define VIRTIO_MAX_TX_QUEUES 128 #define VIRTIO_MAX_MAC_ADDRS 1 #define VIRTIO_MIN_RX_BUFSIZE 64 #define VIRTIO_MAX_RX_PKTLEN 1518 =20 /* Features desired/implemented by this driver. */ #define VTNET_FEATURES \ - (VIRTIO_NET_F_MAC | \ - VIRTIO_NET_F_STATUS | \ - VIRTIO_NET_F_CTRL_VQ | \ - VIRTIO_NET_F_CTRL_RX | \ - VIRTIO_NET_F_CTRL_VLAN | \ - VIRTIO_NET_F_CSUM | \ - VIRTIO_NET_F_HOST_TSO4 | \ - VIRTIO_NET_F_HOST_TSO6 | \ - VIRTIO_NET_F_HOST_ECN | \ - VIRTIO_NET_F_GUEST_CSUM | \ - VIRTIO_NET_F_GUEST_TSO4 | \ - VIRTIO_NET_F_GUEST_TSO6 | \ - VIRTIO_NET_F_GUEST_ECN | \ - VIRTIO_NET_F_MRG_RXBUF | \ - VIRTIO_RING_F_INDIRECT_DESC) + (VIRTIO_NET_F_MAC | \ + VIRTIO_NET_F_STATUS | \ + VIRTIO_NET_F_MQ | \ + VIRTIO_NET_F_CTRL_VQ | \ + VIRTIO_NET_F_CTRL_RX | \ + VIRTIO_NET_F_CTRL_VLAN | \ + VIRTIO_NET_F_CSUM | \ + VIRTIO_NET_F_HOST_TSO4 | \ + VIRTIO_NET_F_HOST_TSO6 | \ + VIRTIO_NET_F_HOST_ECN | \ + VIRTIO_NET_F_GUEST_CSUM | \ + VIRTIO_NET_F_GUEST_TSO4 | \ + VIRTIO_NET_F_GUEST_TSO6 | \ + VIRTIO_NET_F_GUEST_ECN | \ + VIRTIO_NET_F_MRG_RXBUF | \ + VIRTIO_RING_F_INDIRECT_DESC) + +/* + * CQ function prototype + */ +void virtio_dev_cq_start(struct rte_eth_dev *dev); =20 /* * RX/TX function prototypes diff --git a/lib/librte_pmd_virtio/virtio_pci.h b/lib/librte_pmd_virtio/vir= tio_pci.h index f163877..96443c7 100644 --- a/lib/librte_pmd_virtio/virtio_pci.h +++ b/lib/librte_pmd_virtio/virtio_pci.h @@ -192,7 +192,9 @@ struct virtio_net_config { uint8_t mac[ETHER_ADDR_LEN]; /* See VIRTIO_NET_F_STATUS and VIRTIO_NET_S_* above */ uint16_t status; -}; + uint16_t max_virtqueue_pairs; +} __attribute__((packed)); + /* Value indicated in device config */ #define VIRTIO_PCI_FLAG_MSIX 0x0020 /* diff --git a/lib/librte_pmd_virtio/virtio_rxtx.c b/lib/librte_pmd_virtio/vi= rtio_rxtx.c index 2dd2102..1fb5a2b 100644 --- a/lib/librte_pmd_virtio/virtio_rxtx.c +++ b/lib/librte_pmd_virtio/virtio_rxtx.c @@ -82,14 +82,14 @@ virtio_dev_vring_start(struct rte_eth_dev *dev, struct = virtqueue *vq, int queue_ PMD_INIT_FUNC_TRACE(); =20 /* - * Reinitialise since virtio port might have been stopped and restarted - */ + * Reinitialise since virtio port might have been stopped and restarted + */ memset(vq->vq_ring_virt_mem, 0, vq->vq_ring_size); vring_init(vr, size, ring_mem, vq->vq_alignment); vq->vq_used_cons_idx =3D 0; vq->vq_desc_head_idx =3D 0; vq->vq_avail_idx =3D 0; - vq->vq_desc_tail_idx =3D vq->vq_nentries - 1; + vq->vq_desc_tail_idx =3D (uint16_t)(vq->vq_nentries - 1); vq->vq_free_cnt =3D vq->vq_nentries; memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries); =20 @@ -110,37 +110,61 @@ virtio_dev_vring_start(struct rte_eth_dev *dev, struc= t virtqueue *vq, int queue_ /* Only rx virtqueue needs mbufs to be allocated at initialization */ if (queue_type =3D=3D VTNET_RQ) { if (vq->mpool =3D=3D NULL) - rte_exit(EXIT_FAILURE, "Cannot allocate initial mbufs for rx virtqueue= \n"); - /* Allocate blank mbufs for the each rx descriptor */ + rte_exit(EXIT_FAILURE, + "Cannot allocate initial mbufs for rx virtqueue\n"); + + /* Allocate blank mbufs for the each rx descriptor */ nbufs =3D 0; error =3D ENOSPC; while (!virtqueue_full(vq)) { m =3D rte_rxmbuf_alloc(vq->mpool); if (m =3D=3D NULL) break; + /****************************************** * Enqueue allocated buffers * *******************************************/ error =3D virtqueue_enqueue_recv_refill(vq, m); + if (error) { rte_pktmbuf_free_seg(m); break; } nbufs++; } + vq_update_avail_idx(vq); + PMD_INIT_LOG(DEBUG, "Allocated %d bufs\n", nbufs); - VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL, VTNET_SQ_RQ_QUEUE_IDX); + + VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL, + vq->vq_queue_index); + VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN, + vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); + } else if (queue_type =3D=3D VTNET_TQ) { + VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL, + vq->vq_queue_index); VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN, vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); } else { - VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL, VTNET_SQ_TQ_QUEUE_IDX); + VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_SEL, + vq->vq_queue_index); VIRTIO_WRITE_REG_4(vq->hw, VIRTIO_PCI_QUEUE_PFN, vq->mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); } } =20 void +virtio_dev_cq_start(struct rte_eth_dev *dev) +{ + struct virtio_hw *hw + =3D VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + virtio_dev_vring_start(dev, hw->cvq, VTNET_CQ); + VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq); +} + +void virtio_dev_rxtx_start(struct rte_eth_dev *dev) { /* @@ -150,15 +174,20 @@ virtio_dev_rxtx_start(struct rte_eth_dev *dev) * - Allocate blank mbufs for the each rx descriptor * */ + int i; PMD_INIT_FUNC_TRACE(); =20 - /* Start rx vring: by default we have 1 rx virtqueue. */ - virtio_dev_vring_start(dev, dev->data->rx_queues[0], VTNET_RQ); - VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[0]); + /* Start rx vring. */ + for (i =3D 0; i < dev->data->nb_rx_queues; i++) { + virtio_dev_vring_start(dev, dev->data->rx_queues[i], VTNET_RQ); + VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]); + } =20 - /* Start tx vring: by default we have 1 tx virtqueue. */ - virtio_dev_vring_start(dev, dev->data->tx_queues[0], VTNET_TQ); - VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[0]); + /* Start tx vring. */ + for (i =3D 0; i < dev->data->nb_tx_queues; i++) { + virtio_dev_vring_start(dev, dev->data->tx_queues[i], VTNET_TQ); + VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]); + } } =20 int @@ -169,7 +198,7 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, __rte_unused const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) { - uint8_t vtpci_queue_idx =3D VTNET_SQ_RQ_QUEUE_IDX; + uint8_t vtpci_queue_idx =3D 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; struct virtqueue *vq; int ret; =20 @@ -180,6 +209,7 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, PMD_INIT_LOG(ERR, "tvq initialization failed\n"); return ret; } + /* Create mempool for rx mbuf allocation */ vq->mpool =3D mp; =20 @@ -201,7 +231,7 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, unsigned int socket_id, __rte_unused const struct rte_eth_txconf *tx_conf) { - uint8_t vtpci_queue_idx =3D VTNET_SQ_TQ_QUEUE_IDX; + uint8_t vtpci_queue_idx =3D 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; struct virtqueue *vq; int ret; =20 @@ -254,30 +284,44 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx= _pkts, uint16_t nb_pkts) num =3D (uint16_t)(likely(num <=3D VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_M= BUF_BURST_SZ); if (likely(num > DESC_PER_CACHELINE)) num =3D num - ((rxvq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); + if(num =3D=3D 0) return 0; + num =3D virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, num); PMD_RX_LOG(DEBUG, "used:%d dequeue:%d\n", nb_used, num); for (i =3D 0; i < num ; i ++) { rxm =3D rcv_pkts[i]; + PMD_RX_LOG(DEBUG, "packet len:%d\n", len[i]); - if (unlikely(len[i] < (uint32_t)hw->vtnet_hdr_size + ETHER_HDR_LEN)) { + + if (unlikely(len[i] + < (uint32_t)hw->vtnet_hdr_size + ETHER_HDR_LEN)) { PMD_RX_LOG(ERR, "Packet drop\n"); nb_enqueued++; virtio_discard_rxbuf(rxvq, rxm); hw->eth_stats.ierrors++; continue; } + rxm->pkt.in_port =3D rxvq->port_id; rxm->pkt.data =3D (char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM; rxm->pkt.nb_segs =3D 1; rxm->pkt.next =3D NULL; - rxm->pkt.pkt_len =3D (uint32_t)(len[i] - sizeof(struct virtio_net_hdr))= ; - rxm->pkt.data_len =3D (uint16_t)(len[i] - sizeof(struct virtio_net_hdr))= ; + rxm->pkt.pkt_len =3D (uint32_t)(len[i] + - sizeof(struct virtio_net_hdr)); + rxm->pkt.data_len =3D (uint16_t)(len[i] + - sizeof(struct virtio_net_hdr)); + VIRTIO_DUMP_PACKET(rxm, rxm->pkt.data_len); + rx_pkts[nb_rx++] =3D rxm; hw->eth_stats.ibytes +=3D len[i] - sizeof(struct virtio_net_hdr); + hw->eth_stats.q_ibytes[rxvq->queue_id] +=3D len[i] + - sizeof(struct virtio_net_hdr); } + hw->eth_stats.ipackets +=3D nb_rx; + hw->eth_stats.q_ipackets[rxvq->queue_id] +=3D nb_rx; =20 /* Allocate new mbuf for the used descriptor */ error =3D ENOSPC; @@ -294,14 +338,15 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx= _pkts, uint16_t nb_pkts) } nb_enqueued ++; } - if(likely(nb_enqueued)) { - if(unlikely(virtqueue_kick_prepare(rxvq))) { + if (likely(nb_enqueued)) { + if (unlikely(virtqueue_kick_prepare(rxvq))) { virtqueue_notify(rxvq); PMD_RX_LOG(DEBUG, "Notified\n"); } } + vq_update_avail_idx(rxvq); -=09 + return (nb_rx); } =20 @@ -332,6 +377,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_p= kts, uint16_t nb_pkts) virtqueue_dequeue_pkt_tx(txvq); num--; } + if(!virtqueue_full(txvq)) { txm =3D tx_pkts[nb_tx]; /* Enqueue Packet buffers */ @@ -347,6 +393,8 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_p= kts, uint16_t nb_pkts) } nb_tx++; hw->eth_stats.obytes +=3D txm->pkt.data_len; + hw->eth_stats.q_obytes[txvq->queue_id] + +=3D txm->pkt.data_len; } else { PMD_TX_LOG(ERR, "No free tx descriptors to transmit\n"); break; @@ -355,10 +403,12 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx= _pkts, uint16_t nb_pkts) vq_update_avail_idx(txvq); =20 hw->eth_stats.opackets +=3D nb_tx; + hw->eth_stats.q_opackets[txvq->queue_id] +=3D nb_tx; =20 if(unlikely(virtqueue_kick_prepare(txvq))) { virtqueue_notify(txvq); PMD_TX_LOG(DEBUG, "Notified backend after xmit\n"); } + return (nb_tx); } diff --git a/lib/librte_pmd_virtio/virtqueue.h b/lib/librte_pmd_virtio/virt= queue.h index b67c223..7684395 100644 --- a/lib/librte_pmd_virtio/virtqueue.h +++ b/lib/librte_pmd_virtio/virtqueue.h @@ -103,6 +103,24 @@ enum { VTNET_RQ =3D 0, VTNET_TQ =3D 1, VTNET_CQ =3D 2 = }; #define VIRTIO_NET_CTRL_VLAN_ADD 0 #define VIRTIO_NET_CTRL_VLAN_DEL 1 =20 +struct virtio_net_ctrl_hdr { + uint8_t class; + uint8_t cmd; +} __attribute__((packed)); + +typedef uint8_t virtio_net_ctrl_ack; + +#define VIRTIO_NET_OK 0 +#define VIRTIO_NET_ERR 1 + +#define VIRTIO_MAX_CTRL_DATA 128 + +struct virtio_pmd_ctrl { + struct virtio_net_ctrl_hdr hdr; + virtio_net_ctrl_ack status; + uint8_t data[VIRTIO_MAX_CTRL_DATA]; +}; + struct virtqueue { char vq_name[VIRTQUEUE_MAX_NAME_SZ]; struct virtio_hw *hw; /**< virtio_hw structure pointer. */ @@ -142,6 +160,16 @@ struct virtqueue { } vq_descx[0]; }; =20 +/* If multiqueue is provided by host, then we suppport it. */ +#ifndef VIRTIO_NET_F_MQ +/* Device supports Receive Flow Steering */ +#define VIRTIO_NET_F_MQ 0x400000 +#define VIRTIO_NET_CTRL_MQ 4 +#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 +#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 +#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 +#endif + /** * This is the first element of the scatter-gather list. If you don't * specify GSO or CSUM features, you can simply ignore the header. @@ -204,9 +232,10 @@ vq_update_avail_ring(struct virtqueue *vq, uint16_t de= sc_idx) uint16_t avail_idx; /* * Place the head of the descriptor chain into the next slot and make - * it usable to the host. We wait to inform the host until after the burs= t=20 - * is complete to avoid cache alignment issues with descriptors. This=20 - * also helps to avoid any contention on the available index. + * it usable to the host. The chain is made available now rather than + * deferring to virtqueue_notify() in the hopes that if the host is + * currently running on another CPU, we can keep it processing the new + * descriptor. */ avail_idx =3D (uint16_t)(vq->vq_avail_idx & (vq->vq_nentries - 1)); vq->vq_ring.avail->ring[avail_idx] =3D desc_idx; @@ -242,7 +271,7 @@ vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_= idx) vq->vq_free_cnt =3D (uint16_t)(vq->vq_free_cnt + dxp->ndescs); if ((dp->flags & VRING_DESC_F_INDIRECT) =3D=3D 0) { while (dp->flags & VRING_DESC_F_NEXT) { - desc_idx_last =3D dp->next;=20 + desc_idx_last =3D dp->next; dp =3D &vq->vq_ring.desc[dp->next]; } } @@ -259,6 +288,7 @@ vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_= idx) dp_tail =3D &vq->vq_ring.desc[vq->vq_desc_tail_idx]; dp_tail->next =3D desc_idx; } + vq->vq_desc_tail_idx =3D desc_idx_last; dp->next =3D VQ_RING_DESC_CHAIN_END; } @@ -294,7 +324,7 @@ virtqueue_enqueue_recv_refill(struct virtqueue *vq, str= uct rte_mbuf *cookie) idx =3D start_dp[idx].next; vq->vq_desc_head_idx =3D idx; if (vq->vq_desc_head_idx =3D=3D VQ_RING_DESC_CHAIN_END) - vq->vq_desc_tail_idx =3D idx;=20 + vq->vq_desc_tail_idx =3D idx; vq->vq_free_cnt =3D (uint16_t)(vq->vq_free_cnt - needed); vq_update_avail_ring(vq, head_idx); =20 @@ -335,7 +365,7 @@ virtqueue_enqueue_xmit(struct virtqueue *txvq, struct r= te_mbuf *cookie) idx =3D start_dp[idx].next; txvq->vq_desc_head_idx =3D idx; if (txvq->vq_desc_head_idx =3D=3D VQ_RING_DESC_CHAIN_END) - txvq->vq_desc_tail_idx =3D idx;=20 + txvq->vq_desc_tail_idx =3D idx; txvq->vq_free_cnt =3D (uint16_t)(txvq->vq_free_cnt - needed); vq_update_avail_ring(txvq, head_idx); =20 @@ -357,11 +387,13 @@ virtqueue_dequeue_burst_rx(struct virtqueue *vq, stru= ct rte_mbuf **rx_pkts, uint desc_idx =3D (uint16_t) uep->id; len[i] =3D uep->len; cookie =3D (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; + if (unlikely(cookie =3D=3D NULL)) { PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n",=20 vq->vq_used_cons_idx); break; } + rte_prefetch0(cookie); rte_packet_prefetch(cookie->pkt.data); rx_pkts[i] =3D cookie; @@ -369,22 +401,23 @@ virtqueue_dequeue_burst_rx(struct virtqueue *vq, stru= ct rte_mbuf **rx_pkts, uint vq_ring_free_chain(vq, desc_idx); vq->vq_descx[desc_idx].cookie =3D NULL; } + return (i); } =20 static inline uint16_t __attribute__((always_inline)) virtqueue_dequeue_pkt_tx(struct virtqueue *vq) { - struct vring_used_elem *uep; - uint16_t used_idx, desc_idx; + struct vring_used_elem *uep; + uint16_t used_idx, desc_idx; =20 - used_idx =3D (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - = 1)); - uep =3D &vq->vq_ring.used->ring[used_idx]; - desc_idx =3D (uint16_t) uep->id; - vq->vq_used_cons_idx++; - vq_ring_free_chain(vq, desc_idx); + used_idx =3D (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); + uep =3D &vq->vq_ring.used->ring[used_idx]; + desc_idx =3D (uint16_t) uep->id; + vq->vq_used_cons_idx++; + vq_ring_free_chain(vq, desc_idx); =20 - return 0; + return 0; } =20 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP --=20 1.9.0