From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-pd0-f176.google.com (mail-pd0-f176.google.com [209.85.192.176]) by dpdk.org (Postfix) with ESMTP id E61936AB7 for ; Sat, 14 Jun 2014 03:08:35 +0200 (CEST) Received: by mail-pd0-f176.google.com with SMTP id p10so2603918pdj.35 for ; Fri, 13 Jun 2014 18:08:51 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20130820; h=x-gm-message-state:message-id:user-agent:date:from:to:cc:subject :references:mime-version:content-type:content-disposition; bh=T25H/MMa6Z1buxPLgQhHDZcHUol+w96Lc/57fN72+CM=; b=ZLghm5qlf7+sy8yqTI4w+I5djbvDyNEhhKmH2XKyVG/cIAf756WGBJ3ZWL2vwgOOFJ KUjdrNGcCwbJfPyRgyjGfJdP+o5V0RbQvkDOWjiwgaJZE6tjuGXXgns849LMgk3uE8td jz88kg5mWLqx2Wfo/naDXzp7IBcp2Ysu5q9sMU632MppEfRqBX+6vUi+I0xqqKZTj0Ia LZGDjLNwMLtFbQBjpUQ7N2B7rwSbt/0dCrmYC3SOvsEFdSj9cOb9pZXwq/ygjKaJeKky QVPTysCk/LCx1D6tAGfieHMiRatapAtMdLnFnkUwwV33dCEGIR8lgpwJe54QYvQgObzs gU5A== X-Gm-Message-State: ALoCoQl8zjIm0QjKXHSYTaD1IUVCSqjG1dQqwqG0ULKYKFLwwdn64yn3KXVpMgCrnvj+baw0fsRx X-Received: by 10.68.237.228 with SMTP id vf4mr7252797pbc.131.1402708131162; Fri, 13 Jun 2014 18:08:51 -0700 (PDT) Received: from localhost (static-50-53-83-51.bvtn.or.frontiernet.net. [50.53.83.51]) by mx.google.com with ESMTPSA id pb7sm28964290pac.10.2014.06.13.18.08.49 for (version=TLSv1.2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Fri, 13 Jun 2014 18:08:50 -0700 (PDT) Message-Id: <20140614010840.916626751@networkplumber.org> User-Agent: quilt/0.63-1 Date: Fri, 13 Jun 2014 18:06:20 -0700 From: Stephen Hemminger To: dev@dpdk.org References: <20140614010617.902738763@networkplumber.org> MIME-Version: 1.0 Content-Type: text/plain; charset=ISO-8859-15 Content-Disposition: inline; filename=virtio-deinline.patch Subject: [dpdk-dev] [PATCH 3/8] virtio: deinline some code X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 14 Jun 2014 01:08:36 -0000 This driver has lots of functions marked always inline which is actually counterproductive with modern compilers. Better to move the functions to the one file they are used (proper scope) and let compiler decide. For trivial functions leave them as static inline. Signed-off-by: Stephen Hemminger --- lib/librte_pmd_virtio/virtio_rxtx.c | 160 +++++++++++++++++++++++++++++++++ lib/librte_pmd_virtio/virtqueue.h | 170 ------------------------------------ 2 files changed, 164 insertions(+), 166 deletions(-) --- a/lib/librte_pmd_virtio/virtio_rxtx.c 2014-06-13 17:52:48.231990433 -0700 +++ b/lib/librte_pmd_virtio/virtio_rxtx.c 2014-06-13 17:52:48.227990426 -0700 @@ -60,6 +60,166 @@ #define VIRTIO_DUMP_PACKET(m, len) do { } while (0) #endif +static void +vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) +{ + struct vring_desc *dp, *dp_tail; + struct vq_desc_extra *dxp; + uint16_t desc_idx_last = desc_idx; + + dp = &vq->vq_ring.desc[desc_idx]; + dxp = &vq->vq_descx[desc_idx]; + vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs); + if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { + while (dp->flags & VRING_DESC_F_NEXT) { + desc_idx_last = dp->next; + dp = &vq->vq_ring.desc[dp->next]; + } + } + dxp->ndescs = 0; + + /* + * We must append the existing free chain, if any, to the end of + * newly freed chain. If the virtqueue was completely used, then + * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). + */ + if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) { + vq->vq_desc_head_idx = desc_idx; + } else { + dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx]; + dp_tail->next = desc_idx; + } + + vq->vq_desc_tail_idx = desc_idx_last; + dp->next = VQ_RING_DESC_CHAIN_END; +} + +static uint16_t +virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, + uint32_t *len, uint16_t num) +{ + struct vring_used_elem *uep; + struct rte_mbuf *cookie; + uint16_t used_idx, desc_idx; + uint16_t i; + + /* Caller does the check */ + for (i = 0; i < num ; i++) { + used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); + uep = &vq->vq_ring.used->ring[used_idx]; + desc_idx = (uint16_t) uep->id; + len[i] = uep->len; + cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; + + if (unlikely(cookie == NULL)) { + PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n", + vq->vq_used_cons_idx); + break; + } + + rte_prefetch0(cookie); + rte_packet_prefetch(cookie->pkt.data); + rx_pkts[i] = cookie; + vq->vq_used_cons_idx++; + vq_ring_free_chain(vq, desc_idx); + vq->vq_descx[desc_idx].cookie = NULL; + } + + return i; +} + +static void +virtqueue_dequeue_pkt_tx(struct virtqueue *vq) +{ + struct vring_used_elem *uep; + uint16_t used_idx, desc_idx; + + used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); + uep = &vq->vq_ring.used->ring[used_idx]; + desc_idx = (uint16_t) uep->id; + vq->vq_used_cons_idx++; + vq_ring_free_chain(vq, desc_idx); +} + + +static inline int +virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie) +{ + struct vq_desc_extra *dxp; + struct vring_desc *start_dp; + uint16_t needed = 1; + uint16_t head_idx, idx; + + if (unlikely(vq->vq_free_cnt == 0)) + return -ENOSPC; + if (unlikely(vq->vq_free_cnt < needed)) + return -EMSGSIZE; + + head_idx = vq->vq_desc_head_idx; + if (unlikely(head_idx >= vq->vq_nentries)) + return -EFAULT; + + idx = head_idx; + dxp = &vq->vq_descx[idx]; + dxp->cookie = (void *)cookie; + dxp->ndescs = needed; + + start_dp = vq->vq_ring.desc; + start_dp[idx].addr = + (uint64_t) (cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM - sizeof(struct virtio_net_hdr)); + start_dp[idx].len = cookie->buf_len - RTE_PKTMBUF_HEADROOM + sizeof(struct virtio_net_hdr); + start_dp[idx].flags = VRING_DESC_F_WRITE; + idx = start_dp[idx].next; + vq->vq_desc_head_idx = idx; + if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) + vq->vq_desc_tail_idx = idx; + vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); + vq_update_avail_ring(vq, head_idx); + + return 0; +} + +static int +virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie) +{ + struct vq_desc_extra *dxp; + struct vring_desc *start_dp; + uint16_t needed = 2; + uint16_t head_idx, idx; + + if (unlikely(txvq->vq_free_cnt == 0)) + return -ENOSPC; + if (unlikely(txvq->vq_free_cnt < needed)) + return -EMSGSIZE; + head_idx = txvq->vq_desc_head_idx; + if (unlikely(head_idx >= txvq->vq_nentries)) + return -EFAULT; + + idx = head_idx; + dxp = &txvq->vq_descx[idx]; + if (dxp->cookie != NULL) + rte_pktmbuf_free_seg(dxp->cookie); + dxp->cookie = (void *)cookie; + dxp->ndescs = needed; + + start_dp = txvq->vq_ring.desc; + start_dp[idx].addr = (uint64_t)(uintptr_t)txvq->virtio_net_hdr_mem + idx * sizeof(struct virtio_net_hdr); + start_dp[idx].len = sizeof(struct virtio_net_hdr); + start_dp[idx].flags = VRING_DESC_F_NEXT; + idx = start_dp[idx].next; + start_dp[idx].addr = RTE_MBUF_DATA_DMA_ADDR(cookie); + start_dp[idx].len = cookie->pkt.data_len; + start_dp[idx].flags = 0; + idx = start_dp[idx].next; + txvq->vq_desc_head_idx = idx; + if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) + txvq->vq_desc_tail_idx = idx; + txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed); + vq_update_avail_ring(txvq, head_idx); + + return 0; +} + static inline struct rte_mbuf * rte_rxmbuf_alloc(struct rte_mempool *mp) { --- a/lib/librte_pmd_virtio/virtqueue.h 2014-06-13 17:52:48.231990433 -0700 +++ b/lib/librte_pmd_virtio/virtqueue.h 2014-06-13 17:53:20.764051470 -0700 @@ -224,14 +224,14 @@ virtqueue_full(const struct virtqueue *v #define VIRTQUEUE_NUSED(vq) ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx)) -static inline void __attribute__((always_inline)) +static inline void vq_update_avail_idx(struct virtqueue *vq) { rte_compiler_barrier(); vq->vq_ring.avail->idx = vq->vq_avail_idx; } -static inline void __attribute__((always_inline)) +static inline void vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx) { uint16_t avail_idx; @@ -247,13 +247,13 @@ vq_update_avail_ring(struct virtqueue *v vq->vq_avail_idx++; } -static inline int __attribute__((always_inline)) +static inline int virtqueue_kick_prepare(struct virtqueue *vq) { return !(vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY); } -static inline void __attribute__((always_inline)) +static inline void virtqueue_notify(struct virtqueue *vq) { /* @@ -264,166 +264,6 @@ virtqueue_notify(struct virtqueue *vq) VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_NOTIFY, vq->vq_queue_index); } -static inline void __attribute__((always_inline)) -vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) -{ - struct vring_desc *dp, *dp_tail; - struct vq_desc_extra *dxp; - uint16_t desc_idx_last = desc_idx; - - dp = &vq->vq_ring.desc[desc_idx]; - dxp = &vq->vq_descx[desc_idx]; - vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs); - if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { - while (dp->flags & VRING_DESC_F_NEXT) { - desc_idx_last = dp->next; - dp = &vq->vq_ring.desc[dp->next]; - } - } - dxp->ndescs = 0; - - /* - * We must append the existing free chain, if any, to the end of - * newly freed chain. If the virtqueue was completely used, then - * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). - */ - if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) { - vq->vq_desc_head_idx = desc_idx; - } else { - dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx]; - dp_tail->next = desc_idx; - } - - vq->vq_desc_tail_idx = desc_idx_last; - dp->next = VQ_RING_DESC_CHAIN_END; -} - -static inline int __attribute__((always_inline)) -virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie) -{ - struct vq_desc_extra *dxp; - struct vring_desc *start_dp; - uint16_t needed = 1; - uint16_t head_idx, idx; - - if (unlikely(vq->vq_free_cnt == 0)) - return -ENOSPC; - if (unlikely(vq->vq_free_cnt < needed)) - return -EMSGSIZE; - - head_idx = vq->vq_desc_head_idx; - if (unlikely(head_idx >= vq->vq_nentries)) - return -EFAULT; - - idx = head_idx; - dxp = &vq->vq_descx[idx]; - dxp->cookie = (void *)cookie; - dxp->ndescs = needed; - - start_dp = vq->vq_ring.desc; - start_dp[idx].addr = - (uint64_t) (cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM - sizeof(struct virtio_net_hdr)); - start_dp[idx].len = cookie->buf_len - RTE_PKTMBUF_HEADROOM + sizeof(struct virtio_net_hdr); - start_dp[idx].flags = VRING_DESC_F_WRITE; - idx = start_dp[idx].next; - vq->vq_desc_head_idx = idx; - if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) - vq->vq_desc_tail_idx = idx; - vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); - vq_update_avail_ring(vq, head_idx); - - return 0; -} - -static inline int __attribute__((always_inline)) -virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie) -{ - struct vq_desc_extra *dxp; - struct vring_desc *start_dp; - uint16_t needed = 2; - uint16_t head_idx, idx; - - if (unlikely(txvq->vq_free_cnt == 0)) - return -ENOSPC; - if (unlikely(txvq->vq_free_cnt < needed)) - return -EMSGSIZE; - head_idx = txvq->vq_desc_head_idx; - if (unlikely(head_idx >= txvq->vq_nentries)) - return -EFAULT; - - idx = head_idx; - dxp = &txvq->vq_descx[idx]; - if (dxp->cookie != NULL) - rte_pktmbuf_free_seg(dxp->cookie); - dxp->cookie = (void *)cookie; - dxp->ndescs = needed; - - start_dp = txvq->vq_ring.desc; - start_dp[idx].addr = (uint64_t)(uintptr_t)txvq->virtio_net_hdr_mem + idx * sizeof(struct virtio_net_hdr); - start_dp[idx].len = sizeof(struct virtio_net_hdr); - start_dp[idx].flags = VRING_DESC_F_NEXT; - idx = start_dp[idx].next; - start_dp[idx].addr = RTE_MBUF_DATA_DMA_ADDR(cookie); - start_dp[idx].len = cookie->pkt.data_len; - start_dp[idx].flags = 0; - idx = start_dp[idx].next; - txvq->vq_desc_head_idx = idx; - if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) - txvq->vq_desc_tail_idx = idx; - txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed); - vq_update_avail_ring(txvq, head_idx); - - return 0; -} - -static inline uint16_t __attribute__((always_inline)) -virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, uint32_t *len, uint16_t num) -{ - struct vring_used_elem *uep; - struct rte_mbuf *cookie; - uint16_t used_idx, desc_idx; - uint16_t i; - - /* Caller does the check */ - for (i = 0; i < num; i++) { - used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); - uep = &vq->vq_ring.used->ring[used_idx]; - desc_idx = (uint16_t) uep->id; - len[i] = uep->len; - cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; - - if (unlikely(cookie == NULL)) { - PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", - vq->vq_used_cons_idx); - break; - } - - rte_prefetch0(cookie); - rte_packet_prefetch(cookie->pkt.data); - rx_pkts[i] = cookie; - vq->vq_used_cons_idx++; - vq_ring_free_chain(vq, desc_idx); - vq->vq_descx[desc_idx].cookie = NULL; - } - - return i; -} - -static inline uint16_t __attribute__((always_inline)) -virtqueue_dequeue_pkt_tx(struct virtqueue *vq) -{ - struct vring_used_elem *uep; - uint16_t used_idx, desc_idx; - - used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); - uep = &vq->vq_ring.used->ring[used_idx]; - desc_idx = (uint16_t) uep->id; - vq->vq_used_cons_idx++; - vq_ring_free_chain(vq, desc_idx); - - return 0; -} - #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP #define VIRTQUEUE_DUMP(vq) do { \ uint16_t used_idx, nused; \