[dpdk-dev] [PATCH 3/8] virtio: deinline some code

[Stephen Hemminger <stephen@networkplumber.org>]

[-- Attachment #1: virtio-deinline.patch --]
[-- Type: text/plain, Size: 11524 bytes --]

This driver has lots of functions marked always inline which is actually
counterproductive with modern compilers. Better to move the functions to
the one file they are used (proper scope) and let compiler decide.

For trivial functions leave them as static inline.

Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>

---
 lib/librte_pmd_virtio/virtio_rxtx.c |  160 +++++++++++++++++++++++++++++++++
 lib/librte_pmd_virtio/virtqueue.h   |  170 ------------------------------------
 2 files changed, 164 insertions(+), 166 deletions(-)

--- a/lib/librte_pmd_virtio/virtio_rxtx.c	2014-06-13 17:52:48.231990433 -0700
+++ b/lib/librte_pmd_virtio/virtio_rxtx.c	2014-06-13 17:52:48.227990426 -0700
@@ -60,6 +60,166 @@
 #define  VIRTIO_DUMP_PACKET(m, len) do { } while (0)
 #endif
 
+static void
+vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
+{
+	struct vring_desc *dp, *dp_tail;
+	struct vq_desc_extra *dxp;
+	uint16_t desc_idx_last = desc_idx;
+
+	dp  = &vq->vq_ring.desc[desc_idx];
+	dxp = &vq->vq_descx[desc_idx];
+	vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);
+	if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
+		while (dp->flags & VRING_DESC_F_NEXT) {
+			desc_idx_last = dp->next;
+			dp = &vq->vq_ring.desc[dp->next];
+		}
+	}
+	dxp->ndescs = 0;
+
+	/*
+	 * We must append the existing free chain, if any, to the end of
+	 * newly freed chain. If the virtqueue was completely used, then
+	 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
+	 */
+	if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {
+		vq->vq_desc_head_idx = desc_idx;
+	} else {
+		dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx];
+		dp_tail->next = desc_idx;
+	}
+
+	vq->vq_desc_tail_idx = desc_idx_last;
+	dp->next = VQ_RING_DESC_CHAIN_END;
+}
+
+static uint16_t
+virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
+			   uint32_t *len, uint16_t num)
+{
+	struct vring_used_elem *uep;
+	struct rte_mbuf *cookie;
+	uint16_t used_idx, desc_idx;
+	uint16_t i;
+
+	/*  Caller does the check */
+	for (i = 0; i < num ; i++) {
+		used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
+		uep = &vq->vq_ring.used->ring[used_idx];
+		desc_idx = (uint16_t) uep->id;
+		len[i] = uep->len;
+		cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie;
+
+		if (unlikely(cookie == NULL)) {
+			PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n",
+				vq->vq_used_cons_idx);
+			break;
+		}
+
+		rte_prefetch0(cookie);
+		rte_packet_prefetch(cookie->pkt.data);
+		rx_pkts[i]  = cookie;
+		vq->vq_used_cons_idx++;
+		vq_ring_free_chain(vq, desc_idx);
+		vq->vq_descx[desc_idx].cookie = NULL;
+	}
+
+	return i;
+}
+
+static void
+virtqueue_dequeue_pkt_tx(struct virtqueue *vq)
+{
+	struct vring_used_elem *uep;
+	uint16_t used_idx, desc_idx;
+
+	used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
+	uep = &vq->vq_ring.used->ring[used_idx];
+	desc_idx = (uint16_t) uep->id;
+	vq->vq_used_cons_idx++;
+	vq_ring_free_chain(vq, desc_idx);
+}
+
+
+static inline int
+virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
+{
+	struct vq_desc_extra *dxp;
+	struct vring_desc *start_dp;
+	uint16_t needed = 1;
+	uint16_t head_idx, idx;
+
+	if (unlikely(vq->vq_free_cnt == 0))
+		return -ENOSPC;
+	if (unlikely(vq->vq_free_cnt < needed))
+		return -EMSGSIZE;
+
+	head_idx = vq->vq_desc_head_idx;
+	if (unlikely(head_idx >= vq->vq_nentries))
+		return -EFAULT;
+
+	idx = head_idx;
+	dxp = &vq->vq_descx[idx];
+	dxp->cookie = (void *)cookie;
+	dxp->ndescs = needed;
+
+	start_dp = vq->vq_ring.desc;
+	start_dp[idx].addr  =
+		(uint64_t) (cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM - sizeof(struct virtio_net_hdr));
+	start_dp[idx].len   = cookie->buf_len - RTE_PKTMBUF_HEADROOM + sizeof(struct virtio_net_hdr);
+	start_dp[idx].flags =  VRING_DESC_F_WRITE;
+	idx = start_dp[idx].next;
+	vq->vq_desc_head_idx = idx;
+	if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
+		vq->vq_desc_tail_idx = idx;
+	vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
+	vq_update_avail_ring(vq, head_idx);
+
+	return 0;
+}
+
+static int
+virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie)
+{
+	struct vq_desc_extra *dxp;
+	struct vring_desc *start_dp;
+	uint16_t needed = 2;
+	uint16_t head_idx, idx;
+
+	if (unlikely(txvq->vq_free_cnt == 0))
+		return -ENOSPC;
+	if (unlikely(txvq->vq_free_cnt < needed))
+		return -EMSGSIZE;
+	head_idx = txvq->vq_desc_head_idx;
+	if (unlikely(head_idx >= txvq->vq_nentries))
+		return -EFAULT;
+
+	idx = head_idx;
+	dxp = &txvq->vq_descx[idx];
+	if (dxp->cookie != NULL)
+		rte_pktmbuf_free_seg(dxp->cookie);
+	dxp->cookie = (void *)cookie;
+	dxp->ndescs = needed;
+
+	start_dp = txvq->vq_ring.desc;
+	start_dp[idx].addr  = (uint64_t)(uintptr_t)txvq->virtio_net_hdr_mem + idx * sizeof(struct virtio_net_hdr);
+	start_dp[idx].len   = sizeof(struct virtio_net_hdr);
+	start_dp[idx].flags = VRING_DESC_F_NEXT;
+	idx = start_dp[idx].next;
+	start_dp[idx].addr  = RTE_MBUF_DATA_DMA_ADDR(cookie);
+	start_dp[idx].len   = cookie->pkt.data_len;
+	start_dp[idx].flags = 0;
+	idx = start_dp[idx].next;
+	txvq->vq_desc_head_idx = idx;
+	if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
+		txvq->vq_desc_tail_idx = idx;
+	txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed);
+	vq_update_avail_ring(txvq, head_idx);
+
+	return 0;
+}
+
 static inline struct rte_mbuf *
 rte_rxmbuf_alloc(struct rte_mempool *mp)
 {
--- a/lib/librte_pmd_virtio/virtqueue.h	2014-06-13 17:52:48.231990433 -0700
+++ b/lib/librte_pmd_virtio/virtqueue.h	2014-06-13 17:53:20.764051470 -0700
@@ -224,14 +224,14 @@ virtqueue_full(const struct virtqueue *v
 
 #define VIRTQUEUE_NUSED(vq) ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx))
 
-static inline void __attribute__((always_inline))
+static inline void
 vq_update_avail_idx(struct virtqueue *vq)
 {
 	rte_compiler_barrier();
 	vq->vq_ring.avail->idx = vq->vq_avail_idx;
 }
 
-static inline void __attribute__((always_inline))
+static inline void
 vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx)
 {
 	uint16_t avail_idx;
@@ -247,13 +247,13 @@ vq_update_avail_ring(struct virtqueue *v
 	vq->vq_avail_idx++;
 }
 
-static inline int __attribute__((always_inline))
+static inline int
 virtqueue_kick_prepare(struct virtqueue *vq)
 {
 	return !(vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY);
 }
 
-static inline void __attribute__((always_inline))
+static inline void
 virtqueue_notify(struct virtqueue *vq)
 {
 	/*
@@ -264,166 +264,6 @@ virtqueue_notify(struct virtqueue *vq)
 	VIRTIO_WRITE_REG_2(vq->hw, VIRTIO_PCI_QUEUE_NOTIFY, vq->vq_queue_index);
 }
 
-static inline void __attribute__((always_inline))
-vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
-{
-	struct vring_desc *dp, *dp_tail;
-	struct vq_desc_extra *dxp;
-	uint16_t desc_idx_last = desc_idx;
-
-	dp  = &vq->vq_ring.desc[desc_idx];
-	dxp = &vq->vq_descx[desc_idx];
-	vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);
-	if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
-		while (dp->flags & VRING_DESC_F_NEXT) {
-			desc_idx_last = dp->next;
-			dp = &vq->vq_ring.desc[dp->next];
-		}
-	}
-	dxp->ndescs = 0;
-
-	/*
-	 * We must append the existing free chain, if any, to the end of
-	 * newly freed chain. If the virtqueue was completely used, then
-	 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
-	 */
-	if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {
-		vq->vq_desc_head_idx = desc_idx;
-	} else {
-		dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx];
-		dp_tail->next = desc_idx;
-	}
-
-	vq->vq_desc_tail_idx = desc_idx_last;
-	dp->next = VQ_RING_DESC_CHAIN_END;
-}
-
-static inline int __attribute__((always_inline))
-virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
-{
-	struct vq_desc_extra *dxp;
-	struct vring_desc *start_dp;
-	uint16_t needed = 1;
-	uint16_t head_idx, idx;
-
-	if (unlikely(vq->vq_free_cnt == 0))
-		return -ENOSPC;
-	if (unlikely(vq->vq_free_cnt < needed))
-		return -EMSGSIZE;
-
-	head_idx = vq->vq_desc_head_idx;
-	if (unlikely(head_idx >= vq->vq_nentries))
-		return -EFAULT;
-
-	idx = head_idx;
-	dxp = &vq->vq_descx[idx];
-	dxp->cookie = (void *)cookie;
-	dxp->ndescs = needed;
-
-	start_dp = vq->vq_ring.desc;
-	start_dp[idx].addr  =
-		(uint64_t) (cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM - sizeof(struct virtio_net_hdr));
-	start_dp[idx].len   = cookie->buf_len - RTE_PKTMBUF_HEADROOM + sizeof(struct virtio_net_hdr);
-	start_dp[idx].flags =  VRING_DESC_F_WRITE;
-	idx = start_dp[idx].next;
-	vq->vq_desc_head_idx = idx;
-	if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
-		vq->vq_desc_tail_idx = idx;
-	vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
-	vq_update_avail_ring(vq, head_idx);
-
-	return 0;
-}
-
-static inline int __attribute__((always_inline))
-virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie)
-{
-	struct vq_desc_extra *dxp;
-	struct vring_desc *start_dp;
-	uint16_t needed = 2;
-	uint16_t head_idx, idx;
-
-	if (unlikely(txvq->vq_free_cnt == 0))
-		return -ENOSPC;
-	if (unlikely(txvq->vq_free_cnt < needed))
-		return -EMSGSIZE;
-	head_idx = txvq->vq_desc_head_idx;
-	if (unlikely(head_idx >= txvq->vq_nentries))
-		return -EFAULT;
-
-	idx = head_idx;
-	dxp = &txvq->vq_descx[idx];
-	if (dxp->cookie != NULL)
-		rte_pktmbuf_free_seg(dxp->cookie);
-	dxp->cookie = (void *)cookie;
-	dxp->ndescs = needed;
-
-	start_dp = txvq->vq_ring.desc;
-	start_dp[idx].addr  = (uint64_t)(uintptr_t)txvq->virtio_net_hdr_mem + idx * sizeof(struct virtio_net_hdr);
-	start_dp[idx].len   = sizeof(struct virtio_net_hdr);
-	start_dp[idx].flags = VRING_DESC_F_NEXT;
-	idx = start_dp[idx].next;
-	start_dp[idx].addr  = RTE_MBUF_DATA_DMA_ADDR(cookie);
-	start_dp[idx].len   = cookie->pkt.data_len;
-	start_dp[idx].flags = 0;
-	idx = start_dp[idx].next;
-	txvq->vq_desc_head_idx = idx;
-	if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
-		txvq->vq_desc_tail_idx = idx;
-	txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed);
-	vq_update_avail_ring(txvq, head_idx);
-
-	return 0;
-}
-
-static inline uint16_t __attribute__((always_inline))
-virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, uint32_t *len, uint16_t num)
-{
-	struct vring_used_elem *uep;
-	struct rte_mbuf *cookie;
-	uint16_t used_idx, desc_idx;
-	uint16_t i;
-
-	/*  Caller does the check */
-	for (i = 0; i < num; i++) {
-		used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
-		uep = &vq->vq_ring.used->ring[used_idx];
-		desc_idx = (uint16_t) uep->id;
-		len[i] = uep->len;
-		cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie;
-
-		if (unlikely(cookie == NULL)) {
-			PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
-				vq->vq_used_cons_idx);
-			break;
-		}
-
-		rte_prefetch0(cookie);
-		rte_packet_prefetch(cookie->pkt.data);
-		rx_pkts[i]  = cookie;
-		vq->vq_used_cons_idx++;
-		vq_ring_free_chain(vq, desc_idx);
-		vq->vq_descx[desc_idx].cookie = NULL;
-	}
-
-	return i;
-}
-
-static inline uint16_t __attribute__((always_inline))
-virtqueue_dequeue_pkt_tx(struct virtqueue *vq)
-{
-	struct vring_used_elem *uep;
-	uint16_t used_idx, desc_idx;
-
-	used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
-	uep = &vq->vq_ring.used->ring[used_idx];
-	desc_idx = (uint16_t) uep->id;
-	vq->vq_used_cons_idx++;
-	vq_ring_free_chain(vq, desc_idx);
-
-	return 0;
-}
-
 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
 #define VIRTQUEUE_DUMP(vq) do { \
 	uint16_t used_idx, nused; \