DPDK patches and discussions
 help / color / mirror / Atom feed
From: Bruce Richardson <bruce.richardson@intel.com>
To: dev@dpdk.org
Subject: [dpdk-dev] [PATCH v2 12/13] ixgbe: Fix perf regression due to moved pool ptr
Date: Thu, 11 Sep 2014 14:15:46 +0100
Message-ID: <1410441347-22840-13-git-send-email-bruce.richardson@intel.com> (raw)
In-Reply-To: <1410441347-22840-1-git-send-email-bruce.richardson@intel.com>

Adjust the fast-path code to fix the regression caused by the pool
pointer moving to the second cache line. This change adjusts the
prefetching and also the way in which the mbufs are freed back to the
mempool.
Note: slow-path e.g. path supporting jumbo frames, is still slower, but
is dealt with by a later commit

Updates in V2:
* fixup checkpatch issue

Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
---
 lib/librte_pmd_ixgbe/ixgbe_rxtx.c     |  8 ++-
 lib/librte_pmd_ixgbe/ixgbe_rxtx.h     | 14 +-----
 lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c | 92 +++++++++++++----------------------
 3 files changed, 38 insertions(+), 76 deletions(-)

diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx.c b/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
index 1a46393..d6448a4 100644
--- a/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
+++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
@@ -142,10 +142,6 @@ ixgbe_tx_free_bufs(struct igb_tx_queue *txq)
 	 */
 	txep = &(txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]);
 
-	/* prefetch the mbufs that are about to be freed */
-	for (i = 0; i < txq->tx_rs_thresh; ++i)
-		rte_prefetch0((txep + i)->mbuf);
-
 	/* free buffers one at a time */
 	if ((txq->txq_flags & (uint32_t)ETH_TXQ_FLAGS_NOREFCOUNT) != 0) {
 		for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
@@ -186,6 +182,7 @@ tx4(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts)
 				((uint32_t)DCMD_DTYP_FLAGS | pkt_len);
 		txdp->read.olinfo_status =
 				(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
+		rte_prefetch0(&(*pkts)->pool);
 	}
 }
 
@@ -205,6 +202,7 @@ tx1(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts)
 			((uint32_t)DCMD_DTYP_FLAGS | pkt_len);
 	txdp->read.olinfo_status =
 			(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
+	rte_prefetch0(&(*pkts)->pool);
 }
 
 /*
@@ -1875,7 +1873,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
 		PMD_INIT_LOG(INFO, "Using simple tx code path\n");
 #ifdef RTE_IXGBE_INC_VECTOR
 		if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ &&
-		    ixgbe_txq_vec_setup(txq, socket_id) == 0) {
+		    ixgbe_txq_vec_setup(txq) == 0) {
 			PMD_INIT_LOG(INFO, "Vector tx enabled.\n");
 			dev->tx_pkt_burst = ixgbe_xmit_pkts_vec;
 		}
diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx.h b/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
index e92a864..a97fddb 100644
--- a/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
+++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
@@ -96,14 +96,6 @@ struct igb_tx_entry_v {
 };
 
 /**
- * continuous entry sequence, gather by the same mempool
- */
-struct igb_tx_entry_seq {
-	const struct rte_mempool* pool;
-	uint32_t same_pool;
-};
-
-/**
  * Structure associated with each RX queue.
  */
 struct igb_rx_queue {
@@ -190,10 +182,6 @@ struct igb_tx_queue {
 	volatile union ixgbe_adv_tx_desc *tx_ring;
 	uint64_t            tx_ring_phys_addr; /**< TX ring DMA address. */
 	struct igb_tx_entry *sw_ring;      /**< virtual address of SW ring. */
-#ifdef RTE_IXGBE_INC_VECTOR
-	/** continuous tx entry sequence within the same mempool */
-	struct igb_tx_entry_seq *sw_ring_seq;
-#endif
 	volatile uint32_t   *tdt_reg_addr; /**< Address of TDT register. */
 	uint16_t            nb_tx_desc;    /**< number of TX descriptors. */
 	uint16_t            tx_tail;       /**< current value of TDT reg. */
@@ -258,7 +246,7 @@ struct ixgbe_txq_ops {
 #ifdef RTE_IXGBE_INC_VECTOR
 uint16_t ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
 uint16_t ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
-int ixgbe_txq_vec_setup(struct igb_tx_queue *txq, unsigned int socket_id);
+int ixgbe_txq_vec_setup(struct igb_tx_queue *txq);
 int ixgbe_rxq_vec_setup(struct igb_rx_queue *rxq);
 int ixgbe_rx_vec_condition_check(struct rte_eth_dev *dev);
 #endif
diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c b/lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c
index d53e239..8f34f59 100644
--- a/lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c
+++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c
@@ -342,9 +342,8 @@ static inline int __attribute__((always_inline))
 ixgbe_tx_free_bufs(struct igb_tx_queue *txq)
 {
 	struct igb_tx_entry_v *txep;
-	struct igb_tx_entry_seq *txsp;
 	uint32_t status;
-	uint32_t n, k;
+	uint32_t n;
 #ifdef RTE_MBUF_REFCNT
 	uint32_t i;
 	int nb_free = 0;
@@ -364,23 +363,38 @@ ixgbe_tx_free_bufs(struct igb_tx_queue *txq)
 	 */
 	txep = &((struct igb_tx_entry_v *)txq->sw_ring)[txq->tx_next_dd -
 			(n - 1)];
-	txsp = &txq->sw_ring_seq[txq->tx_next_dd - (n - 1)];
-
-	while (n > 0) {
-		k = RTE_MIN(n, txsp[n-1].same_pool);
 #ifdef RTE_MBUF_REFCNT
-		for (i = 0; i < k; i++) {
-			m = __rte_pktmbuf_prefree_seg((txep+n-k+i)->mbuf);
-			if (m != NULL)
-				free[nb_free++] = m;
-		}
-		rte_mempool_put_bulk((void *)txsp[n-1].pool,
-				(void **)free, nb_free);
+	m = __rte_pktmbuf_prefree_seg(txep[0].mbuf);
 #else
-		rte_mempool_put_bulk((void *)txsp[n-1].pool,
-				(void **)(txep+n-k), k);
+	m = txep[0].mbuf;
 #endif
-		n -= k;
+	if (likely(m != NULL)) {
+		free[0] = m;
+		nb_free = 1;
+		for (i = 1; i < n; i++) {
+#ifdef RTE_MBUF_REFCNT
+			m = __rte_pktmbuf_prefree_seg(txep[i].mbuf);
+#else
+			m = txep[i]->mbuf;
+#endif
+			if (likely(m != NULL)) {
+				if (likely(m->pool == free[0]->pool))
+					free[nb_free++] = m;
+				else {
+					rte_mempool_put_bulk(free[0]->pool,
+							(void *)free, nb_free);
+					free[0] = m;
+					nb_free = 1;
+				}
+			}
+		}
+		rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
+	} else {
+		for (i = 1; i < n; i++) {
+			m = __rte_pktmbuf_prefree_seg(txep[i].mbuf);
+			if (m != NULL)
+				rte_mempool_put(m->pool, m);
+		}
 	}
 
 	/* buffers were freed, update counters */
@@ -394,19 +408,11 @@ ixgbe_tx_free_bufs(struct igb_tx_queue *txq)
 
 static inline void __attribute__((always_inline))
 tx_backlog_entry(struct igb_tx_entry_v *txep,
-		 struct igb_tx_entry_seq *txsp,
 		 struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 {
 	int i;
-	for (i = 0; i < (int)nb_pkts; ++i) {
+	for (i = 0; i < (int)nb_pkts; ++i)
 		txep[i].mbuf = tx_pkts[i];
-		/* check and update sequence number */
-		txsp[i].pool = tx_pkts[i]->pool;
-		if (txsp[i-1].pool == tx_pkts[i]->pool)
-			txsp[i].same_pool = txsp[i-1].same_pool + 1;
-		else
-			txsp[i].same_pool = 1;
-	}
 }
 
 uint16_t
@@ -416,7 +422,6 @@ ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 	struct igb_tx_queue *txq = (struct igb_tx_queue *)tx_queue;
 	volatile union ixgbe_adv_tx_desc *txdp;
 	struct igb_tx_entry_v *txep;
-	struct igb_tx_entry_seq *txsp;
 	uint16_t n, nb_commit, tx_id;
 	uint64_t flags = DCMD_DTYP_FLAGS;
 	uint64_t rs = IXGBE_ADVTXD_DCMD_RS|DCMD_DTYP_FLAGS;
@@ -435,14 +440,13 @@ ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 	tx_id = txq->tx_tail;
 	txdp = &txq->tx_ring[tx_id];
 	txep = &((struct igb_tx_entry_v *)txq->sw_ring)[tx_id];
-	txsp = &txq->sw_ring_seq[tx_id];
 
 	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
 
 	n = (uint16_t)(txq->nb_tx_desc - tx_id);
 	if (nb_commit >= n) {
 
-		tx_backlog_entry(txep, txsp, tx_pkts, n);
+		tx_backlog_entry(txep, tx_pkts, n);
 
 		for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp)
 			vtx1(txdp, *tx_pkts, flags);
@@ -457,10 +461,9 @@ ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 		/* avoid reach the end of ring */
 		txdp = &(txq->tx_ring[tx_id]);
 		txep = &(((struct igb_tx_entry_v *)txq->sw_ring)[tx_id]);
-		txsp = &(txq->sw_ring_seq[tx_id]);
 	}
 
-	tx_backlog_entry(txep, txsp, tx_pkts, nb_commit);
+	tx_backlog_entry(txep, tx_pkts, nb_commit);
 
 	vtx(txdp, tx_pkts, nb_commit, flags);
 
@@ -484,7 +487,6 @@ ixgbe_tx_queue_release_mbufs(struct igb_tx_queue *txq)
 {
 	unsigned i;
 	struct igb_tx_entry_v *txe;
-	struct igb_tx_entry_seq *txs;
 	uint16_t nb_free, max_desc;
 
 	if (txq->sw_ring != NULL) {
@@ -502,10 +504,6 @@ ixgbe_tx_queue_release_mbufs(struct igb_tx_queue *txq)
 		for (i = 0; i < txq->nb_tx_desc; i++) {
 			txe = (struct igb_tx_entry_v *)&txq->sw_ring[i];
 			txe->mbuf = NULL;
-
-			txs = &txq->sw_ring_seq[i];
-			txs->pool = NULL;
-			txs->same_pool = 0;
 		}
 	}
 }
@@ -520,11 +518,6 @@ ixgbe_tx_free_swring(struct igb_tx_queue *txq)
 		rte_free((struct igb_rx_entry *)txq->sw_ring - 1);
 		txq->sw_ring = NULL;
 	}
-
-	if (txq->sw_ring_seq != NULL) {
-		rte_free(txq->sw_ring_seq - 1);
-		txq->sw_ring_seq = NULL;
-	}
 }
 
 static void
@@ -533,7 +526,6 @@ ixgbe_reset_tx_queue(struct igb_tx_queue *txq)
 	static const union ixgbe_adv_tx_desc zeroed_desc = { .read = {
 			.buffer_addr = 0} };
 	struct igb_tx_entry_v *txe = (struct igb_tx_entry_v *)txq->sw_ring;
-	struct igb_tx_entry_seq *txs = txq->sw_ring_seq;
 	uint16_t i;
 
 	/* Zero out HW ring memory */
@@ -545,8 +537,6 @@ ixgbe_reset_tx_queue(struct igb_tx_queue *txq)
 		volatile union ixgbe_adv_tx_desc *txd = &txq->tx_ring[i];
 		txd->wb.status = IXGBE_TXD_STAT_DD;
 		txe[i].mbuf = NULL;
-		txs[i].pool = NULL;
-		txs[i].same_pool = 0;
 	}
 
 	txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
@@ -588,28 +578,14 @@ ixgbe_rxq_vec_setup(struct igb_rx_queue *rxq)
 	return 0;
 }
 
-int ixgbe_txq_vec_setup(struct igb_tx_queue *txq,
-			unsigned int socket_id)
+int ixgbe_txq_vec_setup(struct igb_tx_queue *txq)
 {
-	uint16_t nb_desc;
-
 	if (txq->sw_ring == NULL)
 		return -1;
 
-	/* request addtional one entry for continous sequence check */
-	nb_desc = (uint16_t)(txq->nb_tx_desc + 1);
-
-	txq->sw_ring_seq = rte_zmalloc_socket("txq->sw_ring_seq",
-				sizeof(struct igb_tx_entry_seq) * nb_desc,
-				CACHE_LINE_SIZE, socket_id);
-	if (txq->sw_ring_seq == NULL)
-		return -1;
-
-
 	/* leave the first one for overflow */
 	txq->sw_ring = (struct igb_tx_entry *)
 		((struct igb_tx_entry_v *)txq->sw_ring + 1);
-	txq->sw_ring_seq += 1;
 	txq->ops = &vec_txq_ops;
 
 	return 0;
-- 
1.9.3

  parent reply	other threads:[~2014-09-11 13:11 UTC|newest]

Thread overview: 62+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2014-09-03 15:49 [dpdk-dev] [PATCH 00/13] Mbuf Structure Rework, part 2 Bruce Richardson
2014-09-03 15:49 ` [dpdk-dev] [PATCH 01/13] mbuf: replace data pointer by an offset Bruce Richardson
2014-09-08  9:52   ` Olivier MATZ
2014-09-08  9:55     ` Olivier MATZ
2014-09-03 15:49 ` [dpdk-dev] [PATCH 02/13] mbuf: reorder fields by time of use Bruce Richardson
2014-09-08 10:17   ` Olivier MATZ
2014-09-03 15:49 ` [dpdk-dev] [PATCH 03/13] mbuf: add packet_type field Bruce Richardson
2014-09-08 10:17   ` Olivier MATZ
2014-09-08 10:33     ` Yerden Zhumabekov
2014-09-08 11:17       ` Olivier MATZ
2014-09-09  3:59         ` Zhang, Helin
     [not found]           ` <540EB428.9060706@6wind.com>
2014-09-09  8:45             ` Zhang, Helin
2014-09-09  9:47             ` Richardson, Bruce
2014-09-09 15:05         ` Jim Thompson
2014-09-03 15:49 ` [dpdk-dev] [PATCH 04/13] mbuf: expand ol_flags field to 64-bits Bruce Richardson
2014-09-08 10:25   ` Olivier MATZ
2014-09-09  9:00     ` Richardson, Bruce
2014-09-03 15:49 ` [dpdk-dev] [PATCH 05/13] mbuf: introduce a flag to indicate a control mbuf Bruce Richardson
2014-09-08 11:53   ` Olivier MATZ
2014-09-03 15:49 ` [dpdk-dev] [PATCH 06/13] mbuf: minor changes for readability Bruce Richardson
2014-09-08 12:03   ` Olivier MATZ
2014-09-03 15:49 ` [dpdk-dev] [PATCH 07/13] mbuf: use macros only to access the mbuf metadata Bruce Richardson
2014-09-08 12:05   ` Olivier MATZ
2014-09-09  9:01     ` Richardson, Bruce
2014-09-12 16:56       ` Dumitrescu, Cristian
2014-09-12 21:02         ` Olivier MATZ
2014-09-16 20:07           ` Dumitrescu, Cristian
2014-09-16 22:06             ` Ramia, Kannan Babu
2014-09-17 10:31               ` Richardson, Bruce
2014-09-17 14:01                 ` Thomas Monjalon
2014-09-10 15:09     ` Bruce Richardson
2014-09-10 15:31       ` Olivier MATZ
2014-09-03 15:49 ` [dpdk-dev] [PATCH 08/13] mbuf: add named points inside the mbuf structure Bruce Richardson
2014-09-08 12:08   ` Olivier MATZ
2014-09-03 15:49 ` [dpdk-dev] [PATCH 09/13] ixgbe: rework vector pmd following mbuf changes Bruce Richardson
2014-09-03 15:49 ` [dpdk-dev] [PATCH 10/13] mbuf: split mbuf across two cache lines Bruce Richardson
2014-09-08 12:10   ` Olivier MATZ
2014-09-03 15:49 ` [dpdk-dev] [PATCH 11/13] mbuf: move l2_len and l3_len to second cache line Bruce Richardson
2014-09-04  5:08   ` Yerden Zhumabekov
2014-09-04 10:27     ` Bruce Richardson
2014-09-04 11:00       ` Yerden Zhumabekov
2014-09-04 11:55         ` Bruce Richardson
2014-09-03 15:49 ` [dpdk-dev] [PATCH 12/13] ixgbe: Fix perf regression due to moved pool ptr Bruce Richardson
2014-09-03 15:49 ` [dpdk-dev] [PATCH 13/13] ixgbe: Improve slow-path perf: vector scattered RX Bruce Richardson
2014-09-11 13:15 ` [dpdk-dev] [PATCH v2 00/13] Mbuf Structure Rework, part 2 Bruce Richardson
2014-09-11 13:15   ` [dpdk-dev] [PATCH v2 01/13] mbuf: replace data pointer by an offset Bruce Richardson
2014-09-11 13:15   ` [dpdk-dev] [PATCH v2 02/13] mbuf: reorder fields by time of use Bruce Richardson
2014-09-15  7:11     ` Liu, Jijiang
2014-09-15  8:19       ` Richardson, Bruce
2014-09-11 13:15   ` [dpdk-dev] [PATCH v2 03/13] mbuf: expand ol_flags field to 64-bits Bruce Richardson
2014-09-11 13:15   ` [dpdk-dev] [PATCH v2 04/13] mbuf: introduce a flag to indicate a control mbuf Bruce Richardson
2014-09-11 13:15   ` [dpdk-dev] [PATCH v2 05/13] mbuf: minor changes for readability Bruce Richardson
2014-09-11 13:15   ` [dpdk-dev] [PATCH v2 06/13] mbuf: use macros only to access the mbuf metadata Bruce Richardson
2014-09-11 13:15   ` [dpdk-dev] [PATCH v2 07/13] mbuf: move metadata macros to rte_port library Bruce Richardson
2014-09-11 13:15   ` [dpdk-dev] [PATCH v2 08/13] mbuf: add named points inside the mbuf structure Bruce Richardson
2014-09-11 13:15   ` [dpdk-dev] [PATCH v2 09/13] ixgbe: rework vector pmd following mbuf changes Bruce Richardson
2014-09-11 13:15   ` [dpdk-dev] [PATCH v2 10/13] mbuf: split mbuf across two cache lines Bruce Richardson
2014-09-11 13:15   ` [dpdk-dev] [PATCH v2 11/13] mbuf: move l2_len and l3_len to second cache line Bruce Richardson
2014-09-11 13:15   ` Bruce Richardson [this message]
2014-09-15 16:20     ` [dpdk-dev] [PATCH v3 12/13] ixgbe: Fix perf regression due to moved pool ptr Bruce Richardson
2014-09-11 13:15   ` [dpdk-dev] [PATCH v2 13/13] ixgbe: Improve slow-path perf: vector scattered RX Bruce Richardson
2014-09-17 22:35   ` [dpdk-dev] [PATCH v2 00/13] Mbuf Structure Rework, part 2 Thomas Monjalon

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1410441347-22840-13-git-send-email-bruce.richardson@intel.com \
    --to=bruce.richardson@intel.com \
    --cc=dev@dpdk.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

DPDK patches and discussions

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://inbox.dpdk.org/dev/0 dev/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 dev dev/ https://inbox.dpdk.org/dev \
		dev@dpdk.org
	public-inbox-index dev

Example config snippet for mirrors.
Newsgroup available over NNTP:
	nntp://inbox.dpdk.org/inbox.dpdk.dev


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git