From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga14.intel.com (mga14.intel.com [192.55.52.115]) by dpdk.org (Postfix) with ESMTP id 271AF29C7 for ; Mon, 10 Apr 2017 18:00:07 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=intel.com; i=@intel.com; q=dns/txt; s=intel; t=1491840007; x=1523376007; h=from:to:cc:subject:date:message-id:in-reply-to: references; bh=zvY4629XpFF7h0573jsf4+XnUg8G2gShDWULFAHDjeg=; b=CCGZ9uGUUEqyxgvHK87IEFVL2mTHtIstOH5tBx3wtNEVGJxgJUuGq+vW lkokAz3vsT4fUSJuJfTyJbD53vQXnQ==; Received: from fmsmga001.fm.intel.com ([10.253.24.23]) by fmsmga103.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 10 Apr 2017 09:00:05 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.37,182,1488873600"; d="scan'208";a="1133535566" Received: from sivswdev02.ir.intel.com (HELO localhost.localdomain) ([10.237.217.46]) by fmsmga001.fm.intel.com with ESMTP; 10 Apr 2017 09:00:03 -0700 From: Konstantin Ananyev To: dev@dpdk.org Cc: jerin.jacob@caviumnetworks.com, jianbo.liu@linaro.org, Konstantin Ananyev Date: Mon, 10 Apr 2017 16:59:41 +0100 Message-Id: <1491839982-30710-2-git-send-email-konstantin.ananyev@intel.com> X-Mailer: git-send-email 1.7.0.7 In-Reply-To: <1491301790-22457-2-git-send-email-konstantin.ananyev@intel.com> References: <1491301790-22457-2-git-send-email-konstantin.ananyev@intel.com> Subject: [dpdk-dev] [PATCH v2 1/2] net/ixgbe: eliminate mbuf write on rearm X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 10 Apr 2017 16:00:07 -0000 With the mbuf rework, we now have 8 contiguous bytes to be rearmed in the mbuf just before the 8-bytes of olflags. If we don't do the rearm write inside the descriptor ring replenishment function, and delay it to receiving the packet, we can do a single 16B write inside the RX function to set both the rearm data, and the flags together. Signed-off-by: Konstantin Ananyev --- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 72 ++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c index 62afe31..8e19b9d 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c @@ -82,19 +82,23 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq) /* Initialize the mbufs in vector, process 2 mbufs in one loop */ for (i = 0; i < RTE_IXGBE_RXQ_REARM_THRESH; i += 2, rxep += 2) { __m128i vaddr0, vaddr1; - uintptr_t p0, p1; mb0 = rxep[0].mbuf; mb1 = rxep[1].mbuf; - /* - * Flush mbuf with pkt template. - * Data to be rearmed is 6 bytes long. - */ - p0 = (uintptr_t)&mb0->rearm_data; - *(uint64_t *)p0 = rxq->mbuf_initializer; - p1 = (uintptr_t)&mb1->rearm_data; - *(uint64_t *)p1 = rxq->mbuf_initializer; +#ifndef RTE_IXGBE_RX_OLFLAGS_ENABLE + { + uintptr_t p0, p1; + /* + * Flush mbuf with pkt template. + * Data to be rearmed is 6 bytes long. + */ + p0 = (uintptr_t)&mb0->rearm_data; + *(uint64_t *)p0 = rxq->mbuf_initializer; + p1 = (uintptr_t)&mb1->rearm_data; + *(uint64_t *)p1 = rxq->mbuf_initializer; + } +#endif /* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */ vaddr0 = _mm_loadu_si128((__m128i *)&(mb0->buf_addr)); @@ -139,14 +143,11 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq) #ifdef RTE_IXGBE_RX_OLFLAGS_ENABLE static inline void -desc_to_olflags_v(__m128i descs[4], uint8_t vlan_flags, +desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, struct rte_mbuf **rx_pkts) { __m128i ptype0, ptype1, vtag0, vtag1, csum; - union { - uint16_t e[4]; - uint64_t dword; - } vol; + __m128i rearm0, rearm1, rearm2, rearm3; /* mask everything except rss type */ const __m128i rsstype_msk = _mm_set_epi16( @@ -225,12 +226,40 @@ desc_to_olflags_v(__m128i descs[4], uint8_t vlan_flags, vtag1 = _mm_or_si128(vtag0, vtag1); vtag1 = _mm_or_si128(ptype0, vtag1); - vol.dword = _mm_cvtsi128_si64(vtag1); - rx_pkts[0]->ol_flags = vol.e[0]; - rx_pkts[1]->ol_flags = vol.e[1]; - rx_pkts[2]->ol_flags = vol.e[2]; - rx_pkts[3]->ol_flags = vol.e[3]; + /* + * At this point, we have the 4 sets of flags in the low 64-bits + * of vtag1 (4x16). + * We want to extract these, and merge them with the mbuf init data + * so we can do a single 16-byte write to the mbuf to set the flags + * and all the other initialization fields. Extracting the + * appropriate flags means that we have to do a shift and blend for + * each mbuf before we do the write. + */ +#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 + + rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 8), 0x10); + rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 6), 0x10); + rearm2 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 4), 0x10); + rearm3 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 2), 0x10); + +#else + rearm0 = _mm_slli_si128(vtag1, 14); + rearm1 = _mm_slli_si128(vtag1, 12); + rearm2 = _mm_slli_si128(vtag1, 10); + rearm3 = _mm_slli_si128(vtag1, 8); + + rearm0 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm0, 48)); + rearm1 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm1, 48)); + rearm2 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm2, 48)); + rearm3 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm3, 48)); + +#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ + + _mm_store_si128((__m128i *)&rx_pkts[0]->rearm_data, rearm0); + _mm_store_si128((__m128i *)&rx_pkts[1]->rearm_data, rearm1); + _mm_store_si128((__m128i *)&rx_pkts[2]->rearm_data, rearm2); + _mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3); } #else #define desc_to_olflags_v(desc, vlan_flags, rx_pkts) do { \ @@ -265,6 +294,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 0, 0 /* ignore pkt_type field */ ); __m128i dd_check, eop_check; + __m128i mbuf_init; uint8_t vlan_flags; /* nb_pkts shall be less equal than RTE_IXGBE_MAX_RX_BURST */ @@ -310,6 +340,8 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 0xFF, 0xFF ); + mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); + /* Cache is empty -> need to scan the buffer rings, but first move * the next 'n' mbufs into the cache */ @@ -382,7 +414,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, sterr_tmp1 = _mm_unpackhi_epi32(descs[1], descs[0]); /* set ol_flags with vlan packet type */ - desc_to_olflags_v(descs, vlan_flags, &rx_pkts[pos]); + desc_to_olflags_v(descs, mbuf_init, vlan_flags, &rx_pkts[pos]); /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust); -- 2.5.0