From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <dev-bounces@dpdk.org>
Received: from dpdk.org (dpdk.org [92.243.14.124])
by inbox.dpdk.org (Postfix) with ESMTP id 10D95A057D;
Wed, 18 Mar 2020 22:38:57 +0100 (CET)
Received: from [92.243.14.124] (localhost [127.0.0.1])
by dpdk.org (Postfix) with ESMTP id 7A6231C125;
Wed, 18 Mar 2020 22:36:44 +0100 (CET)
Received: from mx0b-0016f401.pphosted.com (mx0b-0016f401.pphosted.com
[67.231.156.173]) by dpdk.org (Postfix) with ESMTP id 0DAE01C125
for <dev@dpdk.org>; Wed, 18 Mar 2020 22:36:42 +0100 (CET)
Received: from pps.filterd (m0045851.ppops.net [127.0.0.1])
by mx0b-0016f401.pphosted.com (8.16.0.42/8.16.0.42) with SMTP id
02ILVTaD013858; Wed, 18 Mar 2020 14:36:41 -0700
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com;
h=from : to : cc :
subject : date : message-id : in-reply-to : references : mime-version :
content-transfer-encoding : content-type; s=pfpt0818;
bh=7KWZPAmFDz79E4WkBA82Zq3frBiX6YFlBtPg219U+Jg=;
b=CfbHK8J337V+NCXrxRd1nh/esz2FjV9HDnivZm81qcAIHd5VBVE3ymRBOgEKnTlyeLfl
mNvHxHdFSvU1obC4xbKKEpawQkBp8ZpelnnjwcQNdd8WO8A+zyKwRcW6sKoZnB4t/MNZ
Qnzx73i1sqbisWflP/Cx2qDwtbt2RytJH6PAU8WCMcpOGjFF41DNsdpmVI3Ix51RyVcj
6IIUNFj95pvyx7zd+qAoavqQXvkroJXV9ftOowkN25s/M5xM+e4HU9IUKsu9/zMqh9dT
68mVOETdWwa/pPVDXSs5lFM0tPEoybkcYqZ5U4LStwzBbwOqDvd0H32vZ3z0Jf+jhkZ/ ZA==
Received: from sc-exch01.marvell.com ([199.233.58.181])
by mx0b-0016f401.pphosted.com with ESMTP id 2yu9rpcc6f-1
(version=TLSv1.2 cipher=ECDHE-RSA-AES256-SHA384 bits=256 verify=NOT);
Wed, 18 Mar 2020 14:36:41 -0700
Received: from DC5-EXCH01.marvell.com (10.69.176.38) by SC-EXCH01.marvell.com
(10.93.176.81) with Microsoft SMTP Server (TLS) id 15.0.1497.2;
Wed, 18 Mar 2020 14:36:38 -0700
Received: from SC-EXCH01.marvell.com (10.93.176.81) by DC5-EXCH01.marvell.com
(10.69.176.38) with Microsoft SMTP Server (TLS) id 15.0.1497.2;
Wed, 18 Mar 2020 14:36:38 -0700
Received: from maili.marvell.com (10.93.176.43) by SC-EXCH01.marvell.com
(10.93.176.81) with Microsoft SMTP Server id 15.0.1497.2 via Frontend
Transport; Wed, 18 Mar 2020 14:36:37 -0700
Received: from jerin-lab.marvell.com (jerin-lab.marvell.com [10.28.34.14])
by maili.marvell.com (Postfix) with ESMTP id 77FF33F703F;
Wed, 18 Mar 2020 14:36:35 -0700 (PDT)
From: <jerinj@marvell.com>
To: Nithin Dabilpuram <ndabilpuram@marvell.com>, Pavan Nikhilesh
<pbhagavatula@marvell.com>
CC: <dev@dpdk.org>, <thomas@monjalon.net>, <david.marchand@redhat.com>,
<mdr@ashroe.eu>, <mattias.ronnblom@ericsson.com>, <kirankumark@marvell.com>
Date: Thu, 19 Mar 2020 03:05:45 +0530
Message-ID: <20200318213551.3489504-21-jerinj@marvell.com>
X-Mailer: git-send-email 2.25.1
In-Reply-To: <20200318213551.3489504-1-jerinj@marvell.com>
References: <20200318213551.3489504-1-jerinj@marvell.com>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
Content-Type: text/plain
X-Proofpoint-Virus-Version: vendor=fsecure engine=2.50.10434:6.0.138, 18.0.645
definitions=2020-03-18_07:2020-03-18,
2020-03-18 signatures=0
Subject: [dpdk-dev] [PATCH v1 20/26] node: ipv4 lookup for x86
X-BeenThere: dev@dpdk.org
X-Mailman-Version: 2.1.15
Precedence: list
List-Id: DPDK patches and discussions <dev.dpdk.org>
List-Unsubscribe: <https://mails.dpdk.org/options/dev>,
<mailto:dev-request@dpdk.org?subject=unsubscribe>
List-Archive: <http://mails.dpdk.org/archives/dev/>
List-Post: <mailto:dev@dpdk.org>
List-Help: <mailto:dev-request@dpdk.org?subject=help>
List-Subscribe: <https://mails.dpdk.org/listinfo/dev>,
<mailto:dev-request@dpdk.org?subject=subscribe>
Errors-To: dev-bounces@dpdk.org
Sender: "dev" <dev-bounces@dpdk.org>
From: Pavan Nikhilesh <pbhagavatula@marvell.com>
Add IPv4 lookup process function for ip4_lookup
rte_node. This node performs LPM lookup using x86_64
vector supported RTE_LPM API on every packet received
and forwards it to a next node that is identified by
lookup result.
Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Signed-off-by: Nithin Dabilpuram <ndabilpuram@marvell.com>
Signed-off-by: Kiran Kumar K <kirankumark@marvell.com>
---
lib/librte_node/ip4_lookup.c | 245 +++++++++++++++++++++++++++++++++++
1 file changed, 245 insertions(+)
diff --git a/lib/librte_node/ip4_lookup.c b/lib/librte_node/ip4_lookup.c
index d7fcd1158..c003e9c91 100644
--- a/lib/librte_node/ip4_lookup.c
+++ b/lib/librte_node/ip4_lookup.c
@@ -264,6 +264,251 @@ ip4_lookup_node_process(struct rte_graph *graph, struct rte_node *node,
return nb_objs;
}
+#elif defined(RTE_ARCH_X86)
+
+/* X86 SSE */
+static uint16_t
+ip4_lookup_node_process(struct rte_graph *graph, struct rte_node *node,
+ void **objs, uint16_t nb_objs)
+{
+ struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3, **pkts;
+ rte_edge_t next0, next1, next2, next3, next_index;
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_ether_hdr *eth_hdr;
+ uint32_t ip0, ip1, ip2, ip3;
+ void **to_next, **from;
+ uint16_t last_spec = 0;
+ uint16_t n_left_from;
+ struct rte_lpm *lpm;
+ uint16_t held = 0;
+ uint32_t drop_nh;
+ rte_xmm_t dst;
+ __m128i dip; /* SSE register */
+ int rc, i;
+
+ /* Speculative next */
+ next_index = RTE_NODE_IP4_LOOKUP_NEXT_REWRITE;
+ /* Drop node */
+ drop_nh = ((uint32_t)RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP) << 16;
+
+ /* Get socket specific LPM from ctx */
+ lpm = *((struct rte_lpm **)node->ctx);
+
+ pkts = (struct rte_mbuf **)objs;
+ from = objs;
+ n_left_from = nb_objs;
+
+ if (n_left_from >= 4) {
+ for (i = 0; i < 4; i++) {
+ rte_prefetch0(rte_pktmbuf_mtod(pkts[i],
+ struct rte_ether_hdr *) +
+ 1);
+ }
+ }
+
+ /* Get stream for the speculated next node */
+ to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs);
+ while (n_left_from >= 4) {
+ /* Prefetch next-next mbufs */
+ if (likely(n_left_from >= 11)) {
+ rte_prefetch0(pkts[8]);
+ rte_prefetch0(pkts[9]);
+ rte_prefetch0(pkts[10]);
+ rte_prefetch0(pkts[11]);
+ }
+
+ /* Prefetch next mbuf data */
+ if (likely(n_left_from >= 7)) {
+ rte_prefetch0(rte_pktmbuf_mtod(pkts[4],
+ struct rte_ether_hdr *) +
+ 1);
+ rte_prefetch0(rte_pktmbuf_mtod(pkts[5],
+ struct rte_ether_hdr *) +
+ 1);
+ rte_prefetch0(rte_pktmbuf_mtod(pkts[6],
+ struct rte_ether_hdr *) +
+ 1);
+ rte_prefetch0(rte_pktmbuf_mtod(pkts[7],
+ struct rte_ether_hdr *) +
+ 1);
+ }
+
+ mbuf0 = pkts[0];
+ mbuf1 = pkts[1];
+ mbuf2 = pkts[2];
+ mbuf3 = pkts[3];
+
+ pkts += 4;
+ n_left_from -= 4;
+
+ /* Extract DIP of mbuf0 */
+ eth_hdr = rte_pktmbuf_mtod(mbuf0, struct rte_ether_hdr *);
+ ipv4_hdr = (struct rte_ipv4_hdr *)(eth_hdr + 1);
+ ip0 = ipv4_hdr->dst_addr;
+ /* Extract cksum, ttl as ipv4 hdr is in cache */
+ rte_node_mbuf_priv1(mbuf0)->cksum = ipv4_hdr->hdr_checksum;
+ rte_node_mbuf_priv1(mbuf0)->ttl = ipv4_hdr->time_to_live;
+
+ /* Extract DIP of mbuf1 */
+ eth_hdr = rte_pktmbuf_mtod(mbuf1, struct rte_ether_hdr *);
+ ipv4_hdr = (struct rte_ipv4_hdr *)(eth_hdr + 1);
+ ip1 = ipv4_hdr->dst_addr;
+ /* Extract cksum, ttl as ipv4 hdr is in cache */
+ rte_node_mbuf_priv1(mbuf1)->cksum = ipv4_hdr->hdr_checksum;
+ rte_node_mbuf_priv1(mbuf1)->ttl = ipv4_hdr->time_to_live;
+
+ /* Extract DIP of mbuf2 */
+ eth_hdr = rte_pktmbuf_mtod(mbuf2, struct rte_ether_hdr *);
+ ipv4_hdr = (struct rte_ipv4_hdr *)(eth_hdr + 1);
+ ip2 = ipv4_hdr->dst_addr;
+ /* Extract cksum, ttl as ipv4 hdr is in cache */
+ rte_node_mbuf_priv1(mbuf2)->cksum = ipv4_hdr->hdr_checksum;
+ rte_node_mbuf_priv1(mbuf2)->ttl = ipv4_hdr->time_to_live;
+
+ /* Extract DIP of mbuf3 */
+ eth_hdr = rte_pktmbuf_mtod(mbuf3, struct rte_ether_hdr *);
+ ipv4_hdr = (struct rte_ipv4_hdr *)(eth_hdr + 1);
+ ip3 = ipv4_hdr->dst_addr;
+
+ /* Prepare for lookup x4 */
+ dip = _mm_set_epi32(ip3, ip2, ip1, ip0);
+
+ /* Byte swap 4 IPV4 addresses. */
+ const __m128i bswap_mask = _mm_set_epi8(
+ 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3);
+ dip = _mm_shuffle_epi8(dip, bswap_mask);
+
+ /* Extract cksum, ttl as ipv4 hdr is in cache */
+ rte_node_mbuf_priv1(mbuf3)->cksum = ipv4_hdr->hdr_checksum;
+ rte_node_mbuf_priv1(mbuf3)->ttl = ipv4_hdr->time_to_live;
+
+ /* Perform LPM lookup to get NH and next node */
+ rte_lpm_lookupx4(lpm, dip, dst.u32, drop_nh);
+
+ /* Extract next node id and NH */
+ rte_node_mbuf_priv1(mbuf0)->nh = dst.u32[0] & 0xFFFF;
+ next0 = (dst.u32[0] >> 16);
+
+ rte_node_mbuf_priv1(mbuf1)->nh = dst.u32[1] & 0xFFFF;
+ next1 = (dst.u32[1] >> 16);
+
+ rte_node_mbuf_priv1(mbuf2)->nh = dst.u32[2] & 0xFFFF;
+ next2 = (dst.u32[2] >> 16);
+
+ rte_node_mbuf_priv1(mbuf3)->nh = dst.u32[3] & 0xFFFF;
+ next3 = (dst.u32[3] >> 16);
+
+ /* Enqueue four to next node */
+ rte_edge_t fix_spec =
+ (next_index ^ next0) | (next_index ^ next1) |
+ (next_index ^ next2) | (next_index ^ next3);
+
+ if (unlikely(fix_spec)) {
+ /* Copy things successfully speculated till now */
+ rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
+ from += last_spec;
+ to_next += last_spec;
+ held += last_spec;
+ last_spec = 0;
+
+ /* Next0 */
+ if (next_index == next0) {
+ to_next[0] = from[0];
+ to_next++;
+ held++;
+ } else {
+ rte_node_enqueue_x1(graph, node, next0,
+ from[0]);
+ }
+
+ /* Next1 */
+ if (next_index == next1) {
+ to_next[0] = from[1];
+ to_next++;
+ held++;
+ } else {
+ rte_node_enqueue_x1(graph, node, next1,
+ from[1]);
+ }
+
+ /* Next2 */
+ if (next_index == next2) {
+ to_next[0] = from[2];
+ to_next++;
+ held++;
+ } else {
+ rte_node_enqueue_x1(graph, node, next2,
+ from[2]);
+ }
+
+ /* Next3 */
+ if (next_index == next3) {
+ to_next[0] = from[3];
+ to_next++;
+ held++;
+ } else {
+ rte_node_enqueue_x1(graph, node, next3,
+ from[3]);
+ }
+
+ from += 4;
+
+ } else {
+ last_spec += 4;
+ }
+ }
+
+ while (n_left_from > 0) {
+ uint32_t next_hop;
+
+ mbuf0 = pkts[0];
+
+ pkts += 1;
+ n_left_from -= 1;
+
+ /* Extract DIP of mbuf0 */
+ eth_hdr = rte_pktmbuf_mtod(mbuf0, struct rte_ether_hdr *);
+ ipv4_hdr = (struct rte_ipv4_hdr *)(eth_hdr + 1);
+ /* Extract cksum, ttl as ipv4 hdr is in cache */
+ rte_node_mbuf_priv1(mbuf0)->cksum = ipv4_hdr->hdr_checksum;
+ rte_node_mbuf_priv1(mbuf0)->ttl = ipv4_hdr->time_to_live;
+
+ rc = rte_lpm_lookup(lpm, rte_be_to_cpu_32(ipv4_hdr->dst_addr),
+ &next_hop);
+ next_hop = (rc == 0) ? next_hop : drop_nh;
+
+ rte_node_mbuf_priv1(mbuf0)->nh = next_hop & 0xFFFF;
+ next0 = (next_hop >> 16);
+
+ if (unlikely(next_index ^ next0)) {
+ /* Copy things successfully speculated till now */
+ rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
+ from += last_spec;
+ to_next += last_spec;
+ held += last_spec;
+ last_spec = 0;
+
+ rte_node_enqueue_x1(graph, node, next0, from[0]);
+ from += 1;
+ } else {
+ last_spec += 1;
+ }
+ }
+
+ /* !!! Home run !!! */
+ if (likely(last_spec == nb_objs)) {
+ rte_node_next_stream_move(graph, node, next_index);
+ return nb_objs;
+ }
+
+ held += last_spec;
+ /* Copy things successfully speculated till now */
+ rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
+ rte_node_next_stream_put(graph, node, next_index, held);
+
+ return nb_objs;
+}
+
#else
static uint16_t
--
2.25.1