* [PATCH] gro : ipv6 changes to support GRO for TCP/ipv6
@ 2022-10-20 18:07 Kumara Parameshwaran
2022-10-20 18:14 ` [PATCH v2] " Kumara Parameshwaran
0 siblings, 1 reply; 32+ messages in thread
From: Kumara Parameshwaran @ 2022-10-20 18:07 UTC (permalink / raw)
To: dev; +Cc: jiayu.hu, Kumara Parameshwaran
From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
The patch adds GRO support for TCP/ipv6 packets. This does not
include the support for vxlan, udp ipv6 packets.
Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
---
v1:
* Changes to support GRO for TCP/ipv6 packets. This does not include
vxlan changes.
* The GRO is performed only for ipv6 packets that does not contain
extension headers.
* The logic for the TCP coalescing remains the same, in ipv6 header
the source address, destination address, flow label, version fields
are expected to be the same.
* Re-organised the code to reuse certain tcp functions for both ipv4 and
ipv6 flows.
lib/gro/gro_tcp.h | 155 ++++++++++++++++
lib/gro/gro_tcp4.c | 7 +-
lib/gro/gro_tcp4.h | 152 +--------------
lib/gro/gro_tcp6.c | 387 +++++++++++++++++++++++++++++++++++++++
lib/gro/gro_tcp6.h | 150 +++++++++++++++
lib/gro/gro_vxlan_tcp4.c | 3 +-
lib/gro/gro_vxlan_tcp4.h | 3 +-
lib/gro/meson.build | 1 +
lib/gro/rte_gro.c | 83 +++++++--
lib/gro/rte_gro.h | 3 +
10 files changed, 774 insertions(+), 170 deletions(-)
create mode 100644 lib/gro/gro_tcp.h
create mode 100644 lib/gro/gro_tcp6.c
create mode 100644 lib/gro/gro_tcp6.h
diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h
new file mode 100644
index 0000000000..c5d248a022
--- /dev/null
+++ b/lib/gro/gro_tcp.h
@@ -0,0 +1,155 @@
+#ifndef _GRO_TCP_H_
+#define _GRO_TCP_H_
+
+#include <rte_tcp.h>
+
+/*
+ * The max length of a IPv4 packet, which includes the length of the L3
+ * header, the L4 header and the data payload.
+ */
+#define MAX_IP_PKT_LENGTH UINT16_MAX
+
+/* The maximum TCP header length */
+#define MAX_TCP_HLEN 60
+#define INVALID_TCP_HDRLEN(len) \
+ (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
+
+struct gro_tcp_item {
+ /*
+ * The first MBUF segment of the packet. If the value
+ * is NULL, it means the item is empty.
+ */
+ struct rte_mbuf *firstseg;
+ /* The last MBUF segment of the packet */
+ struct rte_mbuf *lastseg;
+ /*
+ * The time when the first packet is inserted into the table.
+ * This value won't be updated, even if the packet is merged
+ * with other packets.
+ */
+ uint64_t start_time;
+ /*
+ * next_pkt_idx is used to chain the packets that
+ * are in the same flow but can't be merged together
+ * (e.g. caused by packet reordering).
+ */
+ uint32_t next_pkt_idx;
+ /* TCP sequence number of the packet */
+ uint32_t sent_seq;
+ /* IPv4 ID of the packet */
+ uint16_t ip_id;
+ /* the number of merged packets */
+ uint16_t nb_merged;
+ /* Indicate if IPv4 ID can be ignored */
+ uint8_t is_atomic;
+};
+
+/*
+ * Merge two TCP packets without updating checksums.
+ * If cmp is larger than 0, append the new packet to the
+ * original packet. Otherwise, pre-pend the new packet to
+ * the original packet.
+ */
+static inline int
+merge_two_tcp_packets(struct gro_tcp_item *item,
+ struct rte_mbuf *pkt,
+ int cmp,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t l2_offset)
+{
+ struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
+ uint16_t hdr_len, l2_len;
+
+ if (cmp > 0) {
+ pkt_head = item->firstseg;
+ pkt_tail = pkt;
+ } else {
+ pkt_head = pkt;
+ pkt_tail = item->firstseg;
+ }
+
+ /* check if the IPv4 packet length is greater than the max value */
+ hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
+ pkt_head->l4_len;
+ l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
+ if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
+ hdr_len > MAX_IP_PKT_LENGTH))
+ return 0;
+
+ /* remove the packet header for the tail packet */
+ rte_pktmbuf_adj(pkt_tail, hdr_len);
+
+ /* chain two packets together */
+ if (cmp > 0) {
+ item->lastseg->next = pkt;
+ item->lastseg = rte_pktmbuf_lastseg(pkt);
+ /* update IP ID to the larger value */
+ item->ip_id = ip_id;
+ } else {
+ lastseg = rte_pktmbuf_lastseg(pkt);
+ lastseg->next = item->firstseg;
+ item->firstseg = pkt;
+ /* update sent_seq to the smaller value */
+ item->sent_seq = sent_seq;
+ item->ip_id = ip_id;
+ }
+ item->nb_merged++;
+
+ /* update MBUF metadata for the merged packet */
+ pkt_head->nb_segs += pkt_tail->nb_segs;
+ pkt_head->pkt_len += pkt_tail->pkt_len;
+
+ return 1;
+}
+
+/*
+ * Check if two TCP/IPv4 packets are neighbors.
+ */
+static inline int
+check_seq_option(struct gro_tcp_item *item,
+ struct rte_tcp_hdr *tcph,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t tcp_hl,
+ uint16_t tcp_dl,
+ uint16_t l2_offset,
+ uint8_t is_atomic)
+{
+ struct rte_mbuf *pkt_orig = item->firstseg;
+ char *iph_orig;
+ struct rte_tcp_hdr *tcph_orig;
+ uint16_t len, tcp_hl_orig;
+
+ iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
+ l2_offset + pkt_orig->l2_len);
+ tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
+ tcp_hl_orig = pkt_orig->l4_len;
+
+ /* Check if TCP option fields equal */
+ len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
+ if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
+ (memcmp(tcph + 1, tcph_orig + 1,
+ len) != 0)))
+ return 0;
+
+ /* Don't merge packets whose DF bits are different */
+ if (unlikely(item->is_atomic ^ is_atomic))
+ return 0;
+
+ /* check if the two packets are neighbors */
+ len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
+ pkt_orig->l3_len - tcp_hl_orig;
+ if ((sent_seq == item->sent_seq + len) && (is_atomic ||
+ (ip_id == item->ip_id + 1)))
+ /* append the new packet */
+ return 1;
+ else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
+ (ip_id + item->nb_merged == item->ip_id)))
+ /* pre-pend the new packet */
+ return -1;
+
+ return 0;
+}
+
+#endif
diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c
index 8f5e800250..eea2a72ecd 100644
--- a/lib/gro/gro_tcp4.c
+++ b/lib/gro/gro_tcp4.c
@@ -7,6 +7,7 @@
#include <rte_ethdev.h>
#include "gro_tcp4.h"
+#include "gro_tcp.h"
void *
gro_tcp4_tbl_create(uint16_t socket_id,
@@ -30,7 +31,7 @@ gro_tcp4_tbl_create(uint16_t socket_id,
if (tbl == NULL)
return NULL;
- size = sizeof(struct gro_tcp4_item) * entries_num;
+ size = sizeof(struct gro_tcp_item) * entries_num;
tbl->items = rte_zmalloc_socket(__func__,
size,
RTE_CACHE_LINE_SIZE,
@@ -177,7 +178,7 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
* update the packet length for the flushed packet.
*/
static inline void
-update_header(struct gro_tcp4_item *item)
+update_header(struct gro_tcp_item *item)
{
struct rte_ipv4_hdr *ipv4_hdr;
struct rte_mbuf *pkt = item->firstseg;
@@ -302,7 +303,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
is_atomic);
if (cmp) {
- if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
+ if (merge_two_tcp_packets(&(tbl->items[cur_idx]),
pkt, cmp, sent_seq, ip_id, 0))
return 1;
/*
diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h
index 212f97a042..634a215b98 100644
--- a/lib/gro/gro_tcp4.h
+++ b/lib/gro/gro_tcp4.h
@@ -5,22 +5,9 @@
#ifndef _GRO_TCP4_H_
#define _GRO_TCP4_H_
-#include <rte_tcp.h>
-
#define INVALID_ARRAY_INDEX 0xffffffffUL
#define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
-/*
- * The max length of a IPv4 packet, which includes the length of the L3
- * header, the L4 header and the data payload.
- */
-#define MAX_IPV4_PKT_LENGTH UINT16_MAX
-
-/* The maximum TCP header length */
-#define MAX_TCP_HLEN 60
-#define INVALID_TCP_HDRLEN(len) \
- (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
-
/* Header fields representing a TCP/IPv4 flow */
struct tcp4_flow_key {
struct rte_ether_addr eth_saddr;
@@ -42,42 +29,12 @@ struct gro_tcp4_flow {
uint32_t start_index;
};
-struct gro_tcp4_item {
- /*
- * The first MBUF segment of the packet. If the value
- * is NULL, it means the item is empty.
- */
- struct rte_mbuf *firstseg;
- /* The last MBUF segment of the packet */
- struct rte_mbuf *lastseg;
- /*
- * The time when the first packet is inserted into the table.
- * This value won't be updated, even if the packet is merged
- * with other packets.
- */
- uint64_t start_time;
- /*
- * next_pkt_idx is used to chain the packets that
- * are in the same flow but can't be merged together
- * (e.g. caused by packet reordering).
- */
- uint32_t next_pkt_idx;
- /* TCP sequence number of the packet */
- uint32_t sent_seq;
- /* IPv4 ID of the packet */
- uint16_t ip_id;
- /* the number of merged packets */
- uint16_t nb_merged;
- /* Indicate if IPv4 ID can be ignored */
- uint8_t is_atomic;
-};
-
/*
* TCP/IPv4 reassembly table structure.
*/
struct gro_tcp4_tbl {
/* item array */
- struct gro_tcp4_item *items;
+ struct gro_tcp_item *items;
/* flow array */
struct gro_tcp4_flow *flows;
/* current item number */
@@ -195,111 +152,4 @@ is_same_tcp4_flow(struct tcp4_flow_key k1, struct tcp4_flow_key k2)
(k1.dst_port == k2.dst_port));
}
-/*
- * Merge two TCP/IPv4 packets without updating checksums.
- * If cmp is larger than 0, append the new packet to the
- * original packet. Otherwise, pre-pend the new packet to
- * the original packet.
- */
-static inline int
-merge_two_tcp4_packets(struct gro_tcp4_item *item,
- struct rte_mbuf *pkt,
- int cmp,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t l2_offset)
-{
- struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
- uint16_t hdr_len, l2_len;
-
- if (cmp > 0) {
- pkt_head = item->firstseg;
- pkt_tail = pkt;
- } else {
- pkt_head = pkt;
- pkt_tail = item->firstseg;
- }
-
- /* check if the IPv4 packet length is greater than the max value */
- hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
- pkt_head->l4_len;
- l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
- if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
- hdr_len > MAX_IPV4_PKT_LENGTH))
- return 0;
-
- /* remove the packet header for the tail packet */
- rte_pktmbuf_adj(pkt_tail, hdr_len);
-
- /* chain two packets together */
- if (cmp > 0) {
- item->lastseg->next = pkt;
- item->lastseg = rte_pktmbuf_lastseg(pkt);
- /* update IP ID to the larger value */
- item->ip_id = ip_id;
- } else {
- lastseg = rte_pktmbuf_lastseg(pkt);
- lastseg->next = item->firstseg;
- item->firstseg = pkt;
- /* update sent_seq to the smaller value */
- item->sent_seq = sent_seq;
- item->ip_id = ip_id;
- }
- item->nb_merged++;
-
- /* update MBUF metadata for the merged packet */
- pkt_head->nb_segs += pkt_tail->nb_segs;
- pkt_head->pkt_len += pkt_tail->pkt_len;
-
- return 1;
-}
-
-/*
- * Check if two TCP/IPv4 packets are neighbors.
- */
-static inline int
-check_seq_option(struct gro_tcp4_item *item,
- struct rte_tcp_hdr *tcph,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t tcp_hl,
- uint16_t tcp_dl,
- uint16_t l2_offset,
- uint8_t is_atomic)
-{
- struct rte_mbuf *pkt_orig = item->firstseg;
- struct rte_ipv4_hdr *iph_orig;
- struct rte_tcp_hdr *tcph_orig;
- uint16_t len, tcp_hl_orig;
-
- iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char *) +
- l2_offset + pkt_orig->l2_len);
- tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig + pkt_orig->l3_len);
- tcp_hl_orig = pkt_orig->l4_len;
-
- /* Check if TCP option fields equal */
- len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
- if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
- (memcmp(tcph + 1, tcph_orig + 1,
- len) != 0)))
- return 0;
-
- /* Don't merge packets whose DF bits are different */
- if (unlikely(item->is_atomic ^ is_atomic))
- return 0;
-
- /* check if the two packets are neighbors */
- len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
- pkt_orig->l3_len - tcp_hl_orig;
- if ((sent_seq == item->sent_seq + len) && (is_atomic ||
- (ip_id == item->ip_id + 1)))
- /* append the new packet */
- return 1;
- else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
- (ip_id + item->nb_merged == item->ip_id)))
- /* pre-pend the new packet */
- return -1;
-
- return 0;
-}
#endif
diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c
new file mode 100644
index 0000000000..a37d8e2512
--- /dev/null
+++ b/lib/gro/gro_tcp6.c
@@ -0,0 +1,387 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp6.h"
+#include "gro_tcp.h"
+
+void *
+gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow)
+{
+ struct gro_tcp6_tbl *tbl;
+ size_t size;
+ uint32_t entries_num, i;
+
+ entries_num = max_flow_num * max_item_per_flow;
+ entries_num = RTE_MIN(entries_num, GRO_TCP6_TBL_MAX_ITEM_NUM);
+
+ if (entries_num == 0)
+ return NULL;
+
+ tbl = rte_zmalloc_socket(__func__,
+ sizeof(struct gro_tcp6_tbl),
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl == NULL)
+ return NULL;
+
+ size = sizeof(struct gro_tcp_item) * entries_num;
+ tbl->items = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->items == NULL) {
+ rte_free(tbl);
+ return NULL;
+ }
+ tbl->max_item_num = entries_num;
+
+ size = sizeof(struct gro_tcp6_flow) * entries_num;
+ tbl->flows = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->flows == NULL) {
+ rte_free(tbl->items);
+ rte_free(tbl);
+ return NULL;
+ }
+ /* INVALID_ARRAY_INDEX indicates an empty flow */
+ for (i = 0; i < entries_num; i++)
+ tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
+ tbl->max_flow_num = entries_num;
+
+ return tbl;
+}
+
+void
+gro_tcp6_tbl_destroy(void *tbl)
+{
+ struct gro_tcp6_tbl *tcp_tbl = tbl;
+
+ if (tcp_tbl) {
+ rte_free(tcp_tbl->items);
+ rte_free(tcp_tbl->flows);
+ }
+ rte_free(tcp_tbl);
+}
+
+static inline uint32_t
+find_an_empty_item(struct gro_tcp6_tbl *tbl)
+{
+ uint32_t i;
+ uint32_t max_item_num = tbl->max_item_num;
+
+ for (i = 0; i < max_item_num; i++)
+ if (tbl->items[i].firstseg == NULL)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+find_an_empty_flow(struct gro_tcp6_tbl *tbl)
+{
+ uint32_t i;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++)
+ if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+insert_new_item(struct gro_tcp6_tbl *tbl,
+ struct rte_mbuf *pkt,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint8_t is_atomic)
+{
+ uint32_t item_idx;
+
+ item_idx = find_an_empty_item(tbl);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return INVALID_ARRAY_INDEX;
+
+ tbl->items[item_idx].firstseg = pkt;
+ tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
+ tbl->items[item_idx].start_time = start_time;
+ tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
+ tbl->items[item_idx].sent_seq = sent_seq;
+ tbl->items[item_idx].nb_merged = 1;
+ tbl->items[item_idx].is_atomic = is_atomic;
+ tbl->item_num++;
+
+ /* if the previous packet exists, chain them together. */
+ if (prev_idx != INVALID_ARRAY_INDEX) {
+ tbl->items[item_idx].next_pkt_idx =
+ tbl->items[prev_idx].next_pkt_idx;
+ tbl->items[prev_idx].next_pkt_idx = item_idx;
+ }
+
+ return item_idx;
+}
+
+static inline uint32_t
+delete_item(struct gro_tcp6_tbl *tbl, uint32_t item_idx,
+ uint32_t prev_item_idx)
+{
+ uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
+
+ /* NULL indicates an empty item */
+ tbl->items[item_idx].firstseg = NULL;
+ tbl->item_num--;
+ if (prev_item_idx != INVALID_ARRAY_INDEX)
+ tbl->items[prev_item_idx].next_pkt_idx = next_idx;
+
+ return next_idx;
+}
+
+static inline uint32_t
+insert_new_flow(struct gro_tcp6_tbl *tbl,
+ struct tcp6_flow_key *src,
+ rte_be32_t vtc_flow,
+ uint32_t item_idx)
+{
+ struct tcp6_flow_key *dst;
+ uint32_t flow_idx;
+
+ flow_idx = find_an_empty_flow(tbl);
+ if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
+ return INVALID_ARRAY_INDEX;
+
+ dst = &(tbl->flows[flow_idx].key);
+
+ rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
+ rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
+ memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst->src_addr));
+ memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst->dst_addr));
+ dst->recv_ack = src->recv_ack;
+ dst->src_port = src->src_port;
+ dst->dst_port = src->dst_port;
+
+ tbl->flows[flow_idx].start_index = item_idx;
+ tbl->flow_num++;
+ tbl->flows->vtc_flow = vtc_flow;
+
+ return flow_idx;
+}
+
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
+
+ ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len - pkt->l3_len);
+}
+
+int32_t
+gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time)
+{
+ struct rte_ether_hdr *eth_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_tcp_hdr *tcp_hdr;
+ uint32_t sent_seq;
+ int32_t tcp_dl;
+ uint16_t ip_tlen;
+ struct tcp6_flow_key key;
+ uint32_t cur_idx, prev_idx, item_idx;
+ uint32_t i, max_flow_num, remaining_flow_num;
+ int cmp;
+ uint8_t find;
+ rte_be32_t vtc_flow_diff;
+
+ /*
+ * Don't process the packet whose TCP header length is greater
+ * than 60 bytes or less than 20 bytes.
+ */
+ if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
+ return -1;
+
+ eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
+ ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
+ tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv6_hdr + pkt->l3_len);
+
+ /*
+ * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
+ * or CWR set.
+ */
+ if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
+ return -1;
+
+ ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
+ /*
+ * Trim the tail padding bytes. The IPv6 header is fixed to
+ * 40 bytes unlike IPv4 that is variable. The length in the IPv6 header
+ * contains only length of TCP Header + TCP Payload, whereas IPv4 header contains
+ * length of IP Header + TCP Header + TCP Payload
+ */
+ if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len + pkt->l3_len))
+ rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len - pkt->l3_len);
+ /*
+ * Don't process the packet whose payload length is less than or
+ * equal to 0.
+ */
+ tcp_dl = ip_tlen - pkt->l4_len;
+ if (tcp_dl <= 0)
+ return -1;
+
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
+ memcpy(&key.src_addr[0], &ipv6_hdr->src_addr, sizeof(key.src_addr));
+ memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr, sizeof(key.dst_addr));
+ key.src_port = tcp_hdr->src_port;
+ key.dst_port = tcp_hdr->dst_port;
+ key.recv_ack = tcp_hdr->recv_ack;
+
+ /* Search for a matched flow. */
+ max_flow_num = tbl->max_flow_num;
+ remaining_flow_num = tbl->flow_num;
+ find = 0;
+ for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
+ if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
+ if (is_same_tcp6_flow(tbl->flows[i].key, key)) {
+ /*
+ * IP version (4) Traffic Class (8) Flow Label (20)
+ * All fields except Traffic class should be same
+ */
+ vtc_flow_diff = (ipv6_hdr->vtc_flow ^ tbl->flows->vtc_flow);
+ if (vtc_flow_diff & htonl(0xF00FFFFF))
+ continue;
+ find = 1;
+ break;
+ }
+ remaining_flow_num--;
+ }
+ }
+
+ /*
+ * Fail to find a matched flow. Insert a new flow and store the
+ * packet into the flow.
+ */
+ if (find == 0) {
+ item_idx = insert_new_item(tbl, pkt, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, true);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return -1;
+ if (insert_new_flow(tbl, &key, ipv6_hdr->vtc_flow, item_idx) ==
+ INVALID_ARRAY_INDEX) {
+ /*
+ * Fail to insert a new flow, so delete the
+ * stored packet.
+ */
+ delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
+ return -1;
+ }
+ return 0;
+ }
+
+ /*
+ * Check all packets in the flow and try to find a neighbor for
+ * the input packet.
+ */
+ cur_idx = tbl->flows[i].start_index;
+ prev_idx = cur_idx;
+ do {
+ cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
+ sent_seq, 0, pkt->l4_len, tcp_dl, 0,
+ true);
+ if (cmp) {
+ if (merge_two_tcp_packets(&(tbl->items[cur_idx]),
+ pkt, cmp, sent_seq, 0, 0)) {
+ return 1;
+ }
+
+ /*
+ * Fail to merge the two packets, as the packet
+ * length is greater than the max value. Store
+ * the packet into the flow.
+ */
+ if (insert_new_item(tbl, pkt, start_time, cur_idx,
+ sent_seq, true) ==
+ INVALID_ARRAY_INDEX)
+ return -1;
+ return 0;
+ }
+ prev_idx = cur_idx;
+ cur_idx = tbl->items[cur_idx].next_pkt_idx;
+ } while (cur_idx != INVALID_ARRAY_INDEX);
+
+ /* Fail to find a neighbor, so store the packet into the flow. */
+ if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
+ true) == INVALID_ARRAY_INDEX)
+ return -1;
+
+ return 0;
+}
+
+uint16_t
+gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out)
+{
+ uint16_t k = 0;
+ uint32_t i, j;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++) {
+ if (unlikely(tbl->flow_num == 0))
+ return k;
+
+ j = tbl->flows[i].start_index;
+ while (j != INVALID_ARRAY_INDEX) {
+ if (tbl->items[j].start_time <= flush_timestamp) {
+ out[k++] = tbl->items[j].firstseg;
+ if (tbl->items[j].nb_merged > 1)
+ update_header(&(tbl->items[j]));
+ /*
+ * Delete the packet and get the next
+ * packet in the flow.
+ */
+ j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
+ tbl->flows[i].start_index = j;
+ if (j == INVALID_ARRAY_INDEX)
+ tbl->flow_num--;
+
+ if (unlikely(k == nb_out))
+ return k;
+ } else
+ /*
+ * The left packets in this flow won't be
+ * timeout. Go to check other flows.
+ */
+ break;
+ }
+ }
+ return k;
+}
+
+uint32_t
+gro_tcp6_tbl_pkt_count(void *tbl)
+{
+ struct gro_tcp6_tbl *gro_tbl = tbl;
+
+ if (gro_tbl)
+ return gro_tbl->item_num;
+
+ return 0;
+}
diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h
new file mode 100644
index 0000000000..f37686df3a
--- /dev/null
+++ b/lib/gro/gro_tcp6.h
@@ -0,0 +1,150 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#ifndef _GRO_TCP6_H_
+#define _GRO_TCP6_H_
+
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+#define GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
+
+/* Header fields representing a TCP/IPv6 flow */
+struct tcp6_flow_key {
+ struct rte_ether_addr eth_saddr;
+ struct rte_ether_addr eth_daddr;
+ uint8_t src_addr[16];
+ uint8_t dst_addr[16];
+
+ uint32_t recv_ack;
+ uint16_t src_port;
+ uint16_t dst_port;
+};
+
+struct gro_tcp6_flow {
+ struct tcp6_flow_key key;
+ rte_be32_t vtc_flow;
+ /*
+ * The index of the first packet in the flow.
+ * INVALID_ARRAY_INDEX indicates an empty flow.
+ */
+ uint32_t start_index;
+};
+
+/*
+ * TCP/IPv6 reassembly table structure.
+ */
+struct gro_tcp6_tbl {
+ /* item array */
+ struct gro_tcp_item *items;
+ /* flow array */
+ struct gro_tcp6_flow *flows;
+ /* current item number */
+ uint32_t item_num;
+ /* current flow num */
+ uint32_t flow_num;
+ /* item array size */
+ uint32_t max_item_num;
+ /* flow array size */
+ uint32_t max_flow_num;
+};
+
+/**
+ * This function creates a TCP/IPv6 reassembly table.
+ *
+ * @param socket_id
+ * Socket index for allocating the TCP/IPv6 reassemble table
+ * @param max_flow_num
+ * The maximum number of flows in the TCP/IPv6 GRO table
+ * @param max_item_per_flow
+ * The maximum number of packets per flow
+ *
+ * @return
+ * - Return the table pointer on success.
+ * - Return NULL on failure.
+ */
+void *gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow);
+
+/**
+ * This function destroys a TCP/IPv6 reassembly table.
+ *
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table.
+ */
+void gro_tcp6_tbl_destroy(void *tbl);
+
+/**
+ * This function merges a TCP/IPv6 packet. It doesn't process the packet,
+ * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't have
+ * payload.
+ *
+ * This function doesn't check if the packet has correct checksums and
+ * doesn't re-calculate checksums for the merged packet. Additionally,
+ * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
+ * when IP fragmentation is possible (i.e., DF==0). It returns the
+ * packet, if the packet has invalid parameters (e.g. SYN bit is set)
+ * or there is no available space in the table.
+ *
+ * @param pkt
+ * Packet to reassemble
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table
+ * @start_time
+ * The time when the packet is inserted into the table
+ *
+ * @return
+ * - Return a positive value if the packet is merged.
+ * - Return zero if the packet isn't merged but stored in the table.
+ * - Return a negative value for invalid parameters or no available
+ * space in the table.
+ */
+int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time);
+
+/**
+ * This function flushes timeout packets in a TCP/IPv4 reassembly table,
+ * and without updating checksums.
+ *
+ * @param tbl
+ * TCP/IPv4 reassembly table pointer
+ * @param flush_timestamp
+ * Flush packets which are inserted into the table before or at the
+ * flush_timestamp.
+ * @param out
+ * Pointer array used to keep flushed packets
+ * @param nb_out
+ * The element number in 'out'. It also determines the maximum number of
+ * packets that can be flushed finally.
+ *
+ * @return
+ * The number of flushed packets
+ */
+uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out);
+
+/**
+ * This function returns the number of the packets in a TCP/IPv4
+ * reassembly table.
+ *
+ * @param tbl
+ * TCP/IPv4 reassembly table pointer
+ *
+ * @return
+ * The number of packets in the table
+ */
+uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
+
+/*
+ * Check if two TCP/IPv4 packets belong to the same flow.
+ */
+static inline int
+is_same_tcp6_flow(struct tcp6_flow_key k1, struct tcp6_flow_key k2)
+{
+ return (!memcmp(&k1, &k2, sizeof(struct tcp6_flow_key)));
+}
+
+#endif
diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
index 3be4deb7c7..56b30b8c98 100644
--- a/lib/gro/gro_vxlan_tcp4.c
+++ b/lib/gro/gro_vxlan_tcp4.c
@@ -7,6 +7,7 @@
#include <rte_ethdev.h>
#include <rte_udp.h>
+#include "gro_tcp.h"
#include "gro_vxlan_tcp4.h"
void *
@@ -248,7 +249,7 @@ merge_two_vxlan_tcp4_packets(struct gro_vxlan_tcp4_item *item,
uint16_t outer_ip_id,
uint16_t ip_id)
{
- if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp, sent_seq,
+ if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
ip_id, pkt->outer_l2_len +
pkt->outer_l3_len)) {
/* Update the outer IPv4 ID to the large value. */
diff --git a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h
index 7832942a68..d68d5fcd5b 100644
--- a/lib/gro/gro_vxlan_tcp4.h
+++ b/lib/gro/gro_vxlan_tcp4.h
@@ -5,6 +5,7 @@
#ifndef _GRO_VXLAN_TCP4_H_
#define _GRO_VXLAN_TCP4_H_
+#include "gro_tcp.h"
#include "gro_tcp4.h"
#define GRO_VXLAN_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
@@ -36,7 +37,7 @@ struct gro_vxlan_tcp4_flow {
};
struct gro_vxlan_tcp4_item {
- struct gro_tcp4_item inner_item;
+ struct gro_tcp_item inner_item;
/* IPv4 ID in the outer IPv4 header */
uint16_t outer_ip_id;
/* Indicate if outer IPv4 ID can be ignored */
diff --git a/lib/gro/meson.build b/lib/gro/meson.build
index e4fa2958bd..dbce05220d 100644
--- a/lib/gro/meson.build
+++ b/lib/gro/meson.build
@@ -4,6 +4,7 @@
sources = files(
'rte_gro.c',
'gro_tcp4.c',
+ 'gro_tcp6.c',
'gro_udp4.c',
'gro_vxlan_tcp4.c',
'gro_vxlan_udp4.c',
diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c
index e35399fd42..d824eebd93 100644
--- a/lib/gro/rte_gro.c
+++ b/lib/gro/rte_gro.c
@@ -8,6 +8,7 @@
#include "rte_gro.h"
#include "gro_tcp4.h"
+#include "gro_tcp6.h"
#include "gro_udp4.h"
#include "gro_vxlan_tcp4.h"
#include "gro_vxlan_udp4.h"
@@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
- gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
+ gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, gro_tcp6_tbl_create, NULL};
static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
+ gro_tcp6_tbl_destroy,
NULL};
static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count,
+ gro_tcp6_tbl_pkt_count,
NULL};
#define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
@@ -35,6 +38,12 @@ static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+/* GRO with extension headers is not supported */
+#define IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
+ ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
+ ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
+ (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+
#define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
@@ -147,7 +156,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
/* allocate a reassembly table for TCP/IPv4 GRO */
struct gro_tcp4_tbl tcp_tbl;
struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
- struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+ struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+
+ struct gro_tcp6_tbl tcp6_tbl;
+ struct gro_tcp6_flow tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
+ struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
/* allocate a reassembly table for UDP/IPv4 GRO */
struct gro_udp4_tbl udp_tbl;
@@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
int32_t ret;
uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
- do_vxlan_udp_gro = 0;
+ do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
do_udp4_gro = 1;
}
+ if (param->gro_types & RTE_GRO_TCP_IPV6) {
+ for (i = 0; i < item_num; i++)
+ tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+ tcp6_tbl.flows = tcp6_flows;
+ tcp6_tbl.items = tcp6_items;
+ tcp6_tbl.flow_num = 0;
+ tcp6_tbl.item_num = 0;
+ tcp6_tbl.max_flow_num = item_num;
+ tcp6_tbl.max_item_num = item_num;
+ do_tcp6_gro = 1;
+ }
for (i = 0; i < nb_pkts; i++) {
/*
@@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
nb_after_gro--;
else if (ret < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
+ if (ret > 0)
+ /* merge successfully */
+ nb_after_gro--;
+ else if (ret < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
if ((nb_after_gro < nb_pkts)
|| (unprocess_num < nb_pkts)) {
i = 0;
+ /* Copy unprocessed packets */
+ if (unprocess_num > 0) {
+ memcpy(&pkts[i], unprocess_pkts,
+ sizeof(struct rte_mbuf *) *
+ unprocess_num);
+ i = unprocess_num;
+ }
+
/* Flush all packets from the tables */
if (do_vxlan_tcp_gro) {
- i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
+ i += gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
0, pkts, nb_pkts);
}
@@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
&pkts[i], nb_pkts - i);
}
- /* Copy unprocessed packets */
- if (unprocess_num > 0) {
- memcpy(&pkts[i], unprocess_pkts,
- sizeof(struct rte_mbuf *) *
- unprocess_num);
+
+ if (do_tcp6_gro) {
+ i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
+ &pkts[i], nb_pkts - i);
}
- nb_after_gro = i + unprocess_num;
}
return nb_after_gro;
@@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
{
struct rte_mbuf *unprocess_pkts[nb_pkts];
struct gro_ctx *gro_ctx = ctx;
- void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
+ void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
uint64_t current_time;
uint16_t i, unprocess_num = 0;
- uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro;
+ uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro, do_tcp6_gro;
if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
+ tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
RTE_GRO_TCP_IPV4;
@@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
RTE_GRO_UDP_IPV4;
do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
RTE_GRO_IPV4_VXLAN_UDP_IPV4;
+ do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) == RTE_GRO_TCP_IPV6;
current_time = rte_rdtsc();
@@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
if (gro_udp4_reassemble(pkts[i], udp_tbl,
current_time) < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
+ current_time) < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
flush_timestamp,
&out[num], left_nb_out);
+ left_nb_out = max_nb_out - num;
+ }
+
+ if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
+ num += gro_tcp6_tbl_timeout_flush(
+ gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
+ flush_timestamp,
+ &out[num], left_nb_out);
+
}
return num;
diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h
index 9f9ed4935a..c83dfd9ad1 100644
--- a/lib/gro/rte_gro.h
+++ b/lib/gro/rte_gro.h
@@ -38,6 +38,9 @@ extern "C" {
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL << RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
/**< VxLAN UDP/IPv4 GRO flag. */
+#define RTE_GRO_TCP_IPV6_INDEX 4
+#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX)
+/**< TCP/IPv6 GRO flag. */
/**
* Structure used to create GRO context objects or used to pass
--
2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* [PATCH v2] gro : ipv6 changes to support GRO for TCP/ipv6
2022-10-20 18:07 [PATCH] gro : ipv6 changes to support GRO for TCP/ipv6 Kumara Parameshwaran
@ 2022-10-20 18:14 ` Kumara Parameshwaran
2023-05-12 2:47 ` Hu, Jiayu
` (10 more replies)
0 siblings, 11 replies; 32+ messages in thread
From: Kumara Parameshwaran @ 2022-10-20 18:14 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev, Kumara Parameshwaran
From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
The patch adds GRO support for TCP/ipv6 packets. This does not
include the support for vxlan, udp ipv6 packets.
Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
---
v1:
* Changes to support GRO for TCP/ipv6 packets. This does not include
vxlan changes.
* The GRO is performed only for ipv6 packets that does not contain
extension headers.
* The logic for the TCP coalescing remains the same, in ipv6 header
the source address, destination address, flow label, version fields
are expected to be the same.
* Re-organised the code to reuse certain tcp functions for both ipv4 and
ipv6 flows.
v2:
* Fix comments in gro_tcp6.h header file.
lib/gro/gro_tcp.h | 155 ++++++++++++++++
lib/gro/gro_tcp4.c | 7 +-
lib/gro/gro_tcp4.h | 152 +--------------
lib/gro/gro_tcp6.c | 387 +++++++++++++++++++++++++++++++++++++++
lib/gro/gro_tcp6.h | 150 +++++++++++++++
lib/gro/gro_vxlan_tcp4.c | 3 +-
lib/gro/gro_vxlan_tcp4.h | 3 +-
lib/gro/meson.build | 1 +
lib/gro/rte_gro.c | 83 +++++++--
lib/gro/rte_gro.h | 3 +
10 files changed, 774 insertions(+), 170 deletions(-)
create mode 100644 lib/gro/gro_tcp.h
create mode 100644 lib/gro/gro_tcp6.c
create mode 100644 lib/gro/gro_tcp6.h
diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h
new file mode 100644
index 0000000000..c5d248a022
--- /dev/null
+++ b/lib/gro/gro_tcp.h
@@ -0,0 +1,155 @@
+#ifndef _GRO_TCP_H_
+#define _GRO_TCP_H_
+
+#include <rte_tcp.h>
+
+/*
+ * The max length of a IPv4 packet, which includes the length of the L3
+ * header, the L4 header and the data payload.
+ */
+#define MAX_IP_PKT_LENGTH UINT16_MAX
+
+/* The maximum TCP header length */
+#define MAX_TCP_HLEN 60
+#define INVALID_TCP_HDRLEN(len) \
+ (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
+
+struct gro_tcp_item {
+ /*
+ * The first MBUF segment of the packet. If the value
+ * is NULL, it means the item is empty.
+ */
+ struct rte_mbuf *firstseg;
+ /* The last MBUF segment of the packet */
+ struct rte_mbuf *lastseg;
+ /*
+ * The time when the first packet is inserted into the table.
+ * This value won't be updated, even if the packet is merged
+ * with other packets.
+ */
+ uint64_t start_time;
+ /*
+ * next_pkt_idx is used to chain the packets that
+ * are in the same flow but can't be merged together
+ * (e.g. caused by packet reordering).
+ */
+ uint32_t next_pkt_idx;
+ /* TCP sequence number of the packet */
+ uint32_t sent_seq;
+ /* IPv4 ID of the packet */
+ uint16_t ip_id;
+ /* the number of merged packets */
+ uint16_t nb_merged;
+ /* Indicate if IPv4 ID can be ignored */
+ uint8_t is_atomic;
+};
+
+/*
+ * Merge two TCP packets without updating checksums.
+ * If cmp is larger than 0, append the new packet to the
+ * original packet. Otherwise, pre-pend the new packet to
+ * the original packet.
+ */
+static inline int
+merge_two_tcp_packets(struct gro_tcp_item *item,
+ struct rte_mbuf *pkt,
+ int cmp,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t l2_offset)
+{
+ struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
+ uint16_t hdr_len, l2_len;
+
+ if (cmp > 0) {
+ pkt_head = item->firstseg;
+ pkt_tail = pkt;
+ } else {
+ pkt_head = pkt;
+ pkt_tail = item->firstseg;
+ }
+
+ /* check if the IPv4 packet length is greater than the max value */
+ hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
+ pkt_head->l4_len;
+ l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
+ if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
+ hdr_len > MAX_IP_PKT_LENGTH))
+ return 0;
+
+ /* remove the packet header for the tail packet */
+ rte_pktmbuf_adj(pkt_tail, hdr_len);
+
+ /* chain two packets together */
+ if (cmp > 0) {
+ item->lastseg->next = pkt;
+ item->lastseg = rte_pktmbuf_lastseg(pkt);
+ /* update IP ID to the larger value */
+ item->ip_id = ip_id;
+ } else {
+ lastseg = rte_pktmbuf_lastseg(pkt);
+ lastseg->next = item->firstseg;
+ item->firstseg = pkt;
+ /* update sent_seq to the smaller value */
+ item->sent_seq = sent_seq;
+ item->ip_id = ip_id;
+ }
+ item->nb_merged++;
+
+ /* update MBUF metadata for the merged packet */
+ pkt_head->nb_segs += pkt_tail->nb_segs;
+ pkt_head->pkt_len += pkt_tail->pkt_len;
+
+ return 1;
+}
+
+/*
+ * Check if two TCP/IPv4 packets are neighbors.
+ */
+static inline int
+check_seq_option(struct gro_tcp_item *item,
+ struct rte_tcp_hdr *tcph,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t tcp_hl,
+ uint16_t tcp_dl,
+ uint16_t l2_offset,
+ uint8_t is_atomic)
+{
+ struct rte_mbuf *pkt_orig = item->firstseg;
+ char *iph_orig;
+ struct rte_tcp_hdr *tcph_orig;
+ uint16_t len, tcp_hl_orig;
+
+ iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
+ l2_offset + pkt_orig->l2_len);
+ tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
+ tcp_hl_orig = pkt_orig->l4_len;
+
+ /* Check if TCP option fields equal */
+ len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
+ if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
+ (memcmp(tcph + 1, tcph_orig + 1,
+ len) != 0)))
+ return 0;
+
+ /* Don't merge packets whose DF bits are different */
+ if (unlikely(item->is_atomic ^ is_atomic))
+ return 0;
+
+ /* check if the two packets are neighbors */
+ len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
+ pkt_orig->l3_len - tcp_hl_orig;
+ if ((sent_seq == item->sent_seq + len) && (is_atomic ||
+ (ip_id == item->ip_id + 1)))
+ /* append the new packet */
+ return 1;
+ else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
+ (ip_id + item->nb_merged == item->ip_id)))
+ /* pre-pend the new packet */
+ return -1;
+
+ return 0;
+}
+
+#endif
diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c
index 8f5e800250..eea2a72ecd 100644
--- a/lib/gro/gro_tcp4.c
+++ b/lib/gro/gro_tcp4.c
@@ -7,6 +7,7 @@
#include <rte_ethdev.h>
#include "gro_tcp4.h"
+#include "gro_tcp.h"
void *
gro_tcp4_tbl_create(uint16_t socket_id,
@@ -30,7 +31,7 @@ gro_tcp4_tbl_create(uint16_t socket_id,
if (tbl == NULL)
return NULL;
- size = sizeof(struct gro_tcp4_item) * entries_num;
+ size = sizeof(struct gro_tcp_item) * entries_num;
tbl->items = rte_zmalloc_socket(__func__,
size,
RTE_CACHE_LINE_SIZE,
@@ -177,7 +178,7 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
* update the packet length for the flushed packet.
*/
static inline void
-update_header(struct gro_tcp4_item *item)
+update_header(struct gro_tcp_item *item)
{
struct rte_ipv4_hdr *ipv4_hdr;
struct rte_mbuf *pkt = item->firstseg;
@@ -302,7 +303,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
is_atomic);
if (cmp) {
- if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
+ if (merge_two_tcp_packets(&(tbl->items[cur_idx]),
pkt, cmp, sent_seq, ip_id, 0))
return 1;
/*
diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h
index 212f97a042..634a215b98 100644
--- a/lib/gro/gro_tcp4.h
+++ b/lib/gro/gro_tcp4.h
@@ -5,22 +5,9 @@
#ifndef _GRO_TCP4_H_
#define _GRO_TCP4_H_
-#include <rte_tcp.h>
-
#define INVALID_ARRAY_INDEX 0xffffffffUL
#define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
-/*
- * The max length of a IPv4 packet, which includes the length of the L3
- * header, the L4 header and the data payload.
- */
-#define MAX_IPV4_PKT_LENGTH UINT16_MAX
-
-/* The maximum TCP header length */
-#define MAX_TCP_HLEN 60
-#define INVALID_TCP_HDRLEN(len) \
- (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
-
/* Header fields representing a TCP/IPv4 flow */
struct tcp4_flow_key {
struct rte_ether_addr eth_saddr;
@@ -42,42 +29,12 @@ struct gro_tcp4_flow {
uint32_t start_index;
};
-struct gro_tcp4_item {
- /*
- * The first MBUF segment of the packet. If the value
- * is NULL, it means the item is empty.
- */
- struct rte_mbuf *firstseg;
- /* The last MBUF segment of the packet */
- struct rte_mbuf *lastseg;
- /*
- * The time when the first packet is inserted into the table.
- * This value won't be updated, even if the packet is merged
- * with other packets.
- */
- uint64_t start_time;
- /*
- * next_pkt_idx is used to chain the packets that
- * are in the same flow but can't be merged together
- * (e.g. caused by packet reordering).
- */
- uint32_t next_pkt_idx;
- /* TCP sequence number of the packet */
- uint32_t sent_seq;
- /* IPv4 ID of the packet */
- uint16_t ip_id;
- /* the number of merged packets */
- uint16_t nb_merged;
- /* Indicate if IPv4 ID can be ignored */
- uint8_t is_atomic;
-};
-
/*
* TCP/IPv4 reassembly table structure.
*/
struct gro_tcp4_tbl {
/* item array */
- struct gro_tcp4_item *items;
+ struct gro_tcp_item *items;
/* flow array */
struct gro_tcp4_flow *flows;
/* current item number */
@@ -195,111 +152,4 @@ is_same_tcp4_flow(struct tcp4_flow_key k1, struct tcp4_flow_key k2)
(k1.dst_port == k2.dst_port));
}
-/*
- * Merge two TCP/IPv4 packets without updating checksums.
- * If cmp is larger than 0, append the new packet to the
- * original packet. Otherwise, pre-pend the new packet to
- * the original packet.
- */
-static inline int
-merge_two_tcp4_packets(struct gro_tcp4_item *item,
- struct rte_mbuf *pkt,
- int cmp,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t l2_offset)
-{
- struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
- uint16_t hdr_len, l2_len;
-
- if (cmp > 0) {
- pkt_head = item->firstseg;
- pkt_tail = pkt;
- } else {
- pkt_head = pkt;
- pkt_tail = item->firstseg;
- }
-
- /* check if the IPv4 packet length is greater than the max value */
- hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
- pkt_head->l4_len;
- l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
- if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
- hdr_len > MAX_IPV4_PKT_LENGTH))
- return 0;
-
- /* remove the packet header for the tail packet */
- rte_pktmbuf_adj(pkt_tail, hdr_len);
-
- /* chain two packets together */
- if (cmp > 0) {
- item->lastseg->next = pkt;
- item->lastseg = rte_pktmbuf_lastseg(pkt);
- /* update IP ID to the larger value */
- item->ip_id = ip_id;
- } else {
- lastseg = rte_pktmbuf_lastseg(pkt);
- lastseg->next = item->firstseg;
- item->firstseg = pkt;
- /* update sent_seq to the smaller value */
- item->sent_seq = sent_seq;
- item->ip_id = ip_id;
- }
- item->nb_merged++;
-
- /* update MBUF metadata for the merged packet */
- pkt_head->nb_segs += pkt_tail->nb_segs;
- pkt_head->pkt_len += pkt_tail->pkt_len;
-
- return 1;
-}
-
-/*
- * Check if two TCP/IPv4 packets are neighbors.
- */
-static inline int
-check_seq_option(struct gro_tcp4_item *item,
- struct rte_tcp_hdr *tcph,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t tcp_hl,
- uint16_t tcp_dl,
- uint16_t l2_offset,
- uint8_t is_atomic)
-{
- struct rte_mbuf *pkt_orig = item->firstseg;
- struct rte_ipv4_hdr *iph_orig;
- struct rte_tcp_hdr *tcph_orig;
- uint16_t len, tcp_hl_orig;
-
- iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char *) +
- l2_offset + pkt_orig->l2_len);
- tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig + pkt_orig->l3_len);
- tcp_hl_orig = pkt_orig->l4_len;
-
- /* Check if TCP option fields equal */
- len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
- if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
- (memcmp(tcph + 1, tcph_orig + 1,
- len) != 0)))
- return 0;
-
- /* Don't merge packets whose DF bits are different */
- if (unlikely(item->is_atomic ^ is_atomic))
- return 0;
-
- /* check if the two packets are neighbors */
- len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
- pkt_orig->l3_len - tcp_hl_orig;
- if ((sent_seq == item->sent_seq + len) && (is_atomic ||
- (ip_id == item->ip_id + 1)))
- /* append the new packet */
- return 1;
- else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
- (ip_id + item->nb_merged == item->ip_id)))
- /* pre-pend the new packet */
- return -1;
-
- return 0;
-}
#endif
diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c
new file mode 100644
index 0000000000..a37d8e2512
--- /dev/null
+++ b/lib/gro/gro_tcp6.c
@@ -0,0 +1,387 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp6.h"
+#include "gro_tcp.h"
+
+void *
+gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow)
+{
+ struct gro_tcp6_tbl *tbl;
+ size_t size;
+ uint32_t entries_num, i;
+
+ entries_num = max_flow_num * max_item_per_flow;
+ entries_num = RTE_MIN(entries_num, GRO_TCP6_TBL_MAX_ITEM_NUM);
+
+ if (entries_num == 0)
+ return NULL;
+
+ tbl = rte_zmalloc_socket(__func__,
+ sizeof(struct gro_tcp6_tbl),
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl == NULL)
+ return NULL;
+
+ size = sizeof(struct gro_tcp_item) * entries_num;
+ tbl->items = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->items == NULL) {
+ rte_free(tbl);
+ return NULL;
+ }
+ tbl->max_item_num = entries_num;
+
+ size = sizeof(struct gro_tcp6_flow) * entries_num;
+ tbl->flows = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->flows == NULL) {
+ rte_free(tbl->items);
+ rte_free(tbl);
+ return NULL;
+ }
+ /* INVALID_ARRAY_INDEX indicates an empty flow */
+ for (i = 0; i < entries_num; i++)
+ tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
+ tbl->max_flow_num = entries_num;
+
+ return tbl;
+}
+
+void
+gro_tcp6_tbl_destroy(void *tbl)
+{
+ struct gro_tcp6_tbl *tcp_tbl = tbl;
+
+ if (tcp_tbl) {
+ rte_free(tcp_tbl->items);
+ rte_free(tcp_tbl->flows);
+ }
+ rte_free(tcp_tbl);
+}
+
+static inline uint32_t
+find_an_empty_item(struct gro_tcp6_tbl *tbl)
+{
+ uint32_t i;
+ uint32_t max_item_num = tbl->max_item_num;
+
+ for (i = 0; i < max_item_num; i++)
+ if (tbl->items[i].firstseg == NULL)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+find_an_empty_flow(struct gro_tcp6_tbl *tbl)
+{
+ uint32_t i;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++)
+ if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+insert_new_item(struct gro_tcp6_tbl *tbl,
+ struct rte_mbuf *pkt,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint8_t is_atomic)
+{
+ uint32_t item_idx;
+
+ item_idx = find_an_empty_item(tbl);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return INVALID_ARRAY_INDEX;
+
+ tbl->items[item_idx].firstseg = pkt;
+ tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
+ tbl->items[item_idx].start_time = start_time;
+ tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
+ tbl->items[item_idx].sent_seq = sent_seq;
+ tbl->items[item_idx].nb_merged = 1;
+ tbl->items[item_idx].is_atomic = is_atomic;
+ tbl->item_num++;
+
+ /* if the previous packet exists, chain them together. */
+ if (prev_idx != INVALID_ARRAY_INDEX) {
+ tbl->items[item_idx].next_pkt_idx =
+ tbl->items[prev_idx].next_pkt_idx;
+ tbl->items[prev_idx].next_pkt_idx = item_idx;
+ }
+
+ return item_idx;
+}
+
+static inline uint32_t
+delete_item(struct gro_tcp6_tbl *tbl, uint32_t item_idx,
+ uint32_t prev_item_idx)
+{
+ uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
+
+ /* NULL indicates an empty item */
+ tbl->items[item_idx].firstseg = NULL;
+ tbl->item_num--;
+ if (prev_item_idx != INVALID_ARRAY_INDEX)
+ tbl->items[prev_item_idx].next_pkt_idx = next_idx;
+
+ return next_idx;
+}
+
+static inline uint32_t
+insert_new_flow(struct gro_tcp6_tbl *tbl,
+ struct tcp6_flow_key *src,
+ rte_be32_t vtc_flow,
+ uint32_t item_idx)
+{
+ struct tcp6_flow_key *dst;
+ uint32_t flow_idx;
+
+ flow_idx = find_an_empty_flow(tbl);
+ if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
+ return INVALID_ARRAY_INDEX;
+
+ dst = &(tbl->flows[flow_idx].key);
+
+ rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
+ rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
+ memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst->src_addr));
+ memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst->dst_addr));
+ dst->recv_ack = src->recv_ack;
+ dst->src_port = src->src_port;
+ dst->dst_port = src->dst_port;
+
+ tbl->flows[flow_idx].start_index = item_idx;
+ tbl->flow_num++;
+ tbl->flows->vtc_flow = vtc_flow;
+
+ return flow_idx;
+}
+
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
+
+ ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len - pkt->l3_len);
+}
+
+int32_t
+gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time)
+{
+ struct rte_ether_hdr *eth_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_tcp_hdr *tcp_hdr;
+ uint32_t sent_seq;
+ int32_t tcp_dl;
+ uint16_t ip_tlen;
+ struct tcp6_flow_key key;
+ uint32_t cur_idx, prev_idx, item_idx;
+ uint32_t i, max_flow_num, remaining_flow_num;
+ int cmp;
+ uint8_t find;
+ rte_be32_t vtc_flow_diff;
+
+ /*
+ * Don't process the packet whose TCP header length is greater
+ * than 60 bytes or less than 20 bytes.
+ */
+ if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
+ return -1;
+
+ eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
+ ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
+ tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv6_hdr + pkt->l3_len);
+
+ /*
+ * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
+ * or CWR set.
+ */
+ if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
+ return -1;
+
+ ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
+ /*
+ * Trim the tail padding bytes. The IPv6 header is fixed to
+ * 40 bytes unlike IPv4 that is variable. The length in the IPv6 header
+ * contains only length of TCP Header + TCP Payload, whereas IPv4 header contains
+ * length of IP Header + TCP Header + TCP Payload
+ */
+ if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len + pkt->l3_len))
+ rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len - pkt->l3_len);
+ /*
+ * Don't process the packet whose payload length is less than or
+ * equal to 0.
+ */
+ tcp_dl = ip_tlen - pkt->l4_len;
+ if (tcp_dl <= 0)
+ return -1;
+
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
+ memcpy(&key.src_addr[0], &ipv6_hdr->src_addr, sizeof(key.src_addr));
+ memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr, sizeof(key.dst_addr));
+ key.src_port = tcp_hdr->src_port;
+ key.dst_port = tcp_hdr->dst_port;
+ key.recv_ack = tcp_hdr->recv_ack;
+
+ /* Search for a matched flow. */
+ max_flow_num = tbl->max_flow_num;
+ remaining_flow_num = tbl->flow_num;
+ find = 0;
+ for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
+ if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
+ if (is_same_tcp6_flow(tbl->flows[i].key, key)) {
+ /*
+ * IP version (4) Traffic Class (8) Flow Label (20)
+ * All fields except Traffic class should be same
+ */
+ vtc_flow_diff = (ipv6_hdr->vtc_flow ^ tbl->flows->vtc_flow);
+ if (vtc_flow_diff & htonl(0xF00FFFFF))
+ continue;
+ find = 1;
+ break;
+ }
+ remaining_flow_num--;
+ }
+ }
+
+ /*
+ * Fail to find a matched flow. Insert a new flow and store the
+ * packet into the flow.
+ */
+ if (find == 0) {
+ item_idx = insert_new_item(tbl, pkt, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, true);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return -1;
+ if (insert_new_flow(tbl, &key, ipv6_hdr->vtc_flow, item_idx) ==
+ INVALID_ARRAY_INDEX) {
+ /*
+ * Fail to insert a new flow, so delete the
+ * stored packet.
+ */
+ delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
+ return -1;
+ }
+ return 0;
+ }
+
+ /*
+ * Check all packets in the flow and try to find a neighbor for
+ * the input packet.
+ */
+ cur_idx = tbl->flows[i].start_index;
+ prev_idx = cur_idx;
+ do {
+ cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
+ sent_seq, 0, pkt->l4_len, tcp_dl, 0,
+ true);
+ if (cmp) {
+ if (merge_two_tcp_packets(&(tbl->items[cur_idx]),
+ pkt, cmp, sent_seq, 0, 0)) {
+ return 1;
+ }
+
+ /*
+ * Fail to merge the two packets, as the packet
+ * length is greater than the max value. Store
+ * the packet into the flow.
+ */
+ if (insert_new_item(tbl, pkt, start_time, cur_idx,
+ sent_seq, true) ==
+ INVALID_ARRAY_INDEX)
+ return -1;
+ return 0;
+ }
+ prev_idx = cur_idx;
+ cur_idx = tbl->items[cur_idx].next_pkt_idx;
+ } while (cur_idx != INVALID_ARRAY_INDEX);
+
+ /* Fail to find a neighbor, so store the packet into the flow. */
+ if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
+ true) == INVALID_ARRAY_INDEX)
+ return -1;
+
+ return 0;
+}
+
+uint16_t
+gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out)
+{
+ uint16_t k = 0;
+ uint32_t i, j;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++) {
+ if (unlikely(tbl->flow_num == 0))
+ return k;
+
+ j = tbl->flows[i].start_index;
+ while (j != INVALID_ARRAY_INDEX) {
+ if (tbl->items[j].start_time <= flush_timestamp) {
+ out[k++] = tbl->items[j].firstseg;
+ if (tbl->items[j].nb_merged > 1)
+ update_header(&(tbl->items[j]));
+ /*
+ * Delete the packet and get the next
+ * packet in the flow.
+ */
+ j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
+ tbl->flows[i].start_index = j;
+ if (j == INVALID_ARRAY_INDEX)
+ tbl->flow_num--;
+
+ if (unlikely(k == nb_out))
+ return k;
+ } else
+ /*
+ * The left packets in this flow won't be
+ * timeout. Go to check other flows.
+ */
+ break;
+ }
+ }
+ return k;
+}
+
+uint32_t
+gro_tcp6_tbl_pkt_count(void *tbl)
+{
+ struct gro_tcp6_tbl *gro_tbl = tbl;
+
+ if (gro_tbl)
+ return gro_tbl->item_num;
+
+ return 0;
+}
diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h
new file mode 100644
index 0000000000..91325c0da2
--- /dev/null
+++ b/lib/gro/gro_tcp6.h
@@ -0,0 +1,150 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#ifndef _GRO_TCP6_H_
+#define _GRO_TCP6_H_
+
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+#define GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
+
+/* Header fields representing a TCP/IPv6 flow */
+struct tcp6_flow_key {
+ struct rte_ether_addr eth_saddr;
+ struct rte_ether_addr eth_daddr;
+ uint8_t src_addr[16];
+ uint8_t dst_addr[16];
+
+ uint32_t recv_ack;
+ uint16_t src_port;
+ uint16_t dst_port;
+};
+
+struct gro_tcp6_flow {
+ struct tcp6_flow_key key;
+ rte_be32_t vtc_flow;
+ /*
+ * The index of the first packet in the flow.
+ * INVALID_ARRAY_INDEX indicates an empty flow.
+ */
+ uint32_t start_index;
+};
+
+/*
+ * TCP/IPv6 reassembly table structure.
+ */
+struct gro_tcp6_tbl {
+ /* item array */
+ struct gro_tcp_item *items;
+ /* flow array */
+ struct gro_tcp6_flow *flows;
+ /* current item number */
+ uint32_t item_num;
+ /* current flow num */
+ uint32_t flow_num;
+ /* item array size */
+ uint32_t max_item_num;
+ /* flow array size */
+ uint32_t max_flow_num;
+};
+
+/**
+ * This function creates a TCP/IPv6 reassembly table.
+ *
+ * @param socket_id
+ * Socket index for allocating the TCP/IPv6 reassemble table
+ * @param max_flow_num
+ * The maximum number of flows in the TCP/IPv6 GRO table
+ * @param max_item_per_flow
+ * The maximum number of packets per flow
+ *
+ * @return
+ * - Return the table pointer on success.
+ * - Return NULL on failure.
+ */
+void *gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow);
+
+/**
+ * This function destroys a TCP/IPv6 reassembly table.
+ *
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table.
+ */
+void gro_tcp6_tbl_destroy(void *tbl);
+
+/**
+ * This function merges a TCP/IPv6 packet. It doesn't process the packet,
+ * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't have
+ * payload.
+ *
+ * This function doesn't check if the packet has correct checksums and
+ * doesn't re-calculate checksums for the merged packet. Additionally,
+ * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
+ * when IP fragmentation is possible (i.e., DF==0). It returns the
+ * packet, if the packet has invalid parameters (e.g. SYN bit is set)
+ * or there is no available space in the table.
+ *
+ * @param pkt
+ * Packet to reassemble
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table
+ * @start_time
+ * The time when the packet is inserted into the table
+ *
+ * @return
+ * - Return a positive value if the packet is merged.
+ * - Return zero if the packet isn't merged but stored in the table.
+ * - Return a negative value for invalid parameters or no available
+ * space in the table.
+ */
+int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time);
+
+/**
+ * This function flushes timeout packets in a TCP/IPv6 reassembly table,
+ * and without updating checksums.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ * @param flush_timestamp
+ * Flush packets which are inserted into the table before or at the
+ * flush_timestamp.
+ * @param out
+ * Pointer array used to keep flushed packets
+ * @param nb_out
+ * The element number in 'out'. It also determines the maximum number of
+ * packets that can be flushed finally.
+ *
+ * @return
+ * The number of flushed packets
+ */
+uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out);
+
+/**
+ * This function returns the number of the packets in a TCP/IPv6
+ * reassembly table.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ *
+ * @return
+ * The number of packets in the table
+ */
+uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
+
+/*
+ * Check if two TCP/IPv6 packets belong to the same flow.
+ */
+static inline int
+is_same_tcp6_flow(struct tcp6_flow_key k1, struct tcp6_flow_key k2)
+{
+ return (!memcmp(&k1, &k2, sizeof(struct tcp6_flow_key)));
+}
+
+#endif
diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
index 3be4deb7c7..56b30b8c98 100644
--- a/lib/gro/gro_vxlan_tcp4.c
+++ b/lib/gro/gro_vxlan_tcp4.c
@@ -7,6 +7,7 @@
#include <rte_ethdev.h>
#include <rte_udp.h>
+#include "gro_tcp.h"
#include "gro_vxlan_tcp4.h"
void *
@@ -248,7 +249,7 @@ merge_two_vxlan_tcp4_packets(struct gro_vxlan_tcp4_item *item,
uint16_t outer_ip_id,
uint16_t ip_id)
{
- if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp, sent_seq,
+ if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
ip_id, pkt->outer_l2_len +
pkt->outer_l3_len)) {
/* Update the outer IPv4 ID to the large value. */
diff --git a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h
index 7832942a68..d68d5fcd5b 100644
--- a/lib/gro/gro_vxlan_tcp4.h
+++ b/lib/gro/gro_vxlan_tcp4.h
@@ -5,6 +5,7 @@
#ifndef _GRO_VXLAN_TCP4_H_
#define _GRO_VXLAN_TCP4_H_
+#include "gro_tcp.h"
#include "gro_tcp4.h"
#define GRO_VXLAN_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
@@ -36,7 +37,7 @@ struct gro_vxlan_tcp4_flow {
};
struct gro_vxlan_tcp4_item {
- struct gro_tcp4_item inner_item;
+ struct gro_tcp_item inner_item;
/* IPv4 ID in the outer IPv4 header */
uint16_t outer_ip_id;
/* Indicate if outer IPv4 ID can be ignored */
diff --git a/lib/gro/meson.build b/lib/gro/meson.build
index e4fa2958bd..dbce05220d 100644
--- a/lib/gro/meson.build
+++ b/lib/gro/meson.build
@@ -4,6 +4,7 @@
sources = files(
'rte_gro.c',
'gro_tcp4.c',
+ 'gro_tcp6.c',
'gro_udp4.c',
'gro_vxlan_tcp4.c',
'gro_vxlan_udp4.c',
diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c
index e35399fd42..d824eebd93 100644
--- a/lib/gro/rte_gro.c
+++ b/lib/gro/rte_gro.c
@@ -8,6 +8,7 @@
#include "rte_gro.h"
#include "gro_tcp4.h"
+#include "gro_tcp6.h"
#include "gro_udp4.h"
#include "gro_vxlan_tcp4.h"
#include "gro_vxlan_udp4.h"
@@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
- gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
+ gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, gro_tcp6_tbl_create, NULL};
static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
+ gro_tcp6_tbl_destroy,
NULL};
static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count,
+ gro_tcp6_tbl_pkt_count,
NULL};
#define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
@@ -35,6 +38,12 @@ static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+/* GRO with extension headers is not supported */
+#define IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
+ ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
+ ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
+ (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+
#define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
@@ -147,7 +156,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
/* allocate a reassembly table for TCP/IPv4 GRO */
struct gro_tcp4_tbl tcp_tbl;
struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
- struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+ struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+
+ struct gro_tcp6_tbl tcp6_tbl;
+ struct gro_tcp6_flow tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
+ struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
/* allocate a reassembly table for UDP/IPv4 GRO */
struct gro_udp4_tbl udp_tbl;
@@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
int32_t ret;
uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
- do_vxlan_udp_gro = 0;
+ do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
do_udp4_gro = 1;
}
+ if (param->gro_types & RTE_GRO_TCP_IPV6) {
+ for (i = 0; i < item_num; i++)
+ tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+ tcp6_tbl.flows = tcp6_flows;
+ tcp6_tbl.items = tcp6_items;
+ tcp6_tbl.flow_num = 0;
+ tcp6_tbl.item_num = 0;
+ tcp6_tbl.max_flow_num = item_num;
+ tcp6_tbl.max_item_num = item_num;
+ do_tcp6_gro = 1;
+ }
for (i = 0; i < nb_pkts; i++) {
/*
@@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
nb_after_gro--;
else if (ret < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
+ if (ret > 0)
+ /* merge successfully */
+ nb_after_gro--;
+ else if (ret < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
if ((nb_after_gro < nb_pkts)
|| (unprocess_num < nb_pkts)) {
i = 0;
+ /* Copy unprocessed packets */
+ if (unprocess_num > 0) {
+ memcpy(&pkts[i], unprocess_pkts,
+ sizeof(struct rte_mbuf *) *
+ unprocess_num);
+ i = unprocess_num;
+ }
+
/* Flush all packets from the tables */
if (do_vxlan_tcp_gro) {
- i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
+ i += gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
0, pkts, nb_pkts);
}
@@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
&pkts[i], nb_pkts - i);
}
- /* Copy unprocessed packets */
- if (unprocess_num > 0) {
- memcpy(&pkts[i], unprocess_pkts,
- sizeof(struct rte_mbuf *) *
- unprocess_num);
+
+ if (do_tcp6_gro) {
+ i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
+ &pkts[i], nb_pkts - i);
}
- nb_after_gro = i + unprocess_num;
}
return nb_after_gro;
@@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
{
struct rte_mbuf *unprocess_pkts[nb_pkts];
struct gro_ctx *gro_ctx = ctx;
- void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
+ void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
uint64_t current_time;
uint16_t i, unprocess_num = 0;
- uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro;
+ uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro, do_tcp6_gro;
if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
+ tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
RTE_GRO_TCP_IPV4;
@@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
RTE_GRO_UDP_IPV4;
do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
RTE_GRO_IPV4_VXLAN_UDP_IPV4;
+ do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) == RTE_GRO_TCP_IPV6;
current_time = rte_rdtsc();
@@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
if (gro_udp4_reassemble(pkts[i], udp_tbl,
current_time) < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
+ current_time) < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
flush_timestamp,
&out[num], left_nb_out);
+ left_nb_out = max_nb_out - num;
+ }
+
+ if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
+ num += gro_tcp6_tbl_timeout_flush(
+ gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
+ flush_timestamp,
+ &out[num], left_nb_out);
+
}
return num;
diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h
index 9f9ed4935a..c83dfd9ad1 100644
--- a/lib/gro/rte_gro.h
+++ b/lib/gro/rte_gro.h
@@ -38,6 +38,9 @@ extern "C" {
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL << RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
/**< VxLAN UDP/IPv4 GRO flag. */
+#define RTE_GRO_TCP_IPV6_INDEX 4
+#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX)
+/**< TCP/IPv6 GRO flag. */
/**
* Structure used to create GRO context objects or used to pass
--
2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* RE: [PATCH v2] gro : ipv6 changes to support GRO for TCP/ipv6
2022-10-20 18:14 ` [PATCH v2] " Kumara Parameshwaran
@ 2023-05-12 2:47 ` Hu, Jiayu
2023-05-16 9:28 ` kumaraparameshwaran rathinavel
2023-06-02 6:02 ` [PATCH v3] gro : ipv6-gro review comments to reduce code duplication across v4 and v6 Kumara Parameshwaran
` (9 subsequent siblings)
10 siblings, 1 reply; 32+ messages in thread
From: Hu, Jiayu @ 2023-05-12 2:47 UTC (permalink / raw)
To: Kumara Parameshwaran; +Cc: dev
Hi Kumar,
For TCP/IPv4 and TCP/IPv6, the TCP layer is the same and the difference is
the IP layer. So the code used for TCP layer needs to be shared among gro_tcp6.c
and gro_tcp4.c. But there are too much code duplication in gro_tcp4.c and gro_tcp6.c
in current implementation.
For example, struct tcp6_flow_key and struct tcp4_flow_key share most of fields, except
the IP address type. It's better to have a common TCP flow key structure in gro_tcp.h. In
gro_tcp4.h and gro_tcp6.h, we implement tcp4 and tcp6 flow key structures with using the
common structure.
Thanks,
Jiayu
> -----Original Message-----
> From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> Sent: Friday, October 21, 2022 2:14 AM
> To: Hu, Jiayu <jiayu.hu@intel.com>
> Cc: dev@dpdk.org; Kumara Parameshwaran
> <kumaraparamesh92@gmail.com>
> Subject: [PATCH v2] gro : ipv6 changes to support GRO for TCP/ipv6
>
> From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
>
> The patch adds GRO support for TCP/ipv6 packets. This does not include the
> support for vxlan, udp ipv6 packets.
>
> Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> ---
> v1:
> * Changes to support GRO for TCP/ipv6 packets. This does not
> include
> vxlan changes.
> * The GRO is performed only for ipv6 packets that does not contain
> extension headers.
> * The logic for the TCP coalescing remains the same, in ipv6 header
> the source address, destination address, flow label, version fields
> are expected to be the same.
> * Re-organised the code to reuse certain tcp functions for both ipv4
> and
> ipv6 flows.
> v2:
> * Fix comments in gro_tcp6.h header file.
>
> lib/gro/gro_tcp.h | 155 ++++++++++++++++
> lib/gro/gro_tcp4.c | 7 +-
> lib/gro/gro_tcp4.h | 152 +--------------
> lib/gro/gro_tcp6.c | 387
> +++++++++++++++++++++++++++++++++++++++
> lib/gro/gro_tcp6.h | 150 +++++++++++++++
> lib/gro/gro_vxlan_tcp4.c | 3 +-
> lib/gro/gro_vxlan_tcp4.h | 3 +-
> lib/gro/meson.build | 1 +
> lib/gro/rte_gro.c | 83 +++++++--
> lib/gro/rte_gro.h | 3 +
> 10 files changed, 774 insertions(+), 170 deletions(-) create mode 100644
> lib/gro/gro_tcp.h create mode 100644 lib/gro/gro_tcp6.c create mode
> 100644 lib/gro/gro_tcp6.h
>
> diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h new file mode 100644 index
> 0000000000..c5d248a022
> --- /dev/null
> +++ b/lib/gro/gro_tcp.h
> @@ -0,0 +1,155 @@
> +#ifndef _GRO_TCP_H_
> +#define _GRO_TCP_H_
> +
> +#include <rte_tcp.h>
> +
> +/*
> + * The max length of a IPv4 packet, which includes the length of the L3
> + * header, the L4 header and the data payload.
> + */
> +#define MAX_IP_PKT_LENGTH UINT16_MAX
> +
> +/* The maximum TCP header length */
> +#define MAX_TCP_HLEN 60
> +#define INVALID_TCP_HDRLEN(len) \
> + (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> +
> +struct gro_tcp_item {
> + /*
> + * The first MBUF segment of the packet. If the value
> + * is NULL, it means the item is empty.
> + */
> + struct rte_mbuf *firstseg;
> + /* The last MBUF segment of the packet */
> + struct rte_mbuf *lastseg;
> + /*
> + * The time when the first packet is inserted into the table.
> + * This value won't be updated, even if the packet is merged
> + * with other packets.
> + */
> + uint64_t start_time;
> + /*
> + * next_pkt_idx is used to chain the packets that
> + * are in the same flow but can't be merged together
> + * (e.g. caused by packet reordering).
> + */
> + uint32_t next_pkt_idx;
> + /* TCP sequence number of the packet */
> + uint32_t sent_seq;
> + /* IPv4 ID of the packet */
> + uint16_t ip_id;
> + /* the number of merged packets */
> + uint16_t nb_merged;
> + /* Indicate if IPv4 ID can be ignored */
> + uint8_t is_atomic;
> +};
> +
> +/*
> + * Merge two TCP packets without updating checksums.
> + * If cmp is larger than 0, append the new packet to the
> + * original packet. Otherwise, pre-pend the new packet to
> + * the original packet.
> + */
> +static inline int
> +merge_two_tcp_packets(struct gro_tcp_item *item,
> + struct rte_mbuf *pkt,
> + int cmp,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint16_t l2_offset)
> +{
> + struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
> + uint16_t hdr_len, l2_len;
> +
> + if (cmp > 0) {
> + pkt_head = item->firstseg;
> + pkt_tail = pkt;
> + } else {
> + pkt_head = pkt;
> + pkt_tail = item->firstseg;
> + }
> +
> + /* check if the IPv4 packet length is greater than the max value */
> + hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
> + pkt_head->l4_len;
> + l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
> + if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
> + hdr_len > MAX_IP_PKT_LENGTH))
> + return 0;
> +
> + /* remove the packet header for the tail packet */
> + rte_pktmbuf_adj(pkt_tail, hdr_len);
> +
> + /* chain two packets together */
> + if (cmp > 0) {
> + item->lastseg->next = pkt;
> + item->lastseg = rte_pktmbuf_lastseg(pkt);
> + /* update IP ID to the larger value */
> + item->ip_id = ip_id;
> + } else {
> + lastseg = rte_pktmbuf_lastseg(pkt);
> + lastseg->next = item->firstseg;
> + item->firstseg = pkt;
> + /* update sent_seq to the smaller value */
> + item->sent_seq = sent_seq;
> + item->ip_id = ip_id;
> + }
> + item->nb_merged++;
> +
> + /* update MBUF metadata for the merged packet */
> + pkt_head->nb_segs += pkt_tail->nb_segs;
> + pkt_head->pkt_len += pkt_tail->pkt_len;
> +
> + return 1;
> +}
> +
> +/*
> + * Check if two TCP/IPv4 packets are neighbors.
> + */
> +static inline int
> +check_seq_option(struct gro_tcp_item *item,
> + struct rte_tcp_hdr *tcph,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint16_t tcp_hl,
> + uint16_t tcp_dl,
> + uint16_t l2_offset,
> + uint8_t is_atomic)
> +{
> + struct rte_mbuf *pkt_orig = item->firstseg;
> + char *iph_orig;
> + struct rte_tcp_hdr *tcph_orig;
> + uint16_t len, tcp_hl_orig;
> +
> + iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
> + l2_offset + pkt_orig->l2_len);
> + tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
> + tcp_hl_orig = pkt_orig->l4_len;
> +
> + /* Check if TCP option fields equal */
> + len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> + if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> + (memcmp(tcph + 1, tcph_orig + 1,
> + len) != 0)))
> + return 0;
> +
> + /* Don't merge packets whose DF bits are different */
> + if (unlikely(item->is_atomic ^ is_atomic))
> + return 0;
> +
> + /* check if the two packets are neighbors */
> + len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> + pkt_orig->l3_len - tcp_hl_orig;
> + if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> + (ip_id == item->ip_id + 1)))
> + /* append the new packet */
> + return 1;
> + else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> + (ip_id + item->nb_merged == item->ip_id)))
> + /* pre-pend the new packet */
> + return -1;
> +
> + return 0;
> +}
> +
> +#endif
> diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c index
> 8f5e800250..eea2a72ecd 100644
> --- a/lib/gro/gro_tcp4.c
> +++ b/lib/gro/gro_tcp4.c
> @@ -7,6 +7,7 @@
> #include <rte_ethdev.h>
>
> #include "gro_tcp4.h"
> +#include "gro_tcp.h"
>
> void *
> gro_tcp4_tbl_create(uint16_t socket_id, @@ -30,7 +31,7 @@
> gro_tcp4_tbl_create(uint16_t socket_id,
> if (tbl == NULL)
> return NULL;
>
> - size = sizeof(struct gro_tcp4_item) * entries_num;
> + size = sizeof(struct gro_tcp_item) * entries_num;
> tbl->items = rte_zmalloc_socket(__func__,
> size,
> RTE_CACHE_LINE_SIZE,
> @@ -177,7 +178,7 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
> * update the packet length for the flushed packet.
> */
> static inline void
> -update_header(struct gro_tcp4_item *item)
> +update_header(struct gro_tcp_item *item)
> {
> struct rte_ipv4_hdr *ipv4_hdr;
> struct rte_mbuf *pkt = item->firstseg; @@ -302,7 +303,7 @@
> gro_tcp4_reassemble(struct rte_mbuf *pkt,
> sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> is_atomic);
> if (cmp) {
> - if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
> + if (merge_two_tcp_packets(&(tbl->items[cur_idx]),
> pkt, cmp, sent_seq, ip_id, 0))
> return 1;
> /*
> diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h index
> 212f97a042..634a215b98 100644
> --- a/lib/gro/gro_tcp4.h
> +++ b/lib/gro/gro_tcp4.h
> @@ -5,22 +5,9 @@
> #ifndef _GRO_TCP4_H_
> #define _GRO_TCP4_H_
>
> -#include <rte_tcp.h>
> -
> #define INVALID_ARRAY_INDEX 0xffffffffUL #define
> GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
>
> -/*
> - * The max length of a IPv4 packet, which includes the length of the L3
> - * header, the L4 header and the data payload.
> - */
> -#define MAX_IPV4_PKT_LENGTH UINT16_MAX
> -
> -/* The maximum TCP header length */
> -#define MAX_TCP_HLEN 60
> -#define INVALID_TCP_HDRLEN(len) \
> - (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> -
> /* Header fields representing a TCP/IPv4 flow */ struct tcp4_flow_key {
> struct rte_ether_addr eth_saddr;
> @@ -42,42 +29,12 @@ struct gro_tcp4_flow {
> uint32_t start_index;
> };
>
> -struct gro_tcp4_item {
> - /*
> - * The first MBUF segment of the packet. If the value
> - * is NULL, it means the item is empty.
> - */
> - struct rte_mbuf *firstseg;
> - /* The last MBUF segment of the packet */
> - struct rte_mbuf *lastseg;
> - /*
> - * The time when the first packet is inserted into the table.
> - * This value won't be updated, even if the packet is merged
> - * with other packets.
> - */
> - uint64_t start_time;
> - /*
> - * next_pkt_idx is used to chain the packets that
> - * are in the same flow but can't be merged together
> - * (e.g. caused by packet reordering).
> - */
> - uint32_t next_pkt_idx;
> - /* TCP sequence number of the packet */
> - uint32_t sent_seq;
> - /* IPv4 ID of the packet */
> - uint16_t ip_id;
> - /* the number of merged packets */
> - uint16_t nb_merged;
> - /* Indicate if IPv4 ID can be ignored */
> - uint8_t is_atomic;
> -};
> -
> /*
> * TCP/IPv4 reassembly table structure.
> */
> struct gro_tcp4_tbl {
> /* item array */
> - struct gro_tcp4_item *items;
> + struct gro_tcp_item *items;
> /* flow array */
> struct gro_tcp4_flow *flows;
> /* current item number */
> @@ -195,111 +152,4 @@ is_same_tcp4_flow(struct tcp4_flow_key k1, struct
> tcp4_flow_key k2)
> (k1.dst_port == k2.dst_port));
> }
>
> -/*
> - * Merge two TCP/IPv4 packets without updating checksums.
> - * If cmp is larger than 0, append the new packet to the
> - * original packet. Otherwise, pre-pend the new packet to
> - * the original packet.
> - */
> -static inline int
> -merge_two_tcp4_packets(struct gro_tcp4_item *item,
> - struct rte_mbuf *pkt,
> - int cmp,
> - uint32_t sent_seq,
> - uint16_t ip_id,
> - uint16_t l2_offset)
> -{
> - struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
> - uint16_t hdr_len, l2_len;
> -
> - if (cmp > 0) {
> - pkt_head = item->firstseg;
> - pkt_tail = pkt;
> - } else {
> - pkt_head = pkt;
> - pkt_tail = item->firstseg;
> - }
> -
> - /* check if the IPv4 packet length is greater than the max value */
> - hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
> - pkt_head->l4_len;
> - l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
> - if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
> - hdr_len > MAX_IPV4_PKT_LENGTH))
> - return 0;
> -
> - /* remove the packet header for the tail packet */
> - rte_pktmbuf_adj(pkt_tail, hdr_len);
> -
> - /* chain two packets together */
> - if (cmp > 0) {
> - item->lastseg->next = pkt;
> - item->lastseg = rte_pktmbuf_lastseg(pkt);
> - /* update IP ID to the larger value */
> - item->ip_id = ip_id;
> - } else {
> - lastseg = rte_pktmbuf_lastseg(pkt);
> - lastseg->next = item->firstseg;
> - item->firstseg = pkt;
> - /* update sent_seq to the smaller value */
> - item->sent_seq = sent_seq;
> - item->ip_id = ip_id;
> - }
> - item->nb_merged++;
> -
> - /* update MBUF metadata for the merged packet */
> - pkt_head->nb_segs += pkt_tail->nb_segs;
> - pkt_head->pkt_len += pkt_tail->pkt_len;
> -
> - return 1;
> -}
> -
> -/*
> - * Check if two TCP/IPv4 packets are neighbors.
> - */
> -static inline int
> -check_seq_option(struct gro_tcp4_item *item,
> - struct rte_tcp_hdr *tcph,
> - uint32_t sent_seq,
> - uint16_t ip_id,
> - uint16_t tcp_hl,
> - uint16_t tcp_dl,
> - uint16_t l2_offset,
> - uint8_t is_atomic)
> -{
> - struct rte_mbuf *pkt_orig = item->firstseg;
> - struct rte_ipv4_hdr *iph_orig;
> - struct rte_tcp_hdr *tcph_orig;
> - uint16_t len, tcp_hl_orig;
> -
> - iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char
> *) +
> - l2_offset + pkt_orig->l2_len);
> - tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig + pkt_orig-
> >l3_len);
> - tcp_hl_orig = pkt_orig->l4_len;
> -
> - /* Check if TCP option fields equal */
> - len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> - if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> - (memcmp(tcph + 1, tcph_orig + 1,
> - len) != 0)))
> - return 0;
> -
> - /* Don't merge packets whose DF bits are different */
> - if (unlikely(item->is_atomic ^ is_atomic))
> - return 0;
> -
> - /* check if the two packets are neighbors */
> - len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> - pkt_orig->l3_len - tcp_hl_orig;
> - if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> - (ip_id == item->ip_id + 1)))
> - /* append the new packet */
> - return 1;
> - else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> - (ip_id + item->nb_merged == item->ip_id)))
> - /* pre-pend the new packet */
> - return -1;
> -
> - return 0;
> -}
> #endif
> diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c new file mode 100644
> index 0000000000..a37d8e2512
> --- /dev/null
> +++ b/lib/gro/gro_tcp6.c
> @@ -0,0 +1,387 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
> +
> +#include <rte_malloc.h>
> +#include <rte_mbuf.h>
> +#include <rte_ethdev.h>
> +
> +#include "gro_tcp6.h"
> +#include "gro_tcp.h"
> +
> +void *
> +gro_tcp6_tbl_create(uint16_t socket_id,
> + uint16_t max_flow_num,
> + uint16_t max_item_per_flow)
> +{
> + struct gro_tcp6_tbl *tbl;
> + size_t size;
> + uint32_t entries_num, i;
> +
> + entries_num = max_flow_num * max_item_per_flow;
> + entries_num = RTE_MIN(entries_num,
> GRO_TCP6_TBL_MAX_ITEM_NUM);
> +
> + if (entries_num == 0)
> + return NULL;
> +
> + tbl = rte_zmalloc_socket(__func__,
> + sizeof(struct gro_tcp6_tbl),
> + RTE_CACHE_LINE_SIZE,
> + socket_id);
> + if (tbl == NULL)
> + return NULL;
> +
> + size = sizeof(struct gro_tcp_item) * entries_num;
> + tbl->items = rte_zmalloc_socket(__func__,
> + size,
> + RTE_CACHE_LINE_SIZE,
> + socket_id);
> + if (tbl->items == NULL) {
> + rte_free(tbl);
> + return NULL;
> + }
> + tbl->max_item_num = entries_num;
> +
> + size = sizeof(struct gro_tcp6_flow) * entries_num;
> + tbl->flows = rte_zmalloc_socket(__func__,
> + size,
> + RTE_CACHE_LINE_SIZE,
> + socket_id);
> + if (tbl->flows == NULL) {
> + rte_free(tbl->items);
> + rte_free(tbl);
> + return NULL;
> + }
> + /* INVALID_ARRAY_INDEX indicates an empty flow */
> + for (i = 0; i < entries_num; i++)
> + tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
> + tbl->max_flow_num = entries_num;
> +
> + return tbl;
> +}
> +
> +void
> +gro_tcp6_tbl_destroy(void *tbl)
> +{
> + struct gro_tcp6_tbl *tcp_tbl = tbl;
> +
> + if (tcp_tbl) {
> + rte_free(tcp_tbl->items);
> + rte_free(tcp_tbl->flows);
> + }
> + rte_free(tcp_tbl);
> +}
> +
> +static inline uint32_t
> +find_an_empty_item(struct gro_tcp6_tbl *tbl) {
> + uint32_t i;
> + uint32_t max_item_num = tbl->max_item_num;
> +
> + for (i = 0; i < max_item_num; i++)
> + if (tbl->items[i].firstseg == NULL)
> + return i;
> + return INVALID_ARRAY_INDEX;
> +}
> +
> +static inline uint32_t
> +find_an_empty_flow(struct gro_tcp6_tbl *tbl) {
> + uint32_t i;
> + uint32_t max_flow_num = tbl->max_flow_num;
> +
> + for (i = 0; i < max_flow_num; i++)
> + if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
> + return i;
> + return INVALID_ARRAY_INDEX;
> +}
> +
> +static inline uint32_t
> +insert_new_item(struct gro_tcp6_tbl *tbl,
> + struct rte_mbuf *pkt,
> + uint64_t start_time,
> + uint32_t prev_idx,
> + uint32_t sent_seq,
> + uint8_t is_atomic)
> +{
> + uint32_t item_idx;
> +
> + item_idx = find_an_empty_item(tbl);
> + if (item_idx == INVALID_ARRAY_INDEX)
> + return INVALID_ARRAY_INDEX;
> +
> + tbl->items[item_idx].firstseg = pkt;
> + tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> + tbl->items[item_idx].start_time = start_time;
> + tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> + tbl->items[item_idx].sent_seq = sent_seq;
> + tbl->items[item_idx].nb_merged = 1;
> + tbl->items[item_idx].is_atomic = is_atomic;
> + tbl->item_num++;
> +
> + /* if the previous packet exists, chain them together. */
> + if (prev_idx != INVALID_ARRAY_INDEX) {
> + tbl->items[item_idx].next_pkt_idx =
> + tbl->items[prev_idx].next_pkt_idx;
> + tbl->items[prev_idx].next_pkt_idx = item_idx;
> + }
> +
> + return item_idx;
> +}
> +
> +static inline uint32_t
> +delete_item(struct gro_tcp6_tbl *tbl, uint32_t item_idx,
> + uint32_t prev_item_idx)
> +{
> + uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
> +
> + /* NULL indicates an empty item */
> + tbl->items[item_idx].firstseg = NULL;
> + tbl->item_num--;
> + if (prev_item_idx != INVALID_ARRAY_INDEX)
> + tbl->items[prev_item_idx].next_pkt_idx = next_idx;
> +
> + return next_idx;
> +}
> +
> +static inline uint32_t
> +insert_new_flow(struct gro_tcp6_tbl *tbl,
> + struct tcp6_flow_key *src,
> + rte_be32_t vtc_flow,
> + uint32_t item_idx)
> +{
> + struct tcp6_flow_key *dst;
> + uint32_t flow_idx;
> +
> + flow_idx = find_an_empty_flow(tbl);
> + if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
> + return INVALID_ARRAY_INDEX;
> +
> + dst = &(tbl->flows[flow_idx].key);
> +
> + rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
> + rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
> + memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst-
> >src_addr));
> + memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst-
> >dst_addr));
> + dst->recv_ack = src->recv_ack;
> + dst->src_port = src->src_port;
> + dst->dst_port = src->dst_port;
> +
> + tbl->flows[flow_idx].start_index = item_idx;
> + tbl->flow_num++;
> + tbl->flows->vtc_flow = vtc_flow;
> +
> + return flow_idx;
> +}
> +
> +/*
> + * update the packet length for the flushed packet.
> + */
> +static inline void
> +update_header(struct gro_tcp_item *item) {
> + struct rte_ipv6_hdr *ipv6_hdr;
> + struct rte_mbuf *pkt = item->firstseg;
> +
> + ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> + pkt->l2_len);
> + ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
> + pkt->l2_len - pkt->l3_len);
> +}
> +
> +int32_t
> +gro_tcp6_reassemble(struct rte_mbuf *pkt,
> + struct gro_tcp6_tbl *tbl,
> + uint64_t start_time)
> +{
> + struct rte_ether_hdr *eth_hdr;
> + struct rte_ipv6_hdr *ipv6_hdr;
> + struct rte_tcp_hdr *tcp_hdr;
> + uint32_t sent_seq;
> + int32_t tcp_dl;
> + uint16_t ip_tlen;
> + struct tcp6_flow_key key;
> + uint32_t cur_idx, prev_idx, item_idx;
> + uint32_t i, max_flow_num, remaining_flow_num;
> + int cmp;
> + uint8_t find;
> + rte_be32_t vtc_flow_diff;
> +
> + /*
> + * Don't process the packet whose TCP header length is greater
> + * than 60 bytes or less than 20 bytes.
> + */
> + if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
> + return -1;
> +
> + eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
> + ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
> + tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv6_hdr + pkt->l3_len);
> +
> + /*
> + * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
> + * or CWR set.
> + */
> + if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
> + return -1;
> +
> + ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
> + /*
> + * Trim the tail padding bytes. The IPv6 header is fixed to
> + * 40 bytes unlike IPv4 that is variable. The length in the IPv6 header
> + * contains only length of TCP Header + TCP Payload, whereas IPv4
> header contains
> + * length of IP Header + TCP Header + TCP Payload
> + */
> + if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len + pkt->l3_len))
> + rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len -
> pkt->l3_len);
> + /*
> + * Don't process the packet whose payload length is less than or
> + * equal to 0.
> + */
> + tcp_dl = ip_tlen - pkt->l4_len;
> + if (tcp_dl <= 0)
> + return -1;
> +
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> +
> + rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
> + rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
> + memcpy(&key.src_addr[0], &ipv6_hdr->src_addr,
> sizeof(key.src_addr));
> + memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr,
> sizeof(key.dst_addr));
> + key.src_port = tcp_hdr->src_port;
> + key.dst_port = tcp_hdr->dst_port;
> + key.recv_ack = tcp_hdr->recv_ack;
> +
> + /* Search for a matched flow. */
> + max_flow_num = tbl->max_flow_num;
> + remaining_flow_num = tbl->flow_num;
> + find = 0;
> + for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
> + if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
> + if (is_same_tcp6_flow(tbl->flows[i].key, key)) {
> + /*
> + * IP version (4) Traffic Class (8) Flow Label (20)
> + * All fields except Traffic class should be
> same
> + */
> + vtc_flow_diff = (ipv6_hdr->vtc_flow ^ tbl-
> >flows->vtc_flow);
> + if (vtc_flow_diff & htonl(0xF00FFFFF))
> + continue;
> + find = 1;
> + break;
> + }
> + remaining_flow_num--;
> + }
> + }
> +
> + /*
> + * Fail to find a matched flow. Insert a new flow and store the
> + * packet into the flow.
> + */
> + if (find == 0) {
> + item_idx = insert_new_item(tbl, pkt, start_time,
> + INVALID_ARRAY_INDEX, sent_seq, true);
> + if (item_idx == INVALID_ARRAY_INDEX)
> + return -1;
> + if (insert_new_flow(tbl, &key, ipv6_hdr->vtc_flow, item_idx)
> ==
> + INVALID_ARRAY_INDEX) {
> + /*
> + * Fail to insert a new flow, so delete the
> + * stored packet.
> + */
> + delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
> + return -1;
> + }
> + return 0;
> + }
> +
> + /*
> + * Check all packets in the flow and try to find a neighbor for
> + * the input packet.
> + */
> + cur_idx = tbl->flows[i].start_index;
> + prev_idx = cur_idx;
> + do {
> + cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
> + sent_seq, 0, pkt->l4_len, tcp_dl, 0,
> + true);
> + if (cmp) {
> + if (merge_two_tcp_packets(&(tbl->items[cur_idx]),
> + pkt, cmp, sent_seq, 0, 0)) {
> + return 1;
> + }
> +
> + /*
> + * Fail to merge the two packets, as the packet
> + * length is greater than the max value. Store
> + * the packet into the flow.
> + */
> + if (insert_new_item(tbl, pkt, start_time, cur_idx,
> + sent_seq, true) ==
> + INVALID_ARRAY_INDEX)
> + return -1;
> + return 0;
> + }
> + prev_idx = cur_idx;
> + cur_idx = tbl->items[cur_idx].next_pkt_idx;
> + } while (cur_idx != INVALID_ARRAY_INDEX);
> +
> + /* Fail to find a neighbor, so store the packet into the flow. */
> + if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
> + true) == INVALID_ARRAY_INDEX)
> + return -1;
> +
> + return 0;
> +}
> +
> +uint16_t
> +gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
> + uint64_t flush_timestamp,
> + struct rte_mbuf **out,
> + uint16_t nb_out)
> +{
> + uint16_t k = 0;
> + uint32_t i, j;
> + uint32_t max_flow_num = tbl->max_flow_num;
> +
> + for (i = 0; i < max_flow_num; i++) {
> + if (unlikely(tbl->flow_num == 0))
> + return k;
> +
> + j = tbl->flows[i].start_index;
> + while (j != INVALID_ARRAY_INDEX) {
> + if (tbl->items[j].start_time <= flush_timestamp) {
> + out[k++] = tbl->items[j].firstseg;
> + if (tbl->items[j].nb_merged > 1)
> + update_header(&(tbl->items[j]));
> + /*
> + * Delete the packet and get the next
> + * packet in the flow.
> + */
> + j = delete_item(tbl, j,
> INVALID_ARRAY_INDEX);
> + tbl->flows[i].start_index = j;
> + if (j == INVALID_ARRAY_INDEX)
> + tbl->flow_num--;
> +
> + if (unlikely(k == nb_out))
> + return k;
> + } else
> + /*
> + * The left packets in this flow won't be
> + * timeout. Go to check other flows.
> + */
> + break;
> + }
> + }
> + return k;
> +}
> +
> +uint32_t
> +gro_tcp6_tbl_pkt_count(void *tbl)
> +{
> + struct gro_tcp6_tbl *gro_tbl = tbl;
> +
> + if (gro_tbl)
> + return gro_tbl->item_num;
> +
> + return 0;
> +}
> diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h new file mode 100644
> index 0000000000..91325c0da2
> --- /dev/null
> +++ b/lib/gro/gro_tcp6.h
> @@ -0,0 +1,150 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
> +
> +#ifndef _GRO_TCP6_H_
> +#define _GRO_TCP6_H_
> +
> +#define INVALID_ARRAY_INDEX 0xffffffffUL #define
> +GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
> +
> +/* Header fields representing a TCP/IPv6 flow */ struct tcp6_flow_key {
> + struct rte_ether_addr eth_saddr;
> + struct rte_ether_addr eth_daddr;
> + uint8_t src_addr[16];
> + uint8_t dst_addr[16];
> +
> + uint32_t recv_ack;
> + uint16_t src_port;
> + uint16_t dst_port;
> +};
> +
> +struct gro_tcp6_flow {
> + struct tcp6_flow_key key;
> + rte_be32_t vtc_flow;
> + /*
> + * The index of the first packet in the flow.
> + * INVALID_ARRAY_INDEX indicates an empty flow.
> + */
> + uint32_t start_index;
> +};
> +
> +/*
> + * TCP/IPv6 reassembly table structure.
> + */
> +struct gro_tcp6_tbl {
> + /* item array */
> + struct gro_tcp_item *items;
> + /* flow array */
> + struct gro_tcp6_flow *flows;
> + /* current item number */
> + uint32_t item_num;
> + /* current flow num */
> + uint32_t flow_num;
> + /* item array size */
> + uint32_t max_item_num;
> + /* flow array size */
> + uint32_t max_flow_num;
> +};
> +
> +/**
> + * This function creates a TCP/IPv6 reassembly table.
> + *
> + * @param socket_id
> + * Socket index for allocating the TCP/IPv6 reassemble table
> + * @param max_flow_num
> + * The maximum number of flows in the TCP/IPv6 GRO table
> + * @param max_item_per_flow
> + * The maximum number of packets per flow
> + *
> + * @return
> + * - Return the table pointer on success.
> + * - Return NULL on failure.
> + */
> +void *gro_tcp6_tbl_create(uint16_t socket_id,
> + uint16_t max_flow_num,
> + uint16_t max_item_per_flow);
> +
> +/**
> + * This function destroys a TCP/IPv6 reassembly table.
> + *
> + * @param tbl
> + * Pointer pointing to the TCP/IPv6 reassembly table.
> + */
> +void gro_tcp6_tbl_destroy(void *tbl);
> +
> +/**
> + * This function merges a TCP/IPv6 packet. It doesn't process the
> +packet,
> + * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't have
> + * payload.
> + *
> + * This function doesn't check if the packet has correct checksums and
> + * doesn't re-calculate checksums for the merged packet. Additionally,
> + * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
> + * when IP fragmentation is possible (i.e., DF==0). It returns the
> + * packet, if the packet has invalid parameters (e.g. SYN bit is set)
> + * or there is no available space in the table.
> + *
> + * @param pkt
> + * Packet to reassemble
> + * @param tbl
> + * Pointer pointing to the TCP/IPv6 reassembly table
> + * @start_time
> + * The time when the packet is inserted into the table
> + *
> + * @return
> + * - Return a positive value if the packet is merged.
> + * - Return zero if the packet isn't merged but stored in the table.
> + * - Return a negative value for invalid parameters or no available
> + * space in the table.
> + */
> +int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
> + struct gro_tcp6_tbl *tbl,
> + uint64_t start_time);
> +
> +/**
> + * This function flushes timeout packets in a TCP/IPv6 reassembly
> +table,
> + * and without updating checksums.
> + *
> + * @param tbl
> + * TCP/IPv6 reassembly table pointer
> + * @param flush_timestamp
> + * Flush packets which are inserted into the table before or at the
> + * flush_timestamp.
> + * @param out
> + * Pointer array used to keep flushed packets
> + * @param nb_out
> + * The element number in 'out'. It also determines the maximum number
> +of
> + * packets that can be flushed finally.
> + *
> + * @return
> + * The number of flushed packets
> + */
> +uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
> + uint64_t flush_timestamp,
> + struct rte_mbuf **out,
> + uint16_t nb_out);
> +
> +/**
> + * This function returns the number of the packets in a TCP/IPv6
> + * reassembly table.
> + *
> + * @param tbl
> + * TCP/IPv6 reassembly table pointer
> + *
> + * @return
> + * The number of packets in the table
> + */
> +uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
> +
> +/*
> + * Check if two TCP/IPv6 packets belong to the same flow.
> + */
> +static inline int
> +is_same_tcp6_flow(struct tcp6_flow_key k1, struct tcp6_flow_key k2) {
> + return (!memcmp(&k1, &k2, sizeof(struct tcp6_flow_key))); }
> +
> +#endif
> diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c index
> 3be4deb7c7..56b30b8c98 100644
> --- a/lib/gro/gro_vxlan_tcp4.c
> +++ b/lib/gro/gro_vxlan_tcp4.c
> @@ -7,6 +7,7 @@
> #include <rte_ethdev.h>
> #include <rte_udp.h>
>
> +#include "gro_tcp.h"
> #include "gro_vxlan_tcp4.h"
>
> void *
> @@ -248,7 +249,7 @@ merge_two_vxlan_tcp4_packets(struct
> gro_vxlan_tcp4_item *item,
> uint16_t outer_ip_id,
> uint16_t ip_id)
> {
> - if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp,
> sent_seq,
> + if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
> ip_id, pkt->outer_l2_len +
> pkt->outer_l3_len)) {
> /* Update the outer IPv4 ID to the large value. */ diff --git
> a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h index
> 7832942a68..d68d5fcd5b 100644
> --- a/lib/gro/gro_vxlan_tcp4.h
> +++ b/lib/gro/gro_vxlan_tcp4.h
> @@ -5,6 +5,7 @@
> #ifndef _GRO_VXLAN_TCP4_H_
> #define _GRO_VXLAN_TCP4_H_
>
> +#include "gro_tcp.h"
> #include "gro_tcp4.h"
>
> #define GRO_VXLAN_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL) @@ -
> 36,7 +37,7 @@ struct gro_vxlan_tcp4_flow { };
>
> struct gro_vxlan_tcp4_item {
> - struct gro_tcp4_item inner_item;
> + struct gro_tcp_item inner_item;
> /* IPv4 ID in the outer IPv4 header */
> uint16_t outer_ip_id;
> /* Indicate if outer IPv4 ID can be ignored */ diff --git
> a/lib/gro/meson.build b/lib/gro/meson.build index e4fa2958bd..dbce05220d
> 100644
> --- a/lib/gro/meson.build
> +++ b/lib/gro/meson.build
> @@ -4,6 +4,7 @@
> sources = files(
> 'rte_gro.c',
> 'gro_tcp4.c',
> + 'gro_tcp6.c',
> 'gro_udp4.c',
> 'gro_vxlan_tcp4.c',
> 'gro_vxlan_udp4.c',
> diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c index
> e35399fd42..d824eebd93 100644
> --- a/lib/gro/rte_gro.c
> +++ b/lib/gro/rte_gro.c
> @@ -8,6 +8,7 @@
>
> #include "rte_gro.h"
> #include "gro_tcp4.h"
> +#include "gro_tcp6.h"
> #include "gro_udp4.h"
> #include "gro_vxlan_tcp4.h"
> #include "gro_vxlan_udp4.h"
> @@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
>
> static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
> gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
> - gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
> + gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create,
> gro_tcp6_tbl_create,
> +NULL};
> static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
> gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
> gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
> + gro_tcp6_tbl_destroy,
> NULL};
> static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM]
> = {
> gro_tcp4_tbl_pkt_count,
> gro_vxlan_tcp4_tbl_pkt_count,
> gro_udp4_tbl_pkt_count,
> gro_vxlan_udp4_tbl_pkt_count,
> + gro_tcp6_tbl_pkt_count,
> NULL};
>
> #define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \ @@
> -35,6 +38,12 @@ static gro_tbl_pkt_count_fn
> tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
> ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) &&
> \
> (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
>
> +/* GRO with extension headers is not supported */ #define
> +IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
> + ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
> + ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) &&
> \
> + (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
> +
> #define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
> ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
> (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
> @@ -147,7 +156,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> /* allocate a reassembly table for TCP/IPv4 GRO */
> struct gro_tcp4_tbl tcp_tbl;
> struct gro_tcp4_flow
> tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
> - struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
> = {{0} };
> + struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
> = {{0} };
> +
> + struct gro_tcp6_tbl tcp6_tbl;
> + struct gro_tcp6_flow
> tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
> + struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM]
> = {{0} };
>
> /* allocate a reassembly table for UDP/IPv4 GRO */
> struct gro_udp4_tbl udp_tbl;
> @@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> int32_t ret;
> uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
> uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
> - do_vxlan_udp_gro = 0;
> + do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
>
> if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4
> |
> - RTE_GRO_TCP_IPV4 |
> + RTE_GRO_TCP_IPV4 |
> RTE_GRO_TCP_IPV6 |
> RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
> RTE_GRO_UDP_IPV4)) == 0))
> return nb_pkts;
> @@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> do_udp4_gro = 1;
> }
>
> + if (param->gro_types & RTE_GRO_TCP_IPV6) {
> + for (i = 0; i < item_num; i++)
> + tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
> +
> + tcp6_tbl.flows = tcp6_flows;
> + tcp6_tbl.items = tcp6_items;
> + tcp6_tbl.flow_num = 0;
> + tcp6_tbl.item_num = 0;
> + tcp6_tbl.max_flow_num = item_num;
> + tcp6_tbl.max_item_num = item_num;
> + do_tcp6_gro = 1;
> + }
>
> for (i = 0; i < nb_pkts; i++) {
> /*
> @@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> nb_after_gro--;
> else if (ret < 0)
> unprocess_pkts[unprocess_num++] = pkts[i];
> + } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
> + do_tcp6_gro) {
> + ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
> + if (ret > 0)
> + /* merge successfully */
> + nb_after_gro--;
> + else if (ret < 0)
> + unprocess_pkts[unprocess_num++] = pkts[i];
> } else
> unprocess_pkts[unprocess_num++] = pkts[i];
> }
> @@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> if ((nb_after_gro < nb_pkts)
> || (unprocess_num < nb_pkts)) {
> i = 0;
> + /* Copy unprocessed packets */
> + if (unprocess_num > 0) {
> + memcpy(&pkts[i], unprocess_pkts,
> + sizeof(struct rte_mbuf *) *
> + unprocess_num);
> + i = unprocess_num;
> + }
> +
> /* Flush all packets from the tables */
> if (do_vxlan_tcp_gro) {
> - i =
> gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
> + i +=
> gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
> 0, pkts, nb_pkts);
> }
>
> @@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
> &pkts[i], nb_pkts - i);
> }
> - /* Copy unprocessed packets */
> - if (unprocess_num > 0) {
> - memcpy(&pkts[i], unprocess_pkts,
> - sizeof(struct rte_mbuf *) *
> - unprocess_num);
> +
> + if (do_tcp6_gro) {
> + i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
> + &pkts[i], nb_pkts - i);
> }
> - nb_after_gro = i + unprocess_num;
> }
>
> return nb_after_gro;
> @@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts, {
> struct rte_mbuf *unprocess_pkts[nb_pkts];
> struct gro_ctx *gro_ctx = ctx;
> - void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
> + void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
> uint64_t current_time;
> uint16_t i, unprocess_num = 0;
> - uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro,
> do_vxlan_udp_gro;
> + uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro,
> do_vxlan_udp_gro,
> +do_tcp6_gro;
>
> if (unlikely((gro_ctx->gro_types &
> (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
> - RTE_GRO_TCP_IPV4 |
> + RTE_GRO_TCP_IPV4 |
> RTE_GRO_TCP_IPV6 |
> RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
> RTE_GRO_UDP_IPV4)) == 0))
> return nb_pkts;
> @@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> vxlan_tcp_tbl = gro_ctx-
> >tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
> udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
> vxlan_udp_tbl = gro_ctx-
> >tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
> + tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
>
> do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
> RTE_GRO_TCP_IPV4;
> @@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> RTE_GRO_UDP_IPV4;
> do_vxlan_udp_gro = (gro_ctx->gro_types &
> RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
> RTE_GRO_IPV4_VXLAN_UDP_IPV4;
> + do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) ==
> +RTE_GRO_TCP_IPV6;
>
> current_time = rte_rdtsc();
>
> @@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> if (gro_udp4_reassemble(pkts[i], udp_tbl,
> current_time) < 0)
> unprocess_pkts[unprocess_num++] = pkts[i];
> + } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
> + do_tcp6_gro) {
> + if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
> + current_time) < 0)
> + unprocess_pkts[unprocess_num++] = pkts[i];
> } else
> unprocess_pkts[unprocess_num++] = pkts[i];
> }
> @@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
> gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
> flush_timestamp,
> &out[num], left_nb_out);
> + left_nb_out = max_nb_out - num;
> + }
> +
> + if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
> + num += gro_tcp6_tbl_timeout_flush(
> + gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
> + flush_timestamp,
> + &out[num], left_nb_out);
> +
> }
>
> return num;
> diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h index
> 9f9ed4935a..c83dfd9ad1 100644
> --- a/lib/gro/rte_gro.h
> +++ b/lib/gro/rte_gro.h
> @@ -38,6 +38,9 @@ extern "C" {
> #define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3 #define
> RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL <<
> RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
> /**< VxLAN UDP/IPv4 GRO flag. */
> +#define RTE_GRO_TCP_IPV6_INDEX 4
> +#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX) /**<
> TCP/IPv6
> +GRO flag. */
>
> /**
> * Structure used to create GRO context objects or used to pass
> --
> 2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [PATCH v2] gro : ipv6 changes to support GRO for TCP/ipv6
2023-05-12 2:47 ` Hu, Jiayu
@ 2023-05-16 9:28 ` kumaraparameshwaran rathinavel
2023-05-25 11:22 ` kumaraparameshwaran rathinavel
0 siblings, 1 reply; 32+ messages in thread
From: kumaraparameshwaran rathinavel @ 2023-05-16 9:28 UTC (permalink / raw)
To: Hu, Jiayu; +Cc: dev
[-- Attachment #1: Type: text/plain, Size: 50555 bytes --]
On Fri, May 12, 2023 at 8:17 AM Hu, Jiayu <jiayu.hu@intel.com> wrote:
> Hi Kumar,
>
> For TCP/IPv4 and TCP/IPv6, the TCP layer is the same and the difference is
> the IP layer. So the code used for TCP layer needs to be shared among
> gro_tcp6.c
> and gro_tcp4.c. But there are too much code duplication in gro_tcp4.c and
> gro_tcp6.c
> in current implementation.
>
> Sure, will do it. I had followed the same that was similar to what was
>> done in gro_tcp4.c and gro_vxlan_tcp4.c. Eventhough the TCP could have been
>> reused for vxlan, we had code duplication. I assumed that this was better
>> in terms of performance etc.
>
>
> For example, struct tcp6_flow_key and struct tcp4_flow_key share most of
> fields, except
> the IP address type. It's better to have a common TCP flow key structure
> in gro_tcp.h. In
> gro_tcp4.h and gro_tcp6.h, we implement tcp4 and tcp6 flow key structures
> with using the
> common structure.
>
Thanks,
> Jiayu
>
> > -----Original Message-----
> > From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> > Sent: Friday, October 21, 2022 2:14 AM
> > To: Hu, Jiayu <jiayu.hu@intel.com>
> > Cc: dev@dpdk.org; Kumara Parameshwaran
> > <kumaraparamesh92@gmail.com>
> > Subject: [PATCH v2] gro : ipv6 changes to support GRO for TCP/ipv6
> >
> > From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> >
> > The patch adds GRO support for TCP/ipv6 packets. This does not include
> the
> > support for vxlan, udp ipv6 packets.
> >
> > Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> > ---
> > v1:
> > * Changes to support GRO for TCP/ipv6 packets. This does not
> > include
> > vxlan changes.
> > * The GRO is performed only for ipv6 packets that does not contain
> > extension headers.
> > * The logic for the TCP coalescing remains the same, in ipv6 header
> > the source address, destination address, flow label, version
> fields
> > are expected to be the same.
> > * Re-organised the code to reuse certain tcp functions for both
> ipv4
> > and
> > ipv6 flows.
> > v2:
> > * Fix comments in gro_tcp6.h header file.
> >
> > lib/gro/gro_tcp.h | 155 ++++++++++++++++
> > lib/gro/gro_tcp4.c | 7 +-
> > lib/gro/gro_tcp4.h | 152 +--------------
> > lib/gro/gro_tcp6.c | 387
> > +++++++++++++++++++++++++++++++++++++++
> > lib/gro/gro_tcp6.h | 150 +++++++++++++++
> > lib/gro/gro_vxlan_tcp4.c | 3 +-
> > lib/gro/gro_vxlan_tcp4.h | 3 +-
> > lib/gro/meson.build | 1 +
> > lib/gro/rte_gro.c | 83 +++++++--
> > lib/gro/rte_gro.h | 3 +
> > 10 files changed, 774 insertions(+), 170 deletions(-) create mode
> 100644
> > lib/gro/gro_tcp.h create mode 100644 lib/gro/gro_tcp6.c create mode
> > 100644 lib/gro/gro_tcp6.h
> >
> > diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h new file mode 100644
> index
> > 0000000000..c5d248a022
> > --- /dev/null
> > +++ b/lib/gro/gro_tcp.h
> > @@ -0,0 +1,155 @@
> > +#ifndef _GRO_TCP_H_
> > +#define _GRO_TCP_H_
> > +
> > +#include <rte_tcp.h>
> > +
> > +/*
> > + * The max length of a IPv4 packet, which includes the length of the L3
> > + * header, the L4 header and the data payload.
> > + */
> > +#define MAX_IP_PKT_LENGTH UINT16_MAX
> > +
> > +/* The maximum TCP header length */
> > +#define MAX_TCP_HLEN 60
> > +#define INVALID_TCP_HDRLEN(len) \
> > + (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> > +
> > +struct gro_tcp_item {
> > + /*
> > + * The first MBUF segment of the packet. If the value
> > + * is NULL, it means the item is empty.
> > + */
> > + struct rte_mbuf *firstseg;
> > + /* The last MBUF segment of the packet */
> > + struct rte_mbuf *lastseg;
> > + /*
> > + * The time when the first packet is inserted into the table.
> > + * This value won't be updated, even if the packet is merged
> > + * with other packets.
> > + */
> > + uint64_t start_time;
> > + /*
> > + * next_pkt_idx is used to chain the packets that
> > + * are in the same flow but can't be merged together
> > + * (e.g. caused by packet reordering).
> > + */
> > + uint32_t next_pkt_idx;
> > + /* TCP sequence number of the packet */
> > + uint32_t sent_seq;
> > + /* IPv4 ID of the packet */
> > + uint16_t ip_id;
> > + /* the number of merged packets */
> > + uint16_t nb_merged;
> > + /* Indicate if IPv4 ID can be ignored */
> > + uint8_t is_atomic;
> > +};
> > +
> > +/*
> > + * Merge two TCP packets without updating checksums.
> > + * If cmp is larger than 0, append the new packet to the
> > + * original packet. Otherwise, pre-pend the new packet to
> > + * the original packet.
> > + */
> > +static inline int
> > +merge_two_tcp_packets(struct gro_tcp_item *item,
> > + struct rte_mbuf *pkt,
> > + int cmp,
> > + uint32_t sent_seq,
> > + uint16_t ip_id,
> > + uint16_t l2_offset)
> > +{
> > + struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
> > + uint16_t hdr_len, l2_len;
> > +
> > + if (cmp > 0) {
> > + pkt_head = item->firstseg;
> > + pkt_tail = pkt;
> > + } else {
> > + pkt_head = pkt;
> > + pkt_tail = item->firstseg;
> > + }
> > +
> > + /* check if the IPv4 packet length is greater than the max value */
> > + hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
> > + pkt_head->l4_len;
> > + l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
> > + if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
> > + hdr_len > MAX_IP_PKT_LENGTH))
> > + return 0;
> > +
> > + /* remove the packet header for the tail packet */
> > + rte_pktmbuf_adj(pkt_tail, hdr_len);
> > +
> > + /* chain two packets together */
> > + if (cmp > 0) {
> > + item->lastseg->next = pkt;
> > + item->lastseg = rte_pktmbuf_lastseg(pkt);
> > + /* update IP ID to the larger value */
> > + item->ip_id = ip_id;
> > + } else {
> > + lastseg = rte_pktmbuf_lastseg(pkt);
> > + lastseg->next = item->firstseg;
> > + item->firstseg = pkt;
> > + /* update sent_seq to the smaller value */
> > + item->sent_seq = sent_seq;
> > + item->ip_id = ip_id;
> > + }
> > + item->nb_merged++;
> > +
> > + /* update MBUF metadata for the merged packet */
> > + pkt_head->nb_segs += pkt_tail->nb_segs;
> > + pkt_head->pkt_len += pkt_tail->pkt_len;
> > +
> > + return 1;
> > +}
> > +
> > +/*
> > + * Check if two TCP/IPv4 packets are neighbors.
> > + */
> > +static inline int
> > +check_seq_option(struct gro_tcp_item *item,
> > + struct rte_tcp_hdr *tcph,
> > + uint32_t sent_seq,
> > + uint16_t ip_id,
> > + uint16_t tcp_hl,
> > + uint16_t tcp_dl,
> > + uint16_t l2_offset,
> > + uint8_t is_atomic)
> > +{
> > + struct rte_mbuf *pkt_orig = item->firstseg;
> > + char *iph_orig;
> > + struct rte_tcp_hdr *tcph_orig;
> > + uint16_t len, tcp_hl_orig;
> > +
> > + iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
> > + l2_offset + pkt_orig->l2_len);
> > + tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
> > + tcp_hl_orig = pkt_orig->l4_len;
> > +
> > + /* Check if TCP option fields equal */
> > + len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> > + if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> > + (memcmp(tcph + 1, tcph_orig + 1,
> > + len) != 0)))
> > + return 0;
> > +
> > + /* Don't merge packets whose DF bits are different */
> > + if (unlikely(item->is_atomic ^ is_atomic))
> > + return 0;
> > +
> > + /* check if the two packets are neighbors */
> > + len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> > + pkt_orig->l3_len - tcp_hl_orig;
> > + if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> > + (ip_id == item->ip_id + 1)))
> > + /* append the new packet */
> > + return 1;
> > + else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> > + (ip_id + item->nb_merged == item->ip_id)))
> > + /* pre-pend the new packet */
> > + return -1;
> > +
> > + return 0;
> > +}
> > +
> > +#endif
> > diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c index
> > 8f5e800250..eea2a72ecd 100644
> > --- a/lib/gro/gro_tcp4.c
> > +++ b/lib/gro/gro_tcp4.c
> > @@ -7,6 +7,7 @@
> > #include <rte_ethdev.h>
> >
> > #include "gro_tcp4.h"
> > +#include "gro_tcp.h"
> >
> > void *
> > gro_tcp4_tbl_create(uint16_t socket_id, @@ -30,7 +31,7 @@
> > gro_tcp4_tbl_create(uint16_t socket_id,
> > if (tbl == NULL)
> > return NULL;
> >
> > - size = sizeof(struct gro_tcp4_item) * entries_num;
> > + size = sizeof(struct gro_tcp_item) * entries_num;
> > tbl->items = rte_zmalloc_socket(__func__,
> > size,
> > RTE_CACHE_LINE_SIZE,
> > @@ -177,7 +178,7 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
> > * update the packet length for the flushed packet.
> > */
> > static inline void
> > -update_header(struct gro_tcp4_item *item)
> > +update_header(struct gro_tcp_item *item)
> > {
> > struct rte_ipv4_hdr *ipv4_hdr;
> > struct rte_mbuf *pkt = item->firstseg; @@ -302,7 +303,7 @@
> > gro_tcp4_reassemble(struct rte_mbuf *pkt,
> > sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> > is_atomic);
> > if (cmp) {
> > - if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
> > + if (merge_two_tcp_packets(&(tbl->items[cur_idx]),
> > pkt, cmp, sent_seq, ip_id,
> 0))
> > return 1;
> > /*
> > diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h index
> > 212f97a042..634a215b98 100644
> > --- a/lib/gro/gro_tcp4.h
> > +++ b/lib/gro/gro_tcp4.h
> > @@ -5,22 +5,9 @@
> > #ifndef _GRO_TCP4_H_
> > #define _GRO_TCP4_H_
> >
> > -#include <rte_tcp.h>
> > -
> > #define INVALID_ARRAY_INDEX 0xffffffffUL #define
> > GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
> >
> > -/*
> > - * The max length of a IPv4 packet, which includes the length of the L3
> > - * header, the L4 header and the data payload.
> > - */
> > -#define MAX_IPV4_PKT_LENGTH UINT16_MAX
> > -
> > -/* The maximum TCP header length */
> > -#define MAX_TCP_HLEN 60
> > -#define INVALID_TCP_HDRLEN(len) \
> > - (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> > -
> > /* Header fields representing a TCP/IPv4 flow */ struct tcp4_flow_key {
> > struct rte_ether_addr eth_saddr;
> > @@ -42,42 +29,12 @@ struct gro_tcp4_flow {
> > uint32_t start_index;
> > };
> >
> > -struct gro_tcp4_item {
> > - /*
> > - * The first MBUF segment of the packet. If the value
> > - * is NULL, it means the item is empty.
> > - */
> > - struct rte_mbuf *firstseg;
> > - /* The last MBUF segment of the packet */
> > - struct rte_mbuf *lastseg;
> > - /*
> > - * The time when the first packet is inserted into the table.
> > - * This value won't be updated, even if the packet is merged
> > - * with other packets.
> > - */
> > - uint64_t start_time;
> > - /*
> > - * next_pkt_idx is used to chain the packets that
> > - * are in the same flow but can't be merged together
> > - * (e.g. caused by packet reordering).
> > - */
> > - uint32_t next_pkt_idx;
> > - /* TCP sequence number of the packet */
> > - uint32_t sent_seq;
> > - /* IPv4 ID of the packet */
> > - uint16_t ip_id;
> > - /* the number of merged packets */
> > - uint16_t nb_merged;
> > - /* Indicate if IPv4 ID can be ignored */
> > - uint8_t is_atomic;
> > -};
> > -
> > /*
> > * TCP/IPv4 reassembly table structure.
> > */
> > struct gro_tcp4_tbl {
> > /* item array */
> > - struct gro_tcp4_item *items;
> > + struct gro_tcp_item *items;
> > /* flow array */
> > struct gro_tcp4_flow *flows;
> > /* current item number */
> > @@ -195,111 +152,4 @@ is_same_tcp4_flow(struct tcp4_flow_key k1, struct
> > tcp4_flow_key k2)
> > (k1.dst_port == k2.dst_port));
> > }
> >
> > -/*
> > - * Merge two TCP/IPv4 packets without updating checksums.
> > - * If cmp is larger than 0, append the new packet to the
> > - * original packet. Otherwise, pre-pend the new packet to
> > - * the original packet.
> > - */
> > -static inline int
> > -merge_two_tcp4_packets(struct gro_tcp4_item *item,
> > - struct rte_mbuf *pkt,
> > - int cmp,
> > - uint32_t sent_seq,
> > - uint16_t ip_id,
> > - uint16_t l2_offset)
> > -{
> > - struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
> > - uint16_t hdr_len, l2_len;
> > -
> > - if (cmp > 0) {
> > - pkt_head = item->firstseg;
> > - pkt_tail = pkt;
> > - } else {
> > - pkt_head = pkt;
> > - pkt_tail = item->firstseg;
> > - }
> > -
> > - /* check if the IPv4 packet length is greater than the max value */
> > - hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
> > - pkt_head->l4_len;
> > - l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
> > - if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
> > - hdr_len > MAX_IPV4_PKT_LENGTH))
> > - return 0;
> > -
> > - /* remove the packet header for the tail packet */
> > - rte_pktmbuf_adj(pkt_tail, hdr_len);
> > -
> > - /* chain two packets together */
> > - if (cmp > 0) {
> > - item->lastseg->next = pkt;
> > - item->lastseg = rte_pktmbuf_lastseg(pkt);
> > - /* update IP ID to the larger value */
> > - item->ip_id = ip_id;
> > - } else {
> > - lastseg = rte_pktmbuf_lastseg(pkt);
> > - lastseg->next = item->firstseg;
> > - item->firstseg = pkt;
> > - /* update sent_seq to the smaller value */
> > - item->sent_seq = sent_seq;
> > - item->ip_id = ip_id;
> > - }
> > - item->nb_merged++;
> > -
> > - /* update MBUF metadata for the merged packet */
> > - pkt_head->nb_segs += pkt_tail->nb_segs;
> > - pkt_head->pkt_len += pkt_tail->pkt_len;
> > -
> > - return 1;
> > -}
> > -
> > -/*
> > - * Check if two TCP/IPv4 packets are neighbors.
> > - */
> > -static inline int
> > -check_seq_option(struct gro_tcp4_item *item,
> > - struct rte_tcp_hdr *tcph,
> > - uint32_t sent_seq,
> > - uint16_t ip_id,
> > - uint16_t tcp_hl,
> > - uint16_t tcp_dl,
> > - uint16_t l2_offset,
> > - uint8_t is_atomic)
> > -{
> > - struct rte_mbuf *pkt_orig = item->firstseg;
> > - struct rte_ipv4_hdr *iph_orig;
> > - struct rte_tcp_hdr *tcph_orig;
> > - uint16_t len, tcp_hl_orig;
> > -
> > - iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char
> > *) +
> > - l2_offset + pkt_orig->l2_len);
> > - tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig + pkt_orig-
> > >l3_len);
> > - tcp_hl_orig = pkt_orig->l4_len;
> > -
> > - /* Check if TCP option fields equal */
> > - len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> > - if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> > - (memcmp(tcph + 1, tcph_orig + 1,
> > - len) != 0)))
> > - return 0;
> > -
> > - /* Don't merge packets whose DF bits are different */
> > - if (unlikely(item->is_atomic ^ is_atomic))
> > - return 0;
> > -
> > - /* check if the two packets are neighbors */
> > - len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> > - pkt_orig->l3_len - tcp_hl_orig;
> > - if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> > - (ip_id == item->ip_id + 1)))
> > - /* append the new packet */
> > - return 1;
> > - else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> > - (ip_id + item->nb_merged == item->ip_id)))
> > - /* pre-pend the new packet */
> > - return -1;
> > -
> > - return 0;
> > -}
> > #endif
> > diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c new file mode 100644
> > index 0000000000..a37d8e2512
> > --- /dev/null
> > +++ b/lib/gro/gro_tcp6.c
> > @@ -0,0 +1,387 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2017 Intel Corporation
> > + */
> > +
> > +#include <rte_malloc.h>
> > +#include <rte_mbuf.h>
> > +#include <rte_ethdev.h>
> > +
> > +#include "gro_tcp6.h"
> > +#include "gro_tcp.h"
> > +
> > +void *
> > +gro_tcp6_tbl_create(uint16_t socket_id,
> > + uint16_t max_flow_num,
> > + uint16_t max_item_per_flow)
> > +{
> > + struct gro_tcp6_tbl *tbl;
> > + size_t size;
> > + uint32_t entries_num, i;
> > +
> > + entries_num = max_flow_num * max_item_per_flow;
> > + entries_num = RTE_MIN(entries_num,
> > GRO_TCP6_TBL_MAX_ITEM_NUM);
> > +
> > + if (entries_num == 0)
> > + return NULL;
> > +
> > + tbl = rte_zmalloc_socket(__func__,
> > + sizeof(struct gro_tcp6_tbl),
> > + RTE_CACHE_LINE_SIZE,
> > + socket_id);
> > + if (tbl == NULL)
> > + return NULL;
> > +
> > + size = sizeof(struct gro_tcp_item) * entries_num;
> > + tbl->items = rte_zmalloc_socket(__func__,
> > + size,
> > + RTE_CACHE_LINE_SIZE,
> > + socket_id);
> > + if (tbl->items == NULL) {
> > + rte_free(tbl);
> > + return NULL;
> > + }
> > + tbl->max_item_num = entries_num;
> > +
> > + size = sizeof(struct gro_tcp6_flow) * entries_num;
> > + tbl->flows = rte_zmalloc_socket(__func__,
> > + size,
> > + RTE_CACHE_LINE_SIZE,
> > + socket_id);
> > + if (tbl->flows == NULL) {
> > + rte_free(tbl->items);
> > + rte_free(tbl);
> > + return NULL;
> > + }
> > + /* INVALID_ARRAY_INDEX indicates an empty flow */
> > + for (i = 0; i < entries_num; i++)
> > + tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
> > + tbl->max_flow_num = entries_num;
> > +
> > + return tbl;
> > +}
> > +
> > +void
> > +gro_tcp6_tbl_destroy(void *tbl)
> > +{
> > + struct gro_tcp6_tbl *tcp_tbl = tbl;
> > +
> > + if (tcp_tbl) {
> > + rte_free(tcp_tbl->items);
> > + rte_free(tcp_tbl->flows);
> > + }
> > + rte_free(tcp_tbl);
> > +}
> > +
> > +static inline uint32_t
> > +find_an_empty_item(struct gro_tcp6_tbl *tbl) {
> > + uint32_t i;
> > + uint32_t max_item_num = tbl->max_item_num;
> > +
> > + for (i = 0; i < max_item_num; i++)
> > + if (tbl->items[i].firstseg == NULL)
> > + return i;
> > + return INVALID_ARRAY_INDEX;
> > +}
> > +
> > +static inline uint32_t
> > +find_an_empty_flow(struct gro_tcp6_tbl *tbl) {
> > + uint32_t i;
> > + uint32_t max_flow_num = tbl->max_flow_num;
> > +
> > + for (i = 0; i < max_flow_num; i++)
> > + if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
> > + return i;
> > + return INVALID_ARRAY_INDEX;
> > +}
> > +
> > +static inline uint32_t
> > +insert_new_item(struct gro_tcp6_tbl *tbl,
> > + struct rte_mbuf *pkt,
> > + uint64_t start_time,
> > + uint32_t prev_idx,
> > + uint32_t sent_seq,
> > + uint8_t is_atomic)
> > +{
> > + uint32_t item_idx;
> > +
> > + item_idx = find_an_empty_item(tbl);
> > + if (item_idx == INVALID_ARRAY_INDEX)
> > + return INVALID_ARRAY_INDEX;
> > +
> > + tbl->items[item_idx].firstseg = pkt;
> > + tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> > + tbl->items[item_idx].start_time = start_time;
> > + tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> > + tbl->items[item_idx].sent_seq = sent_seq;
> > + tbl->items[item_idx].nb_merged = 1;
> > + tbl->items[item_idx].is_atomic = is_atomic;
> > + tbl->item_num++;
> > +
> > + /* if the previous packet exists, chain them together. */
> > + if (prev_idx != INVALID_ARRAY_INDEX) {
> > + tbl->items[item_idx].next_pkt_idx =
> > + tbl->items[prev_idx].next_pkt_idx;
> > + tbl->items[prev_idx].next_pkt_idx = item_idx;
> > + }
> > +
> > + return item_idx;
> > +}
> > +
> > +static inline uint32_t
> > +delete_item(struct gro_tcp6_tbl *tbl, uint32_t item_idx,
> > + uint32_t prev_item_idx)
> > +{
> > + uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
> > +
> > + /* NULL indicates an empty item */
> > + tbl->items[item_idx].firstseg = NULL;
> > + tbl->item_num--;
> > + if (prev_item_idx != INVALID_ARRAY_INDEX)
> > + tbl->items[prev_item_idx].next_pkt_idx = next_idx;
> > +
> > + return next_idx;
> > +}
> > +
> > +static inline uint32_t
> > +insert_new_flow(struct gro_tcp6_tbl *tbl,
> > + struct tcp6_flow_key *src,
> > + rte_be32_t vtc_flow,
> > + uint32_t item_idx)
> > +{
> > + struct tcp6_flow_key *dst;
> > + uint32_t flow_idx;
> > +
> > + flow_idx = find_an_empty_flow(tbl);
> > + if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
> > + return INVALID_ARRAY_INDEX;
> > +
> > + dst = &(tbl->flows[flow_idx].key);
> > +
> > + rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
> > + rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
> > + memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst-
> > >src_addr));
> > + memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst-
> > >dst_addr));
> > + dst->recv_ack = src->recv_ack;
> > + dst->src_port = src->src_port;
> > + dst->dst_port = src->dst_port;
> > +
> > + tbl->flows[flow_idx].start_index = item_idx;
> > + tbl->flow_num++;
> > + tbl->flows->vtc_flow = vtc_flow;
> > +
> > + return flow_idx;
> > +}
> > +
> > +/*
> > + * update the packet length for the flushed packet.
> > + */
> > +static inline void
> > +update_header(struct gro_tcp_item *item) {
> > + struct rte_ipv6_hdr *ipv6_hdr;
> > + struct rte_mbuf *pkt = item->firstseg;
> > +
> > + ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> > + pkt->l2_len);
> > + ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
> > + pkt->l2_len - pkt->l3_len);
> > +}
> > +
> > +int32_t
> > +gro_tcp6_reassemble(struct rte_mbuf *pkt,
> > + struct gro_tcp6_tbl *tbl,
> > + uint64_t start_time)
> > +{
> > + struct rte_ether_hdr *eth_hdr;
> > + struct rte_ipv6_hdr *ipv6_hdr;
> > + struct rte_tcp_hdr *tcp_hdr;
> > + uint32_t sent_seq;
> > + int32_t tcp_dl;
> > + uint16_t ip_tlen;
> > + struct tcp6_flow_key key;
> > + uint32_t cur_idx, prev_idx, item_idx;
> > + uint32_t i, max_flow_num, remaining_flow_num;
> > + int cmp;
> > + uint8_t find;
> > + rte_be32_t vtc_flow_diff;
> > +
> > + /*
> > + * Don't process the packet whose TCP header length is greater
> > + * than 60 bytes or less than 20 bytes.
> > + */
> > + if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
> > + return -1;
> > +
> > + eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
> > + ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
> > + tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv6_hdr + pkt->l3_len);
> > +
> > + /*
> > + * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
> > + * or CWR set.
> > + */
> > + if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
> > + return -1;
> > +
> > + ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
> > + /*
> > + * Trim the tail padding bytes. The IPv6 header is fixed to
> > + * 40 bytes unlike IPv4 that is variable. The length in the IPv6
> header
> > + * contains only length of TCP Header + TCP Payload, whereas IPv4
> > header contains
> > + * length of IP Header + TCP Header + TCP Payload
> > + */
> > + if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len + pkt->l3_len))
> > + rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len
> -
> > pkt->l3_len);
> > + /*
> > + * Don't process the packet whose payload length is less than or
> > + * equal to 0.
> > + */
> > + tcp_dl = ip_tlen - pkt->l4_len;
> > + if (tcp_dl <= 0)
> > + return -1;
> > +
> > + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> > +
> > + rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
> > + rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
> > + memcpy(&key.src_addr[0], &ipv6_hdr->src_addr,
> > sizeof(key.src_addr));
> > + memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr,
> > sizeof(key.dst_addr));
> > + key.src_port = tcp_hdr->src_port;
> > + key.dst_port = tcp_hdr->dst_port;
> > + key.recv_ack = tcp_hdr->recv_ack;
> > +
> > + /* Search for a matched flow. */
> > + max_flow_num = tbl->max_flow_num;
> > + remaining_flow_num = tbl->flow_num;
> > + find = 0;
> > + for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
> > + if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
> > + if (is_same_tcp6_flow(tbl->flows[i].key, key)) {
> > + /*
> > + * IP version (4) Traffic Class (8) Flow
> Label (20)
> > + * All fields except Traffic class should
> be
> > same
> > + */
> > + vtc_flow_diff = (ipv6_hdr->vtc_flow ^ tbl-
> > >flows->vtc_flow);
> > + if (vtc_flow_diff & htonl(0xF00FFFFF))
> > + continue;
> > + find = 1;
> > + break;
> > + }
> > + remaining_flow_num--;
> > + }
> > + }
> > +
> > + /*
> > + * Fail to find a matched flow. Insert a new flow and store the
> > + * packet into the flow.
> > + */
> > + if (find == 0) {
> > + item_idx = insert_new_item(tbl, pkt, start_time,
> > + INVALID_ARRAY_INDEX, sent_seq, true);
> > + if (item_idx == INVALID_ARRAY_INDEX)
> > + return -1;
> > + if (insert_new_flow(tbl, &key, ipv6_hdr->vtc_flow,
> item_idx)
> > ==
> > + INVALID_ARRAY_INDEX) {
> > + /*
> > + * Fail to insert a new flow, so delete the
> > + * stored packet.
> > + */
> > + delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
> > + return -1;
> > + }
> > + return 0;
> > + }
> > +
> > + /*
> > + * Check all packets in the flow and try to find a neighbor for
> > + * the input packet.
> > + */
> > + cur_idx = tbl->flows[i].start_index;
> > + prev_idx = cur_idx;
> > + do {
> > + cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
> > + sent_seq, 0, pkt->l4_len, tcp_dl, 0,
> > + true);
> > + if (cmp) {
> > + if (merge_two_tcp_packets(&(tbl->items[cur_idx]),
> > + pkt, cmp, sent_seq, 0, 0))
> {
> > + return 1;
> > + }
> > +
> > + /*
> > + * Fail to merge the two packets, as the packet
> > + * length is greater than the max value. Store
> > + * the packet into the flow.
> > + */
> > + if (insert_new_item(tbl, pkt, start_time, cur_idx,
> > + sent_seq, true) ==
> > + INVALID_ARRAY_INDEX)
> > + return -1;
> > + return 0;
> > + }
> > + prev_idx = cur_idx;
> > + cur_idx = tbl->items[cur_idx].next_pkt_idx;
> > + } while (cur_idx != INVALID_ARRAY_INDEX);
> > +
> > + /* Fail to find a neighbor, so store the packet into the flow. */
> > + if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
> > + true) == INVALID_ARRAY_INDEX)
> > + return -1;
> > +
> > + return 0;
> > +}
> > +
> > +uint16_t
> > +gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
> > + uint64_t flush_timestamp,
> > + struct rte_mbuf **out,
> > + uint16_t nb_out)
> > +{
> > + uint16_t k = 0;
> > + uint32_t i, j;
> > + uint32_t max_flow_num = tbl->max_flow_num;
> > +
> > + for (i = 0; i < max_flow_num; i++) {
> > + if (unlikely(tbl->flow_num == 0))
> > + return k;
> > +
> > + j = tbl->flows[i].start_index;
> > + while (j != INVALID_ARRAY_INDEX) {
> > + if (tbl->items[j].start_time <= flush_timestamp) {
> > + out[k++] = tbl->items[j].firstseg;
> > + if (tbl->items[j].nb_merged > 1)
> > + update_header(&(tbl->items[j]));
> > + /*
> > + * Delete the packet and get the next
> > + * packet in the flow.
> > + */
> > + j = delete_item(tbl, j,
> > INVALID_ARRAY_INDEX);
> > + tbl->flows[i].start_index = j;
> > + if (j == INVALID_ARRAY_INDEX)
> > + tbl->flow_num--;
> > +
> > + if (unlikely(k == nb_out))
> > + return k;
> > + } else
> > + /*
> > + * The left packets in this flow won't be
> > + * timeout. Go to check other flows.
> > + */
> > + break;
> > + }
> > + }
> > + return k;
> > +}
> > +
> > +uint32_t
> > +gro_tcp6_tbl_pkt_count(void *tbl)
> > +{
> > + struct gro_tcp6_tbl *gro_tbl = tbl;
> > +
> > + if (gro_tbl)
> > + return gro_tbl->item_num;
> > +
> > + return 0;
> > +}
> > diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h new file mode 100644
> > index 0000000000..91325c0da2
> > --- /dev/null
> > +++ b/lib/gro/gro_tcp6.h
> > @@ -0,0 +1,150 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2017 Intel Corporation
> > + */
> > +
> > +#ifndef _GRO_TCP6_H_
> > +#define _GRO_TCP6_H_
> > +
> > +#define INVALID_ARRAY_INDEX 0xffffffffUL #define
> > +GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
> > +
> > +/* Header fields representing a TCP/IPv6 flow */ struct tcp6_flow_key {
> > + struct rte_ether_addr eth_saddr;
> > + struct rte_ether_addr eth_daddr;
> > + uint8_t src_addr[16];
> > + uint8_t dst_addr[16];
> > +
> > + uint32_t recv_ack;
> > + uint16_t src_port;
> > + uint16_t dst_port;
> > +};
> > +
> > +struct gro_tcp6_flow {
> > + struct tcp6_flow_key key;
> > + rte_be32_t vtc_flow;
> > + /*
> > + * The index of the first packet in the flow.
> > + * INVALID_ARRAY_INDEX indicates an empty flow.
> > + */
> > + uint32_t start_index;
> > +};
> > +
> > +/*
> > + * TCP/IPv6 reassembly table structure.
> > + */
> > +struct gro_tcp6_tbl {
> > + /* item array */
> > + struct gro_tcp_item *items;
> > + /* flow array */
> > + struct gro_tcp6_flow *flows;
> > + /* current item number */
> > + uint32_t item_num;
> > + /* current flow num */
> > + uint32_t flow_num;
> > + /* item array size */
> > + uint32_t max_item_num;
> > + /* flow array size */
> > + uint32_t max_flow_num;
> > +};
> > +
> > +/**
> > + * This function creates a TCP/IPv6 reassembly table.
> > + *
> > + * @param socket_id
> > + * Socket index for allocating the TCP/IPv6 reassemble table
> > + * @param max_flow_num
> > + * The maximum number of flows in the TCP/IPv6 GRO table
> > + * @param max_item_per_flow
> > + * The maximum number of packets per flow
> > + *
> > + * @return
> > + * - Return the table pointer on success.
> > + * - Return NULL on failure.
> > + */
> > +void *gro_tcp6_tbl_create(uint16_t socket_id,
> > + uint16_t max_flow_num,
> > + uint16_t max_item_per_flow);
> > +
> > +/**
> > + * This function destroys a TCP/IPv6 reassembly table.
> > + *
> > + * @param tbl
> > + * Pointer pointing to the TCP/IPv6 reassembly table.
> > + */
> > +void gro_tcp6_tbl_destroy(void *tbl);
> > +
> > +/**
> > + * This function merges a TCP/IPv6 packet. It doesn't process the
> > +packet,
> > + * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't have
> > + * payload.
> > + *
> > + * This function doesn't check if the packet has correct checksums and
> > + * doesn't re-calculate checksums for the merged packet. Additionally,
> > + * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
> > + * when IP fragmentation is possible (i.e., DF==0). It returns the
> > + * packet, if the packet has invalid parameters (e.g. SYN bit is set)
> > + * or there is no available space in the table.
> > + *
> > + * @param pkt
> > + * Packet to reassemble
> > + * @param tbl
> > + * Pointer pointing to the TCP/IPv6 reassembly table
> > + * @start_time
> > + * The time when the packet is inserted into the table
> > + *
> > + * @return
> > + * - Return a positive value if the packet is merged.
> > + * - Return zero if the packet isn't merged but stored in the table.
> > + * - Return a negative value for invalid parameters or no available
> > + * space in the table.
> > + */
> > +int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
> > + struct gro_tcp6_tbl *tbl,
> > + uint64_t start_time);
> > +
> > +/**
> > + * This function flushes timeout packets in a TCP/IPv6 reassembly
> > +table,
> > + * and without updating checksums.
> > + *
> > + * @param tbl
> > + * TCP/IPv6 reassembly table pointer
> > + * @param flush_timestamp
> > + * Flush packets which are inserted into the table before or at the
> > + * flush_timestamp.
> > + * @param out
> > + * Pointer array used to keep flushed packets
> > + * @param nb_out
> > + * The element number in 'out'. It also determines the maximum number
> > +of
> > + * packets that can be flushed finally.
> > + *
> > + * @return
> > + * The number of flushed packets
> > + */
> > +uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
> > + uint64_t flush_timestamp,
> > + struct rte_mbuf **out,
> > + uint16_t nb_out);
> > +
> > +/**
> > + * This function returns the number of the packets in a TCP/IPv6
> > + * reassembly table.
> > + *
> > + * @param tbl
> > + * TCP/IPv6 reassembly table pointer
> > + *
> > + * @return
> > + * The number of packets in the table
> > + */
> > +uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
> > +
> > +/*
> > + * Check if two TCP/IPv6 packets belong to the same flow.
> > + */
> > +static inline int
> > +is_same_tcp6_flow(struct tcp6_flow_key k1, struct tcp6_flow_key k2) {
> > + return (!memcmp(&k1, &k2, sizeof(struct tcp6_flow_key))); }
> > +
> > +#endif
> > diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c index
> > 3be4deb7c7..56b30b8c98 100644
> > --- a/lib/gro/gro_vxlan_tcp4.c
> > +++ b/lib/gro/gro_vxlan_tcp4.c
> > @@ -7,6 +7,7 @@
> > #include <rte_ethdev.h>
> > #include <rte_udp.h>
> >
> > +#include "gro_tcp.h"
> > #include "gro_vxlan_tcp4.h"
> >
> > void *
> > @@ -248,7 +249,7 @@ merge_two_vxlan_tcp4_packets(struct
> > gro_vxlan_tcp4_item *item,
> > uint16_t outer_ip_id,
> > uint16_t ip_id)
> > {
> > - if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp,
> > sent_seq,
> > + if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
> > ip_id, pkt->outer_l2_len +
> > pkt->outer_l3_len)) {
> > /* Update the outer IPv4 ID to the large value. */ diff
> --git
> > a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h index
> > 7832942a68..d68d5fcd5b 100644
> > --- a/lib/gro/gro_vxlan_tcp4.h
> > +++ b/lib/gro/gro_vxlan_tcp4.h
> > @@ -5,6 +5,7 @@
> > #ifndef _GRO_VXLAN_TCP4_H_
> > #define _GRO_VXLAN_TCP4_H_
> >
> > +#include "gro_tcp.h"
> > #include "gro_tcp4.h"
> >
> > #define GRO_VXLAN_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL) @@ -
> > 36,7 +37,7 @@ struct gro_vxlan_tcp4_flow { };
> >
> > struct gro_vxlan_tcp4_item {
> > - struct gro_tcp4_item inner_item;
> > + struct gro_tcp_item inner_item;
> > /* IPv4 ID in the outer IPv4 header */
> > uint16_t outer_ip_id;
> > /* Indicate if outer IPv4 ID can be ignored */ diff --git
> > a/lib/gro/meson.build b/lib/gro/meson.build index e4fa2958bd..dbce05220d
> > 100644
> > --- a/lib/gro/meson.build
> > +++ b/lib/gro/meson.build
> > @@ -4,6 +4,7 @@
> > sources = files(
> > 'rte_gro.c',
> > 'gro_tcp4.c',
> > + 'gro_tcp6.c',
> > 'gro_udp4.c',
> > 'gro_vxlan_tcp4.c',
> > 'gro_vxlan_udp4.c',
> > diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c index
> > e35399fd42..d824eebd93 100644
> > --- a/lib/gro/rte_gro.c
> > +++ b/lib/gro/rte_gro.c
> > @@ -8,6 +8,7 @@
> >
> > #include "rte_gro.h"
> > #include "gro_tcp4.h"
> > +#include "gro_tcp6.h"
> > #include "gro_udp4.h"
> > #include "gro_vxlan_tcp4.h"
> > #include "gro_vxlan_udp4.h"
> > @@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
> >
> > static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
> > gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
> > - gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
> > + gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create,
> > gro_tcp6_tbl_create,
> > +NULL};
> > static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
> > gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
> > gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
> > + gro_tcp6_tbl_destroy,
> > NULL};
> > static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM]
> > = {
> > gro_tcp4_tbl_pkt_count,
> > gro_vxlan_tcp4_tbl_pkt_count,
> > gro_udp4_tbl_pkt_count,
> > gro_vxlan_udp4_tbl_pkt_count,
> > + gro_tcp6_tbl_pkt_count,
> > NULL};
> >
> > #define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \ @@
> > -35,6 +38,12 @@ static gro_tbl_pkt_count_fn
> > tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
> > ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) &&
> > \
> > (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
> >
> > +/* GRO with extension headers is not supported */ #define
> > +IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
> > + ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
> > + ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) &&
> > \
> > + (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
> > +
> > #define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
> > ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
> > (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
> > @@ -147,7 +156,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> > /* allocate a reassembly table for TCP/IPv4 GRO */
> > struct gro_tcp4_tbl tcp_tbl;
> > struct gro_tcp4_flow
> > tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
> > - struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
> > = {{0} };
> > + struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
> > = {{0} };
> > +
> > + struct gro_tcp6_tbl tcp6_tbl;
> > + struct gro_tcp6_flow
> > tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
> > + struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM]
> > = {{0} };
> >
> > /* allocate a reassembly table for UDP/IPv4 GRO */
> > struct gro_udp4_tbl udp_tbl;
> > @@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> > int32_t ret;
> > uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
> > uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
> > - do_vxlan_udp_gro = 0;
> > + do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
> >
> > if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4
> > |
> > - RTE_GRO_TCP_IPV4 |
> > + RTE_GRO_TCP_IPV4 |
> > RTE_GRO_TCP_IPV6 |
> > RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
> > RTE_GRO_UDP_IPV4)) == 0))
> > return nb_pkts;
> > @@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> > do_udp4_gro = 1;
> > }
> >
> > + if (param->gro_types & RTE_GRO_TCP_IPV6) {
> > + for (i = 0; i < item_num; i++)
> > + tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
> > +
> > + tcp6_tbl.flows = tcp6_flows;
> > + tcp6_tbl.items = tcp6_items;
> > + tcp6_tbl.flow_num = 0;
> > + tcp6_tbl.item_num = 0;
> > + tcp6_tbl.max_flow_num = item_num;
> > + tcp6_tbl.max_item_num = item_num;
> > + do_tcp6_gro = 1;
> > + }
> >
> > for (i = 0; i < nb_pkts; i++) {
> > /*
> > @@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> > nb_after_gro--;
> > else if (ret < 0)
> > unprocess_pkts[unprocess_num++] = pkts[i];
> > + } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
> > + do_tcp6_gro) {
> > + ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
> > + if (ret > 0)
> > + /* merge successfully */
> > + nb_after_gro--;
> > + else if (ret < 0)
> > + unprocess_pkts[unprocess_num++] = pkts[i];
> > } else
> > unprocess_pkts[unprocess_num++] = pkts[i];
> > }
> > @@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> > if ((nb_after_gro < nb_pkts)
> > || (unprocess_num < nb_pkts)) {
> > i = 0;
> > + /* Copy unprocessed packets */
> > + if (unprocess_num > 0) {
> > + memcpy(&pkts[i], unprocess_pkts,
> > + sizeof(struct rte_mbuf *) *
> > + unprocess_num);
> > + i = unprocess_num;
> > + }
> > +
> > /* Flush all packets from the tables */
> > if (do_vxlan_tcp_gro) {
> > - i =
> > gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
> > + i +=
> > gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
> > 0, pkts, nb_pkts);
> > }
> >
> > @@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> > i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
> > &pkts[i], nb_pkts - i);
> > }
> > - /* Copy unprocessed packets */
> > - if (unprocess_num > 0) {
> > - memcpy(&pkts[i], unprocess_pkts,
> > - sizeof(struct rte_mbuf *) *
> > - unprocess_num);
> > +
> > + if (do_tcp6_gro) {
> > + i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
> > + &pkts[i], nb_pkts - i);
> > }
> > - nb_after_gro = i + unprocess_num;
> > }
> >
> > return nb_after_gro;
> > @@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts, {
> > struct rte_mbuf *unprocess_pkts[nb_pkts];
> > struct gro_ctx *gro_ctx = ctx;
> > - void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
> > + void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
> > uint64_t current_time;
> > uint16_t i, unprocess_num = 0;
> > - uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro,
> > do_vxlan_udp_gro;
> > + uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro,
> > do_vxlan_udp_gro,
> > +do_tcp6_gro;
> >
> > if (unlikely((gro_ctx->gro_types &
> > (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
> > - RTE_GRO_TCP_IPV4 |
> > + RTE_GRO_TCP_IPV4 |
> > RTE_GRO_TCP_IPV6 |
> > RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
> > RTE_GRO_UDP_IPV4)) == 0))
> > return nb_pkts;
> > @@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> > vxlan_tcp_tbl = gro_ctx-
> > >tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
> > udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
> > vxlan_udp_tbl = gro_ctx-
> > >tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
> > + tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
> >
> > do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
> > RTE_GRO_TCP_IPV4;
> > @@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> > RTE_GRO_UDP_IPV4;
> > do_vxlan_udp_gro = (gro_ctx->gro_types &
> > RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
> > RTE_GRO_IPV4_VXLAN_UDP_IPV4;
> > + do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) ==
> > +RTE_GRO_TCP_IPV6;
> >
> > current_time = rte_rdtsc();
> >
> > @@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> > if (gro_udp4_reassemble(pkts[i], udp_tbl,
> > current_time) < 0)
> > unprocess_pkts[unprocess_num++] = pkts[i];
> > + } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
> > + do_tcp6_gro) {
> > + if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
> > + current_time) < 0)
> > + unprocess_pkts[unprocess_num++] = pkts[i];
> > } else
> > unprocess_pkts[unprocess_num++] = pkts[i];
> > }
> > @@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
> > gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
> > flush_timestamp,
> > &out[num], left_nb_out);
> > + left_nb_out = max_nb_out - num;
> > + }
> > +
> > + if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
> > + num += gro_tcp6_tbl_timeout_flush(
> > + gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
> > + flush_timestamp,
> > + &out[num], left_nb_out);
> > +
> > }
> >
> > return num;
> > diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h index
> > 9f9ed4935a..c83dfd9ad1 100644
> > --- a/lib/gro/rte_gro.h
> > +++ b/lib/gro/rte_gro.h
> > @@ -38,6 +38,9 @@ extern "C" {
> > #define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3 #define
> > RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL <<
> > RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
> > /**< VxLAN UDP/IPv4 GRO flag. */
> > +#define RTE_GRO_TCP_IPV6_INDEX 4
> > +#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX) /**<
> > TCP/IPv6
> > +GRO flag. */
> >
> > /**
> > * Structure used to create GRO context objects or used to pass
> > --
> > 2.25.1
>
>
[-- Attachment #2: Type: text/html, Size: 64451 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [PATCH v2] gro : ipv6 changes to support GRO for TCP/ipv6
2023-05-16 9:28 ` kumaraparameshwaran rathinavel
@ 2023-05-25 11:22 ` kumaraparameshwaran rathinavel
2023-05-31 8:20 ` Hu, Jiayu
0 siblings, 1 reply; 32+ messages in thread
From: kumaraparameshwaran rathinavel @ 2023-05-25 11:22 UTC (permalink / raw)
To: Hu, Jiayu; +Cc: dev
[-- Attachment #1: Type: text/plain, Size: 57180 bytes --]
On Tue, May 16, 2023 at 2:58 PM kumaraparameshwaran rathinavel <
kumaraparamesh92@gmail.com> wrote:
>
>
> On Fri, May 12, 2023 at 8:17 AM Hu, Jiayu <jiayu.hu@intel.com> wrote:
>
>> Hi Kumar,
>>
>> For TCP/IPv4 and TCP/IPv6, the TCP layer is the same and the difference is
>> the IP layer. So the code used for TCP layer needs to be shared among
>> gro_tcp6.c
>> and gro_tcp4.c. But there are too much code duplication in gro_tcp4.c
>> and gro_tcp6.c
>> in current implementation.
>>
>
>
>> Sure, will do it. I had followed the same that was similar to what was
>>> done in gro_tcp4.c and gro_vxlan_tcp4.c. Eventhough the TCP could have been
>>> reused for vxlan, we had code duplication. I assumed that this was better
>>> in terms of performance etc.
>>
>>
>>>>
>>> I am thinking two have two functions gro_tcp4_reassemble and
>>>> gro_tcp6_reassemble which would perform the validations for the respective
>>>> IP types. The GRO table is different for V4 and V6 but the itmes array in
>>>> the gro table is common to both V4 and V6 and hence we can move insert_item
>>>> and delete_item to gro_tcp.c/h file. Introduce a new function
>>>> gro_tcp_reassemble which would perform the logic of lookup/create/merge
>>>> operations. Since the table and the key would be different for V4/V6
>>>> introduce a new structure which would hold the lookup/create flow function
>>>> pointers for each of the IP type. This way we would reduce the code
>>>> duplication in gro_tcp4_reassmeble and gro_tcp6_reassemble. Pleae let me
>>>> know your thoughts. We can get on a call if require to discuss further.
>>>>
>>>
> For example, struct tcp6_flow_key and struct tcp4_flow_key share most of
>>>>> fields, except
>>>>> the IP address type. It's better to have a common TCP flow key
>>>>> structure in gro_tcp.h. In
>>>>> gro_tcp4.h and gro_tcp6.h, we implement tcp4 and tcp6 flow key
>>>>> structures with using the
>>>>> common structure.
>>>>>
>>>> Thanks,
>>>>> Jiayu
>>>>>
>>>>> > -----Original Message-----
>>>>> > From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
>>>>> > Sent: Friday, October 21, 2022 2:14 AM
>>>>> > To: Hu, Jiayu <jiayu.hu@intel.com>
>>>>> > Cc: dev@dpdk.org; Kumara Parameshwaran
>>>>> > <kumaraparamesh92@gmail.com>
>>>>> > Subject: [PATCH v2] gro : ipv6 changes to support GRO for TCP/ipv6
>>>>> >
>>>>> > From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
>>>>> >
>>>>> > The patch adds GRO support for TCP/ipv6 packets. This does not
>>>>> include the
>>>>> > support for vxlan, udp ipv6 packets.
>>>>> >
>>>>> > Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
>>>>> > ---
>>>>> > v1:
>>>>> > * Changes to support GRO for TCP/ipv6 packets. This does not
>>>>> > include
>>>>> > vxlan changes.
>>>>> > * The GRO is performed only for ipv6 packets that does not
>>>>> contain
>>>>> > extension headers.
>>>>> > * The logic for the TCP coalescing remains the same, in ipv6
>>>>> header
>>>>> > the source address, destination address, flow label, version
>>>>> fields
>>>>> > are expected to be the same.
>>>>> > * Re-organised the code to reuse certain tcp functions for
>>>>> both ipv4
>>>>> > and
>>>>> > ipv6 flows.
>>>>> > v2:
>>>>> > * Fix comments in gro_tcp6.h header file.
>>>>> >
>>>>> > lib/gro/gro_tcp.h | 155 ++++++++++++++++
>>>>> > lib/gro/gro_tcp4.c | 7 +-
>>>>> > lib/gro/gro_tcp4.h | 152 +--------------
>>>>> > lib/gro/gro_tcp6.c | 387
>>>>> > +++++++++++++++++++++++++++++++++++++++
>>>>> > lib/gro/gro_tcp6.h | 150 +++++++++++++++
>>>>> > lib/gro/gro_vxlan_tcp4.c | 3 +-
>>>>> > lib/gro/gro_vxlan_tcp4.h | 3 +-
>>>>> > lib/gro/meson.build | 1 +
>>>>> > lib/gro/rte_gro.c | 83 +++++++--
>>>>> > lib/gro/rte_gro.h | 3 +
>>>>> > 10 files changed, 774 insertions(+), 170 deletions(-) create mode
>>>>> 100644
>>>>> > lib/gro/gro_tcp.h create mode 100644 lib/gro/gro_tcp6.c create mode
>>>>> > 100644 lib/gro/gro_tcp6.h
>>>>> >
>>>>> > diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h new file mode
>>>>> 100644 index
>>>>> > 0000000000..c5d248a022
>>>>> > --- /dev/null
>>>>> > +++ b/lib/gro/gro_tcp.h
>>>>> > @@ -0,0 +1,155 @@
>>>>> > +#ifndef _GRO_TCP_H_
>>>>> > +#define _GRO_TCP_H_
>>>>> > +
>>>>> > +#include <rte_tcp.h>
>>>>> > +
>>>>> > +/*
>>>>> > + * The max length of a IPv4 packet, which includes the length of
>>>>> the L3
>>>>> > + * header, the L4 header and the data payload.
>>>>> > + */
>>>>> > +#define MAX_IP_PKT_LENGTH UINT16_MAX
>>>>> > +
>>>>> > +/* The maximum TCP header length */
>>>>> > +#define MAX_TCP_HLEN 60
>>>>> > +#define INVALID_TCP_HDRLEN(len) \
>>>>> > + (((len) < sizeof(struct rte_tcp_hdr)) || ((len) >
>>>>> MAX_TCP_HLEN))
>>>>> > +
>>>>> > +struct gro_tcp_item {
>>>>> > + /*
>>>>> > + * The first MBUF segment of the packet. If the value
>>>>> > + * is NULL, it means the item is empty.
>>>>> > + */
>>>>> > + struct rte_mbuf *firstseg;
>>>>> > + /* The last MBUF segment of the packet */
>>>>> > + struct rte_mbuf *lastseg;
>>>>> > + /*
>>>>> > + * The time when the first packet is inserted into the table.
>>>>> > + * This value won't be updated, even if the packet is merged
>>>>> > + * with other packets.
>>>>> > + */
>>>>> > + uint64_t start_time;
>>>>> > + /*
>>>>> > + * next_pkt_idx is used to chain the packets that
>>>>> > + * are in the same flow but can't be merged together
>>>>> > + * (e.g. caused by packet reordering).
>>>>> > + */
>>>>> > + uint32_t next_pkt_idx;
>>>>> > + /* TCP sequence number of the packet */
>>>>> > + uint32_t sent_seq;
>>>>> > + /* IPv4 ID of the packet */
>>>>> > + uint16_t ip_id;
>>>>> > + /* the number of merged packets */
>>>>> > + uint16_t nb_merged;
>>>>> > + /* Indicate if IPv4 ID can be ignored */
>>>>> > + uint8_t is_atomic;
>>>>> > +};
>>>>> > +
>>>>> > +/*
>>>>> > + * Merge two TCP packets without updating checksums.
>>>>> > + * If cmp is larger than 0, append the new packet to the
>>>>> > + * original packet. Otherwise, pre-pend the new packet to
>>>>> > + * the original packet.
>>>>> > + */
>>>>> > +static inline int
>>>>> > +merge_two_tcp_packets(struct gro_tcp_item *item,
>>>>> > + struct rte_mbuf *pkt,
>>>>> > + int cmp,
>>>>> > + uint32_t sent_seq,
>>>>> > + uint16_t ip_id,
>>>>> > + uint16_t l2_offset)
>>>>> > +{
>>>>> > + struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
>>>>> > + uint16_t hdr_len, l2_len;
>>>>> > +
>>>>> > + if (cmp > 0) {
>>>>> > + pkt_head = item->firstseg;
>>>>> > + pkt_tail = pkt;
>>>>> > + } else {
>>>>> > + pkt_head = pkt;
>>>>> > + pkt_tail = item->firstseg;
>>>>> > + }
>>>>> > +
>>>>> > + /* check if the IPv4 packet length is greater than the max
>>>>> value */
>>>>> > + hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
>>>>> > + pkt_head->l4_len;
>>>>> > + l2_len = l2_offset > 0 ? pkt_head->outer_l2_len :
>>>>> pkt_head->l2_len;
>>>>> > + if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
>>>>> > + hdr_len > MAX_IP_PKT_LENGTH))
>>>>> > + return 0;
>>>>> > +
>>>>> > + /* remove the packet header for the tail packet */
>>>>> > + rte_pktmbuf_adj(pkt_tail, hdr_len);
>>>>> > +
>>>>> > + /* chain two packets together */
>>>>> > + if (cmp > 0) {
>>>>> > + item->lastseg->next = pkt;
>>>>> > + item->lastseg = rte_pktmbuf_lastseg(pkt);
>>>>> > + /* update IP ID to the larger value */
>>>>> > + item->ip_id = ip_id;
>>>>> > + } else {
>>>>> > + lastseg = rte_pktmbuf_lastseg(pkt);
>>>>> > + lastseg->next = item->firstseg;
>>>>> > + item->firstseg = pkt;
>>>>> > + /* update sent_seq to the smaller value */
>>>>> > + item->sent_seq = sent_seq;
>>>>> > + item->ip_id = ip_id;
>>>>> > + }
>>>>> > + item->nb_merged++;
>>>>> > +
>>>>> > + /* update MBUF metadata for the merged packet */
>>>>> > + pkt_head->nb_segs += pkt_tail->nb_segs;
>>>>> > + pkt_head->pkt_len += pkt_tail->pkt_len;
>>>>> > +
>>>>> > + return 1;
>>>>> > +}
>>>>> > +
>>>>> > +/*
>>>>> > + * Check if two TCP/IPv4 packets are neighbors.
>>>>> > + */
>>>>> > +static inline int
>>>>> > +check_seq_option(struct gro_tcp_item *item,
>>>>> > + struct rte_tcp_hdr *tcph,
>>>>> > + uint32_t sent_seq,
>>>>> > + uint16_t ip_id,
>>>>> > + uint16_t tcp_hl,
>>>>> > + uint16_t tcp_dl,
>>>>> > + uint16_t l2_offset,
>>>>> > + uint8_t is_atomic)
>>>>> > +{
>>>>> > + struct rte_mbuf *pkt_orig = item->firstseg;
>>>>> > + char *iph_orig;
>>>>> > + struct rte_tcp_hdr *tcph_orig;
>>>>> > + uint16_t len, tcp_hl_orig;
>>>>> > +
>>>>> > + iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
>>>>> > + l2_offset + pkt_orig->l2_len);
>>>>> > + tcph_orig = (struct rte_tcp_hdr *)(iph_orig +
>>>>> pkt_orig->l3_len);
>>>>> > + tcp_hl_orig = pkt_orig->l4_len;
>>>>> > +
>>>>> > + /* Check if TCP option fields equal */
>>>>> > + len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct
>>>>> rte_tcp_hdr);
>>>>> > + if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
>>>>> > + (memcmp(tcph + 1, tcph_orig + 1,
>>>>> > + len) != 0)))
>>>>> > + return 0;
>>>>> > +
>>>>> > + /* Don't merge packets whose DF bits are different */
>>>>> > + if (unlikely(item->is_atomic ^ is_atomic))
>>>>> > + return 0;
>>>>> > +
>>>>> > + /* check if the two packets are neighbors */
>>>>> > + len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
>>>>> > + pkt_orig->l3_len - tcp_hl_orig;
>>>>> > + if ((sent_seq == item->sent_seq + len) && (is_atomic ||
>>>>> > + (ip_id == item->ip_id + 1)))
>>>>> > + /* append the new packet */
>>>>> > + return 1;
>>>>> > + else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
>>>>> > + (ip_id + item->nb_merged ==
>>>>> item->ip_id)))
>>>>> > + /* pre-pend the new packet */
>>>>> > + return -1;
>>>>> > +
>>>>> > + return 0;
>>>>> > +}
>>>>> > +
>>>>> > +#endif
>>>>> > diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c index
>>>>> > 8f5e800250..eea2a72ecd 100644
>>>>> > --- a/lib/gro/gro_tcp4.c
>>>>> > +++ b/lib/gro/gro_tcp4.c
>>>>> > @@ -7,6 +7,7 @@
>>>>> > #include <rte_ethdev.h>
>>>>> >
>>>>> > #include "gro_tcp4.h"
>>>>> > +#include "gro_tcp.h"
>>>>> >
>>>>> > void *
>>>>> > gro_tcp4_tbl_create(uint16_t socket_id, @@ -30,7 +31,7 @@
>>>>> > gro_tcp4_tbl_create(uint16_t socket_id,
>>>>> > if (tbl == NULL)
>>>>> > return NULL;
>>>>> >
>>>>> > - size = sizeof(struct gro_tcp4_item) * entries_num;
>>>>> > + size = sizeof(struct gro_tcp_item) * entries_num;
>>>>> > tbl->items = rte_zmalloc_socket(__func__,
>>>>> > size,
>>>>> > RTE_CACHE_LINE_SIZE,
>>>>> > @@ -177,7 +178,7 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
>>>>> > * update the packet length for the flushed packet.
>>>>> > */
>>>>> > static inline void
>>>>> > -update_header(struct gro_tcp4_item *item)
>>>>> > +update_header(struct gro_tcp_item *item)
>>>>> > {
>>>>> > struct rte_ipv4_hdr *ipv4_hdr;
>>>>> > struct rte_mbuf *pkt = item->firstseg; @@ -302,7 +303,7 @@
>>>>> > gro_tcp4_reassemble(struct rte_mbuf *pkt,
>>>>> > sent_seq, ip_id, pkt->l4_len, tcp_dl,
>>>>> 0,
>>>>> > is_atomic);
>>>>> > if (cmp) {
>>>>> > - if
>>>>> (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
>>>>> > + if
>>>>> (merge_two_tcp_packets(&(tbl->items[cur_idx]),
>>>>> > pkt, cmp, sent_seq,
>>>>> ip_id, 0))
>>>>> > return 1;
>>>>> > /*
>>>>> > diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h index
>>>>> > 212f97a042..634a215b98 100644
>>>>> > --- a/lib/gro/gro_tcp4.h
>>>>> > +++ b/lib/gro/gro_tcp4.h
>>>>> > @@ -5,22 +5,9 @@
>>>>> > #ifndef _GRO_TCP4_H_
>>>>> > #define _GRO_TCP4_H_
>>>>> >
>>>>> > -#include <rte_tcp.h>
>>>>> > -
>>>>> > #define INVALID_ARRAY_INDEX 0xffffffffUL #define
>>>>> > GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
>>>>> >
>>>>> > -/*
>>>>> > - * The max length of a IPv4 packet, which includes the length of
>>>>> the L3
>>>>> > - * header, the L4 header and the data payload.
>>>>> > - */
>>>>> > -#define MAX_IPV4_PKT_LENGTH UINT16_MAX
>>>>> > -
>>>>> > -/* The maximum TCP header length */
>>>>> > -#define MAX_TCP_HLEN 60
>>>>> > -#define INVALID_TCP_HDRLEN(len) \
>>>>> > - (((len) < sizeof(struct rte_tcp_hdr)) || ((len) >
>>>>> MAX_TCP_HLEN))
>>>>> > -
>>>>> > /* Header fields representing a TCP/IPv4 flow */ struct
>>>>> tcp4_flow_key {
>>>>> > struct rte_ether_addr eth_saddr;
>>>>> > @@ -42,42 +29,12 @@ struct gro_tcp4_flow {
>>>>> > uint32_t start_index;
>>>>> > };
>>>>> >
>>>>> > -struct gro_tcp4_item {
>>>>> > - /*
>>>>> > - * The first MBUF segment of the packet. If the value
>>>>> > - * is NULL, it means the item is empty.
>>>>> > - */
>>>>> > - struct rte_mbuf *firstseg;
>>>>> > - /* The last MBUF segment of the packet */
>>>>> > - struct rte_mbuf *lastseg;
>>>>> > - /*
>>>>> > - * The time when the first packet is inserted into the table.
>>>>> > - * This value won't be updated, even if the packet is merged
>>>>> > - * with other packets.
>>>>> > - */
>>>>> > - uint64_t start_time;
>>>>> > - /*
>>>>> > - * next_pkt_idx is used to chain the packets that
>>>>> > - * are in the same flow but can't be merged together
>>>>> > - * (e.g. caused by packet reordering).
>>>>> > - */
>>>>> > - uint32_t next_pkt_idx;
>>>>> > - /* TCP sequence number of the packet */
>>>>> > - uint32_t sent_seq;
>>>>> > - /* IPv4 ID of the packet */
>>>>> > - uint16_t ip_id;
>>>>> > - /* the number of merged packets */
>>>>> > - uint16_t nb_merged;
>>>>> > - /* Indicate if IPv4 ID can be ignored */
>>>>> > - uint8_t is_atomic;
>>>>> > -};
>>>>> > -
>>>>> > /*
>>>>> > * TCP/IPv4 reassembly table structure.
>>>>> > */
>>>>> > struct gro_tcp4_tbl {
>>>>> > /* item array */
>>>>> > - struct gro_tcp4_item *items;
>>>>> > + struct gro_tcp_item *items;
>>>>> > /* flow array */
>>>>> > struct gro_tcp4_flow *flows;
>>>>> > /* current item number */
>>>>> > @@ -195,111 +152,4 @@ is_same_tcp4_flow(struct tcp4_flow_key k1,
>>>>> struct
>>>>> > tcp4_flow_key k2)
>>>>> > (k1.dst_port == k2.dst_port));
>>>>> > }
>>>>> >
>>>>> > -/*
>>>>> > - * Merge two TCP/IPv4 packets without updating checksums.
>>>>> > - * If cmp is larger than 0, append the new packet to the
>>>>> > - * original packet. Otherwise, pre-pend the new packet to
>>>>> > - * the original packet.
>>>>> > - */
>>>>> > -static inline int
>>>>> > -merge_two_tcp4_packets(struct gro_tcp4_item *item,
>>>>> > - struct rte_mbuf *pkt,
>>>>> > - int cmp,
>>>>> > - uint32_t sent_seq,
>>>>> > - uint16_t ip_id,
>>>>> > - uint16_t l2_offset)
>>>>> > -{
>>>>> > - struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
>>>>> > - uint16_t hdr_len, l2_len;
>>>>> > -
>>>>> > - if (cmp > 0) {
>>>>> > - pkt_head = item->firstseg;
>>>>> > - pkt_tail = pkt;
>>>>> > - } else {
>>>>> > - pkt_head = pkt;
>>>>> > - pkt_tail = item->firstseg;
>>>>> > - }
>>>>> > -
>>>>> > - /* check if the IPv4 packet length is greater than the max
>>>>> value */
>>>>> > - hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
>>>>> > - pkt_head->l4_len;
>>>>> > - l2_len = l2_offset > 0 ? pkt_head->outer_l2_len :
>>>>> pkt_head->l2_len;
>>>>> > - if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
>>>>> > - hdr_len > MAX_IPV4_PKT_LENGTH))
>>>>> > - return 0;
>>>>> > -
>>>>> > - /* remove the packet header for the tail packet */
>>>>> > - rte_pktmbuf_adj(pkt_tail, hdr_len);
>>>>> > -
>>>>> > - /* chain two packets together */
>>>>> > - if (cmp > 0) {
>>>>> > - item->lastseg->next = pkt;
>>>>> > - item->lastseg = rte_pktmbuf_lastseg(pkt);
>>>>> > - /* update IP ID to the larger value */
>>>>> > - item->ip_id = ip_id;
>>>>> > - } else {
>>>>> > - lastseg = rte_pktmbuf_lastseg(pkt);
>>>>> > - lastseg->next = item->firstseg;
>>>>> > - item->firstseg = pkt;
>>>>> > - /* update sent_seq to the smaller value */
>>>>> > - item->sent_seq = sent_seq;
>>>>> > - item->ip_id = ip_id;
>>>>> > - }
>>>>> > - item->nb_merged++;
>>>>> > -
>>>>> > - /* update MBUF metadata for the merged packet */
>>>>> > - pkt_head->nb_segs += pkt_tail->nb_segs;
>>>>> > - pkt_head->pkt_len += pkt_tail->pkt_len;
>>>>> > -
>>>>> > - return 1;
>>>>> > -}
>>>>> > -
>>>>> > -/*
>>>>> > - * Check if two TCP/IPv4 packets are neighbors.
>>>>> > - */
>>>>> > -static inline int
>>>>> > -check_seq_option(struct gro_tcp4_item *item,
>>>>> > - struct rte_tcp_hdr *tcph,
>>>>> > - uint32_t sent_seq,
>>>>> > - uint16_t ip_id,
>>>>> > - uint16_t tcp_hl,
>>>>> > - uint16_t tcp_dl,
>>>>> > - uint16_t l2_offset,
>>>>> > - uint8_t is_atomic)
>>>>> > -{
>>>>> > - struct rte_mbuf *pkt_orig = item->firstseg;
>>>>> > - struct rte_ipv4_hdr *iph_orig;
>>>>> > - struct rte_tcp_hdr *tcph_orig;
>>>>> > - uint16_t len, tcp_hl_orig;
>>>>> > -
>>>>> > - iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig,
>>>>> char
>>>>> > *) +
>>>>> > - l2_offset + pkt_orig->l2_len);
>>>>> > - tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig + pkt_orig-
>>>>> > >l3_len);
>>>>> > - tcp_hl_orig = pkt_orig->l4_len;
>>>>> > -
>>>>> > - /* Check if TCP option fields equal */
>>>>> > - len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct
>>>>> rte_tcp_hdr);
>>>>> > - if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
>>>>> > - (memcmp(tcph + 1, tcph_orig + 1,
>>>>> > - len) != 0)))
>>>>> > - return 0;
>>>>> > -
>>>>> > - /* Don't merge packets whose DF bits are different */
>>>>> > - if (unlikely(item->is_atomic ^ is_atomic))
>>>>> > - return 0;
>>>>> > -
>>>>> > - /* check if the two packets are neighbors */
>>>>> > - len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
>>>>> > - pkt_orig->l3_len - tcp_hl_orig;
>>>>> > - if ((sent_seq == item->sent_seq + len) && (is_atomic ||
>>>>> > - (ip_id == item->ip_id + 1)))
>>>>> > - /* append the new packet */
>>>>> > - return 1;
>>>>> > - else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
>>>>> > - (ip_id + item->nb_merged ==
>>>>> item->ip_id)))
>>>>> > - /* pre-pend the new packet */
>>>>> > - return -1;
>>>>> > -
>>>>> > - return 0;
>>>>> > -}
>>>>> > #endif
>>>>> > diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c new file mode
>>>>> 100644
>>>>> > index 0000000000..a37d8e2512
>>>>> > --- /dev/null
>>>>> > +++ b/lib/gro/gro_tcp6.c
>>>>> > @@ -0,0 +1,387 @@
>>>>> > +/* SPDX-License-Identifier: BSD-3-Clause
>>>>> > + * Copyright(c) 2017 Intel Corporation
>>>>> > + */
>>>>> > +
>>>>> > +#include <rte_malloc.h>
>>>>> > +#include <rte_mbuf.h>
>>>>> > +#include <rte_ethdev.h>
>>>>> > +
>>>>> > +#include "gro_tcp6.h"
>>>>> > +#include "gro_tcp.h"
>>>>> > +
>>>>> > +void *
>>>>> > +gro_tcp6_tbl_create(uint16_t socket_id,
>>>>> > + uint16_t max_flow_num,
>>>>> > + uint16_t max_item_per_flow)
>>>>> > +{
>>>>> > + struct gro_tcp6_tbl *tbl;
>>>>> > + size_t size;
>>>>> > + uint32_t entries_num, i;
>>>>> > +
>>>>> > + entries_num = max_flow_num * max_item_per_flow;
>>>>> > + entries_num = RTE_MIN(entries_num,
>>>>> > GRO_TCP6_TBL_MAX_ITEM_NUM);
>>>>> > +
>>>>> > + if (entries_num == 0)
>>>>> > + return NULL;
>>>>> > +
>>>>> > + tbl = rte_zmalloc_socket(__func__,
>>>>> > + sizeof(struct gro_tcp6_tbl),
>>>>> > + RTE_CACHE_LINE_SIZE,
>>>>> > + socket_id);
>>>>> > + if (tbl == NULL)
>>>>> > + return NULL;
>>>>> > +
>>>>> > + size = sizeof(struct gro_tcp_item) * entries_num;
>>>>> > + tbl->items = rte_zmalloc_socket(__func__,
>>>>> > + size,
>>>>> > + RTE_CACHE_LINE_SIZE,
>>>>> > + socket_id);
>>>>> > + if (tbl->items == NULL) {
>>>>> > + rte_free(tbl);
>>>>> > + return NULL;
>>>>> > + }
>>>>> > + tbl->max_item_num = entries_num;
>>>>> > +
>>>>> > + size = sizeof(struct gro_tcp6_flow) * entries_num;
>>>>> > + tbl->flows = rte_zmalloc_socket(__func__,
>>>>> > + size,
>>>>> > + RTE_CACHE_LINE_SIZE,
>>>>> > + socket_id);
>>>>> > + if (tbl->flows == NULL) {
>>>>> > + rte_free(tbl->items);
>>>>> > + rte_free(tbl);
>>>>> > + return NULL;
>>>>> > + }
>>>>> > + /* INVALID_ARRAY_INDEX indicates an empty flow */
>>>>> > + for (i = 0; i < entries_num; i++)
>>>>> > + tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
>>>>> > + tbl->max_flow_num = entries_num;
>>>>> > +
>>>>> > + return tbl;
>>>>> > +}
>>>>> > +
>>>>> > +void
>>>>> > +gro_tcp6_tbl_destroy(void *tbl)
>>>>> > +{
>>>>> > + struct gro_tcp6_tbl *tcp_tbl = tbl;
>>>>> > +
>>>>> > + if (tcp_tbl) {
>>>>> > + rte_free(tcp_tbl->items);
>>>>> > + rte_free(tcp_tbl->flows);
>>>>> > + }
>>>>> > + rte_free(tcp_tbl);
>>>>> > +}
>>>>> > +
>>>>> > +static inline uint32_t
>>>>> > +find_an_empty_item(struct gro_tcp6_tbl *tbl) {
>>>>> > + uint32_t i;
>>>>> > + uint32_t max_item_num = tbl->max_item_num;
>>>>> > +
>>>>> > + for (i = 0; i < max_item_num; i++)
>>>>> > + if (tbl->items[i].firstseg == NULL)
>>>>> > + return i;
>>>>> > + return INVALID_ARRAY_INDEX;
>>>>> > +}
>>>>> > +
>>>>> > +static inline uint32_t
>>>>> > +find_an_empty_flow(struct gro_tcp6_tbl *tbl) {
>>>>> > + uint32_t i;
>>>>> > + uint32_t max_flow_num = tbl->max_flow_num;
>>>>> > +
>>>>> > + for (i = 0; i < max_flow_num; i++)
>>>>> > + if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
>>>>> > + return i;
>>>>> > + return INVALID_ARRAY_INDEX;
>>>>> > +}
>>>>> > +
>>>>> > +static inline uint32_t
>>>>> > +insert_new_item(struct gro_tcp6_tbl *tbl,
>>>>> > + struct rte_mbuf *pkt,
>>>>> > + uint64_t start_time,
>>>>> > + uint32_t prev_idx,
>>>>> > + uint32_t sent_seq,
>>>>> > + uint8_t is_atomic)
>>>>> > +{
>>>>> > + uint32_t item_idx;
>>>>> > +
>>>>> > + item_idx = find_an_empty_item(tbl);
>>>>> > + if (item_idx == INVALID_ARRAY_INDEX)
>>>>> > + return INVALID_ARRAY_INDEX;
>>>>> > +
>>>>> > + tbl->items[item_idx].firstseg = pkt;
>>>>> > + tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
>>>>> > + tbl->items[item_idx].start_time = start_time;
>>>>> > + tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
>>>>> > + tbl->items[item_idx].sent_seq = sent_seq;
>>>>> > + tbl->items[item_idx].nb_merged = 1;
>>>>> > + tbl->items[item_idx].is_atomic = is_atomic;
>>>>> > + tbl->item_num++;
>>>>> > +
>>>>> > + /* if the previous packet exists, chain them together. */
>>>>> > + if (prev_idx != INVALID_ARRAY_INDEX) {
>>>>> > + tbl->items[item_idx].next_pkt_idx =
>>>>> > + tbl->items[prev_idx].next_pkt_idx;
>>>>> > + tbl->items[prev_idx].next_pkt_idx = item_idx;
>>>>> > + }
>>>>> > +
>>>>> > + return item_idx;
>>>>> > +}
>>>>> > +
>>>>> > +static inline uint32_t
>>>>> > +delete_item(struct gro_tcp6_tbl *tbl, uint32_t item_idx,
>>>>> > + uint32_t prev_item_idx)
>>>>> > +{
>>>>> > + uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
>>>>> > +
>>>>> > + /* NULL indicates an empty item */
>>>>> > + tbl->items[item_idx].firstseg = NULL;
>>>>> > + tbl->item_num--;
>>>>> > + if (prev_item_idx != INVALID_ARRAY_INDEX)
>>>>> > + tbl->items[prev_item_idx].next_pkt_idx = next_idx;
>>>>> > +
>>>>> > + return next_idx;
>>>>> > +}
>>>>> > +
>>>>> > +static inline uint32_t
>>>>> > +insert_new_flow(struct gro_tcp6_tbl *tbl,
>>>>> > + struct tcp6_flow_key *src,
>>>>> > + rte_be32_t vtc_flow,
>>>>> > + uint32_t item_idx)
>>>>> > +{
>>>>> > + struct tcp6_flow_key *dst;
>>>>> > + uint32_t flow_idx;
>>>>> > +
>>>>> > + flow_idx = find_an_empty_flow(tbl);
>>>>> > + if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
>>>>> > + return INVALID_ARRAY_INDEX;
>>>>> > +
>>>>> > + dst = &(tbl->flows[flow_idx].key);
>>>>> > +
>>>>> > + rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
>>>>> > + rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
>>>>> > + memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst-
>>>>> > >src_addr));
>>>>> > + memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst-
>>>>> > >dst_addr));
>>>>> > + dst->recv_ack = src->recv_ack;
>>>>> > + dst->src_port = src->src_port;
>>>>> > + dst->dst_port = src->dst_port;
>>>>> > +
>>>>> > + tbl->flows[flow_idx].start_index = item_idx;
>>>>> > + tbl->flow_num++;
>>>>> > + tbl->flows->vtc_flow = vtc_flow;
>>>>> > +
>>>>> > + return flow_idx;
>>>>> > +}
>>>>> > +
>>>>> > +/*
>>>>> > + * update the packet length for the flushed packet.
>>>>> > + */
>>>>> > +static inline void
>>>>> > +update_header(struct gro_tcp_item *item) {
>>>>> > + struct rte_ipv6_hdr *ipv6_hdr;
>>>>> > + struct rte_mbuf *pkt = item->firstseg;
>>>>> > +
>>>>> > + ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char
>>>>> *) +
>>>>> > + pkt->l2_len);
>>>>> > + ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
>>>>> > + pkt->l2_len - pkt->l3_len);
>>>>> > +}
>>>>> > +
>>>>> > +int32_t
>>>>> > +gro_tcp6_reassemble(struct rte_mbuf *pkt,
>>>>> > + struct gro_tcp6_tbl *tbl,
>>>>> > + uint64_t start_time)
>>>>> > +{
>>>>> > + struct rte_ether_hdr *eth_hdr;
>>>>> > + struct rte_ipv6_hdr *ipv6_hdr;
>>>>> > + struct rte_tcp_hdr *tcp_hdr;
>>>>> > + uint32_t sent_seq;
>>>>> > + int32_t tcp_dl;
>>>>> > + uint16_t ip_tlen;
>>>>> > + struct tcp6_flow_key key;
>>>>> > + uint32_t cur_idx, prev_idx, item_idx;
>>>>> > + uint32_t i, max_flow_num, remaining_flow_num;
>>>>> > + int cmp;
>>>>> > + uint8_t find;
>>>>> > + rte_be32_t vtc_flow_diff;
>>>>> > +
>>>>> > + /*
>>>>> > + * Don't process the packet whose TCP header length is greater
>>>>> > + * than 60 bytes or less than 20 bytes.
>>>>> > + */
>>>>> > + if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
>>>>> > + return -1;
>>>>> > +
>>>>> > + eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
>>>>> > + ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr +
>>>>> pkt->l2_len);
>>>>> > + tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv6_hdr +
>>>>> pkt->l3_len);
>>>>> > +
>>>>> > + /*
>>>>> > + * Don't process the packet which has FIN, SYN, RST, PSH, URG,
>>>>> ECE
>>>>> > + * or CWR set.
>>>>> > + */
>>>>> > + if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
>>>>> > + return -1;
>>>>> > +
>>>>> > + ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
>>>>> > + /*
>>>>> > + * Trim the tail padding bytes. The IPv6 header is fixed to
>>>>> > + * 40 bytes unlike IPv4 that is variable. The length in the
>>>>> IPv6 header
>>>>> > + * contains only length of TCP Header + TCP Payload, whereas
>>>>> IPv4
>>>>> > header contains
>>>>> > + * length of IP Header + TCP Header + TCP Payload
>>>>> > + */
>>>>> > + if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len +
>>>>> pkt->l3_len))
>>>>> > + rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen -
>>>>> pkt->l2_len -
>>>>> > pkt->l3_len);
>>>>> > + /*
>>>>> > + * Don't process the packet whose payload length is less than
>>>>> or
>>>>> > + * equal to 0.
>>>>> > + */
>>>>> > + tcp_dl = ip_tlen - pkt->l4_len;
>>>>> > + if (tcp_dl <= 0)
>>>>> > + return -1;
>>>>> > +
>>>>> > + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
>>>>> > +
>>>>> > + rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
>>>>> > + rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
>>>>> > + memcpy(&key.src_addr[0], &ipv6_hdr->src_addr,
>>>>> > sizeof(key.src_addr));
>>>>> > + memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr,
>>>>> > sizeof(key.dst_addr));
>>>>> > + key.src_port = tcp_hdr->src_port;
>>>>> > + key.dst_port = tcp_hdr->dst_port;
>>>>> > + key.recv_ack = tcp_hdr->recv_ack;
>>>>> > +
>>>>> > + /* Search for a matched flow. */
>>>>> > + max_flow_num = tbl->max_flow_num;
>>>>> > + remaining_flow_num = tbl->flow_num;
>>>>> > + find = 0;
>>>>> > + for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
>>>>> > + if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
>>>>> > + if (is_same_tcp6_flow(tbl->flows[i].key, key))
>>>>> {
>>>>> > + /*
>>>>> > + * IP version (4) Traffic Class (8)
>>>>> Flow Label (20)
>>>>> > + * All fields except Traffic class
>>>>> should be
>>>>> > same
>>>>> > + */
>>>>> > + vtc_flow_diff = (ipv6_hdr->vtc_flow ^
>>>>> tbl-
>>>>> > >flows->vtc_flow);
>>>>> > + if (vtc_flow_diff & htonl(0xF00FFFFF))
>>>>> > + continue;
>>>>> > + find = 1;
>>>>> > + break;
>>>>> > + }
>>>>> > + remaining_flow_num--;
>>>>> > + }
>>>>> > + }
>>>>> > +
>>>>> > + /*
>>>>> > + * Fail to find a matched flow. Insert a new flow and store the
>>>>> > + * packet into the flow.
>>>>> > + */
>>>>> > + if (find == 0) {
>>>>> > + item_idx = insert_new_item(tbl, pkt, start_time,
>>>>> > + INVALID_ARRAY_INDEX, sent_seq, true);
>>>>> > + if (item_idx == INVALID_ARRAY_INDEX)
>>>>> > + return -1;
>>>>> > + if (insert_new_flow(tbl, &key, ipv6_hdr->vtc_flow,
>>>>> item_idx)
>>>>> > ==
>>>>> > + INVALID_ARRAY_INDEX) {
>>>>> > + /*
>>>>> > + * Fail to insert a new flow, so delete the
>>>>> > + * stored packet.
>>>>> > + */
>>>>> > + delete_item(tbl, item_idx,
>>>>> INVALID_ARRAY_INDEX);
>>>>> > + return -1;
>>>>> > + }
>>>>> > + return 0;
>>>>> > + }
>>>>> > +
>>>>> > + /*
>>>>> > + * Check all packets in the flow and try to find a neighbor for
>>>>> > + * the input packet.
>>>>> > + */
>>>>> > + cur_idx = tbl->flows[i].start_index;
>>>>> > + prev_idx = cur_idx;
>>>>> > + do {
>>>>> > + cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
>>>>> > + sent_seq, 0, pkt->l4_len, tcp_dl, 0,
>>>>> > + true);
>>>>> > + if (cmp) {
>>>>> > + if
>>>>> (merge_two_tcp_packets(&(tbl->items[cur_idx]),
>>>>> > + pkt, cmp, sent_seq, 0,
>>>>> 0)) {
>>>>> > + return 1;
>>>>> > + }
>>>>> > +
>>>>> > + /*
>>>>> > + * Fail to merge the two packets, as the packet
>>>>> > + * length is greater than the max value. Store
>>>>> > + * the packet into the flow.
>>>>> > + */
>>>>> > + if (insert_new_item(tbl, pkt, start_time,
>>>>> cur_idx,
>>>>> > + sent_seq, true) ==
>>>>> > + INVALID_ARRAY_INDEX)
>>>>> > + return -1;
>>>>> > + return 0;
>>>>> > + }
>>>>> > + prev_idx = cur_idx;
>>>>> > + cur_idx = tbl->items[cur_idx].next_pkt_idx;
>>>>> > + } while (cur_idx != INVALID_ARRAY_INDEX);
>>>>> > +
>>>>> > + /* Fail to find a neighbor, so store the packet into the flow.
>>>>> */
>>>>> > + if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
>>>>> > + true) == INVALID_ARRAY_INDEX)
>>>>> > + return -1;
>>>>> > +
>>>>> > + return 0;
>>>>> > +}
>>>>> > +
>>>>> > +uint16_t
>>>>> > +gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
>>>>> > + uint64_t flush_timestamp,
>>>>> > + struct rte_mbuf **out,
>>>>> > + uint16_t nb_out)
>>>>> > +{
>>>>> > + uint16_t k = 0;
>>>>> > + uint32_t i, j;
>>>>> > + uint32_t max_flow_num = tbl->max_flow_num;
>>>>> > +
>>>>> > + for (i = 0; i < max_flow_num; i++) {
>>>>> > + if (unlikely(tbl->flow_num == 0))
>>>>> > + return k;
>>>>> > +
>>>>> > + j = tbl->flows[i].start_index;
>>>>> > + while (j != INVALID_ARRAY_INDEX) {
>>>>> > + if (tbl->items[j].start_time <=
>>>>> flush_timestamp) {
>>>>> > + out[k++] = tbl->items[j].firstseg;
>>>>> > + if (tbl->items[j].nb_merged > 1)
>>>>> > +
>>>>> update_header(&(tbl->items[j]));
>>>>> > + /*
>>>>> > + * Delete the packet and get the next
>>>>> > + * packet in the flow.
>>>>> > + */
>>>>> > + j = delete_item(tbl, j,
>>>>> > INVALID_ARRAY_INDEX);
>>>>> > + tbl->flows[i].start_index = j;
>>>>> > + if (j == INVALID_ARRAY_INDEX)
>>>>> > + tbl->flow_num--;
>>>>> > +
>>>>> > + if (unlikely(k == nb_out))
>>>>> > + return k;
>>>>> > + } else
>>>>> > + /*
>>>>> > + * The left packets in this flow won't
>>>>> be
>>>>> > + * timeout. Go to check other flows.
>>>>> > + */
>>>>> > + break;
>>>>> > + }
>>>>> > + }
>>>>> > + return k;
>>>>> > +}
>>>>> > +
>>>>> > +uint32_t
>>>>> > +gro_tcp6_tbl_pkt_count(void *tbl)
>>>>> > +{
>>>>> > + struct gro_tcp6_tbl *gro_tbl = tbl;
>>>>> > +
>>>>> > + if (gro_tbl)
>>>>> > + return gro_tbl->item_num;
>>>>> > +
>>>>> > + return 0;
>>>>> > +}
>>>>> > diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h new file mode
>>>>> 100644
>>>>> > index 0000000000..91325c0da2
>>>>> > --- /dev/null
>>>>> > +++ b/lib/gro/gro_tcp6.h
>>>>> > @@ -0,0 +1,150 @@
>>>>> > +/* SPDX-License-Identifier: BSD-3-Clause
>>>>> > + * Copyright(c) 2017 Intel Corporation
>>>>> > + */
>>>>> > +
>>>>> > +#ifndef _GRO_TCP6_H_
>>>>> > +#define _GRO_TCP6_H_
>>>>> > +
>>>>> > +#define INVALID_ARRAY_INDEX 0xffffffffUL #define
>>>>> > +GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
>>>>> > +
>>>>> > +/* Header fields representing a TCP/IPv6 flow */ struct
>>>>> tcp6_flow_key {
>>>>> > + struct rte_ether_addr eth_saddr;
>>>>> > + struct rte_ether_addr eth_daddr;
>>>>> > + uint8_t src_addr[16];
>>>>> > + uint8_t dst_addr[16];
>>>>> > +
>>>>> > + uint32_t recv_ack;
>>>>> > + uint16_t src_port;
>>>>> > + uint16_t dst_port;
>>>>> > +};
>>>>> > +
>>>>> > +struct gro_tcp6_flow {
>>>>> > + struct tcp6_flow_key key;
>>>>> > + rte_be32_t vtc_flow;
>>>>> > + /*
>>>>> > + * The index of the first packet in the flow.
>>>>> > + * INVALID_ARRAY_INDEX indicates an empty flow.
>>>>> > + */
>>>>> > + uint32_t start_index;
>>>>> > +};
>>>>> > +
>>>>> > +/*
>>>>> > + * TCP/IPv6 reassembly table structure.
>>>>> > + */
>>>>> > +struct gro_tcp6_tbl {
>>>>> > + /* item array */
>>>>> > + struct gro_tcp_item *items;
>>>>> > + /* flow array */
>>>>> > + struct gro_tcp6_flow *flows;
>>>>> > + /* current item number */
>>>>> > + uint32_t item_num;
>>>>> > + /* current flow num */
>>>>> > + uint32_t flow_num;
>>>>> > + /* item array size */
>>>>> > + uint32_t max_item_num;
>>>>> > + /* flow array size */
>>>>> > + uint32_t max_flow_num;
>>>>> > +};
>>>>> > +
>>>>> > +/**
>>>>> > + * This function creates a TCP/IPv6 reassembly table.
>>>>> > + *
>>>>> > + * @param socket_id
>>>>> > + * Socket index for allocating the TCP/IPv6 reassemble table
>>>>> > + * @param max_flow_num
>>>>> > + * The maximum number of flows in the TCP/IPv6 GRO table
>>>>> > + * @param max_item_per_flow
>>>>> > + * The maximum number of packets per flow
>>>>> > + *
>>>>> > + * @return
>>>>> > + * - Return the table pointer on success.
>>>>> > + * - Return NULL on failure.
>>>>> > + */
>>>>> > +void *gro_tcp6_tbl_create(uint16_t socket_id,
>>>>> > + uint16_t max_flow_num,
>>>>> > + uint16_t max_item_per_flow);
>>>>> > +
>>>>> > +/**
>>>>> > + * This function destroys a TCP/IPv6 reassembly table.
>>>>> > + *
>>>>> > + * @param tbl
>>>>> > + * Pointer pointing to the TCP/IPv6 reassembly table.
>>>>> > + */
>>>>> > +void gro_tcp6_tbl_destroy(void *tbl);
>>>>> > +
>>>>> > +/**
>>>>> > + * This function merges a TCP/IPv6 packet. It doesn't process the
>>>>> > +packet,
>>>>> > + * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't
>>>>> have
>>>>> > + * payload.
>>>>> > + *
>>>>> > + * This function doesn't check if the packet has correct checksums
>>>>> and
>>>>> > + * doesn't re-calculate checksums for the merged packet.
>>>>> Additionally,
>>>>> > + * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
>>>>> > + * when IP fragmentation is possible (i.e., DF==0). It returns the
>>>>> > + * packet, if the packet has invalid parameters (e.g. SYN bit is
>>>>> set)
>>>>> > + * or there is no available space in the table.
>>>>> > + *
>>>>> > + * @param pkt
>>>>> > + * Packet to reassemble
>>>>> > + * @param tbl
>>>>> > + * Pointer pointing to the TCP/IPv6 reassembly table
>>>>> > + * @start_time
>>>>> > + * The time when the packet is inserted into the table
>>>>> > + *
>>>>> > + * @return
>>>>> > + * - Return a positive value if the packet is merged.
>>>>> > + * - Return zero if the packet isn't merged but stored in the
>>>>> table.
>>>>> > + * - Return a negative value for invalid parameters or no available
>>>>> > + * space in the table.
>>>>> > + */
>>>>> > +int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
>>>>> > + struct gro_tcp6_tbl *tbl,
>>>>> > + uint64_t start_time);
>>>>> > +
>>>>> > +/**
>>>>> > + * This function flushes timeout packets in a TCP/IPv6 reassembly
>>>>> > +table,
>>>>> > + * and without updating checksums.
>>>>> > + *
>>>>> > + * @param tbl
>>>>> > + * TCP/IPv6 reassembly table pointer
>>>>> > + * @param flush_timestamp
>>>>> > + * Flush packets which are inserted into the table before or at the
>>>>> > + * flush_timestamp.
>>>>> > + * @param out
>>>>> > + * Pointer array used to keep flushed packets
>>>>> > + * @param nb_out
>>>>> > + * The element number in 'out'. It also determines the maximum
>>>>> number
>>>>> > +of
>>>>> > + * packets that can be flushed finally.
>>>>> > + *
>>>>> > + * @return
>>>>> > + * The number of flushed packets
>>>>> > + */
>>>>> > +uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
>>>>> > + uint64_t flush_timestamp,
>>>>> > + struct rte_mbuf **out,
>>>>> > + uint16_t nb_out);
>>>>> > +
>>>>> > +/**
>>>>> > + * This function returns the number of the packets in a TCP/IPv6
>>>>> > + * reassembly table.
>>>>> > + *
>>>>> > + * @param tbl
>>>>> > + * TCP/IPv6 reassembly table pointer
>>>>> > + *
>>>>> > + * @return
>>>>> > + * The number of packets in the table
>>>>> > + */
>>>>> > +uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
>>>>> > +
>>>>> > +/*
>>>>> > + * Check if two TCP/IPv6 packets belong to the same flow.
>>>>> > + */
>>>>> > +static inline int
>>>>> > +is_same_tcp6_flow(struct tcp6_flow_key k1, struct tcp6_flow_key k2)
>>>>> {
>>>>> > + return (!memcmp(&k1, &k2, sizeof(struct tcp6_flow_key))); }
>>>>> > +
>>>>> > +#endif
>>>>> > diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
>>>>> index
>>>>> > 3be4deb7c7..56b30b8c98 100644
>>>>> > --- a/lib/gro/gro_vxlan_tcp4.c
>>>>> > +++ b/lib/gro/gro_vxlan_tcp4.c
>>>>> > @@ -7,6 +7,7 @@
>>>>> > #include <rte_ethdev.h>
>>>>> > #include <rte_udp.h>
>>>>> >
>>>>> > +#include "gro_tcp.h"
>>>>> > #include "gro_vxlan_tcp4.h"
>>>>> >
>>>>> > void *
>>>>> > @@ -248,7 +249,7 @@ merge_two_vxlan_tcp4_packets(struct
>>>>> > gro_vxlan_tcp4_item *item,
>>>>> > uint16_t outer_ip_id,
>>>>> > uint16_t ip_id)
>>>>> > {
>>>>> > - if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp,
>>>>> > sent_seq,
>>>>> > + if (merge_two_tcp_packets(&item->inner_item, pkt, cmp,
>>>>> sent_seq,
>>>>> > ip_id, pkt->outer_l2_len +
>>>>> > pkt->outer_l3_len)) {
>>>>> > /* Update the outer IPv4 ID to the large value. */
>>>>> diff --git
>>>>> > a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h index
>>>>> > 7832942a68..d68d5fcd5b 100644
>>>>> > --- a/lib/gro/gro_vxlan_tcp4.h
>>>>> > +++ b/lib/gro/gro_vxlan_tcp4.h
>>>>> > @@ -5,6 +5,7 @@
>>>>> > #ifndef _GRO_VXLAN_TCP4_H_
>>>>> > #define _GRO_VXLAN_TCP4_H_
>>>>> >
>>>>> > +#include "gro_tcp.h"
>>>>> > #include "gro_tcp4.h"
>>>>> >
>>>>> > #define GRO_VXLAN_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL) @@ -
>>>>> > 36,7 +37,7 @@ struct gro_vxlan_tcp4_flow { };
>>>>> >
>>>>> > struct gro_vxlan_tcp4_item {
>>>>> > - struct gro_tcp4_item inner_item;
>>>>> > + struct gro_tcp_item inner_item;
>>>>> > /* IPv4 ID in the outer IPv4 header */
>>>>> > uint16_t outer_ip_id;
>>>>> > /* Indicate if outer IPv4 ID can be ignored */ diff --git
>>>>> > a/lib/gro/meson.build b/lib/gro/meson.build index
>>>>> e4fa2958bd..dbce05220d
>>>>> > 100644
>>>>> > --- a/lib/gro/meson.build
>>>>> > +++ b/lib/gro/meson.build
>>>>> > @@ -4,6 +4,7 @@
>>>>> > sources = files(
>>>>> > 'rte_gro.c',
>>>>> > 'gro_tcp4.c',
>>>>> > + 'gro_tcp6.c',
>>>>> > 'gro_udp4.c',
>>>>> > 'gro_vxlan_tcp4.c',
>>>>> > 'gro_vxlan_udp4.c',
>>>>> > diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c index
>>>>> > e35399fd42..d824eebd93 100644
>>>>> > --- a/lib/gro/rte_gro.c
>>>>> > +++ b/lib/gro/rte_gro.c
>>>>> > @@ -8,6 +8,7 @@
>>>>> >
>>>>> > #include "rte_gro.h"
>>>>> > #include "gro_tcp4.h"
>>>>> > +#include "gro_tcp6.h"
>>>>> > #include "gro_udp4.h"
>>>>> > #include "gro_vxlan_tcp4.h"
>>>>> > #include "gro_vxlan_udp4.h"
>>>>> > @@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void
>>>>> *tbl);
>>>>> >
>>>>> > static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
>>>>> > gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
>>>>> > - gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
>>>>> > + gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create,
>>>>> > gro_tcp6_tbl_create,
>>>>> > +NULL};
>>>>> > static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
>>>>> > gro_tcp4_tbl_destroy,
>>>>> gro_vxlan_tcp4_tbl_destroy,
>>>>> > gro_udp4_tbl_destroy,
>>>>> gro_vxlan_udp4_tbl_destroy,
>>>>> > + gro_tcp6_tbl_destroy,
>>>>> > NULL};
>>>>> > static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM]
>>>>> > = {
>>>>> > gro_tcp4_tbl_pkt_count,
>>>>> > gro_vxlan_tcp4_tbl_pkt_count,
>>>>> > gro_udp4_tbl_pkt_count,
>>>>> > gro_vxlan_udp4_tbl_pkt_count,
>>>>> > + gro_tcp6_tbl_pkt_count,
>>>>> > NULL};
>>>>> >
>>>>> > #define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \ @@
>>>>> > -35,6 +38,12 @@ static gro_tbl_pkt_count_fn
>>>>> > tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
>>>>> > ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) &&
>>>>> > \
>>>>> > (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
>>>>> >
>>>>> > +/* GRO with extension headers is not supported */ #define
>>>>> > +IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
>>>>> > + ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
>>>>> > + ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) &&
>>>>> > \
>>>>> > + (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
>>>>> > +
>>>>> > #define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
>>>>> > ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
>>>>> > (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
>>>>> > @@ -147,7 +156,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
>>>>> > /* allocate a reassembly table for TCP/IPv4 GRO */
>>>>> > struct gro_tcp4_tbl tcp_tbl;
>>>>> > struct gro_tcp4_flow
>>>>> > tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
>>>>> > - struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
>>>>> > = {{0} };
>>>>> > + struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
>>>>> > = {{0} };
>>>>> > +
>>>>> > + struct gro_tcp6_tbl tcp6_tbl;
>>>>> > + struct gro_tcp6_flow
>>>>> > tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
>>>>> > + struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM]
>>>>> > = {{0} };
>>>>> >
>>>>> > /* allocate a reassembly table for UDP/IPv4 GRO */
>>>>> > struct gro_udp4_tbl udp_tbl;
>>>>> > @@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf
>>>>> **pkts,
>>>>> > int32_t ret;
>>>>> > uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
>>>>> > uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
>>>>> > - do_vxlan_udp_gro = 0;
>>>>> > + do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
>>>>> >
>>>>> > if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4
>>>>> > |
>>>>> > - RTE_GRO_TCP_IPV4 |
>>>>> > + RTE_GRO_TCP_IPV4 |
>>>>> > RTE_GRO_TCP_IPV6 |
>>>>> > RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
>>>>> > RTE_GRO_UDP_IPV4)) == 0))
>>>>> > return nb_pkts;
>>>>> > @@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
>>>>> > do_udp4_gro = 1;
>>>>> > }
>>>>> >
>>>>> > + if (param->gro_types & RTE_GRO_TCP_IPV6) {
>>>>> > + for (i = 0; i < item_num; i++)
>>>>> > + tcp6_flows[i].start_index =
>>>>> INVALID_ARRAY_INDEX;
>>>>> > +
>>>>> > + tcp6_tbl.flows = tcp6_flows;
>>>>> > + tcp6_tbl.items = tcp6_items;
>>>>> > + tcp6_tbl.flow_num = 0;
>>>>> > + tcp6_tbl.item_num = 0;
>>>>> > + tcp6_tbl.max_flow_num = item_num;
>>>>> > + tcp6_tbl.max_item_num = item_num;
>>>>> > + do_tcp6_gro = 1;
>>>>> > + }
>>>>> >
>>>>> > for (i = 0; i < nb_pkts; i++) {
>>>>> > /*
>>>>> > @@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
>>>>> > nb_after_gro--;
>>>>> > else if (ret < 0)
>>>>> > unprocess_pkts[unprocess_num++] =
>>>>> pkts[i];
>>>>> > + } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
>>>>> > + do_tcp6_gro) {
>>>>> > + ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl,
>>>>> 0);
>>>>> > + if (ret > 0)
>>>>> > + /* merge successfully */
>>>>> > + nb_after_gro--;
>>>>> > + else if (ret < 0)
>>>>> > + unprocess_pkts[unprocess_num++] =
>>>>> pkts[i];
>>>>> > } else
>>>>> > unprocess_pkts[unprocess_num++] = pkts[i];
>>>>> > }
>>>>> > @@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
>>>>> > if ((nb_after_gro < nb_pkts)
>>>>> > || (unprocess_num < nb_pkts)) {
>>>>> > i = 0;
>>>>> > + /* Copy unprocessed packets */
>>>>> > + if (unprocess_num > 0) {
>>>>> > + memcpy(&pkts[i], unprocess_pkts,
>>>>> > + sizeof(struct rte_mbuf *) *
>>>>> > + unprocess_num);
>>>>> > + i = unprocess_num;
>>>>> > + }
>>>>> > +
>>>>> > /* Flush all packets from the tables */
>>>>> > if (do_vxlan_tcp_gro) {
>>>>> > - i =
>>>>> > gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
>>>>> > + i +=
>>>>> > gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
>>>>> > 0, pkts, nb_pkts);
>>>>> > }
>>>>> >
>>>>> > @@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf
>>>>> **pkts,
>>>>> > i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
>>>>> > &pkts[i], nb_pkts - i);
>>>>> > }
>>>>> > - /* Copy unprocessed packets */
>>>>> > - if (unprocess_num > 0) {
>>>>> > - memcpy(&pkts[i], unprocess_pkts,
>>>>> > - sizeof(struct rte_mbuf *) *
>>>>> > - unprocess_num);
>>>>> > +
>>>>> > + if (do_tcp6_gro) {
>>>>> > + i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
>>>>> > + &pkts[i], nb_pkts - i);
>>>>> > }
>>>>> > - nb_after_gro = i + unprocess_num;
>>>>> > }
>>>>> >
>>>>> > return nb_after_gro;
>>>>> > @@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts, {
>>>>> > struct rte_mbuf *unprocess_pkts[nb_pkts];
>>>>> > struct gro_ctx *gro_ctx = ctx;
>>>>> > - void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
>>>>> > + void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl,
>>>>> *tcp6_tbl;
>>>>> > uint64_t current_time;
>>>>> > uint16_t i, unprocess_num = 0;
>>>>> > - uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro,
>>>>> > do_vxlan_udp_gro;
>>>>> > + uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro,
>>>>> > do_vxlan_udp_gro,
>>>>> > +do_tcp6_gro;
>>>>> >
>>>>> > if (unlikely((gro_ctx->gro_types &
>>>>> > (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
>>>>> > - RTE_GRO_TCP_IPV4 |
>>>>> > + RTE_GRO_TCP_IPV4 |
>>>>> > RTE_GRO_TCP_IPV6 |
>>>>> > RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
>>>>> > RTE_GRO_UDP_IPV4)) == 0))
>>>>> > return nb_pkts;
>>>>> > @@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
>>>>> > vxlan_tcp_tbl = gro_ctx-
>>>>> > >tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
>>>>> > udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
>>>>> > vxlan_udp_tbl = gro_ctx-
>>>>> > >tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
>>>>> > + tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
>>>>> >
>>>>> > do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
>>>>> > RTE_GRO_TCP_IPV4;
>>>>> > @@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
>>>>> > RTE_GRO_UDP_IPV4;
>>>>> > do_vxlan_udp_gro = (gro_ctx->gro_types &
>>>>> > RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
>>>>> > RTE_GRO_IPV4_VXLAN_UDP_IPV4;
>>>>> > + do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) ==
>>>>> > +RTE_GRO_TCP_IPV6;
>>>>> >
>>>>> > current_time = rte_rdtsc();
>>>>> >
>>>>> > @@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
>>>>> > if (gro_udp4_reassemble(pkts[i], udp_tbl,
>>>>> > current_time) < 0)
>>>>> > unprocess_pkts[unprocess_num++] =
>>>>> pkts[i];
>>>>> > + } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
>>>>> > + do_tcp6_gro) {
>>>>> > + if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
>>>>> > + current_time) < 0)
>>>>> > + unprocess_pkts[unprocess_num++] =
>>>>> pkts[i];
>>>>> > } else
>>>>> > unprocess_pkts[unprocess_num++] = pkts[i];
>>>>> > }
>>>>> > @@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
>>>>> > gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
>>>>> > flush_timestamp,
>>>>> > &out[num], left_nb_out);
>>>>> > + left_nb_out = max_nb_out - num;
>>>>> > + }
>>>>> > +
>>>>> > + if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
>>>>> > + num += gro_tcp6_tbl_timeout_flush(
>>>>> > + gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
>>>>> > + flush_timestamp,
>>>>> > + &out[num], left_nb_out);
>>>>> > +
>>>>> > }
>>>>> >
>>>>> > return num;
>>>>> > diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h index
>>>>> > 9f9ed4935a..c83dfd9ad1 100644
>>>>> > --- a/lib/gro/rte_gro.h
>>>>> > +++ b/lib/gro/rte_gro.h
>>>>> > @@ -38,6 +38,9 @@ extern "C" {
>>>>> > #define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3 #define
>>>>> > RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL <<
>>>>> > RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
>>>>> > /**< VxLAN UDP/IPv4 GRO flag. */
>>>>> > +#define RTE_GRO_TCP_IPV6_INDEX 4
>>>>> > +#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX) /**<
>>>>> > TCP/IPv6
>>>>> > +GRO flag. */
>>>>> >
>>>>> > /**
>>>>> > * Structure used to create GRO context objects or used to pass
>>>>> > --
>>>>> > 2.25.1
>>>>>
>>>>>
[-- Attachment #2: Type: text/html, Size: 67466 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread
* RE: [PATCH v2] gro : ipv6 changes to support GRO for TCP/ipv6
2023-05-25 11:22 ` kumaraparameshwaran rathinavel
@ 2023-05-31 8:20 ` Hu, Jiayu
0 siblings, 0 replies; 32+ messages in thread
From: Hu, Jiayu @ 2023-05-31 8:20 UTC (permalink / raw)
To: kumaraparameshwaran rathinavel; +Cc: dev
[-- Attachment #1: Type: text/plain, Size: 49368 bytes --]
Hi Kumara,
For the idea you proposed, it’s better to send the patch and I can review later.
Thanks,
Jiayu
From: kumaraparameshwaran rathinavel <kumaraparamesh92@gmail.com>
Sent: Thursday, May 25, 2023 7:23 PM
To: Hu, Jiayu <jiayu.hu@intel.com>
Cc: dev@dpdk.org
Subject: Re: [PATCH v2] gro : ipv6 changes to support GRO for TCP/ipv6
On Tue, May 16, 2023 at 2:58 PM kumaraparameshwaran rathinavel <kumaraparamesh92@gmail.com<mailto:kumaraparamesh92@gmail.com>> wrote:
On Fri, May 12, 2023 at 8:17 AM Hu, Jiayu <jiayu.hu@intel.com<mailto:jiayu.hu@intel.com>> wrote:
Hi Kumar,
For TCP/IPv4 and TCP/IPv6, the TCP layer is the same and the difference is
the IP layer. So the code used for TCP layer needs to be shared among gro_tcp6.c
and gro_tcp4.c. But there are too much code duplication in gro_tcp4.c and gro_tcp6.c
in current implementation.
Sure, will do it. I had followed the same that was similar to what was done in gro_tcp4.c and gro_vxlan_tcp4.c. Eventhough the TCP could have been reused for vxlan, we had code duplication. I assumed that this was better in terms of performance etc.
I am thinking two have two functions gro_tcp4_reassemble and gro_tcp6_reassemble which would perform the validations for the respective IP types. The GRO table is different for V4 and V6 but the itmes array in the gro table is common to both V4 and V6 and hence we can move insert_item and delete_item to gro_tcp.c/h file. Introduce a new function gro_tcp_reassemble which would perform the logic of lookup/create/merge operations. Since the table and the key would be different for V4/V6 introduce a new structure which would hold the lookup/create flow function pointers for each of the IP type. This way we would reduce the code duplication in gro_tcp4_reassmeble and gro_tcp6_reassemble. Pleae let me know your thoughts. We can get on a call if require to discuss further.
For example, struct tcp6_flow_key and struct tcp4_flow_key share most of fields, except
the IP address type. It's better to have a common TCP flow key structure in gro_tcp.h. In
gro_tcp4.h and gro_tcp6.h, we implement tcp4 and tcp6 flow key structures with using the
common structure.
Thanks,
Jiayu
> -----Original Message-----
> From: Kumara Parameshwaran <kumaraparamesh92@gmail.com<mailto:kumaraparamesh92@gmail.com>>
> Sent: Friday, October 21, 2022 2:14 AM
> To: Hu, Jiayu <jiayu.hu@intel.com<mailto:jiayu.hu@intel.com>>
> Cc: dev@dpdk.org<mailto:dev@dpdk.org>; Kumara Parameshwaran
> <kumaraparamesh92@gmail.com<mailto:kumaraparamesh92@gmail.com>>
> Subject: [PATCH v2] gro : ipv6 changes to support GRO for TCP/ipv6
>
> From: Kumara Parameshwaran <kumaraparamesh92@gmail.com<mailto:kumaraparamesh92@gmail.com>>
>
> The patch adds GRO support for TCP/ipv6 packets. This does not include the
> support for vxlan, udp ipv6 packets.
>
> Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com<mailto:kumaraparamesh92@gmail.com>>
> ---
> v1:
> * Changes to support GRO for TCP/ipv6 packets. This does not
> include
> vxlan changes.
> * The GRO is performed only for ipv6 packets that does not contain
> extension headers.
> * The logic for the TCP coalescing remains the same, in ipv6 header
> the source address, destination address, flow label, version fields
> are expected to be the same.
> * Re-organised the code to reuse certain tcp functions for both ipv4
> and
> ipv6 flows.
> v2:
> * Fix comments in gro_tcp6.h header file.
>
> lib/gro/gro_tcp.h | 155 ++++++++++++++++
> lib/gro/gro_tcp4.c | 7 +-
> lib/gro/gro_tcp4.h | 152 +--------------
> lib/gro/gro_tcp6.c | 387
> +++++++++++++++++++++++++++++++++++++++
> lib/gro/gro_tcp6.h | 150 +++++++++++++++
> lib/gro/gro_vxlan_tcp4.c | 3 +-
> lib/gro/gro_vxlan_tcp4.h | 3 +-
> lib/gro/meson.build | 1 +
> lib/gro/rte_gro.c | 83 +++++++--
> lib/gro/rte_gro.h | 3 +
> 10 files changed, 774 insertions(+), 170 deletions(-) create mode 100644
> lib/gro/gro_tcp.h create mode 100644 lib/gro/gro_tcp6.c create mode
> 100644 lib/gro/gro_tcp6.h
>
> diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h new file mode 100644 index
> 0000000000..c5d248a022
> --- /dev/null
> +++ b/lib/gro/gro_tcp.h
> @@ -0,0 +1,155 @@
> +#ifndef _GRO_TCP_H_
> +#define _GRO_TCP_H_
> +
> +#include <rte_tcp.h>
> +
> +/*
> + * The max length of a IPv4 packet, which includes the length of the L3
> + * header, the L4 header and the data payload.
> + */
> +#define MAX_IP_PKT_LENGTH UINT16_MAX
> +
> +/* The maximum TCP header length */
> +#define MAX_TCP_HLEN 60
> +#define INVALID_TCP_HDRLEN(len) \
> + (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> +
> +struct gro_tcp_item {
> + /*
> + * The first MBUF segment of the packet. If the value
> + * is NULL, it means the item is empty.
> + */
> + struct rte_mbuf *firstseg;
> + /* The last MBUF segment of the packet */
> + struct rte_mbuf *lastseg;
> + /*
> + * The time when the first packet is inserted into the table.
> + * This value won't be updated, even if the packet is merged
> + * with other packets.
> + */
> + uint64_t start_time;
> + /*
> + * next_pkt_idx is used to chain the packets that
> + * are in the same flow but can't be merged together
> + * (e.g. caused by packet reordering).
> + */
> + uint32_t next_pkt_idx;
> + /* TCP sequence number of the packet */
> + uint32_t sent_seq;
> + /* IPv4 ID of the packet */
> + uint16_t ip_id;
> + /* the number of merged packets */
> + uint16_t nb_merged;
> + /* Indicate if IPv4 ID can be ignored */
> + uint8_t is_atomic;
> +};
> +
> +/*
> + * Merge two TCP packets without updating checksums.
> + * If cmp is larger than 0, append the new packet to the
> + * original packet. Otherwise, pre-pend the new packet to
> + * the original packet.
> + */
> +static inline int
> +merge_two_tcp_packets(struct gro_tcp_item *item,
> + struct rte_mbuf *pkt,
> + int cmp,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint16_t l2_offset)
> +{
> + struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
> + uint16_t hdr_len, l2_len;
> +
> + if (cmp > 0) {
> + pkt_head = item->firstseg;
> + pkt_tail = pkt;
> + } else {
> + pkt_head = pkt;
> + pkt_tail = item->firstseg;
> + }
> +
> + /* check if the IPv4 packet length is greater than the max value */
> + hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
> + pkt_head->l4_len;
> + l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
> + if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
> + hdr_len > MAX_IP_PKT_LENGTH))
> + return 0;
> +
> + /* remove the packet header for the tail packet */
> + rte_pktmbuf_adj(pkt_tail, hdr_len);
> +
> + /* chain two packets together */
> + if (cmp > 0) {
> + item->lastseg->next = pkt;
> + item->lastseg = rte_pktmbuf_lastseg(pkt);
> + /* update IP ID to the larger value */
> + item->ip_id = ip_id;
> + } else {
> + lastseg = rte_pktmbuf_lastseg(pkt);
> + lastseg->next = item->firstseg;
> + item->firstseg = pkt;
> + /* update sent_seq to the smaller value */
> + item->sent_seq = sent_seq;
> + item->ip_id = ip_id;
> + }
> + item->nb_merged++;
> +
> + /* update MBUF metadata for the merged packet */
> + pkt_head->nb_segs += pkt_tail->nb_segs;
> + pkt_head->pkt_len += pkt_tail->pkt_len;
> +
> + return 1;
> +}
> +
> +/*
> + * Check if two TCP/IPv4 packets are neighbors.
> + */
> +static inline int
> +check_seq_option(struct gro_tcp_item *item,
> + struct rte_tcp_hdr *tcph,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint16_t tcp_hl,
> + uint16_t tcp_dl,
> + uint16_t l2_offset,
> + uint8_t is_atomic)
> +{
> + struct rte_mbuf *pkt_orig = item->firstseg;
> + char *iph_orig;
> + struct rte_tcp_hdr *tcph_orig;
> + uint16_t len, tcp_hl_orig;
> +
> + iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
> + l2_offset + pkt_orig->l2_len);
> + tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
> + tcp_hl_orig = pkt_orig->l4_len;
> +
> + /* Check if TCP option fields equal */
> + len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> + if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> + (memcmp(tcph + 1, tcph_orig + 1,
> + len) != 0)))
> + return 0;
> +
> + /* Don't merge packets whose DF bits are different */
> + if (unlikely(item->is_atomic ^ is_atomic))
> + return 0;
> +
> + /* check if the two packets are neighbors */
> + len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> + pkt_orig->l3_len - tcp_hl_orig;
> + if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> + (ip_id == item->ip_id + 1)))
> + /* append the new packet */
> + return 1;
> + else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> + (ip_id + item->nb_merged == item->ip_id)))
> + /* pre-pend the new packet */
> + return -1;
> +
> + return 0;
> +}
> +
> +#endif
> diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c index
> 8f5e800250..eea2a72ecd 100644
> --- a/lib/gro/gro_tcp4.c
> +++ b/lib/gro/gro_tcp4.c
> @@ -7,6 +7,7 @@
> #include <rte_ethdev.h>
>
> #include "gro_tcp4.h"
> +#include "gro_tcp.h"
>
> void *
> gro_tcp4_tbl_create(uint16_t socket_id, @@ -30,7 +31,7 @@
> gro_tcp4_tbl_create(uint16_t socket_id,
> if (tbl == NULL)
> return NULL;
>
> - size = sizeof(struct gro_tcp4_item) * entries_num;
> + size = sizeof(struct gro_tcp_item) * entries_num;
> tbl->items = rte_zmalloc_socket(__func__,
> size,
> RTE_CACHE_LINE_SIZE,
> @@ -177,7 +178,7 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
> * update the packet length for the flushed packet.
> */
> static inline void
> -update_header(struct gro_tcp4_item *item)
> +update_header(struct gro_tcp_item *item)
> {
> struct rte_ipv4_hdr *ipv4_hdr;
> struct rte_mbuf *pkt = item->firstseg; @@ -302,7 +303,7 @@
> gro_tcp4_reassemble(struct rte_mbuf *pkt,
> sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> is_atomic);
> if (cmp) {
> - if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
> + if (merge_two_tcp_packets(&(tbl->items[cur_idx]),
> pkt, cmp, sent_seq, ip_id, 0))
> return 1;
> /*
> diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h index
> 212f97a042..634a215b98 100644
> --- a/lib/gro/gro_tcp4.h
> +++ b/lib/gro/gro_tcp4.h
> @@ -5,22 +5,9 @@
> #ifndef _GRO_TCP4_H_
> #define _GRO_TCP4_H_
>
> -#include <rte_tcp.h>
> -
> #define INVALID_ARRAY_INDEX 0xffffffffUL #define
> GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
>
> -/*
> - * The max length of a IPv4 packet, which includes the length of the L3
> - * header, the L4 header and the data payload.
> - */
> -#define MAX_IPV4_PKT_LENGTH UINT16_MAX
> -
> -/* The maximum TCP header length */
> -#define MAX_TCP_HLEN 60
> -#define INVALID_TCP_HDRLEN(len) \
> - (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> -
> /* Header fields representing a TCP/IPv4 flow */ struct tcp4_flow_key {
> struct rte_ether_addr eth_saddr;
> @@ -42,42 +29,12 @@ struct gro_tcp4_flow {
> uint32_t start_index;
> };
>
> -struct gro_tcp4_item {
> - /*
> - * The first MBUF segment of the packet. If the value
> - * is NULL, it means the item is empty.
> - */
> - struct rte_mbuf *firstseg;
> - /* The last MBUF segment of the packet */
> - struct rte_mbuf *lastseg;
> - /*
> - * The time when the first packet is inserted into the table.
> - * This value won't be updated, even if the packet is merged
> - * with other packets.
> - */
> - uint64_t start_time;
> - /*
> - * next_pkt_idx is used to chain the packets that
> - * are in the same flow but can't be merged together
> - * (e.g. caused by packet reordering).
> - */
> - uint32_t next_pkt_idx;
> - /* TCP sequence number of the packet */
> - uint32_t sent_seq;
> - /* IPv4 ID of the packet */
> - uint16_t ip_id;
> - /* the number of merged packets */
> - uint16_t nb_merged;
> - /* Indicate if IPv4 ID can be ignored */
> - uint8_t is_atomic;
> -};
> -
> /*
> * TCP/IPv4 reassembly table structure.
> */
> struct gro_tcp4_tbl {
> /* item array */
> - struct gro_tcp4_item *items;
> + struct gro_tcp_item *items;
> /* flow array */
> struct gro_tcp4_flow *flows;
> /* current item number */
> @@ -195,111 +152,4 @@ is_same_tcp4_flow(struct tcp4_flow_key k1, struct
> tcp4_flow_key k2)
> (k1.dst_port == k2.dst_port));
> }
>
> -/*
> - * Merge two TCP/IPv4 packets without updating checksums.
> - * If cmp is larger than 0, append the new packet to the
> - * original packet. Otherwise, pre-pend the new packet to
> - * the original packet.
> - */
> -static inline int
> -merge_two_tcp4_packets(struct gro_tcp4_item *item,
> - struct rte_mbuf *pkt,
> - int cmp,
> - uint32_t sent_seq,
> - uint16_t ip_id,
> - uint16_t l2_offset)
> -{
> - struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
> - uint16_t hdr_len, l2_len;
> -
> - if (cmp > 0) {
> - pkt_head = item->firstseg;
> - pkt_tail = pkt;
> - } else {
> - pkt_head = pkt;
> - pkt_tail = item->firstseg;
> - }
> -
> - /* check if the IPv4 packet length is greater than the max value */
> - hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
> - pkt_head->l4_len;
> - l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
> - if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
> - hdr_len > MAX_IPV4_PKT_LENGTH))
> - return 0;
> -
> - /* remove the packet header for the tail packet */
> - rte_pktmbuf_adj(pkt_tail, hdr_len);
> -
> - /* chain two packets together */
> - if (cmp > 0) {
> - item->lastseg->next = pkt;
> - item->lastseg = rte_pktmbuf_lastseg(pkt);
> - /* update IP ID to the larger value */
> - item->ip_id = ip_id;
> - } else {
> - lastseg = rte_pktmbuf_lastseg(pkt);
> - lastseg->next = item->firstseg;
> - item->firstseg = pkt;
> - /* update sent_seq to the smaller value */
> - item->sent_seq = sent_seq;
> - item->ip_id = ip_id;
> - }
> - item->nb_merged++;
> -
> - /* update MBUF metadata for the merged packet */
> - pkt_head->nb_segs += pkt_tail->nb_segs;
> - pkt_head->pkt_len += pkt_tail->pkt_len;
> -
> - return 1;
> -}
> -
> -/*
> - * Check if two TCP/IPv4 packets are neighbors.
> - */
> -static inline int
> -check_seq_option(struct gro_tcp4_item *item,
> - struct rte_tcp_hdr *tcph,
> - uint32_t sent_seq,
> - uint16_t ip_id,
> - uint16_t tcp_hl,
> - uint16_t tcp_dl,
> - uint16_t l2_offset,
> - uint8_t is_atomic)
> -{
> - struct rte_mbuf *pkt_orig = item->firstseg;
> - struct rte_ipv4_hdr *iph_orig;
> - struct rte_tcp_hdr *tcph_orig;
> - uint16_t len, tcp_hl_orig;
> -
> - iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char
> *) +
> - l2_offset + pkt_orig->l2_len);
> - tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig + pkt_orig-
> >l3_len);
> - tcp_hl_orig = pkt_orig->l4_len;
> -
> - /* Check if TCP option fields equal */
> - len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> - if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> - (memcmp(tcph + 1, tcph_orig + 1,
> - len) != 0)))
> - return 0;
> -
> - /* Don't merge packets whose DF bits are different */
> - if (unlikely(item->is_atomic ^ is_atomic))
> - return 0;
> -
> - /* check if the two packets are neighbors */
> - len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> - pkt_orig->l3_len - tcp_hl_orig;
> - if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> - (ip_id == item->ip_id + 1)))
> - /* append the new packet */
> - return 1;
> - else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> - (ip_id + item->nb_merged == item->ip_id)))
> - /* pre-pend the new packet */
> - return -1;
> -
> - return 0;
> -}
> #endif
> diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c new file mode 100644
> index 0000000000..a37d8e2512
> --- /dev/null
> +++ b/lib/gro/gro_tcp6.c
> @@ -0,0 +1,387 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
> +
> +#include <rte_malloc.h>
> +#include <rte_mbuf.h>
> +#include <rte_ethdev.h>
> +
> +#include "gro_tcp6.h"
> +#include "gro_tcp.h"
> +
> +void *
> +gro_tcp6_tbl_create(uint16_t socket_id,
> + uint16_t max_flow_num,
> + uint16_t max_item_per_flow)
> +{
> + struct gro_tcp6_tbl *tbl;
> + size_t size;
> + uint32_t entries_num, i;
> +
> + entries_num = max_flow_num * max_item_per_flow;
> + entries_num = RTE_MIN(entries_num,
> GRO_TCP6_TBL_MAX_ITEM_NUM);
> +
> + if (entries_num == 0)
> + return NULL;
> +
> + tbl = rte_zmalloc_socket(__func__,
> + sizeof(struct gro_tcp6_tbl),
> + RTE_CACHE_LINE_SIZE,
> + socket_id);
> + if (tbl == NULL)
> + return NULL;
> +
> + size = sizeof(struct gro_tcp_item) * entries_num;
> + tbl->items = rte_zmalloc_socket(__func__,
> + size,
> + RTE_CACHE_LINE_SIZE,
> + socket_id);
> + if (tbl->items == NULL) {
> + rte_free(tbl);
> + return NULL;
> + }
> + tbl->max_item_num = entries_num;
> +
> + size = sizeof(struct gro_tcp6_flow) * entries_num;
> + tbl->flows = rte_zmalloc_socket(__func__,
> + size,
> + RTE_CACHE_LINE_SIZE,
> + socket_id);
> + if (tbl->flows == NULL) {
> + rte_free(tbl->items);
> + rte_free(tbl);
> + return NULL;
> + }
> + /* INVALID_ARRAY_INDEX indicates an empty flow */
> + for (i = 0; i < entries_num; i++)
> + tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
> + tbl->max_flow_num = entries_num;
> +
> + return tbl;
> +}
> +
> +void
> +gro_tcp6_tbl_destroy(void *tbl)
> +{
> + struct gro_tcp6_tbl *tcp_tbl = tbl;
> +
> + if (tcp_tbl) {
> + rte_free(tcp_tbl->items);
> + rte_free(tcp_tbl->flows);
> + }
> + rte_free(tcp_tbl);
> +}
> +
> +static inline uint32_t
> +find_an_empty_item(struct gro_tcp6_tbl *tbl) {
> + uint32_t i;
> + uint32_t max_item_num = tbl->max_item_num;
> +
> + for (i = 0; i < max_item_num; i++)
> + if (tbl->items[i].firstseg == NULL)
> + return i;
> + return INVALID_ARRAY_INDEX;
> +}
> +
> +static inline uint32_t
> +find_an_empty_flow(struct gro_tcp6_tbl *tbl) {
> + uint32_t i;
> + uint32_t max_flow_num = tbl->max_flow_num;
> +
> + for (i = 0; i < max_flow_num; i++)
> + if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
> + return i;
> + return INVALID_ARRAY_INDEX;
> +}
> +
> +static inline uint32_t
> +insert_new_item(struct gro_tcp6_tbl *tbl,
> + struct rte_mbuf *pkt,
> + uint64_t start_time,
> + uint32_t prev_idx,
> + uint32_t sent_seq,
> + uint8_t is_atomic)
> +{
> + uint32_t item_idx;
> +
> + item_idx = find_an_empty_item(tbl);
> + if (item_idx == INVALID_ARRAY_INDEX)
> + return INVALID_ARRAY_INDEX;
> +
> + tbl->items[item_idx].firstseg = pkt;
> + tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> + tbl->items[item_idx].start_time = start_time;
> + tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> + tbl->items[item_idx].sent_seq = sent_seq;
> + tbl->items[item_idx].nb_merged = 1;
> + tbl->items[item_idx].is_atomic = is_atomic;
> + tbl->item_num++;
> +
> + /* if the previous packet exists, chain them together. */
> + if (prev_idx != INVALID_ARRAY_INDEX) {
> + tbl->items[item_idx].next_pkt_idx =
> + tbl->items[prev_idx].next_pkt_idx;
> + tbl->items[prev_idx].next_pkt_idx = item_idx;
> + }
> +
> + return item_idx;
> +}
> +
> +static inline uint32_t
> +delete_item(struct gro_tcp6_tbl *tbl, uint32_t item_idx,
> + uint32_t prev_item_idx)
> +{
> + uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
> +
> + /* NULL indicates an empty item */
> + tbl->items[item_idx].firstseg = NULL;
> + tbl->item_num--;
> + if (prev_item_idx != INVALID_ARRAY_INDEX)
> + tbl->items[prev_item_idx].next_pkt_idx = next_idx;
> +
> + return next_idx;
> +}
> +
> +static inline uint32_t
> +insert_new_flow(struct gro_tcp6_tbl *tbl,
> + struct tcp6_flow_key *src,
> + rte_be32_t vtc_flow,
> + uint32_t item_idx)
> +{
> + struct tcp6_flow_key *dst;
> + uint32_t flow_idx;
> +
> + flow_idx = find_an_empty_flow(tbl);
> + if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
> + return INVALID_ARRAY_INDEX;
> +
> + dst = &(tbl->flows[flow_idx].key);
> +
> + rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
> + rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
> + memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst-
> >src_addr));
> + memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst-
> >dst_addr));
> + dst->recv_ack = src->recv_ack;
> + dst->src_port = src->src_port;
> + dst->dst_port = src->dst_port;
> +
> + tbl->flows[flow_idx].start_index = item_idx;
> + tbl->flow_num++;
> + tbl->flows->vtc_flow = vtc_flow;
> +
> + return flow_idx;
> +}
> +
> +/*
> + * update the packet length for the flushed packet.
> + */
> +static inline void
> +update_header(struct gro_tcp_item *item) {
> + struct rte_ipv6_hdr *ipv6_hdr;
> + struct rte_mbuf *pkt = item->firstseg;
> +
> + ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> + pkt->l2_len);
> + ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
> + pkt->l2_len - pkt->l3_len);
> +}
> +
> +int32_t
> +gro_tcp6_reassemble(struct rte_mbuf *pkt,
> + struct gro_tcp6_tbl *tbl,
> + uint64_t start_time)
> +{
> + struct rte_ether_hdr *eth_hdr;
> + struct rte_ipv6_hdr *ipv6_hdr;
> + struct rte_tcp_hdr *tcp_hdr;
> + uint32_t sent_seq;
> + int32_t tcp_dl;
> + uint16_t ip_tlen;
> + struct tcp6_flow_key key;
> + uint32_t cur_idx, prev_idx, item_idx;
> + uint32_t i, max_flow_num, remaining_flow_num;
> + int cmp;
> + uint8_t find;
> + rte_be32_t vtc_flow_diff;
> +
> + /*
> + * Don't process the packet whose TCP header length is greater
> + * than 60 bytes or less than 20 bytes.
> + */
> + if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
> + return -1;
> +
> + eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
> + ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
> + tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv6_hdr + pkt->l3_len);
> +
> + /*
> + * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
> + * or CWR set.
> + */
> + if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
> + return -1;
> +
> + ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
> + /*
> + * Trim the tail padding bytes. The IPv6 header is fixed to
> + * 40 bytes unlike IPv4 that is variable. The length in the IPv6 header
> + * contains only length of TCP Header + TCP Payload, whereas IPv4
> header contains
> + * length of IP Header + TCP Header + TCP Payload
> + */
> + if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len + pkt->l3_len))
> + rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len -
> pkt->l3_len);
> + /*
> + * Don't process the packet whose payload length is less than or
> + * equal to 0.
> + */
> + tcp_dl = ip_tlen - pkt->l4_len;
> + if (tcp_dl <= 0)
> + return -1;
> +
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> +
> + rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
> + rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
> + memcpy(&key.src_addr[0], &ipv6_hdr->src_addr,
> sizeof(key.src_addr));
> + memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr,
> sizeof(key.dst_addr));
> + key.src_port = tcp_hdr->src_port;
> + key.dst_port = tcp_hdr->dst_port;
> + key.recv_ack = tcp_hdr->recv_ack;
> +
> + /* Search for a matched flow. */
> + max_flow_num = tbl->max_flow_num;
> + remaining_flow_num = tbl->flow_num;
> + find = 0;
> + for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
> + if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
> + if (is_same_tcp6_flow(tbl->flows[i].key, key)) {
> + /*
> + * IP version (4) Traffic Class (8) Flow Label (20)
> + * All fields except Traffic class should be
> same
> + */
> + vtc_flow_diff = (ipv6_hdr->vtc_flow ^ tbl-
> >flows->vtc_flow);
> + if (vtc_flow_diff & htonl(0xF00FFFFF))
> + continue;
> + find = 1;
> + break;
> + }
> + remaining_flow_num--;
> + }
> + }
> +
> + /*
> + * Fail to find a matched flow. Insert a new flow and store the
> + * packet into the flow.
> + */
> + if (find == 0) {
> + item_idx = insert_new_item(tbl, pkt, start_time,
> + INVALID_ARRAY_INDEX, sent_seq, true);
> + if (item_idx == INVALID_ARRAY_INDEX)
> + return -1;
> + if (insert_new_flow(tbl, &key, ipv6_hdr->vtc_flow, item_idx)
> ==
> + INVALID_ARRAY_INDEX) {
> + /*
> + * Fail to insert a new flow, so delete the
> + * stored packet.
> + */
> + delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
> + return -1;
> + }
> + return 0;
> + }
> +
> + /*
> + * Check all packets in the flow and try to find a neighbor for
> + * the input packet.
> + */
> + cur_idx = tbl->flows[i].start_index;
> + prev_idx = cur_idx;
> + do {
> + cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
> + sent_seq, 0, pkt->l4_len, tcp_dl, 0,
> + true);
> + if (cmp) {
> + if (merge_two_tcp_packets(&(tbl->items[cur_idx]),
> + pkt, cmp, sent_seq, 0, 0)) {
> + return 1;
> + }
> +
> + /*
> + * Fail to merge the two packets, as the packet
> + * length is greater than the max value. Store
> + * the packet into the flow.
> + */
> + if (insert_new_item(tbl, pkt, start_time, cur_idx,
> + sent_seq, true) ==
> + INVALID_ARRAY_INDEX)
> + return -1;
> + return 0;
> + }
> + prev_idx = cur_idx;
> + cur_idx = tbl->items[cur_idx].next_pkt_idx;
> + } while (cur_idx != INVALID_ARRAY_INDEX);
> +
> + /* Fail to find a neighbor, so store the packet into the flow. */
> + if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
> + true) == INVALID_ARRAY_INDEX)
> + return -1;
> +
> + return 0;
> +}
> +
> +uint16_t
> +gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
> + uint64_t flush_timestamp,
> + struct rte_mbuf **out,
> + uint16_t nb_out)
> +{
> + uint16_t k = 0;
> + uint32_t i, j;
> + uint32_t max_flow_num = tbl->max_flow_num;
> +
> + for (i = 0; i < max_flow_num; i++) {
> + if (unlikely(tbl->flow_num == 0))
> + return k;
> +
> + j = tbl->flows[i].start_index;
> + while (j != INVALID_ARRAY_INDEX) {
> + if (tbl->items[j].start_time <= flush_timestamp) {
> + out[k++] = tbl->items[j].firstseg;
> + if (tbl->items[j].nb_merged > 1)
> + update_header(&(tbl->items[j]));
> + /*
> + * Delete the packet and get the next
> + * packet in the flow.
> + */
> + j = delete_item(tbl, j,
> INVALID_ARRAY_INDEX);
> + tbl->flows[i].start_index = j;
> + if (j == INVALID_ARRAY_INDEX)
> + tbl->flow_num--;
> +
> + if (unlikely(k == nb_out))
> + return k;
> + } else
> + /*
> + * The left packets in this flow won't be
> + * timeout. Go to check other flows.
> + */
> + break;
> + }
> + }
> + return k;
> +}
> +
> +uint32_t
> +gro_tcp6_tbl_pkt_count(void *tbl)
> +{
> + struct gro_tcp6_tbl *gro_tbl = tbl;
> +
> + if (gro_tbl)
> + return gro_tbl->item_num;
> +
> + return 0;
> +}
> diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h new file mode 100644
> index 0000000000..91325c0da2
> --- /dev/null
> +++ b/lib/gro/gro_tcp6.h
> @@ -0,0 +1,150 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
> +
> +#ifndef _GRO_TCP6_H_
> +#define _GRO_TCP6_H_
> +
> +#define INVALID_ARRAY_INDEX 0xffffffffUL #define
> +GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
> +
> +/* Header fields representing a TCP/IPv6 flow */ struct tcp6_flow_key {
> + struct rte_ether_addr eth_saddr;
> + struct rte_ether_addr eth_daddr;
> + uint8_t src_addr[16];
> + uint8_t dst_addr[16];
> +
> + uint32_t recv_ack;
> + uint16_t src_port;
> + uint16_t dst_port;
> +};
> +
> +struct gro_tcp6_flow {
> + struct tcp6_flow_key key;
> + rte_be32_t vtc_flow;
> + /*
> + * The index of the first packet in the flow.
> + * INVALID_ARRAY_INDEX indicates an empty flow.
> + */
> + uint32_t start_index;
> +};
> +
> +/*
> + * TCP/IPv6 reassembly table structure.
> + */
> +struct gro_tcp6_tbl {
> + /* item array */
> + struct gro_tcp_item *items;
> + /* flow array */
> + struct gro_tcp6_flow *flows;
> + /* current item number */
> + uint32_t item_num;
> + /* current flow num */
> + uint32_t flow_num;
> + /* item array size */
> + uint32_t max_item_num;
> + /* flow array size */
> + uint32_t max_flow_num;
> +};
> +
> +/**
> + * This function creates a TCP/IPv6 reassembly table.
> + *
> + * @param socket_id
> + * Socket index for allocating the TCP/IPv6 reassemble table
> + * @param max_flow_num
> + * The maximum number of flows in the TCP/IPv6 GRO table
> + * @param max_item_per_flow
> + * The maximum number of packets per flow
> + *
> + * @return
> + * - Return the table pointer on success.
> + * - Return NULL on failure.
> + */
> +void *gro_tcp6_tbl_create(uint16_t socket_id,
> + uint16_t max_flow_num,
> + uint16_t max_item_per_flow);
> +
> +/**
> + * This function destroys a TCP/IPv6 reassembly table.
> + *
> + * @param tbl
> + * Pointer pointing to the TCP/IPv6 reassembly table.
> + */
> +void gro_tcp6_tbl_destroy(void *tbl);
> +
> +/**
> + * This function merges a TCP/IPv6 packet. It doesn't process the
> +packet,
> + * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't have
> + * payload.
> + *
> + * This function doesn't check if the packet has correct checksums and
> + * doesn't re-calculate checksums for the merged packet. Additionally,
> + * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
> + * when IP fragmentation is possible (i.e., DF==0). It returns the
> + * packet, if the packet has invalid parameters (e.g. SYN bit is set)
> + * or there is no available space in the table.
> + *
> + * @param pkt
> + * Packet to reassemble
> + * @param tbl
> + * Pointer pointing to the TCP/IPv6 reassembly table
> + * @start_time
> + * The time when the packet is inserted into the table
> + *
> + * @return
> + * - Return a positive value if the packet is merged.
> + * - Return zero if the packet isn't merged but stored in the table.
> + * - Return a negative value for invalid parameters or no available
> + * space in the table.
> + */
> +int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
> + struct gro_tcp6_tbl *tbl,
> + uint64_t start_time);
> +
> +/**
> + * This function flushes timeout packets in a TCP/IPv6 reassembly
> +table,
> + * and without updating checksums.
> + *
> + * @param tbl
> + * TCP/IPv6 reassembly table pointer
> + * @param flush_timestamp
> + * Flush packets which are inserted into the table before or at the
> + * flush_timestamp.
> + * @param out
> + * Pointer array used to keep flushed packets
> + * @param nb_out
> + * The element number in 'out'. It also determines the maximum number
> +of
> + * packets that can be flushed finally.
> + *
> + * @return
> + * The number of flushed packets
> + */
> +uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
> + uint64_t flush_timestamp,
> + struct rte_mbuf **out,
> + uint16_t nb_out);
> +
> +/**
> + * This function returns the number of the packets in a TCP/IPv6
> + * reassembly table.
> + *
> + * @param tbl
> + * TCP/IPv6 reassembly table pointer
> + *
> + * @return
> + * The number of packets in the table
> + */
> +uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
> +
> +/*
> + * Check if two TCP/IPv6 packets belong to the same flow.
> + */
> +static inline int
> +is_same_tcp6_flow(struct tcp6_flow_key k1, struct tcp6_flow_key k2) {
> + return (!memcmp(&k1, &k2, sizeof(struct tcp6_flow_key))); }
> +
> +#endif
> diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c index
> 3be4deb7c7..56b30b8c98 100644
> --- a/lib/gro/gro_vxlan_tcp4.c
> +++ b/lib/gro/gro_vxlan_tcp4.c
> @@ -7,6 +7,7 @@
> #include <rte_ethdev.h>
> #include <rte_udp.h>
>
> +#include "gro_tcp.h"
> #include "gro_vxlan_tcp4.h"
>
> void *
> @@ -248,7 +249,7 @@ merge_two_vxlan_tcp4_packets(struct
> gro_vxlan_tcp4_item *item,
> uint16_t outer_ip_id,
> uint16_t ip_id)
> {
> - if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp,
> sent_seq,
> + if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
> ip_id, pkt->outer_l2_len +
> pkt->outer_l3_len)) {
> /* Update the outer IPv4 ID to the large value. */ diff --git
> a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h index
> 7832942a68..d68d5fcd5b 100644
> --- a/lib/gro/gro_vxlan_tcp4.h
> +++ b/lib/gro/gro_vxlan_tcp4.h
> @@ -5,6 +5,7 @@
> #ifndef _GRO_VXLAN_TCP4_H_
> #define _GRO_VXLAN_TCP4_H_
>
> +#include "gro_tcp.h"
> #include "gro_tcp4.h"
>
> #define GRO_VXLAN_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL) @@ -
> 36,7 +37,7 @@ struct gro_vxlan_tcp4_flow { };
>
> struct gro_vxlan_tcp4_item {
> - struct gro_tcp4_item inner_item;
> + struct gro_tcp_item inner_item;
> /* IPv4 ID in the outer IPv4 header */
> uint16_t outer_ip_id;
> /* Indicate if outer IPv4 ID can be ignored */ diff --git
> a/lib/gro/meson.build b/lib/gro/meson.build index e4fa2958bd..dbce05220d
> 100644
> --- a/lib/gro/meson.build
> +++ b/lib/gro/meson.build
> @@ -4,6 +4,7 @@
> sources = files(
> 'rte_gro.c',
> 'gro_tcp4.c',
> + 'gro_tcp6.c',
> 'gro_udp4.c',
> 'gro_vxlan_tcp4.c',
> 'gro_vxlan_udp4.c',
> diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c index
> e35399fd42..d824eebd93 100644
> --- a/lib/gro/rte_gro.c
> +++ b/lib/gro/rte_gro.c
> @@ -8,6 +8,7 @@
>
> #include "rte_gro.h"
> #include "gro_tcp4.h"
> +#include "gro_tcp6.h"
> #include "gro_udp4.h"
> #include "gro_vxlan_tcp4.h"
> #include "gro_vxlan_udp4.h"
> @@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
>
> static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
> gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
> - gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
> + gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create,
> gro_tcp6_tbl_create,
> +NULL};
> static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
> gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
> gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
> + gro_tcp6_tbl_destroy,
> NULL};
> static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM]
> = {
> gro_tcp4_tbl_pkt_count,
> gro_vxlan_tcp4_tbl_pkt_count,
> gro_udp4_tbl_pkt_count,
> gro_vxlan_udp4_tbl_pkt_count,
> + gro_tcp6_tbl_pkt_count,
> NULL};
>
> #define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \ @@
> -35,6 +38,12 @@ static gro_tbl_pkt_count_fn
> tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
> ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) &&
> \
> (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
>
> +/* GRO with extension headers is not supported */ #define
> +IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
> + ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
> + ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) &&
> \
> + (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
> +
> #define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
> ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
> (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
> @@ -147,7 +156,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> /* allocate a reassembly table for TCP/IPv4 GRO */
> struct gro_tcp4_tbl tcp_tbl;
> struct gro_tcp4_flow
> tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
> - struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
> = {{0} };
> + struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
> = {{0} };
> +
> + struct gro_tcp6_tbl tcp6_tbl;
> + struct gro_tcp6_flow
> tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
> + struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM]
> = {{0} };
>
> /* allocate a reassembly table for UDP/IPv4 GRO */
> struct gro_udp4_tbl udp_tbl;
> @@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> int32_t ret;
> uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
> uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
> - do_vxlan_udp_gro = 0;
> + do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
>
> if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4
> |
> - RTE_GRO_TCP_IPV4 |
> + RTE_GRO_TCP_IPV4 |
> RTE_GRO_TCP_IPV6 |
> RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
> RTE_GRO_UDP_IPV4)) == 0))
> return nb_pkts;
> @@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> do_udp4_gro = 1;
> }
>
> + if (param->gro_types & RTE_GRO_TCP_IPV6) {
> + for (i = 0; i < item_num; i++)
> + tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
> +
> + tcp6_tbl.flows = tcp6_flows;
> + tcp6_tbl.items = tcp6_items;
> + tcp6_tbl.flow_num = 0;
> + tcp6_tbl.item_num = 0;
> + tcp6_tbl.max_flow_num = item_num;
> + tcp6_tbl.max_item_num = item_num;
> + do_tcp6_gro = 1;
> + }
>
> for (i = 0; i < nb_pkts; i++) {
> /*
> @@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> nb_after_gro--;
> else if (ret < 0)
> unprocess_pkts[unprocess_num++] = pkts[i];
> + } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
> + do_tcp6_gro) {
> + ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
> + if (ret > 0)
> + /* merge successfully */
> + nb_after_gro--;
> + else if (ret < 0)
> + unprocess_pkts[unprocess_num++] = pkts[i];
> } else
> unprocess_pkts[unprocess_num++] = pkts[i];
> }
> @@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> if ((nb_after_gro < nb_pkts)
> || (unprocess_num < nb_pkts)) {
> i = 0;
> + /* Copy unprocessed packets */
> + if (unprocess_num > 0) {
> + memcpy(&pkts[i], unprocess_pkts,
> + sizeof(struct rte_mbuf *) *
> + unprocess_num);
> + i = unprocess_num;
> + }
> +
> /* Flush all packets from the tables */
> if (do_vxlan_tcp_gro) {
> - i =
> gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
> + i +=
> gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
> 0, pkts, nb_pkts);
> }
>
> @@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
> &pkts[i], nb_pkts - i);
> }
> - /* Copy unprocessed packets */
> - if (unprocess_num > 0) {
> - memcpy(&pkts[i], unprocess_pkts,
> - sizeof(struct rte_mbuf *) *
> - unprocess_num);
> +
> + if (do_tcp6_gro) {
> + i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
> + &pkts[i], nb_pkts - i);
> }
> - nb_after_gro = i + unprocess_num;
> }
>
> return nb_after_gro;
> @@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts, {
> struct rte_mbuf *unprocess_pkts[nb_pkts];
> struct gro_ctx *gro_ctx = ctx;
> - void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
> + void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
> uint64_t current_time;
> uint16_t i, unprocess_num = 0;
> - uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro,
> do_vxlan_udp_gro;
> + uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro,
> do_vxlan_udp_gro,
> +do_tcp6_gro;
>
> if (unlikely((gro_ctx->gro_types &
> (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
> - RTE_GRO_TCP_IPV4 |
> + RTE_GRO_TCP_IPV4 |
> RTE_GRO_TCP_IPV6 |
> RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
> RTE_GRO_UDP_IPV4)) == 0))
> return nb_pkts;
> @@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> vxlan_tcp_tbl = gro_ctx-
> >tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
> udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
> vxlan_udp_tbl = gro_ctx-
> >tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
> + tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
>
> do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
> RTE_GRO_TCP_IPV4;
> @@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> RTE_GRO_UDP_IPV4;
> do_vxlan_udp_gro = (gro_ctx->gro_types &
> RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
> RTE_GRO_IPV4_VXLAN_UDP_IPV4;
> + do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) ==
> +RTE_GRO_TCP_IPV6;
>
> current_time = rte_rdtsc();
>
> @@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> if (gro_udp4_reassemble(pkts[i], udp_tbl,
> current_time) < 0)
> unprocess_pkts[unprocess_num++] = pkts[i];
> + } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
> + do_tcp6_gro) {
> + if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
> + current_time) < 0)
> + unprocess_pkts[unprocess_num++] = pkts[i];
> } else
> unprocess_pkts[unprocess_num++] = pkts[i];
> }
> @@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
> gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
> flush_timestamp,
> &out[num], left_nb_out);
> + left_nb_out = max_nb_out - num;
> + }
> +
> + if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
> + num += gro_tcp6_tbl_timeout_flush(
> + gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
> + flush_timestamp,
> + &out[num], left_nb_out);
> +
> }
>
> return num;
> diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h index
> 9f9ed4935a..c83dfd9ad1 100644
> --- a/lib/gro/rte_gro.h
> +++ b/lib/gro/rte_gro.h
> @@ -38,6 +38,9 @@ extern "C" {
> #define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3 #define
> RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL <<
> RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
> /**< VxLAN UDP/IPv4 GRO flag. */
> +#define RTE_GRO_TCP_IPV6_INDEX 4
> +#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX) /**<
> TCP/IPv6
> +GRO flag. */
>
> /**
> * Structure used to create GRO context objects or used to pass
> --
> 2.25.1
[-- Attachment #2: Type: text/html, Size: 89858 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread
* [PATCH v3] gro : ipv6-gro review comments to reduce code duplication across v4 and v6
2022-10-20 18:14 ` [PATCH v2] " Kumara Parameshwaran
2023-05-12 2:47 ` Hu, Jiayu
@ 2023-06-02 6:02 ` Kumara Parameshwaran
2023-06-02 6:34 ` [PATCH v3] gro : ipv6 changes to support GRO for TCP/ipv6 Kumara Parameshwaran
` (8 subsequent siblings)
10 siblings, 0 replies; 32+ messages in thread
From: Kumara Parameshwaran @ 2023-06-02 6:02 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev, Kumara Parameshwaran
Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
---
v1:
* Changes to support GRO for TCP/ipv6 packets. This does not include
vxlan changes.
* The GRO is performed only for ipv6 packets that does not contain
extension headers.
* The logic for the TCP coalescing remains the same, in ipv6 header
the source address, destination address, flow label, version fields
are expected to be the same.
* Re-organised the code to reuse certain tcp functions for both ipv4 and
ipv6 flows.
v2:
* Fix comments in gro_tcp6.h header file.
v3:
* Adderess review comments to fix code duplication for v4 and v6
lib/gro/gro_tcp.c | 160 ++++++++++++++++++++++++
lib/gro/gro_tcp.h | 63 ++++++++++
lib/gro/gro_tcp4.c | 255 ++++++++++++---------------------------
lib/gro/gro_tcp4.h | 18 +--
lib/gro/gro_tcp6.c | 243 ++++++++++---------------------------
lib/gro/gro_tcp6.h | 31 +++--
lib/gro/gro_vxlan_tcp4.c | 18 +--
lib/gro/meson.build | 1 +
8 files changed, 396 insertions(+), 393 deletions(-)
create mode 100644 lib/gro/gro_tcp.c
diff --git a/lib/gro/gro_tcp.c b/lib/gro/gro_tcp.c
new file mode 100644
index 0000000000..6a5aaada58
--- /dev/null
+++ b/lib/gro/gro_tcp.c
@@ -0,0 +1,160 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp.h"
+
+static inline uint32_t
+find_an_empty_item(struct gro_tcp_item *items,
+ uint32_t table_size)
+{
+ uint32_t i;
+
+ for (i = 0; i < table_size; i++)
+ if (items[i].firstseg == NULL)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t table_size,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic)
+{
+ uint32_t item_idx;
+
+ item_idx = find_an_empty_item(items, table_size);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return INVALID_ARRAY_INDEX;
+
+ items[item_idx].firstseg = pkt;
+ items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
+ items[item_idx].start_time = start_time;
+ items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
+ items[item_idx].sent_seq = sent_seq;
+ items[item_idx].ip_id = ip_id;
+ items[item_idx].nb_merged = 1;
+ items[item_idx].is_atomic = is_atomic;
+ (*item_num) += 1;
+
+ /* if the previous packet exists, chain them together. */
+ if (prev_idx != INVALID_ARRAY_INDEX) {
+ items[item_idx].next_pkt_idx =
+ items[prev_idx].next_pkt_idx;
+ items[prev_idx].next_pkt_idx = item_idx;
+ }
+
+ return item_idx;
+}
+
+uint32_t
+delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx)
+{
+ uint32_t next_idx = items[item_idx].next_pkt_idx;
+
+ /* NULL indicates an empty item */
+ items[item_idx].firstseg = NULL;
+ (*item_num) -= 1;
+ if (prev_item_idx != INVALID_ARRAY_INDEX)
+ items[prev_item_idx].next_pkt_idx = next_idx;
+
+ return next_idx;
+}
+
+int32_t
+gro_tcp_reassemble(struct rte_mbuf *pkt,
+ void *tbl,
+ void *key,
+ int32_t tcp_dl,
+ struct gro_tcp_flow_ops *ops,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t table_size,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time)
+{
+ uint32_t item_idx;
+ uint32_t cur_idx;
+ uint32_t prev_idx;
+ struct rte_tcp_hdr *tcp_hdr;
+ int cmp;
+ uint32_t sent_seq;
+
+ tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt->l2_len + pkt->l3_len);
+ /*
+ * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
+ * or CWR set.
+ */
+ if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
+ return -1;
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+
+ ops->tcp_flow_key_init(key, tcp_hdr);
+
+ item_idx = ops->tcp_flow_lookup(tbl, key);
+ if (item_idx == INVALID_ARRAY_INDEX) {
+ item_idx = insert_new_tcp_item(pkt, items, item_num, table_size, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, ip_id,
+ is_atomic);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return -1;
+ if (ops->tcp_flow_insert(tbl, key, item_idx) ==
+ INVALID_ARRAY_INDEX) {
+ /*
+ * Fail to insert a new flow, so delete the
+ * stored packet.
+ */
+ delete_tcp_item(items, item_idx, item_num, INVALID_ARRAY_INDEX);
+ return -1;
+ }
+ return 0;
+ }
+ /*
+ * Check all packets in the flow and try to find a neighbor for
+ * the input packet.
+ */
+ cur_idx = item_idx;
+ prev_idx = cur_idx;
+ do {
+ cmp = check_seq_option(&items[cur_idx], tcp_hdr,
+ sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
+ is_atomic);
+ if (cmp) {
+ if (merge_two_tcp_packets(&items[cur_idx],
+ pkt, cmp, sent_seq, ip_id, 0))
+ return 1;
+ /*
+ * Fail to merge the two packets, as the packet
+ * length is greater than the max value. Store
+ * the packet into the flow.
+ */
+ if (insert_new_tcp_item(pkt, items, item_num, table_size, start_time, cur_idx,
+ sent_seq, ip_id, is_atomic) ==
+ INVALID_ARRAY_INDEX)
+ return -1;
+ return 0;
+ }
+ prev_idx = cur_idx;
+ cur_idx = items[cur_idx].next_pkt_idx;
+ } while (cur_idx != INVALID_ARRAY_INDEX);
+
+ /* Fail to find a neighbor, so store the packet into the flow. */
+ if (insert_new_tcp_item(pkt, items, item_num, table_size, start_time, prev_idx, sent_seq,
+ ip_id, is_atomic) == INVALID_ARRAY_INDEX)
+ return -1;
+
+ return 0;
+
+}
diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h
index c5d248a022..202f485c18 100644
--- a/lib/gro/gro_tcp.h
+++ b/lib/gro/gro_tcp.h
@@ -1,6 +1,8 @@
#ifndef _GRO_TCP_H_
#define _GRO_TCP_H_
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+
#include <rte_tcp.h>
/*
@@ -14,6 +16,31 @@
#define INVALID_TCP_HDRLEN(len) \
(((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
+struct gro_tcp_flow {
+ struct rte_ether_addr eth_saddr;
+ struct rte_ether_addr eth_daddr;
+ uint32_t recv_ack;
+ uint16_t src_port;
+ uint16_t dst_port;
+};
+
+#define ASSIGN_TCP_FLOW_KEY(k1, k2) \
+ rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
+ rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr)); \
+ k2->recv_ack = k1->recv_ack; \
+ k2->src_port = k1->src_port; \
+ k2->dst_port = k1->dst_port;
+
+typedef uint32_t (*gro_tcp_flow_lookup)(void *table, void *key);
+typedef uint32_t (*gro_tcp_flow_insert)(void *table, void *key, uint32_t item_idx);
+typedef void (*gro_tcp_flow_key_init)(void *key, struct rte_tcp_hdr *tcp_hdr);
+
+struct gro_tcp_flow_ops {
+ gro_tcp_flow_lookup tcp_flow_lookup;
+ gro_tcp_flow_insert tcp_flow_insert;
+ gro_tcp_flow_key_init tcp_flow_key_init;
+};
+
struct gro_tcp_item {
/*
* The first MBUF segment of the packet. If the value
@@ -44,6 +71,36 @@ struct gro_tcp_item {
uint8_t is_atomic;
};
+uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t table_size,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic);
+
+uint32_t
+delete_tcp_item(struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx);
+
+int32_t
+gro_tcp_reassemble(struct rte_mbuf *pkt,
+ void *tbl,
+ void *key,
+ int32_t tcp_dl,
+ struct gro_tcp_flow_ops *ops,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t table_size,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time);
+
/*
* Merge two TCP packets without updating checksums.
* If cmp is larger than 0, append the new packet to the
@@ -152,4 +209,10 @@ check_seq_option(struct gro_tcp_item *item,
return 0;
}
+static inline int
+is_same_tcp_flow(struct gro_tcp_flow *k1, struct gro_tcp_flow *k2)
+{
+ return (!memcmp(k1, k2, sizeof(struct gro_tcp_flow)));
+}
+
#endif
diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c
index 2cfb524689..cd510d3a7c 100644
--- a/lib/gro/gro_tcp4.c
+++ b/lib/gro/gro_tcp4.c
@@ -72,18 +72,6 @@ gro_tcp4_tbl_destroy(void *tbl)
rte_free(tcp_tbl);
}
-static inline uint32_t
-find_an_empty_item(struct gro_tcp4_tbl *tbl)
-{
- uint32_t i;
- uint32_t max_item_num = tbl->max_item_num;
-
- for (i = 0; i < max_item_num; i++)
- if (tbl->items[i].firstseg == NULL)
- return i;
- return INVALID_ARRAY_INDEX;
-}
-
static inline uint32_t
find_an_empty_flow(struct gro_tcp4_tbl *tbl)
{
@@ -97,98 +85,76 @@ find_an_empty_flow(struct gro_tcp4_tbl *tbl)
}
static inline uint32_t
-insert_new_item(struct gro_tcp4_tbl *tbl,
- struct rte_mbuf *pkt,
- uint64_t start_time,
- uint32_t prev_idx,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint8_t is_atomic)
+gro_tcp4_insert_flow(void *tbl,
+ void *key,
+ uint32_t item_idx)
{
- uint32_t item_idx;
+ struct gro_tcp4_tbl *tcp4_tbl = (struct gro_tcp4_tbl *)tbl;
+ struct tcp4_flow_key *src_key = (struct tcp4_flow_key *)key;
+ struct tcp4_flow_key *dst_key;
+ struct gro_tcp_flow *src_flow;
+ struct gro_tcp_flow *dst_flow;
+ uint32_t flow_idx;
- item_idx = find_an_empty_item(tbl);
- if (item_idx == INVALID_ARRAY_INDEX)
+ flow_idx = find_an_empty_flow(tcp4_tbl);
+ if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
return INVALID_ARRAY_INDEX;
- tbl->items[item_idx].firstseg = pkt;
- tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
- tbl->items[item_idx].start_time = start_time;
- tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
- tbl->items[item_idx].sent_seq = sent_seq;
- tbl->items[item_idx].ip_id = ip_id;
- tbl->items[item_idx].nb_merged = 1;
- tbl->items[item_idx].is_atomic = is_atomic;
- tbl->item_num++;
-
- /* if the previous packet exists, chain them together. */
- if (prev_idx != INVALID_ARRAY_INDEX) {
- tbl->items[item_idx].next_pkt_idx =
- tbl->items[prev_idx].next_pkt_idx;
- tbl->items[prev_idx].next_pkt_idx = item_idx;
- }
+ dst_key = &(tcp4_tbl->flows[flow_idx].key);
+ dst_flow = &dst_key->tcp_flow;
+ src_flow = &src_key->tcp_flow;
- return item_idx;
-}
+ ASSIGN_TCP_FLOW_KEY(src_flow, dst_flow)
-static inline uint32_t
-delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
- uint32_t prev_item_idx)
-{
- uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
+ dst_key->ip_src_addr = src_key->ip_src_addr;
+ dst_key->ip_dst_addr = src_key->ip_dst_addr;
- /* NULL indicates an empty item */
- tbl->items[item_idx].firstseg = NULL;
- tbl->item_num--;
- if (prev_item_idx != INVALID_ARRAY_INDEX)
- tbl->items[prev_item_idx].next_pkt_idx = next_idx;
+ tcp4_tbl->flows[flow_idx].start_index = item_idx;
+ tcp4_tbl->flow_num++;
- return next_idx;
+ return flow_idx;
}
-static inline uint32_t
-insert_new_flow(struct gro_tcp4_tbl *tbl,
- struct tcp4_flow_key *src,
- uint32_t item_idx)
-{
- struct tcp4_flow_key *dst;
- uint32_t flow_idx;
-
- flow_idx = find_an_empty_flow(tbl);
- if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
- return INVALID_ARRAY_INDEX;
-
- dst = &(tbl->flows[flow_idx].key);
-
- rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
- rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
- dst->ip_src_addr = src->ip_src_addr;
- dst->ip_dst_addr = src->ip_dst_addr;
- dst->recv_ack = src->recv_ack;
- dst->src_port = src->src_port;
- dst->dst_port = src->dst_port;
- tbl->flows[flow_idx].start_index = item_idx;
- tbl->flow_num++;
+static uint32_t
+gro_tcp4_flow_lookup(void *tbl, void *key)
+{
+ struct gro_tcp4_tbl *tcp4_tbl = (struct gro_tcp4_tbl *)tbl;
+ struct tcp4_flow_key *tcp4_key = (struct tcp4_flow_key *)key;
+ uint32_t max_flow_num = tcp4_tbl->max_flow_num;
+ uint32_t remaining_flow_num = tcp4_tbl->flow_num;
+ uint32_t i;
- return flow_idx;
+ /* Search for a matched flow. */
+ for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
+ if (tcp4_tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
+ if (is_same_tcp4_flow(tcp4_tbl->flows[i].key, *tcp4_key)) {
+ return i;
+ }
+ remaining_flow_num--;
+ }
+ }
+ return INVALID_ARRAY_INDEX;
}
-/*
- * update the packet length for the flushed packet.
- */
-static inline void
-update_header(struct gro_tcp_item *item)
+static void
+gro_tcp4_flow_key_init(void *key, struct rte_tcp_hdr *tcp_hdr)
{
- struct rte_ipv4_hdr *ipv4_hdr;
- struct rte_mbuf *pkt = item->firstseg;
+ struct tcp4_flow_key *tcp4_key = (struct tcp4_flow_key *)key;
+ struct gro_tcp_flow *tcp_flow = &tcp4_key->tcp_flow;
+
+ tcp_flow->src_port = tcp_hdr->src_port;
+ tcp_flow->dst_port = tcp_hdr->dst_port;
+ tcp_flow->recv_ack = tcp_hdr->recv_ack;
- ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
- pkt->l2_len);
- ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
- pkt->l2_len);
}
+struct gro_tcp_flow_ops gro_tcp4_flows_ops = {
+ .tcp_flow_lookup = gro_tcp4_flow_lookup,
+ .tcp_flow_insert = gro_tcp4_insert_flow,
+ .tcp_flow_key_init = gro_tcp4_flow_key_init
+};
+
int32_t
gro_tcp4_reassemble(struct rte_mbuf *pkt,
struct gro_tcp4_tbl *tbl,
@@ -196,17 +162,10 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
{
struct rte_ether_hdr *eth_hdr;
struct rte_ipv4_hdr *ipv4_hdr;
- struct rte_tcp_hdr *tcp_hdr;
- uint32_t sent_seq;
int32_t tcp_dl;
uint16_t ip_id, hdr_len, frag_off, ip_tlen;
uint8_t is_atomic;
-
struct tcp4_flow_key key;
- uint32_t cur_idx, prev_idx, item_idx;
- uint32_t i, max_flow_num, remaining_flow_num;
- int cmp;
- uint8_t find;
/*
* Don't process the packet whose TCP header length is greater
@@ -217,21 +176,13 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
- tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
- hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
-
- /*
- * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
- * or CWR set.
- */
- if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
- return -1;
/* trim the tail padding bytes */
ip_tlen = rte_be_to_cpu_16(ipv4_hdr->total_length);
if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len))
rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len);
+ hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
/*
* Don't process the packet whose payload length is less than or
* equal to 0.
@@ -240,6 +191,10 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
if (tcp_dl <= 0)
return -1;
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.tcp_flow.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.tcp_flow.eth_daddr));
+ key.ip_src_addr = ipv4_hdr->src_addr;
+ key.ip_dst_addr = ipv4_hdr->dst_addr;
/*
* Save IPv4 ID for the packet whose DF bit is 0. For the packet
* whose DF bit is 1, IPv4 ID is ignored.
@@ -247,87 +202,25 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) == RTE_IPV4_HDR_DF_FLAG;
ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
- sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
- key.ip_src_addr = ipv4_hdr->src_addr;
- key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.src_port = tcp_hdr->src_port;
- key.dst_port = tcp_hdr->dst_port;
- key.recv_ack = tcp_hdr->recv_ack;
-
- /* Search for a matched flow. */
- max_flow_num = tbl->max_flow_num;
- remaining_flow_num = tbl->flow_num;
- find = 0;
- for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
- if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
- if (is_same_tcp4_flow(tbl->flows[i].key, key)) {
- find = 1;
- break;
- }
- remaining_flow_num--;
- }
- }
-
- /*
- * Fail to find a matched flow. Insert a new flow and store the
- * packet into the flow.
- */
- if (find == 0) {
- item_idx = insert_new_item(tbl, pkt, start_time,
- INVALID_ARRAY_INDEX, sent_seq, ip_id,
- is_atomic);
- if (item_idx == INVALID_ARRAY_INDEX)
- return -1;
- if (insert_new_flow(tbl, &key, item_idx) ==
- INVALID_ARRAY_INDEX) {
- /*
- * Fail to insert a new flow, so delete the
- * stored packet.
- */
- delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
- return -1;
- }
- return 0;
- }
-
- /*
- * Check all packets in the flow and try to find a neighbor for
- * the input packet.
- */
- cur_idx = tbl->flows[i].start_index;
- prev_idx = cur_idx;
- do {
- cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
- sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
- is_atomic);
- if (cmp) {
- if (merge_two_tcp_packets(&(tbl->items[cur_idx]),
- pkt, cmp, sent_seq, ip_id, 0))
- return 1;
- /*
- * Fail to merge the two packets, as the packet
- * length is greater than the max value. Store
- * the packet into the flow.
- */
- if (insert_new_item(tbl, pkt, start_time, cur_idx,
- sent_seq, ip_id, is_atomic) ==
- INVALID_ARRAY_INDEX)
- return -1;
- return 0;
- }
- prev_idx = cur_idx;
- cur_idx = tbl->items[cur_idx].next_pkt_idx;
- } while (cur_idx != INVALID_ARRAY_INDEX);
+ return gro_tcp_reassemble(pkt, tbl, &key, tcp_dl, &gro_tcp4_flows_ops, tbl->items,
+ &tbl->item_num, tbl->max_item_num,
+ ip_id, is_atomic, start_time);
+}
- /* Fail to find a neighbor, so store the packet into the flow. */
- if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
- ip_id, is_atomic) == INVALID_ARRAY_INDEX)
- return -1;
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
- return 0;
+ ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len);
}
uint16_t
@@ -354,7 +247,7 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
* Delete the packet and get the next
* packet in the flow.
*/
- j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
+ j = delete_tcp_item(tbl->items, j, &tbl->item_num, INVALID_ARRAY_INDEX);
tbl->flows[i].start_index = j;
if (j == INVALID_ARRAY_INDEX)
tbl->flow_num--;
diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h
index 634a215b98..8ff1eed096 100644
--- a/lib/gro/gro_tcp4.h
+++ b/lib/gro/gro_tcp4.h
@@ -5,19 +5,15 @@
#ifndef _GRO_TCP4_H_
#define _GRO_TCP4_H_
-#define INVALID_ARRAY_INDEX 0xffffffffUL
+#include <gro_tcp.h>
+
#define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
/* Header fields representing a TCP/IPv4 flow */
struct tcp4_flow_key {
- struct rte_ether_addr eth_saddr;
- struct rte_ether_addr eth_daddr;
+ struct gro_tcp_flow tcp_flow;
uint32_t ip_src_addr;
uint32_t ip_dst_addr;
-
- uint32_t recv_ack;
- uint16_t src_port;
- uint16_t dst_port;
};
struct gro_tcp4_flow {
@@ -143,13 +139,9 @@ uint32_t gro_tcp4_tbl_pkt_count(void *tbl);
static inline int
is_same_tcp4_flow(struct tcp4_flow_key k1, struct tcp4_flow_key k2)
{
- return (rte_is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) &&
- rte_is_same_ether_addr(&k1.eth_daddr, &k2.eth_daddr) &&
- (k1.ip_src_addr == k2.ip_src_addr) &&
+ return ((k1.ip_src_addr == k2.ip_src_addr) &&
(k1.ip_dst_addr == k2.ip_dst_addr) &&
- (k1.recv_ack == k2.recv_ack) &&
- (k1.src_port == k2.src_port) &&
- (k1.dst_port == k2.dst_port));
+ is_same_tcp_flow(&k1.tcp_flow, &k2.tcp_flow));
}
#endif
diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c
index a37d8e2512..da8d1f2252 100644
--- a/lib/gro/gro_tcp6.c
+++ b/lib/gro/gro_tcp6.c
@@ -72,18 +72,6 @@ gro_tcp6_tbl_destroy(void *tbl)
rte_free(tcp_tbl);
}
-static inline uint32_t
-find_an_empty_item(struct gro_tcp6_tbl *tbl)
-{
- uint32_t i;
- uint32_t max_item_num = tbl->max_item_num;
-
- for (i = 0; i < max_item_num; i++)
- if (tbl->items[i].firstseg == NULL)
- return i;
- return INVALID_ARRAY_INDEX;
-}
-
static inline uint32_t
find_an_empty_flow(struct gro_tcp6_tbl *tbl)
{
@@ -97,79 +85,32 @@ find_an_empty_flow(struct gro_tcp6_tbl *tbl)
}
static inline uint32_t
-insert_new_item(struct gro_tcp6_tbl *tbl,
- struct rte_mbuf *pkt,
- uint64_t start_time,
- uint32_t prev_idx,
- uint32_t sent_seq,
- uint8_t is_atomic)
-{
- uint32_t item_idx;
-
- item_idx = find_an_empty_item(tbl);
- if (item_idx == INVALID_ARRAY_INDEX)
- return INVALID_ARRAY_INDEX;
-
- tbl->items[item_idx].firstseg = pkt;
- tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
- tbl->items[item_idx].start_time = start_time;
- tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
- tbl->items[item_idx].sent_seq = sent_seq;
- tbl->items[item_idx].nb_merged = 1;
- tbl->items[item_idx].is_atomic = is_atomic;
- tbl->item_num++;
-
- /* if the previous packet exists, chain them together. */
- if (prev_idx != INVALID_ARRAY_INDEX) {
- tbl->items[item_idx].next_pkt_idx =
- tbl->items[prev_idx].next_pkt_idx;
- tbl->items[prev_idx].next_pkt_idx = item_idx;
- }
-
- return item_idx;
-}
-
-static inline uint32_t
-delete_item(struct gro_tcp6_tbl *tbl, uint32_t item_idx,
- uint32_t prev_item_idx)
-{
- uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
-
- /* NULL indicates an empty item */
- tbl->items[item_idx].firstseg = NULL;
- tbl->item_num--;
- if (prev_item_idx != INVALID_ARRAY_INDEX)
- tbl->items[prev_item_idx].next_pkt_idx = next_idx;
-
- return next_idx;
-}
-
-static inline uint32_t
-insert_new_flow(struct gro_tcp6_tbl *tbl,
- struct tcp6_flow_key *src,
- rte_be32_t vtc_flow,
+gro_tcp6_insert_flow(void *tbl,
+ void *key,
uint32_t item_idx)
{
- struct tcp6_flow_key *dst;
+ struct gro_tcp6_tbl *tcp6_tbl = (struct gro_tcp6_tbl *)tbl;
+ struct tcp6_flow_key *dst_key;
+ struct tcp6_flow_key *src_key = (struct tcp6_flow_key *)key;
uint32_t flow_idx;
+ struct gro_tcp_flow *src_flow;
+ struct gro_tcp_flow *dst_flow;
- flow_idx = find_an_empty_flow(tbl);
+ flow_idx = find_an_empty_flow(tcp6_tbl);
if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
return INVALID_ARRAY_INDEX;
- dst = &(tbl->flows[flow_idx].key);
+ dst_key = &(tcp6_tbl->flows[flow_idx].key);
+ dst_flow = &dst_key->tcp_flow;
+ src_flow = &src_key->tcp_flow;
- rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
- rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
- memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst->src_addr));
- memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst->dst_addr));
- dst->recv_ack = src->recv_ack;
- dst->src_port = src->src_port;
- dst->dst_port = src->dst_port;
+ ASSIGN_TCP_FLOW_KEY(src_flow, dst_flow)
+ memcpy(&dst_key->src_addr[0], &src_key->src_addr[0], sizeof(dst_key->src_addr));
+ memcpy(&dst_key->dst_addr[0], &src_key->dst_addr[0], sizeof(dst_key->dst_addr));
+ dst_key->vtc_flow = src_key->vtc_flow;
- tbl->flows[flow_idx].start_index = item_idx;
- tbl->flow_num++;
- tbl->flows->vtc_flow = vtc_flow;
+ tcp6_tbl->flows[flow_idx].start_index = item_idx;
+ tcp6_tbl->flow_num++;
return flow_idx;
}
@@ -189,6 +130,47 @@ update_header(struct gro_tcp_item *item)
pkt->l2_len - pkt->l3_len);
}
+static uint32_t
+gro_tcp6_flow_lookup(void *tbl, void *key)
+{
+ struct gro_tcp6_tbl *tcp6_tbl = (struct gro_tcp6_tbl *)tbl;
+ struct tcp6_flow_key *tcp6_key = (struct tcp6_flow_key *)key;
+ uint32_t max_flow_num = tcp6_tbl->max_flow_num;
+ uint32_t remaining_flow_num = tcp6_tbl->flow_num;
+ uint32_t i;
+
+ /* Search for a matched flow. */
+ max_flow_num = tcp6_tbl->max_flow_num;
+ remaining_flow_num = tcp6_tbl->flow_num;
+ for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
+ if (tcp6_tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
+ if (is_same_tcp6_flow(&tcp6_tbl->flows[i].key, tcp6_key)) {
+ return i;
+ }
+ remaining_flow_num--;
+ }
+ }
+
+ return INVALID_ARRAY_INDEX;
+}
+
+static void
+gro_tcp6_flow_key_init(void *key, struct rte_tcp_hdr *tcp_hdr)
+{
+ struct tcp6_flow_key *tcp6_key = (struct tcp6_flow_key *)key;
+ struct gro_tcp_flow *tcp_flow = &tcp6_key->tcp_flow;
+
+ tcp_flow->src_port = tcp_hdr->src_port;
+ tcp_flow->dst_port = tcp_hdr->dst_port;
+ tcp_flow->recv_ack = tcp_hdr->recv_ack;
+}
+
+struct gro_tcp_flow_ops gro_tcp6_flows_ops = {
+ .tcp_flow_lookup = gro_tcp6_flow_lookup,
+ .tcp_flow_insert = gro_tcp6_insert_flow,
+ .tcp_flow_key_init = gro_tcp6_flow_key_init
+};
+
int32_t
gro_tcp6_reassemble(struct rte_mbuf *pkt,
struct gro_tcp6_tbl *tbl,
@@ -196,16 +178,9 @@ gro_tcp6_reassemble(struct rte_mbuf *pkt,
{
struct rte_ether_hdr *eth_hdr;
struct rte_ipv6_hdr *ipv6_hdr;
- struct rte_tcp_hdr *tcp_hdr;
- uint32_t sent_seq;
int32_t tcp_dl;
uint16_t ip_tlen;
struct tcp6_flow_key key;
- uint32_t cur_idx, prev_idx, item_idx;
- uint32_t i, max_flow_num, remaining_flow_num;
- int cmp;
- uint8_t find;
- rte_be32_t vtc_flow_diff;
/*
* Don't process the packet whose TCP header length is greater
@@ -216,14 +191,6 @@ gro_tcp6_reassemble(struct rte_mbuf *pkt,
eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
- tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv6_hdr + pkt->l3_len);
-
- /*
- * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
- * or CWR set.
- */
- if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
- return -1;
ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
/*
@@ -242,95 +209,15 @@ gro_tcp6_reassemble(struct rte_mbuf *pkt,
if (tcp_dl <= 0)
return -1;
- sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
-
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.tcp_flow.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.tcp_flow.eth_daddr));
memcpy(&key.src_addr[0], &ipv6_hdr->src_addr, sizeof(key.src_addr));
memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr, sizeof(key.dst_addr));
- key.src_port = tcp_hdr->src_port;
- key.dst_port = tcp_hdr->dst_port;
- key.recv_ack = tcp_hdr->recv_ack;
+ key.vtc_flow = ipv6_hdr->vtc_flow;
- /* Search for a matched flow. */
- max_flow_num = tbl->max_flow_num;
- remaining_flow_num = tbl->flow_num;
- find = 0;
- for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
- if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
- if (is_same_tcp6_flow(tbl->flows[i].key, key)) {
- /*
- * IP version (4) Traffic Class (8) Flow Label (20)
- * All fields except Traffic class should be same
- */
- vtc_flow_diff = (ipv6_hdr->vtc_flow ^ tbl->flows->vtc_flow);
- if (vtc_flow_diff & htonl(0xF00FFFFF))
- continue;
- find = 1;
- break;
- }
- remaining_flow_num--;
- }
- }
-
- /*
- * Fail to find a matched flow. Insert a new flow and store the
- * packet into the flow.
- */
- if (find == 0) {
- item_idx = insert_new_item(tbl, pkt, start_time,
- INVALID_ARRAY_INDEX, sent_seq, true);
- if (item_idx == INVALID_ARRAY_INDEX)
- return -1;
- if (insert_new_flow(tbl, &key, ipv6_hdr->vtc_flow, item_idx) ==
- INVALID_ARRAY_INDEX) {
- /*
- * Fail to insert a new flow, so delete the
- * stored packet.
- */
- delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
- return -1;
- }
- return 0;
- }
-
- /*
- * Check all packets in the flow and try to find a neighbor for
- * the input packet.
- */
- cur_idx = tbl->flows[i].start_index;
- prev_idx = cur_idx;
- do {
- cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
- sent_seq, 0, pkt->l4_len, tcp_dl, 0,
- true);
- if (cmp) {
- if (merge_two_tcp_packets(&(tbl->items[cur_idx]),
- pkt, cmp, sent_seq, 0, 0)) {
- return 1;
- }
-
- /*
- * Fail to merge the two packets, as the packet
- * length is greater than the max value. Store
- * the packet into the flow.
- */
- if (insert_new_item(tbl, pkt, start_time, cur_idx,
- sent_seq, true) ==
- INVALID_ARRAY_INDEX)
- return -1;
- return 0;
- }
- prev_idx = cur_idx;
- cur_idx = tbl->items[cur_idx].next_pkt_idx;
- } while (cur_idx != INVALID_ARRAY_INDEX);
-
- /* Fail to find a neighbor, so store the packet into the flow. */
- if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
- true) == INVALID_ARRAY_INDEX)
- return -1;
-
- return 0;
+ return gro_tcp_reassemble(pkt, tbl, &key, tcp_dl, &gro_tcp6_flows_ops, tbl->items,
+ &tbl->item_num, tbl->max_item_num,
+ 0, true, start_time);
}
uint16_t
@@ -357,7 +244,7 @@ gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
* Delete the packet and get the next
* packet in the flow.
*/
- j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
+ j = delete_tcp_item(tbl->items, j, &tbl->item_num, INVALID_ARRAY_INDEX);
tbl->flows[i].start_index = j;
if (j == INVALID_ARRAY_INDEX)
tbl->flow_num--;
diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h
index 91325c0da2..9eb6be2dfe 100644
--- a/lib/gro/gro_tcp6.h
+++ b/lib/gro/gro_tcp6.h
@@ -5,24 +5,21 @@
#ifndef _GRO_TCP6_H_
#define _GRO_TCP6_H_
+#include <gro_tcp.h>
+
#define INVALID_ARRAY_INDEX 0xffffffffUL
#define GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
/* Header fields representing a TCP/IPv6 flow */
struct tcp6_flow_key {
- struct rte_ether_addr eth_saddr;
- struct rte_ether_addr eth_daddr;
+ struct gro_tcp_flow tcp_flow;
uint8_t src_addr[16];
uint8_t dst_addr[16];
-
- uint32_t recv_ack;
- uint16_t src_port;
- uint16_t dst_port;
+ rte_be32_t vtc_flow;
};
struct gro_tcp6_flow {
struct tcp6_flow_key key;
- rte_be32_t vtc_flow;
/*
* The index of the first packet in the flow.
* INVALID_ARRAY_INDEX indicates an empty flow.
@@ -142,9 +139,25 @@ uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
* Check if two TCP/IPv6 packets belong to the same flow.
*/
static inline int
-is_same_tcp6_flow(struct tcp6_flow_key k1, struct tcp6_flow_key k2)
+is_same_tcp6_flow(struct tcp6_flow_key *k1, struct tcp6_flow_key *k2)
{
- return (!memcmp(&k1, &k2, sizeof(struct tcp6_flow_key)));
+ rte_be32_t vtc_flow_diff;
+
+ if (memcmp(&k1->src_addr, &k2->src_addr, 16)) {
+ return 0;
+ }
+ if (memcmp(&k1->dst_addr, &k2->dst_addr, 16)) {
+ return 0;
+ }
+ /*
+ * IP version (4) Traffic Class (8) Flow Label (20)
+ * All fields except Traffic class should be same
+ */
+ vtc_flow_diff = (k1->vtc_flow ^ k2->vtc_flow);
+ if (vtc_flow_diff & htonl(0xF00FFFFF))
+ return 0;
+
+ return is_same_tcp_flow(&k1->tcp_flow, &k2->tcp_flow);
}
#endif
diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
index 56b30b8c98..16aea7049f 100644
--- a/lib/gro/gro_vxlan_tcp4.c
+++ b/lib/gro/gro_vxlan_tcp4.c
@@ -164,15 +164,9 @@ insert_new_flow(struct gro_vxlan_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->inner_key.eth_saddr),
- &(dst->inner_key.eth_saddr));
- rte_ether_addr_copy(&(src->inner_key.eth_daddr),
- &(dst->inner_key.eth_daddr));
+ ASSIGN_TCP_FLOW_KEY((&(src->inner_key.tcp_flow)), (&(dst->inner_key.tcp_flow)));
dst->inner_key.ip_src_addr = src->inner_key.ip_src_addr;
dst->inner_key.ip_dst_addr = src->inner_key.ip_dst_addr;
- dst->inner_key.recv_ack = src->inner_key.recv_ack;
- dst->inner_key.src_port = src->inner_key.src_port;
- dst->inner_key.dst_port = src->inner_key.dst_port;
dst->vxlan_hdr.vx_flags = src->vxlan_hdr.vx_flags;
dst->vxlan_hdr.vx_vni = src->vxlan_hdr.vx_vni;
@@ -358,13 +352,13 @@ gro_vxlan_tcp4_reassemble(struct rte_mbuf *pkt,
sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.eth_daddr));
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.tcp_flow.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.tcp_flow.eth_daddr));
key.inner_key.ip_src_addr = ipv4_hdr->src_addr;
key.inner_key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.inner_key.recv_ack = tcp_hdr->recv_ack;
- key.inner_key.src_port = tcp_hdr->src_port;
- key.inner_key.dst_port = tcp_hdr->dst_port;
+ key.inner_key.tcp_flow.recv_ack = tcp_hdr->recv_ack;
+ key.inner_key.tcp_flow.src_port = tcp_hdr->src_port;
+ key.inner_key.tcp_flow.dst_port = tcp_hdr->dst_port;
key.vxlan_hdr.vx_flags = vxlan_hdr->vx_flags;
key.vxlan_hdr.vx_vni = vxlan_hdr->vx_vni;
diff --git a/lib/gro/meson.build b/lib/gro/meson.build
index dbce05220d..a7d73f21ec 100644
--- a/lib/gro/meson.build
+++ b/lib/gro/meson.build
@@ -3,6 +3,7 @@
sources = files(
'rte_gro.c',
+ 'gro_tcp.c',
'gro_tcp4.c',
'gro_tcp6.c',
'gro_udp4.c',
--
2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* [PATCH v3] gro : ipv6 changes to support GRO for TCP/ipv6
2022-10-20 18:14 ` [PATCH v2] " Kumara Parameshwaran
2023-05-12 2:47 ` Hu, Jiayu
2023-06-02 6:02 ` [PATCH v3] gro : ipv6-gro review comments to reduce code duplication across v4 and v6 Kumara Parameshwaran
@ 2023-06-02 6:34 ` Kumara Parameshwaran
2023-06-06 4:35 ` Hu, Jiayu
2023-06-06 14:58 ` [PATCH v4] " Kumara Parameshwaran
` (7 subsequent siblings)
10 siblings, 1 reply; 32+ messages in thread
From: Kumara Parameshwaran @ 2023-06-02 6:34 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev, Kumara Parameshwaran
The patch adds GRO support for TCP/ipv6 packets. This does not
include the support for vxlan, udp ipv6 packets.
Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
---
v1:
* Changes to support GRO for TCP/ipv6 packets. This does not include
vxlan changes.
* The GRO is performed only for ipv6 packets that does not contain
extension headers.
* The logic for the TCP coalescing remains the same, in ipv6 header
the source address, destination address, flow label, version fields
are expected to be the same.
* Re-organised the code to reuse certain tcp functions for both ipv4 and
ipv6 flows.
v2:
* Fix comments in gro_tcp6.h header file.
v3:
* Adderess review comments to fix code duplication for v4 and v6
lib/gro/gro_tcp.c | 160 ++++++++++++++++++++++++
lib/gro/gro_tcp.h | 63 ++++++++++
lib/gro/gro_tcp4.c | 255 ++++++++++++---------------------------
lib/gro/gro_tcp4.h | 18 +--
lib/gro/gro_tcp6.c | 243 ++++++++++---------------------------
lib/gro/gro_tcp6.h | 31 +++--
lib/gro/gro_vxlan_tcp4.c | 18 +--
lib/gro/meson.build | 1 +
8 files changed, 396 insertions(+), 393 deletions(-)
create mode 100644 lib/gro/gro_tcp.c
diff --git a/lib/gro/gro_tcp.c b/lib/gro/gro_tcp.c
new file mode 100644
index 0000000000..6a5aaada58
--- /dev/null
+++ b/lib/gro/gro_tcp.c
@@ -0,0 +1,160 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp.h"
+
+static inline uint32_t
+find_an_empty_item(struct gro_tcp_item *items,
+ uint32_t table_size)
+{
+ uint32_t i;
+
+ for (i = 0; i < table_size; i++)
+ if (items[i].firstseg == NULL)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t table_size,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic)
+{
+ uint32_t item_idx;
+
+ item_idx = find_an_empty_item(items, table_size);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return INVALID_ARRAY_INDEX;
+
+ items[item_idx].firstseg = pkt;
+ items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
+ items[item_idx].start_time = start_time;
+ items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
+ items[item_idx].sent_seq = sent_seq;
+ items[item_idx].ip_id = ip_id;
+ items[item_idx].nb_merged = 1;
+ items[item_idx].is_atomic = is_atomic;
+ (*item_num) += 1;
+
+ /* if the previous packet exists, chain them together. */
+ if (prev_idx != INVALID_ARRAY_INDEX) {
+ items[item_idx].next_pkt_idx =
+ items[prev_idx].next_pkt_idx;
+ items[prev_idx].next_pkt_idx = item_idx;
+ }
+
+ return item_idx;
+}
+
+uint32_t
+delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx)
+{
+ uint32_t next_idx = items[item_idx].next_pkt_idx;
+
+ /* NULL indicates an empty item */
+ items[item_idx].firstseg = NULL;
+ (*item_num) -= 1;
+ if (prev_item_idx != INVALID_ARRAY_INDEX)
+ items[prev_item_idx].next_pkt_idx = next_idx;
+
+ return next_idx;
+}
+
+int32_t
+gro_tcp_reassemble(struct rte_mbuf *pkt,
+ void *tbl,
+ void *key,
+ int32_t tcp_dl,
+ struct gro_tcp_flow_ops *ops,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t table_size,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time)
+{
+ uint32_t item_idx;
+ uint32_t cur_idx;
+ uint32_t prev_idx;
+ struct rte_tcp_hdr *tcp_hdr;
+ int cmp;
+ uint32_t sent_seq;
+
+ tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt->l2_len + pkt->l3_len);
+ /*
+ * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
+ * or CWR set.
+ */
+ if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
+ return -1;
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+
+ ops->tcp_flow_key_init(key, tcp_hdr);
+
+ item_idx = ops->tcp_flow_lookup(tbl, key);
+ if (item_idx == INVALID_ARRAY_INDEX) {
+ item_idx = insert_new_tcp_item(pkt, items, item_num, table_size, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, ip_id,
+ is_atomic);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return -1;
+ if (ops->tcp_flow_insert(tbl, key, item_idx) ==
+ INVALID_ARRAY_INDEX) {
+ /*
+ * Fail to insert a new flow, so delete the
+ * stored packet.
+ */
+ delete_tcp_item(items, item_idx, item_num, INVALID_ARRAY_INDEX);
+ return -1;
+ }
+ return 0;
+ }
+ /*
+ * Check all packets in the flow and try to find a neighbor for
+ * the input packet.
+ */
+ cur_idx = item_idx;
+ prev_idx = cur_idx;
+ do {
+ cmp = check_seq_option(&items[cur_idx], tcp_hdr,
+ sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
+ is_atomic);
+ if (cmp) {
+ if (merge_two_tcp_packets(&items[cur_idx],
+ pkt, cmp, sent_seq, ip_id, 0))
+ return 1;
+ /*
+ * Fail to merge the two packets, as the packet
+ * length is greater than the max value. Store
+ * the packet into the flow.
+ */
+ if (insert_new_tcp_item(pkt, items, item_num, table_size, start_time, cur_idx,
+ sent_seq, ip_id, is_atomic) ==
+ INVALID_ARRAY_INDEX)
+ return -1;
+ return 0;
+ }
+ prev_idx = cur_idx;
+ cur_idx = items[cur_idx].next_pkt_idx;
+ } while (cur_idx != INVALID_ARRAY_INDEX);
+
+ /* Fail to find a neighbor, so store the packet into the flow. */
+ if (insert_new_tcp_item(pkt, items, item_num, table_size, start_time, prev_idx, sent_seq,
+ ip_id, is_atomic) == INVALID_ARRAY_INDEX)
+ return -1;
+
+ return 0;
+
+}
diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h
index c5d248a022..202f485c18 100644
--- a/lib/gro/gro_tcp.h
+++ b/lib/gro/gro_tcp.h
@@ -1,6 +1,8 @@
#ifndef _GRO_TCP_H_
#define _GRO_TCP_H_
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+
#include <rte_tcp.h>
/*
@@ -14,6 +16,31 @@
#define INVALID_TCP_HDRLEN(len) \
(((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
+struct gro_tcp_flow {
+ struct rte_ether_addr eth_saddr;
+ struct rte_ether_addr eth_daddr;
+ uint32_t recv_ack;
+ uint16_t src_port;
+ uint16_t dst_port;
+};
+
+#define ASSIGN_TCP_FLOW_KEY(k1, k2) \
+ rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
+ rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr)); \
+ k2->recv_ack = k1->recv_ack; \
+ k2->src_port = k1->src_port; \
+ k2->dst_port = k1->dst_port;
+
+typedef uint32_t (*gro_tcp_flow_lookup)(void *table, void *key);
+typedef uint32_t (*gro_tcp_flow_insert)(void *table, void *key, uint32_t item_idx);
+typedef void (*gro_tcp_flow_key_init)(void *key, struct rte_tcp_hdr *tcp_hdr);
+
+struct gro_tcp_flow_ops {
+ gro_tcp_flow_lookup tcp_flow_lookup;
+ gro_tcp_flow_insert tcp_flow_insert;
+ gro_tcp_flow_key_init tcp_flow_key_init;
+};
+
struct gro_tcp_item {
/*
* The first MBUF segment of the packet. If the value
@@ -44,6 +71,36 @@ struct gro_tcp_item {
uint8_t is_atomic;
};
+uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t table_size,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic);
+
+uint32_t
+delete_tcp_item(struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx);
+
+int32_t
+gro_tcp_reassemble(struct rte_mbuf *pkt,
+ void *tbl,
+ void *key,
+ int32_t tcp_dl,
+ struct gro_tcp_flow_ops *ops,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t table_size,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time);
+
/*
* Merge two TCP packets without updating checksums.
* If cmp is larger than 0, append the new packet to the
@@ -152,4 +209,10 @@ check_seq_option(struct gro_tcp_item *item,
return 0;
}
+static inline int
+is_same_tcp_flow(struct gro_tcp_flow *k1, struct gro_tcp_flow *k2)
+{
+ return (!memcmp(k1, k2, sizeof(struct gro_tcp_flow)));
+}
+
#endif
diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c
index 2cfb524689..cd510d3a7c 100644
--- a/lib/gro/gro_tcp4.c
+++ b/lib/gro/gro_tcp4.c
@@ -72,18 +72,6 @@ gro_tcp4_tbl_destroy(void *tbl)
rte_free(tcp_tbl);
}
-static inline uint32_t
-find_an_empty_item(struct gro_tcp4_tbl *tbl)
-{
- uint32_t i;
- uint32_t max_item_num = tbl->max_item_num;
-
- for (i = 0; i < max_item_num; i++)
- if (tbl->items[i].firstseg == NULL)
- return i;
- return INVALID_ARRAY_INDEX;
-}
-
static inline uint32_t
find_an_empty_flow(struct gro_tcp4_tbl *tbl)
{
@@ -97,98 +85,76 @@ find_an_empty_flow(struct gro_tcp4_tbl *tbl)
}
static inline uint32_t
-insert_new_item(struct gro_tcp4_tbl *tbl,
- struct rte_mbuf *pkt,
- uint64_t start_time,
- uint32_t prev_idx,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint8_t is_atomic)
+gro_tcp4_insert_flow(void *tbl,
+ void *key,
+ uint32_t item_idx)
{
- uint32_t item_idx;
+ struct gro_tcp4_tbl *tcp4_tbl = (struct gro_tcp4_tbl *)tbl;
+ struct tcp4_flow_key *src_key = (struct tcp4_flow_key *)key;
+ struct tcp4_flow_key *dst_key;
+ struct gro_tcp_flow *src_flow;
+ struct gro_tcp_flow *dst_flow;
+ uint32_t flow_idx;
- item_idx = find_an_empty_item(tbl);
- if (item_idx == INVALID_ARRAY_INDEX)
+ flow_idx = find_an_empty_flow(tcp4_tbl);
+ if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
return INVALID_ARRAY_INDEX;
- tbl->items[item_idx].firstseg = pkt;
- tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
- tbl->items[item_idx].start_time = start_time;
- tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
- tbl->items[item_idx].sent_seq = sent_seq;
- tbl->items[item_idx].ip_id = ip_id;
- tbl->items[item_idx].nb_merged = 1;
- tbl->items[item_idx].is_atomic = is_atomic;
- tbl->item_num++;
-
- /* if the previous packet exists, chain them together. */
- if (prev_idx != INVALID_ARRAY_INDEX) {
- tbl->items[item_idx].next_pkt_idx =
- tbl->items[prev_idx].next_pkt_idx;
- tbl->items[prev_idx].next_pkt_idx = item_idx;
- }
+ dst_key = &(tcp4_tbl->flows[flow_idx].key);
+ dst_flow = &dst_key->tcp_flow;
+ src_flow = &src_key->tcp_flow;
- return item_idx;
-}
+ ASSIGN_TCP_FLOW_KEY(src_flow, dst_flow)
-static inline uint32_t
-delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
- uint32_t prev_item_idx)
-{
- uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
+ dst_key->ip_src_addr = src_key->ip_src_addr;
+ dst_key->ip_dst_addr = src_key->ip_dst_addr;
- /* NULL indicates an empty item */
- tbl->items[item_idx].firstseg = NULL;
- tbl->item_num--;
- if (prev_item_idx != INVALID_ARRAY_INDEX)
- tbl->items[prev_item_idx].next_pkt_idx = next_idx;
+ tcp4_tbl->flows[flow_idx].start_index = item_idx;
+ tcp4_tbl->flow_num++;
- return next_idx;
+ return flow_idx;
}
-static inline uint32_t
-insert_new_flow(struct gro_tcp4_tbl *tbl,
- struct tcp4_flow_key *src,
- uint32_t item_idx)
-{
- struct tcp4_flow_key *dst;
- uint32_t flow_idx;
-
- flow_idx = find_an_empty_flow(tbl);
- if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
- return INVALID_ARRAY_INDEX;
-
- dst = &(tbl->flows[flow_idx].key);
-
- rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
- rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
- dst->ip_src_addr = src->ip_src_addr;
- dst->ip_dst_addr = src->ip_dst_addr;
- dst->recv_ack = src->recv_ack;
- dst->src_port = src->src_port;
- dst->dst_port = src->dst_port;
- tbl->flows[flow_idx].start_index = item_idx;
- tbl->flow_num++;
+static uint32_t
+gro_tcp4_flow_lookup(void *tbl, void *key)
+{
+ struct gro_tcp4_tbl *tcp4_tbl = (struct gro_tcp4_tbl *)tbl;
+ struct tcp4_flow_key *tcp4_key = (struct tcp4_flow_key *)key;
+ uint32_t max_flow_num = tcp4_tbl->max_flow_num;
+ uint32_t remaining_flow_num = tcp4_tbl->flow_num;
+ uint32_t i;
- return flow_idx;
+ /* Search for a matched flow. */
+ for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
+ if (tcp4_tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
+ if (is_same_tcp4_flow(tcp4_tbl->flows[i].key, *tcp4_key)) {
+ return i;
+ }
+ remaining_flow_num--;
+ }
+ }
+ return INVALID_ARRAY_INDEX;
}
-/*
- * update the packet length for the flushed packet.
- */
-static inline void
-update_header(struct gro_tcp_item *item)
+static void
+gro_tcp4_flow_key_init(void *key, struct rte_tcp_hdr *tcp_hdr)
{
- struct rte_ipv4_hdr *ipv4_hdr;
- struct rte_mbuf *pkt = item->firstseg;
+ struct tcp4_flow_key *tcp4_key = (struct tcp4_flow_key *)key;
+ struct gro_tcp_flow *tcp_flow = &tcp4_key->tcp_flow;
+
+ tcp_flow->src_port = tcp_hdr->src_port;
+ tcp_flow->dst_port = tcp_hdr->dst_port;
+ tcp_flow->recv_ack = tcp_hdr->recv_ack;
- ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
- pkt->l2_len);
- ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
- pkt->l2_len);
}
+struct gro_tcp_flow_ops gro_tcp4_flows_ops = {
+ .tcp_flow_lookup = gro_tcp4_flow_lookup,
+ .tcp_flow_insert = gro_tcp4_insert_flow,
+ .tcp_flow_key_init = gro_tcp4_flow_key_init
+};
+
int32_t
gro_tcp4_reassemble(struct rte_mbuf *pkt,
struct gro_tcp4_tbl *tbl,
@@ -196,17 +162,10 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
{
struct rte_ether_hdr *eth_hdr;
struct rte_ipv4_hdr *ipv4_hdr;
- struct rte_tcp_hdr *tcp_hdr;
- uint32_t sent_seq;
int32_t tcp_dl;
uint16_t ip_id, hdr_len, frag_off, ip_tlen;
uint8_t is_atomic;
-
struct tcp4_flow_key key;
- uint32_t cur_idx, prev_idx, item_idx;
- uint32_t i, max_flow_num, remaining_flow_num;
- int cmp;
- uint8_t find;
/*
* Don't process the packet whose TCP header length is greater
@@ -217,21 +176,13 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
- tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
- hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
-
- /*
- * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
- * or CWR set.
- */
- if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
- return -1;
/* trim the tail padding bytes */
ip_tlen = rte_be_to_cpu_16(ipv4_hdr->total_length);
if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len))
rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len);
+ hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
/*
* Don't process the packet whose payload length is less than or
* equal to 0.
@@ -240,6 +191,10 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
if (tcp_dl <= 0)
return -1;
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.tcp_flow.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.tcp_flow.eth_daddr));
+ key.ip_src_addr = ipv4_hdr->src_addr;
+ key.ip_dst_addr = ipv4_hdr->dst_addr;
/*
* Save IPv4 ID for the packet whose DF bit is 0. For the packet
* whose DF bit is 1, IPv4 ID is ignored.
@@ -247,87 +202,25 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) == RTE_IPV4_HDR_DF_FLAG;
ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
- sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
- key.ip_src_addr = ipv4_hdr->src_addr;
- key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.src_port = tcp_hdr->src_port;
- key.dst_port = tcp_hdr->dst_port;
- key.recv_ack = tcp_hdr->recv_ack;
-
- /* Search for a matched flow. */
- max_flow_num = tbl->max_flow_num;
- remaining_flow_num = tbl->flow_num;
- find = 0;
- for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
- if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
- if (is_same_tcp4_flow(tbl->flows[i].key, key)) {
- find = 1;
- break;
- }
- remaining_flow_num--;
- }
- }
-
- /*
- * Fail to find a matched flow. Insert a new flow and store the
- * packet into the flow.
- */
- if (find == 0) {
- item_idx = insert_new_item(tbl, pkt, start_time,
- INVALID_ARRAY_INDEX, sent_seq, ip_id,
- is_atomic);
- if (item_idx == INVALID_ARRAY_INDEX)
- return -1;
- if (insert_new_flow(tbl, &key, item_idx) ==
- INVALID_ARRAY_INDEX) {
- /*
- * Fail to insert a new flow, so delete the
- * stored packet.
- */
- delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
- return -1;
- }
- return 0;
- }
-
- /*
- * Check all packets in the flow and try to find a neighbor for
- * the input packet.
- */
- cur_idx = tbl->flows[i].start_index;
- prev_idx = cur_idx;
- do {
- cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
- sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
- is_atomic);
- if (cmp) {
- if (merge_two_tcp_packets(&(tbl->items[cur_idx]),
- pkt, cmp, sent_seq, ip_id, 0))
- return 1;
- /*
- * Fail to merge the two packets, as the packet
- * length is greater than the max value. Store
- * the packet into the flow.
- */
- if (insert_new_item(tbl, pkt, start_time, cur_idx,
- sent_seq, ip_id, is_atomic) ==
- INVALID_ARRAY_INDEX)
- return -1;
- return 0;
- }
- prev_idx = cur_idx;
- cur_idx = tbl->items[cur_idx].next_pkt_idx;
- } while (cur_idx != INVALID_ARRAY_INDEX);
+ return gro_tcp_reassemble(pkt, tbl, &key, tcp_dl, &gro_tcp4_flows_ops, tbl->items,
+ &tbl->item_num, tbl->max_item_num,
+ ip_id, is_atomic, start_time);
+}
- /* Fail to find a neighbor, so store the packet into the flow. */
- if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
- ip_id, is_atomic) == INVALID_ARRAY_INDEX)
- return -1;
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
- return 0;
+ ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len);
}
uint16_t
@@ -354,7 +247,7 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
* Delete the packet and get the next
* packet in the flow.
*/
- j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
+ j = delete_tcp_item(tbl->items, j, &tbl->item_num, INVALID_ARRAY_INDEX);
tbl->flows[i].start_index = j;
if (j == INVALID_ARRAY_INDEX)
tbl->flow_num--;
diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h
index 634a215b98..8ff1eed096 100644
--- a/lib/gro/gro_tcp4.h
+++ b/lib/gro/gro_tcp4.h
@@ -5,19 +5,15 @@
#ifndef _GRO_TCP4_H_
#define _GRO_TCP4_H_
-#define INVALID_ARRAY_INDEX 0xffffffffUL
+#include <gro_tcp.h>
+
#define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
/* Header fields representing a TCP/IPv4 flow */
struct tcp4_flow_key {
- struct rte_ether_addr eth_saddr;
- struct rte_ether_addr eth_daddr;
+ struct gro_tcp_flow tcp_flow;
uint32_t ip_src_addr;
uint32_t ip_dst_addr;
-
- uint32_t recv_ack;
- uint16_t src_port;
- uint16_t dst_port;
};
struct gro_tcp4_flow {
@@ -143,13 +139,9 @@ uint32_t gro_tcp4_tbl_pkt_count(void *tbl);
static inline int
is_same_tcp4_flow(struct tcp4_flow_key k1, struct tcp4_flow_key k2)
{
- return (rte_is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) &&
- rte_is_same_ether_addr(&k1.eth_daddr, &k2.eth_daddr) &&
- (k1.ip_src_addr == k2.ip_src_addr) &&
+ return ((k1.ip_src_addr == k2.ip_src_addr) &&
(k1.ip_dst_addr == k2.ip_dst_addr) &&
- (k1.recv_ack == k2.recv_ack) &&
- (k1.src_port == k2.src_port) &&
- (k1.dst_port == k2.dst_port));
+ is_same_tcp_flow(&k1.tcp_flow, &k2.tcp_flow));
}
#endif
diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c
index a37d8e2512..da8d1f2252 100644
--- a/lib/gro/gro_tcp6.c
+++ b/lib/gro/gro_tcp6.c
@@ -72,18 +72,6 @@ gro_tcp6_tbl_destroy(void *tbl)
rte_free(tcp_tbl);
}
-static inline uint32_t
-find_an_empty_item(struct gro_tcp6_tbl *tbl)
-{
- uint32_t i;
- uint32_t max_item_num = tbl->max_item_num;
-
- for (i = 0; i < max_item_num; i++)
- if (tbl->items[i].firstseg == NULL)
- return i;
- return INVALID_ARRAY_INDEX;
-}
-
static inline uint32_t
find_an_empty_flow(struct gro_tcp6_tbl *tbl)
{
@@ -97,79 +85,32 @@ find_an_empty_flow(struct gro_tcp6_tbl *tbl)
}
static inline uint32_t
-insert_new_item(struct gro_tcp6_tbl *tbl,
- struct rte_mbuf *pkt,
- uint64_t start_time,
- uint32_t prev_idx,
- uint32_t sent_seq,
- uint8_t is_atomic)
-{
- uint32_t item_idx;
-
- item_idx = find_an_empty_item(tbl);
- if (item_idx == INVALID_ARRAY_INDEX)
- return INVALID_ARRAY_INDEX;
-
- tbl->items[item_idx].firstseg = pkt;
- tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
- tbl->items[item_idx].start_time = start_time;
- tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
- tbl->items[item_idx].sent_seq = sent_seq;
- tbl->items[item_idx].nb_merged = 1;
- tbl->items[item_idx].is_atomic = is_atomic;
- tbl->item_num++;
-
- /* if the previous packet exists, chain them together. */
- if (prev_idx != INVALID_ARRAY_INDEX) {
- tbl->items[item_idx].next_pkt_idx =
- tbl->items[prev_idx].next_pkt_idx;
- tbl->items[prev_idx].next_pkt_idx = item_idx;
- }
-
- return item_idx;
-}
-
-static inline uint32_t
-delete_item(struct gro_tcp6_tbl *tbl, uint32_t item_idx,
- uint32_t prev_item_idx)
-{
- uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
-
- /* NULL indicates an empty item */
- tbl->items[item_idx].firstseg = NULL;
- tbl->item_num--;
- if (prev_item_idx != INVALID_ARRAY_INDEX)
- tbl->items[prev_item_idx].next_pkt_idx = next_idx;
-
- return next_idx;
-}
-
-static inline uint32_t
-insert_new_flow(struct gro_tcp6_tbl *tbl,
- struct tcp6_flow_key *src,
- rte_be32_t vtc_flow,
+gro_tcp6_insert_flow(void *tbl,
+ void *key,
uint32_t item_idx)
{
- struct tcp6_flow_key *dst;
+ struct gro_tcp6_tbl *tcp6_tbl = (struct gro_tcp6_tbl *)tbl;
+ struct tcp6_flow_key *dst_key;
+ struct tcp6_flow_key *src_key = (struct tcp6_flow_key *)key;
uint32_t flow_idx;
+ struct gro_tcp_flow *src_flow;
+ struct gro_tcp_flow *dst_flow;
- flow_idx = find_an_empty_flow(tbl);
+ flow_idx = find_an_empty_flow(tcp6_tbl);
if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
return INVALID_ARRAY_INDEX;
- dst = &(tbl->flows[flow_idx].key);
+ dst_key = &(tcp6_tbl->flows[flow_idx].key);
+ dst_flow = &dst_key->tcp_flow;
+ src_flow = &src_key->tcp_flow;
- rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
- rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
- memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst->src_addr));
- memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst->dst_addr));
- dst->recv_ack = src->recv_ack;
- dst->src_port = src->src_port;
- dst->dst_port = src->dst_port;
+ ASSIGN_TCP_FLOW_KEY(src_flow, dst_flow)
+ memcpy(&dst_key->src_addr[0], &src_key->src_addr[0], sizeof(dst_key->src_addr));
+ memcpy(&dst_key->dst_addr[0], &src_key->dst_addr[0], sizeof(dst_key->dst_addr));
+ dst_key->vtc_flow = src_key->vtc_flow;
- tbl->flows[flow_idx].start_index = item_idx;
- tbl->flow_num++;
- tbl->flows->vtc_flow = vtc_flow;
+ tcp6_tbl->flows[flow_idx].start_index = item_idx;
+ tcp6_tbl->flow_num++;
return flow_idx;
}
@@ -189,6 +130,47 @@ update_header(struct gro_tcp_item *item)
pkt->l2_len - pkt->l3_len);
}
+static uint32_t
+gro_tcp6_flow_lookup(void *tbl, void *key)
+{
+ struct gro_tcp6_tbl *tcp6_tbl = (struct gro_tcp6_tbl *)tbl;
+ struct tcp6_flow_key *tcp6_key = (struct tcp6_flow_key *)key;
+ uint32_t max_flow_num = tcp6_tbl->max_flow_num;
+ uint32_t remaining_flow_num = tcp6_tbl->flow_num;
+ uint32_t i;
+
+ /* Search for a matched flow. */
+ max_flow_num = tcp6_tbl->max_flow_num;
+ remaining_flow_num = tcp6_tbl->flow_num;
+ for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
+ if (tcp6_tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
+ if (is_same_tcp6_flow(&tcp6_tbl->flows[i].key, tcp6_key)) {
+ return i;
+ }
+ remaining_flow_num--;
+ }
+ }
+
+ return INVALID_ARRAY_INDEX;
+}
+
+static void
+gro_tcp6_flow_key_init(void *key, struct rte_tcp_hdr *tcp_hdr)
+{
+ struct tcp6_flow_key *tcp6_key = (struct tcp6_flow_key *)key;
+ struct gro_tcp_flow *tcp_flow = &tcp6_key->tcp_flow;
+
+ tcp_flow->src_port = tcp_hdr->src_port;
+ tcp_flow->dst_port = tcp_hdr->dst_port;
+ tcp_flow->recv_ack = tcp_hdr->recv_ack;
+}
+
+struct gro_tcp_flow_ops gro_tcp6_flows_ops = {
+ .tcp_flow_lookup = gro_tcp6_flow_lookup,
+ .tcp_flow_insert = gro_tcp6_insert_flow,
+ .tcp_flow_key_init = gro_tcp6_flow_key_init
+};
+
int32_t
gro_tcp6_reassemble(struct rte_mbuf *pkt,
struct gro_tcp6_tbl *tbl,
@@ -196,16 +178,9 @@ gro_tcp6_reassemble(struct rte_mbuf *pkt,
{
struct rte_ether_hdr *eth_hdr;
struct rte_ipv6_hdr *ipv6_hdr;
- struct rte_tcp_hdr *tcp_hdr;
- uint32_t sent_seq;
int32_t tcp_dl;
uint16_t ip_tlen;
struct tcp6_flow_key key;
- uint32_t cur_idx, prev_idx, item_idx;
- uint32_t i, max_flow_num, remaining_flow_num;
- int cmp;
- uint8_t find;
- rte_be32_t vtc_flow_diff;
/*
* Don't process the packet whose TCP header length is greater
@@ -216,14 +191,6 @@ gro_tcp6_reassemble(struct rte_mbuf *pkt,
eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
- tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv6_hdr + pkt->l3_len);
-
- /*
- * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
- * or CWR set.
- */
- if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
- return -1;
ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
/*
@@ -242,95 +209,15 @@ gro_tcp6_reassemble(struct rte_mbuf *pkt,
if (tcp_dl <= 0)
return -1;
- sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
-
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.tcp_flow.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.tcp_flow.eth_daddr));
memcpy(&key.src_addr[0], &ipv6_hdr->src_addr, sizeof(key.src_addr));
memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr, sizeof(key.dst_addr));
- key.src_port = tcp_hdr->src_port;
- key.dst_port = tcp_hdr->dst_port;
- key.recv_ack = tcp_hdr->recv_ack;
+ key.vtc_flow = ipv6_hdr->vtc_flow;
- /* Search for a matched flow. */
- max_flow_num = tbl->max_flow_num;
- remaining_flow_num = tbl->flow_num;
- find = 0;
- for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
- if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
- if (is_same_tcp6_flow(tbl->flows[i].key, key)) {
- /*
- * IP version (4) Traffic Class (8) Flow Label (20)
- * All fields except Traffic class should be same
- */
- vtc_flow_diff = (ipv6_hdr->vtc_flow ^ tbl->flows->vtc_flow);
- if (vtc_flow_diff & htonl(0xF00FFFFF))
- continue;
- find = 1;
- break;
- }
- remaining_flow_num--;
- }
- }
-
- /*
- * Fail to find a matched flow. Insert a new flow and store the
- * packet into the flow.
- */
- if (find == 0) {
- item_idx = insert_new_item(tbl, pkt, start_time,
- INVALID_ARRAY_INDEX, sent_seq, true);
- if (item_idx == INVALID_ARRAY_INDEX)
- return -1;
- if (insert_new_flow(tbl, &key, ipv6_hdr->vtc_flow, item_idx) ==
- INVALID_ARRAY_INDEX) {
- /*
- * Fail to insert a new flow, so delete the
- * stored packet.
- */
- delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
- return -1;
- }
- return 0;
- }
-
- /*
- * Check all packets in the flow and try to find a neighbor for
- * the input packet.
- */
- cur_idx = tbl->flows[i].start_index;
- prev_idx = cur_idx;
- do {
- cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
- sent_seq, 0, pkt->l4_len, tcp_dl, 0,
- true);
- if (cmp) {
- if (merge_two_tcp_packets(&(tbl->items[cur_idx]),
- pkt, cmp, sent_seq, 0, 0)) {
- return 1;
- }
-
- /*
- * Fail to merge the two packets, as the packet
- * length is greater than the max value. Store
- * the packet into the flow.
- */
- if (insert_new_item(tbl, pkt, start_time, cur_idx,
- sent_seq, true) ==
- INVALID_ARRAY_INDEX)
- return -1;
- return 0;
- }
- prev_idx = cur_idx;
- cur_idx = tbl->items[cur_idx].next_pkt_idx;
- } while (cur_idx != INVALID_ARRAY_INDEX);
-
- /* Fail to find a neighbor, so store the packet into the flow. */
- if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
- true) == INVALID_ARRAY_INDEX)
- return -1;
-
- return 0;
+ return gro_tcp_reassemble(pkt, tbl, &key, tcp_dl, &gro_tcp6_flows_ops, tbl->items,
+ &tbl->item_num, tbl->max_item_num,
+ 0, true, start_time);
}
uint16_t
@@ -357,7 +244,7 @@ gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
* Delete the packet and get the next
* packet in the flow.
*/
- j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
+ j = delete_tcp_item(tbl->items, j, &tbl->item_num, INVALID_ARRAY_INDEX);
tbl->flows[i].start_index = j;
if (j == INVALID_ARRAY_INDEX)
tbl->flow_num--;
diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h
index 91325c0da2..9eb6be2dfe 100644
--- a/lib/gro/gro_tcp6.h
+++ b/lib/gro/gro_tcp6.h
@@ -5,24 +5,21 @@
#ifndef _GRO_TCP6_H_
#define _GRO_TCP6_H_
+#include <gro_tcp.h>
+
#define INVALID_ARRAY_INDEX 0xffffffffUL
#define GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
/* Header fields representing a TCP/IPv6 flow */
struct tcp6_flow_key {
- struct rte_ether_addr eth_saddr;
- struct rte_ether_addr eth_daddr;
+ struct gro_tcp_flow tcp_flow;
uint8_t src_addr[16];
uint8_t dst_addr[16];
-
- uint32_t recv_ack;
- uint16_t src_port;
- uint16_t dst_port;
+ rte_be32_t vtc_flow;
};
struct gro_tcp6_flow {
struct tcp6_flow_key key;
- rte_be32_t vtc_flow;
/*
* The index of the first packet in the flow.
* INVALID_ARRAY_INDEX indicates an empty flow.
@@ -142,9 +139,25 @@ uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
* Check if two TCP/IPv6 packets belong to the same flow.
*/
static inline int
-is_same_tcp6_flow(struct tcp6_flow_key k1, struct tcp6_flow_key k2)
+is_same_tcp6_flow(struct tcp6_flow_key *k1, struct tcp6_flow_key *k2)
{
- return (!memcmp(&k1, &k2, sizeof(struct tcp6_flow_key)));
+ rte_be32_t vtc_flow_diff;
+
+ if (memcmp(&k1->src_addr, &k2->src_addr, 16)) {
+ return 0;
+ }
+ if (memcmp(&k1->dst_addr, &k2->dst_addr, 16)) {
+ return 0;
+ }
+ /*
+ * IP version (4) Traffic Class (8) Flow Label (20)
+ * All fields except Traffic class should be same
+ */
+ vtc_flow_diff = (k1->vtc_flow ^ k2->vtc_flow);
+ if (vtc_flow_diff & htonl(0xF00FFFFF))
+ return 0;
+
+ return is_same_tcp_flow(&k1->tcp_flow, &k2->tcp_flow);
}
#endif
diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
index 56b30b8c98..16aea7049f 100644
--- a/lib/gro/gro_vxlan_tcp4.c
+++ b/lib/gro/gro_vxlan_tcp4.c
@@ -164,15 +164,9 @@ insert_new_flow(struct gro_vxlan_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->inner_key.eth_saddr),
- &(dst->inner_key.eth_saddr));
- rte_ether_addr_copy(&(src->inner_key.eth_daddr),
- &(dst->inner_key.eth_daddr));
+ ASSIGN_TCP_FLOW_KEY((&(src->inner_key.tcp_flow)), (&(dst->inner_key.tcp_flow)));
dst->inner_key.ip_src_addr = src->inner_key.ip_src_addr;
dst->inner_key.ip_dst_addr = src->inner_key.ip_dst_addr;
- dst->inner_key.recv_ack = src->inner_key.recv_ack;
- dst->inner_key.src_port = src->inner_key.src_port;
- dst->inner_key.dst_port = src->inner_key.dst_port;
dst->vxlan_hdr.vx_flags = src->vxlan_hdr.vx_flags;
dst->vxlan_hdr.vx_vni = src->vxlan_hdr.vx_vni;
@@ -358,13 +352,13 @@ gro_vxlan_tcp4_reassemble(struct rte_mbuf *pkt,
sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.eth_daddr));
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.tcp_flow.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.tcp_flow.eth_daddr));
key.inner_key.ip_src_addr = ipv4_hdr->src_addr;
key.inner_key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.inner_key.recv_ack = tcp_hdr->recv_ack;
- key.inner_key.src_port = tcp_hdr->src_port;
- key.inner_key.dst_port = tcp_hdr->dst_port;
+ key.inner_key.tcp_flow.recv_ack = tcp_hdr->recv_ack;
+ key.inner_key.tcp_flow.src_port = tcp_hdr->src_port;
+ key.inner_key.tcp_flow.dst_port = tcp_hdr->dst_port;
key.vxlan_hdr.vx_flags = vxlan_hdr->vx_flags;
key.vxlan_hdr.vx_vni = vxlan_hdr->vx_vni;
diff --git a/lib/gro/meson.build b/lib/gro/meson.build
index dbce05220d..a7d73f21ec 100644
--- a/lib/gro/meson.build
+++ b/lib/gro/meson.build
@@ -3,6 +3,7 @@
sources = files(
'rte_gro.c',
+ 'gro_tcp.c',
'gro_tcp4.c',
'gro_tcp6.c',
'gro_udp4.c',
--
2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* RE: [PATCH v3] gro : ipv6 changes to support GRO for TCP/ipv6
2023-06-02 6:34 ` [PATCH v3] gro : ipv6 changes to support GRO for TCP/ipv6 Kumara Parameshwaran
@ 2023-06-06 4:35 ` Hu, Jiayu
2023-06-06 9:31 ` kumaraparameshwaran rathinavel
0 siblings, 1 reply; 32+ messages in thread
From: Hu, Jiayu @ 2023-06-06 4:35 UTC (permalink / raw)
To: Kumara Parameshwaran; +Cc: dev
Hi Kumara,
The v3 patch is not complete and it seems to be a patch based on v2.
In addition, did you test the code for tcp4 and tcp6 after your change?
For others, please see replies inline.
Thanks,
Jiayu
> -----Original Message-----
> From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> Sent: Friday, June 2, 2023 2:34 PM
> To: Hu, Jiayu <jiayu.hu@intel.com>
> Cc: dev@dpdk.org; Kumara Parameshwaran
> <kumaraparamesh92@gmail.com>
> Subject: [PATCH v3] gro : ipv6 changes to support GRO for TCP/ipv6
>
> The patch adds GRO support for TCP/ipv6 packets. This does not include the
> support for vxlan, udp ipv6 packets.
>
> Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> ---
> v1:
> * Changes to support GRO for TCP/ipv6 packets. This does not
> include
> vxlan changes.
> * The GRO is performed only for ipv6 packets that does not contain
> extension headers.
> * The logic for the TCP coalescing remains the same, in ipv6 header
> the source address, destination address, flow label, version fields
> are expected to be the same.
> * Re-organised the code to reuse certain tcp functions for both ipv4
> and
> ipv6 flows.
> v2:
> * Fix comments in gro_tcp6.h header file.
>
> v3:
> * Adderess review comments to fix code duplication for v4 and v6
>
> lib/gro/gro_tcp.c | 160 ++++++++++++++++++++++++
> lib/gro/gro_tcp.h | 63 ++++++++++
> lib/gro/gro_tcp4.c | 255 ++++++++++++---------------------------
> lib/gro/gro_tcp4.h | 18 +--
> lib/gro/gro_tcp6.c | 243 ++++++++++---------------------------
> lib/gro/gro_tcp6.h | 31 +++--
> lib/gro/gro_vxlan_tcp4.c | 18 +--
> lib/gro/meson.build | 1 +
> 8 files changed, 396 insertions(+), 393 deletions(-) create mode 100644
> lib/gro/gro_tcp.c
>
> diff --git a/lib/gro/gro_tcp.c b/lib/gro/gro_tcp.c new file mode 100644 index
> 0000000000..6a5aaada58
> --- /dev/null
> +++ b/lib/gro/gro_tcp.c
> @@ -0,0 +1,160 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
> +#include <rte_malloc.h>
> +#include <rte_mbuf.h>
> +#include <rte_ethdev.h>
> +
> +#include "gro_tcp.h"
> +
> +static inline uint32_t
> +find_an_empty_item(struct gro_tcp_item *items,
> + uint32_t table_size)
> +{
> + uint32_t i;
> +
> + for (i = 0; i < table_size; i++)
> + if (items[i].firstseg == NULL)
> + return i;
> + return INVALID_ARRAY_INDEX;
> +}
> +
> +uint32_t
> +insert_new_tcp_item(struct rte_mbuf *pkt,
> + struct gro_tcp_item *items,
> + uint32_t *item_num,
> + uint32_t table_size,
> + uint64_t start_time,
> + uint32_t prev_idx,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint8_t is_atomic)
> +{
> + uint32_t item_idx;
> +
> + item_idx = find_an_empty_item(items, table_size);
> + if (item_idx == INVALID_ARRAY_INDEX)
> + return INVALID_ARRAY_INDEX;
> +
> + items[item_idx].firstseg = pkt;
> + items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> + items[item_idx].start_time = start_time;
> + items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> + items[item_idx].sent_seq = sent_seq;
> + items[item_idx].ip_id = ip_id;
> + items[item_idx].nb_merged = 1;
> + items[item_idx].is_atomic = is_atomic;
> + (*item_num) += 1;
> +
> + /* if the previous packet exists, chain them together. */
> + if (prev_idx != INVALID_ARRAY_INDEX) {
> + items[item_idx].next_pkt_idx =
> + items[prev_idx].next_pkt_idx;
> + items[prev_idx].next_pkt_idx = item_idx;
> + }
> +
> + return item_idx;
> +}
> +
> +uint32_t
> +delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t prev_item_idx)
> +{
> + uint32_t next_idx = items[item_idx].next_pkt_idx;
> +
> + /* NULL indicates an empty item */
> + items[item_idx].firstseg = NULL;
> + (*item_num) -= 1;
> + if (prev_item_idx != INVALID_ARRAY_INDEX)
> + items[prev_item_idx].next_pkt_idx = next_idx;
> +
> + return next_idx;
> +}
> +
> +int32_t
> +gro_tcp_reassemble(struct rte_mbuf *pkt,
> + void *tbl,
> + void *key,
> + int32_t tcp_dl,
> + struct gro_tcp_flow_ops *ops,
> + struct gro_tcp_item *items,
> + uint32_t *item_num,
> + uint32_t table_size,
> + uint16_t ip_id,
> + uint8_t is_atomic,
> + uint64_t start_time)
In general, TCP4 and TCP6 share struct gro_tcp_item and have private flow structures,
i.e., struct gro_tcp4/6_flow, and I like this abstraction. IMO, the code processing
struct gro_tcp_item should be implemented as common functions shared by
gro_tcp4.c and gro_tcp6.c. The code processing struct gro_tcp4/6_flow is tcp4 and
tcp6 dependent and no need to abstract and share.
In gro_tcp_reassemble(), it uses callback functions defined in struct gro_tcp_flow_ops
to provide the different operations on struct gro_tcp4/6_flow. I don't think it's necessary
for abstraction purpose as gro_tcp4/6_flows_ops implementations are alike and it also
introduces extra cost on function calls.
The gro_tcp_reassemble() has two parts: the common part to process struct gro_tcp_item
and the private part to process struct gro_tcp4/6_flow. I think a better way is to remove
gro_tcp_reassemble() and struct gro_tcp_flow_ops, and implement the common part as
an internal function in gro_tcp.c/gro_tcp.h, and make both gro_tcp4/6_reassemble()
implement own private part and invoke the common function to process struct gro_tcp_item.
With this change, there is no callback cost anymore and tcp4/6 can share the common function.
> +{
> + uint32_t item_idx;
> + uint32_t cur_idx;
> + uint32_t prev_idx;
> + struct rte_tcp_hdr *tcp_hdr;
> + int cmp;
> + uint32_t sent_seq;
> +
> + tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt-
> >l2_len + pkt->l3_len);
> + /*
> + * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
> + * or CWR set.
> + */
> + if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
> + return -1;
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> +
> + ops->tcp_flow_key_init(key, tcp_hdr);
> +
> + item_idx = ops->tcp_flow_lookup(tbl, key);
> + if (item_idx == INVALID_ARRAY_INDEX) {
> + item_idx = insert_new_tcp_item(pkt, items, item_num,
> table_size, start_time,
> +
> INVALID_ARRAY_INDEX, sent_seq, ip_id,
> + is_atomic);
> + if (item_idx == INVALID_ARRAY_INDEX)
> + return -1;
> + if (ops->tcp_flow_insert(tbl, key, item_idx) ==
> + INVALID_ARRAY_INDEX) {
> + /*
> + * Fail to insert a new flow, so delete the
> + * stored packet.
> + */
> + delete_tcp_item(items, item_idx, item_num,
> INVALID_ARRAY_INDEX);
> + return -1;
> + }
> + return 0;
> + }
> + /*
> + * Check all packets in the flow and try to find a neighbor for
> + * the input packet.
> + */
> + cur_idx = item_idx;
> + prev_idx = cur_idx;
> + do {
> + cmp = check_seq_option(&items[cur_idx], tcp_hdr,
> + sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> + is_atomic);
> + if (cmp) {
> + if (merge_two_tcp_packets(&items[cur_idx],
> + pkt, cmp, sent_seq, ip_id, 0))
> + return 1;
> + /*
> + * Fail to merge the two packets, as the packet
> + * length is greater than the max value. Store
> + * the packet into the flow.
> + */
> + if (insert_new_tcp_item(pkt, items, item_num,
> table_size, start_time, cur_idx,
> + sent_seq, ip_id, is_atomic) ==
> + INVALID_ARRAY_INDEX)
> + return -1;
> + return 0;
> + }
> + prev_idx = cur_idx;
> + cur_idx = items[cur_idx].next_pkt_idx;
> + } while (cur_idx != INVALID_ARRAY_INDEX);
> +
> + /* Fail to find a neighbor, so store the packet into the flow. */
> + if (insert_new_tcp_item(pkt, items, item_num, table_size, start_time,
> prev_idx, sent_seq,
> + ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> + return -1;
> +
> + return 0;
> +
> +}
> diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h index
> c5d248a022..202f485c18 100644
> --- a/lib/gro/gro_tcp.h
> +++ b/lib/gro/gro_tcp.h
> @@ -1,6 +1,8 @@
> #ifndef _GRO_TCP_H_
> #define _GRO_TCP_H_
>
> +#define INVALID_ARRAY_INDEX 0xffffffffUL
> +
> #include <rte_tcp.h>
>
> /*
> @@ -14,6 +16,31 @@
> #define INVALID_TCP_HDRLEN(len) \
> (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
>
> +struct gro_tcp_flow {
> + struct rte_ether_addr eth_saddr;
> + struct rte_ether_addr eth_daddr;
> + uint32_t recv_ack;
> + uint16_t src_port;
> + uint16_t dst_port;
> +};
> +
> +#define ASSIGN_TCP_FLOW_KEY(k1, k2) \
> + rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
> + rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr)); \
> + k2->recv_ack = k1->recv_ack; \
> + k2->src_port = k1->src_port; \
> + k2->dst_port = k1->dst_port;
For multiline macro, it's better to use do{...}while(0) to avoid unexpected errors
in the future.
> +
> +typedef uint32_t (*gro_tcp_flow_lookup)(void *table, void *key);
> +typedef uint32_t (*gro_tcp_flow_insert)(void *table, void *key,
> +uint32_t item_idx); typedef void (*gro_tcp_flow_key_init)(void *key,
> +struct rte_tcp_hdr *tcp_hdr);
> +
> +struct gro_tcp_flow_ops {
> + gro_tcp_flow_lookup tcp_flow_lookup;
> + gro_tcp_flow_insert tcp_flow_insert;
> + gro_tcp_flow_key_init tcp_flow_key_init; };
> +
> struct gro_tcp_item {
> /*
> * The first MBUF segment of the packet. If the value @@ -44,6
> +71,36 @@ struct gro_tcp_item {
> uint8_t is_atomic;
> };
>
> +uint32_t
> +insert_new_tcp_item(struct rte_mbuf *pkt,
> + struct gro_tcp_item *items,
> + uint32_t *item_num,
> + uint32_t table_size,
> + uint64_t start_time,
> + uint32_t prev_idx,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint8_t is_atomic);
> +
> +uint32_t
> +delete_tcp_item(struct gro_tcp_item *items,
> + uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t prev_item_idx);
> +
> +int32_t
> +gro_tcp_reassemble(struct rte_mbuf *pkt,
> + void *tbl,
> + void *key,
> + int32_t tcp_dl,
> + struct gro_tcp_flow_ops *ops,
> + struct gro_tcp_item *items,
> + uint32_t *item_num,
> + uint32_t table_size,
> + uint16_t ip_id,
> + uint8_t is_atomic,
> + uint64_t start_time);
> +
> /*
> * Merge two TCP packets without updating checksums.
> * If cmp is larger than 0, append the new packet to the @@ -152,4 +209,10
> @@ check_seq_option(struct gro_tcp_item *item,
> return 0;
> }
>
> +static inline int
> +is_same_tcp_flow(struct gro_tcp_flow *k1, struct gro_tcp_flow *k2) {
> + return (!memcmp(k1, k2, sizeof(struct gro_tcp_flow))); }
> +
> #endif
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [PATCH v3] gro : ipv6 changes to support GRO for TCP/ipv6
2023-06-06 4:35 ` Hu, Jiayu
@ 2023-06-06 9:31 ` kumaraparameshwaran rathinavel
0 siblings, 0 replies; 32+ messages in thread
From: kumaraparameshwaran rathinavel @ 2023-06-06 9:31 UTC (permalink / raw)
To: Hu, Jiayu; +Cc: dev
[-- Attachment #1: Type: text/plain, Size: 13020 bytes --]
On Tue, Jun 6, 2023 at 10:05 AM Hu, Jiayu <jiayu.hu@intel.com> wrote:
> Hi Kumara,
>
> The v3 patch is not complete and it seems to be a patch based on v2.
> In addition, did you test the code for tcp4 and tcp6 after your change?
>
>> Will make sure that next patch contains the entire diff.
>>
> For others, please see replies inline.
>
> Thanks,
> Jiayu
>
> > -----Original Message-----
> > From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> > Sent: Friday, June 2, 2023 2:34 PM
> > To: Hu, Jiayu <jiayu.hu@intel.com>
> > Cc: dev@dpdk.org; Kumara Parameshwaran
> > <kumaraparamesh92@gmail.com>
> > Subject: [PATCH v3] gro : ipv6 changes to support GRO for TCP/ipv6
> >
> > The patch adds GRO support for TCP/ipv6 packets. This does not include
> the
> > support for vxlan, udp ipv6 packets.
> >
> > Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> > ---
> > v1:
> > * Changes to support GRO for TCP/ipv6 packets. This does not
> > include
> > vxlan changes.
> > * The GRO is performed only for ipv6 packets that does not contain
> > extension headers.
> > * The logic for the TCP coalescing remains the same, in ipv6 header
> > the source address, destination address, flow label, version
> fields
> > are expected to be the same.
> > * Re-organised the code to reuse certain tcp functions for both
> ipv4
> > and
> > ipv6 flows.
> > v2:
> > * Fix comments in gro_tcp6.h header file.
> >
> > v3:
> > * Adderess review comments to fix code duplication for v4 and v6
> >
> > lib/gro/gro_tcp.c | 160 ++++++++++++++++++++++++
> > lib/gro/gro_tcp.h | 63 ++++++++++
> > lib/gro/gro_tcp4.c | 255 ++++++++++++---------------------------
> > lib/gro/gro_tcp4.h | 18 +--
> > lib/gro/gro_tcp6.c | 243 ++++++++++---------------------------
> > lib/gro/gro_tcp6.h | 31 +++--
> > lib/gro/gro_vxlan_tcp4.c | 18 +--
> > lib/gro/meson.build | 1 +
> > 8 files changed, 396 insertions(+), 393 deletions(-) create mode 100644
> > lib/gro/gro_tcp.c
> >
> > diff --git a/lib/gro/gro_tcp.c b/lib/gro/gro_tcp.c new file mode 100644
> index
> > 0000000000..6a5aaada58
> > --- /dev/null
> > +++ b/lib/gro/gro_tcp.c
> > @@ -0,0 +1,160 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2017 Intel Corporation
> > + */
> > +#include <rte_malloc.h>
> > +#include <rte_mbuf.h>
> > +#include <rte_ethdev.h>
> > +
> > +#include "gro_tcp.h"
> > +
> > +static inline uint32_t
> > +find_an_empty_item(struct gro_tcp_item *items,
> > + uint32_t table_size)
> > +{
> > + uint32_t i;
> > +
> > + for (i = 0; i < table_size; i++)
> > + if (items[i].firstseg == NULL)
> > + return i;
> > + return INVALID_ARRAY_INDEX;
> > +}
> > +
> > +uint32_t
> > +insert_new_tcp_item(struct rte_mbuf *pkt,
> > + struct gro_tcp_item *items,
> > + uint32_t *item_num,
> > + uint32_t table_size,
> > + uint64_t start_time,
> > + uint32_t prev_idx,
> > + uint32_t sent_seq,
> > + uint16_t ip_id,
> > + uint8_t is_atomic)
> > +{
> > + uint32_t item_idx;
> > +
> > + item_idx = find_an_empty_item(items, table_size);
> > + if (item_idx == INVALID_ARRAY_INDEX)
> > + return INVALID_ARRAY_INDEX;
> > +
> > + items[item_idx].firstseg = pkt;
> > + items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> > + items[item_idx].start_time = start_time;
> > + items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> > + items[item_idx].sent_seq = sent_seq;
> > + items[item_idx].ip_id = ip_id;
> > + items[item_idx].nb_merged = 1;
> > + items[item_idx].is_atomic = is_atomic;
> > + (*item_num) += 1;
> > +
> > + /* if the previous packet exists, chain them together. */
> > + if (prev_idx != INVALID_ARRAY_INDEX) {
> > + items[item_idx].next_pkt_idx =
> > + items[prev_idx].next_pkt_idx;
> > + items[prev_idx].next_pkt_idx = item_idx;
> > + }
> > +
> > + return item_idx;
> > +}
> > +
> > +uint32_t
> > +delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
> > + uint32_t *item_num,
> > + uint32_t prev_item_idx)
> > +{
> > + uint32_t next_idx = items[item_idx].next_pkt_idx;
> > +
> > + /* NULL indicates an empty item */
> > + items[item_idx].firstseg = NULL;
> > + (*item_num) -= 1;
> > + if (prev_item_idx != INVALID_ARRAY_INDEX)
> > + items[prev_item_idx].next_pkt_idx = next_idx;
> > +
> > + return next_idx;
> > +}
> > +
> > +int32_t
> > +gro_tcp_reassemble(struct rte_mbuf *pkt,
> > + void *tbl,
> > + void *key,
> > + int32_t tcp_dl,
> > + struct gro_tcp_flow_ops *ops,
> > + struct gro_tcp_item *items,
> > + uint32_t *item_num,
> > + uint32_t table_size,
> > + uint16_t ip_id,
> > + uint8_t is_atomic,
> > + uint64_t start_time)
>
> In general, TCP4 and TCP6 share struct gro_tcp_item and have private flow
> structures,
> i.e., struct gro_tcp4/6_flow, and I like this abstraction. IMO, the code
> processing
> struct gro_tcp_item should be implemented as common functions shared by
> gro_tcp4.c and gro_tcp6.c. The code processing struct gro_tcp4/6_flow is
> tcp4 and
> tcp6 dependent and no need to abstract and share.
>
> In gro_tcp_reassemble(), it uses callback functions defined in struct
> gro_tcp_flow_ops
> to provide the different operations on struct gro_tcp4/6_flow. I don't
> think it's necessary
> for abstraction purpose as gro_tcp4/6_flows_ops implementations are alike
> and it also
> introduces extra cost on function calls.
>
>> Sure, would make the changes accordingly.
>>
>
> The gro_tcp_reassemble() has two parts: the common part to process struct
> gro_tcp_item
> and the private part to process struct gro_tcp4/6_flow. I think a better
> way is to remove
> gro_tcp_reassemble() and struct gro_tcp_flow_ops, and implement the common
> part as
> an internal function in gro_tcp.c/gro_tcp.h, and make both
> gro_tcp4/6_reassemble()
> implement own private part and invoke the common function to process
> struct gro_tcp_item.
> With this change, there is no callback cost anymore and tcp4/6 can share
> the common function.
>
>> Sure, got it.
>>
>> > +{
> > + uint32_t item_idx;
> > + uint32_t cur_idx;
> > + uint32_t prev_idx;
> > + struct rte_tcp_hdr *tcp_hdr;
> > + int cmp;
> > + uint32_t sent_seq;
> > +
> > + tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt-
> > >l2_len + pkt->l3_len);
> > + /*
> > + * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
> > + * or CWR set.
> > + */
> > + if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
> > + return -1;
> > + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> > +
> > + ops->tcp_flow_key_init(key, tcp_hdr);
> > +
> > + item_idx = ops->tcp_flow_lookup(tbl, key);
> > + if (item_idx == INVALID_ARRAY_INDEX) {
> > + item_idx = insert_new_tcp_item(pkt, items, item_num,
> > table_size, start_time,
> > +
> > INVALID_ARRAY_INDEX, sent_seq, ip_id,
> > + is_atomic);
> > + if (item_idx == INVALID_ARRAY_INDEX)
> > + return -1;
> > + if (ops->tcp_flow_insert(tbl, key, item_idx) ==
> > + INVALID_ARRAY_INDEX) {
> > + /*
> > + * Fail to insert a new flow, so delete the
> > + * stored packet.
> > + */
> > + delete_tcp_item(items, item_idx, item_num,
> > INVALID_ARRAY_INDEX);
> > + return -1;
> > + }
> > + return 0;
> > + }
> > + /*
> > + * Check all packets in the flow and try to find a neighbor for
> > + * the input packet.
> > + */
> > + cur_idx = item_idx;
> > + prev_idx = cur_idx;
> > + do {
> > + cmp = check_seq_option(&items[cur_idx], tcp_hdr,
> > + sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> > + is_atomic);
> > + if (cmp) {
> > + if (merge_two_tcp_packets(&items[cur_idx],
> > + pkt, cmp, sent_seq, ip_id,
> 0))
> > + return 1;
> > + /*
> > + * Fail to merge the two packets, as the packet
> > + * length is greater than the max value. Store
> > + * the packet into the flow.
> > + */
> > + if (insert_new_tcp_item(pkt, items, item_num,
> > table_size, start_time, cur_idx,
> > + sent_seq, ip_id,
> is_atomic) ==
> > + INVALID_ARRAY_INDEX)
> > + return -1;
> > + return 0;
> > + }
> > + prev_idx = cur_idx;
> > + cur_idx = items[cur_idx].next_pkt_idx;
> > + } while (cur_idx != INVALID_ARRAY_INDEX);
> > +
> > + /* Fail to find a neighbor, so store the packet into the flow. */
> > + if (insert_new_tcp_item(pkt, items, item_num, table_size,
> start_time,
> > prev_idx, sent_seq,
> > + ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> > + return -1;
> > +
> > + return 0;
> > +
> > +}
> > diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h index
> > c5d248a022..202f485c18 100644
> > --- a/lib/gro/gro_tcp.h
> > +++ b/lib/gro/gro_tcp.h
> > @@ -1,6 +1,8 @@
> > #ifndef _GRO_TCP_H_
> > #define _GRO_TCP_H_
> >
> > +#define INVALID_ARRAY_INDEX 0xffffffffUL
> > +
> > #include <rte_tcp.h>
> >
> > /*
> > @@ -14,6 +16,31 @@
> > #define INVALID_TCP_HDRLEN(len) \
> > (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> >
> > +struct gro_tcp_flow {
> > + struct rte_ether_addr eth_saddr;
> > + struct rte_ether_addr eth_daddr;
> > + uint32_t recv_ack;
> > + uint16_t src_port;
> > + uint16_t dst_port;
> > +};
> > +
> > +#define ASSIGN_TCP_FLOW_KEY(k1, k2) \
> > + rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
> > + rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr)); \
> > + k2->recv_ack = k1->recv_ack; \
> > + k2->src_port = k1->src_port; \
> > + k2->dst_port = k1->dst_port;
>
> For multiline macro, it's better to use do{...}while(0) to avoid
> unexpected errors
> in the future.
>
> > +
> > +typedef uint32_t (*gro_tcp_flow_lookup)(void *table, void *key);
> > +typedef uint32_t (*gro_tcp_flow_insert)(void *table, void *key,
> > +uint32_t item_idx); typedef void (*gro_tcp_flow_key_init)(void *key,
> > +struct rte_tcp_hdr *tcp_hdr);
> > +
> > +struct gro_tcp_flow_ops {
> > + gro_tcp_flow_lookup tcp_flow_lookup;
> > + gro_tcp_flow_insert tcp_flow_insert;
> > + gro_tcp_flow_key_init tcp_flow_key_init; };
> > +
> > struct gro_tcp_item {
> > /*
> > * The first MBUF segment of the packet. If the value @@ -44,6
> > +71,36 @@ struct gro_tcp_item {
> > uint8_t is_atomic;
> > };
> >
> > +uint32_t
> > +insert_new_tcp_item(struct rte_mbuf *pkt,
> > + struct gro_tcp_item *items,
> > + uint32_t *item_num,
> > + uint32_t table_size,
> > + uint64_t start_time,
> > + uint32_t prev_idx,
> > + uint32_t sent_seq,
> > + uint16_t ip_id,
> > + uint8_t is_atomic);
> > +
> > +uint32_t
> > +delete_tcp_item(struct gro_tcp_item *items,
> > + uint32_t item_idx,
> > + uint32_t *item_num,
> > + uint32_t prev_item_idx);
> > +
> > +int32_t
> > +gro_tcp_reassemble(struct rte_mbuf *pkt,
> > + void *tbl,
> > + void *key,
> > + int32_t tcp_dl,
> > + struct gro_tcp_flow_ops *ops,
> > + struct gro_tcp_item *items,
> > + uint32_t *item_num,
> > + uint32_t table_size,
> > + uint16_t ip_id,
> > + uint8_t is_atomic,
> > + uint64_t start_time);
> > +
> > /*
> > * Merge two TCP packets without updating checksums.
> > * If cmp is larger than 0, append the new packet to the @@ -152,4
> +209,10
> > @@ check_seq_option(struct gro_tcp_item *item,
> > return 0;
> > }
> >
> > +static inline int
> > +is_same_tcp_flow(struct gro_tcp_flow *k1, struct gro_tcp_flow *k2) {
> > + return (!memcmp(k1, k2, sizeof(struct gro_tcp_flow))); }
> > +
> > #endif
>
>
[-- Attachment #2: Type: text/html, Size: 17276 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread
* [PATCH v4] gro : ipv6 changes to support GRO for TCP/ipv6
2022-10-20 18:14 ` [PATCH v2] " Kumara Parameshwaran
` (2 preceding siblings ...)
2023-06-02 6:34 ` [PATCH v3] gro : ipv6 changes to support GRO for TCP/ipv6 Kumara Parameshwaran
@ 2023-06-06 14:58 ` Kumara Parameshwaran
2023-06-08 4:05 ` Hu, Jiayu
2023-06-12 11:05 ` [PATCH v5] " Kumara Parameshwaran
` (6 subsequent siblings)
10 siblings, 1 reply; 32+ messages in thread
From: Kumara Parameshwaran @ 2023-06-06 14:58 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev, Kumara Parameshwaran
The patch adds GRO support for TCP/ipv6 packets. This does not
include the support for vxlan, udp ipv6 packets.
Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
---
v1:
* Changes to support GRO for TCP/ipv6 packets. This does not include
vxlan changes.
* The GRO is performed only for ipv6 packets that does not contain
extension headers.
* The logic for the TCP coalescing remains the same, in ipv6 header
the source address, destination address, flow label, version fields
are expected to be the same.
* Re-organised the code to reuse certain tcp functions for both ipv4 and
ipv6 flows.
v2:
* Fix comments in gro_tcp6.h header file.
v3:
* Adderess review comments to fix code duplication for v4 and v6
v4:
* Addresses review comments for v3, do not use callbacks
lib/gro/gro_tcp.c | 128 +++++++++++++++++++
lib/gro/gro_tcp.h | 209 ++++++++++++++++++++++++++++++
lib/gro/gro_tcp4.c | 182 ++++++---------------------
lib/gro/gro_tcp4.h | 168 +------------------------
lib/gro/gro_tcp6.c | 266 +++++++++++++++++++++++++++++++++++++++
lib/gro/gro_tcp6.h | 163 ++++++++++++++++++++++++
lib/gro/gro_vxlan_tcp4.c | 21 ++--
lib/gro/gro_vxlan_tcp4.h | 3 +-
lib/gro/meson.build | 2 +
lib/gro/rte_gro.c | 83 +++++++++---
lib/gro/rte_gro.h | 3 +
11 files changed, 895 insertions(+), 333 deletions(-)
create mode 100644 lib/gro/gro_tcp.c
create mode 100644 lib/gro/gro_tcp.h
create mode 100644 lib/gro/gro_tcp6.c
create mode 100644 lib/gro/gro_tcp6.h
diff --git a/lib/gro/gro_tcp.c b/lib/gro/gro_tcp.c
new file mode 100644
index 0000000000..02a7d0f8c5
--- /dev/null
+++ b/lib/gro/gro_tcp.c
@@ -0,0 +1,128 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp.h"
+
+static inline uint32_t
+find_an_empty_item(struct gro_tcp_item *items,
+ uint32_t max_item_num)
+{
+ uint32_t i;
+
+ for (i = 0; i < max_item_num; i++)
+ if (items[i].firstseg == NULL)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic)
+{
+ uint32_t item_idx;
+
+ item_idx = find_an_empty_item(items, max_item_num);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return INVALID_ARRAY_INDEX;
+
+ items[item_idx].firstseg = pkt;
+ items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
+ items[item_idx].start_time = start_time;
+ items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
+ items[item_idx].sent_seq = sent_seq;
+ items[item_idx].ip_id = ip_id;
+ items[item_idx].nb_merged = 1;
+ items[item_idx].is_atomic = is_atomic;
+ (*item_num) += 1;
+
+ /* if the previous packet exists, chain them together. */
+ if (prev_idx != INVALID_ARRAY_INDEX) {
+ items[item_idx].next_pkt_idx =
+ items[prev_idx].next_pkt_idx;
+ items[prev_idx].next_pkt_idx = item_idx;
+ }
+
+ return item_idx;
+}
+
+uint32_t
+delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx)
+{
+ uint32_t next_idx = items[item_idx].next_pkt_idx;
+
+ /* NULL indicates an empty item */
+ items[item_idx].firstseg = NULL;
+ (*item_num) -= 1;
+ if (prev_item_idx != INVALID_ARRAY_INDEX)
+ items[prev_item_idx].next_pkt_idx = next_idx;
+
+ return next_idx;
+}
+
+int32_t
+gro_process_tcp_item(struct rte_mbuf *pkt,
+ struct rte_tcp_hdr *tcp_hdr,
+ int32_t tcp_dl,
+ struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time)
+{
+ uint32_t cur_idx;
+ uint32_t prev_idx;
+ int cmp;
+ uint32_t sent_seq;
+
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ /*
+ * Check all packets in the flow and try to find a neighbor for
+ * the input packet.
+ */
+ cur_idx = item_idx;
+ prev_idx = cur_idx;
+ do {
+ cmp = check_seq_option(&items[cur_idx], tcp_hdr,
+ sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
+ is_atomic);
+ if (cmp) {
+ if (merge_two_tcp_packets(&items[cur_idx],
+ pkt, cmp, sent_seq, ip_id, 0))
+ return 1;
+ /*
+ * Fail to merge the two packets, as the packet
+ * length is greater than the max value. Store
+ * the packet into the flow.
+ */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num, start_time, cur_idx,
+ sent_seq, ip_id, is_atomic) ==
+ INVALID_ARRAY_INDEX)
+ return -1;
+ return 0;
+ }
+ prev_idx = cur_idx;
+ cur_idx = items[cur_idx].next_pkt_idx;
+ } while (cur_idx != INVALID_ARRAY_INDEX);
+
+ /* Fail to find a neighbor, so store the packet into the flow. */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num, start_time, prev_idx, sent_seq,
+ ip_id, is_atomic) == INVALID_ARRAY_INDEX)
+ return -1;
+
+ return 0;
+}
diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h
new file mode 100644
index 0000000000..4b5b4eda9c
--- /dev/null
+++ b/lib/gro/gro_tcp.h
@@ -0,0 +1,209 @@
+#ifndef _GRO_TCP_H_
+#define _GRO_TCP_H_
+
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+
+#include <rte_tcp.h>
+
+/*
+ * The max length of a IPv4 packet, which includes the length of the L3
+ * header, the L4 header and the data payload.
+ */
+#define MAX_IP_PKT_LENGTH UINT16_MAX
+
+/* The maximum TCP header length */
+#define MAX_TCP_HLEN 60
+#define INVALID_TCP_HDRLEN(len) \
+ (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
+
+struct gro_tcp_flow {
+ struct rte_ether_addr eth_saddr;
+ struct rte_ether_addr eth_daddr;
+ uint32_t recv_ack;
+ uint16_t src_port;
+ uint16_t dst_port;
+};
+
+#define ASSIGN_TCP_FLOW_KEY(k1, k2) \
+ do {\
+ rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
+ rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr)); \
+ k2->recv_ack = k1->recv_ack; \
+ k2->src_port = k1->src_port; \
+ k2->dst_port = k1->dst_port; \
+ } while (0)
+
+struct gro_tcp_item {
+ /*
+ * The first MBUF segment of the packet. If the value
+ * is NULL, it means the item is empty.
+ */
+ struct rte_mbuf *firstseg;
+ /* The last MBUF segment of the packet */
+ struct rte_mbuf *lastseg;
+ /*
+ * The time when the first packet is inserted into the table.
+ * This value won't be updated, even if the packet is merged
+ * with other packets.
+ */
+ uint64_t start_time;
+ /*
+ * next_pkt_idx is used to chain the packets that
+ * are in the same flow but can't be merged together
+ * (e.g. caused by packet reordering).
+ */
+ uint32_t next_pkt_idx;
+ /* TCP sequence number of the packet */
+ uint32_t sent_seq;
+ /* IPv4 ID of the packet */
+ uint16_t ip_id;
+ /* the number of merged packets */
+ uint16_t nb_merged;
+ /* Indicate if IPv4 ID can be ignored */
+ uint8_t is_atomic;
+};
+
+uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t table_size,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic);
+
+uint32_t
+delete_tcp_item(struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx);
+
+int32_t
+gro_process_tcp_item(struct rte_mbuf *pkt,
+ struct rte_tcp_hdr *tcp_hdr,
+ int32_t tcp_dl,
+ struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t table_size,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time);
+
+/*
+ * Merge two TCP packets without updating checksums.
+ * If cmp is larger than 0, append the new packet to the
+ * original packet. Otherwise, pre-pend the new packet to
+ * the original packet.
+ */
+static inline int
+merge_two_tcp_packets(struct gro_tcp_item *item,
+ struct rte_mbuf *pkt,
+ int cmp,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t l2_offset)
+{
+ struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
+ uint16_t hdr_len, l2_len;
+
+ if (cmp > 0) {
+ pkt_head = item->firstseg;
+ pkt_tail = pkt;
+ } else {
+ pkt_head = pkt;
+ pkt_tail = item->firstseg;
+ }
+
+ /* check if the IPv4 packet length is greater than the max value */
+ hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
+ pkt_head->l4_len;
+ l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
+ if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
+ hdr_len > MAX_IP_PKT_LENGTH))
+ return 0;
+
+ /* remove the packet header for the tail packet */
+ rte_pktmbuf_adj(pkt_tail, hdr_len);
+
+ /* chain two packets together */
+ if (cmp > 0) {
+ item->lastseg->next = pkt;
+ item->lastseg = rte_pktmbuf_lastseg(pkt);
+ /* update IP ID to the larger value */
+ item->ip_id = ip_id;
+ } else {
+ lastseg = rte_pktmbuf_lastseg(pkt);
+ lastseg->next = item->firstseg;
+ item->firstseg = pkt;
+ /* update sent_seq to the smaller value */
+ item->sent_seq = sent_seq;
+ item->ip_id = ip_id;
+ }
+ item->nb_merged++;
+
+ /* update MBUF metadata for the merged packet */
+ pkt_head->nb_segs += pkt_tail->nb_segs;
+ pkt_head->pkt_len += pkt_tail->pkt_len;
+
+ return 1;
+}
+
+/*
+ * Check if two TCP/IPv4 packets are neighbors.
+ */
+static inline int
+check_seq_option(struct gro_tcp_item *item,
+ struct rte_tcp_hdr *tcph,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t tcp_hl,
+ uint16_t tcp_dl,
+ uint16_t l2_offset,
+ uint8_t is_atomic)
+{
+ struct rte_mbuf *pkt_orig = item->firstseg;
+ char *iph_orig;
+ struct rte_tcp_hdr *tcph_orig;
+ uint16_t len, tcp_hl_orig;
+
+ iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
+ l2_offset + pkt_orig->l2_len);
+ tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
+ tcp_hl_orig = pkt_orig->l4_len;
+
+ /* Check if TCP option fields equal */
+ len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
+ if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
+ (memcmp(tcph + 1, tcph_orig + 1,
+ len) != 0)))
+ return 0;
+
+ /* Don't merge packets whose DF bits are different */
+ if (unlikely(item->is_atomic ^ is_atomic))
+ return 0;
+
+ /* check if the two packets are neighbors */
+ len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
+ pkt_orig->l3_len - tcp_hl_orig;
+ if ((sent_seq == item->sent_seq + len) && (is_atomic ||
+ (ip_id == item->ip_id + 1)))
+ /* append the new packet */
+ return 1;
+ else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
+ (ip_id + item->nb_merged == item->ip_id)))
+ /* pre-pend the new packet */
+ return -1;
+
+ return 0;
+}
+
+static inline int
+is_same_tcp_flow(struct gro_tcp_flow *k1, struct gro_tcp_flow *k2)
+{
+ return (!memcmp(k1, k2, sizeof(struct gro_tcp_flow)));
+}
+
+#endif
diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c
index 0014096e63..ffc33747c4 100644
--- a/lib/gro/gro_tcp4.c
+++ b/lib/gro/gro_tcp4.c
@@ -7,6 +7,7 @@
#include <rte_ethdev.h>
#include "gro_tcp4.h"
+#include "gro_tcp.h"
void *
gro_tcp4_tbl_create(uint16_t socket_id,
@@ -30,7 +31,7 @@ gro_tcp4_tbl_create(uint16_t socket_id,
if (tbl == NULL)
return NULL;
- size = sizeof(struct gro_tcp4_item) * entries_num;
+ size = sizeof(struct gro_tcp_item) * entries_num;
tbl->items = rte_zmalloc_socket(__func__,
size,
RTE_CACHE_LINE_SIZE,
@@ -71,18 +72,6 @@ gro_tcp4_tbl_destroy(void *tbl)
rte_free(tcp_tbl);
}
-static inline uint32_t
-find_an_empty_item(struct gro_tcp4_tbl *tbl)
-{
- uint32_t i;
- uint32_t max_item_num = tbl->max_item_num;
-
- for (i = 0; i < max_item_num; i++)
- if (tbl->items[i].firstseg == NULL)
- return i;
- return INVALID_ARRAY_INDEX;
-}
-
static inline uint32_t
find_an_empty_flow(struct gro_tcp4_tbl *tbl)
{
@@ -95,56 +84,6 @@ find_an_empty_flow(struct gro_tcp4_tbl *tbl)
return INVALID_ARRAY_INDEX;
}
-static inline uint32_t
-insert_new_item(struct gro_tcp4_tbl *tbl,
- struct rte_mbuf *pkt,
- uint64_t start_time,
- uint32_t prev_idx,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint8_t is_atomic)
-{
- uint32_t item_idx;
-
- item_idx = find_an_empty_item(tbl);
- if (item_idx == INVALID_ARRAY_INDEX)
- return INVALID_ARRAY_INDEX;
-
- tbl->items[item_idx].firstseg = pkt;
- tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
- tbl->items[item_idx].start_time = start_time;
- tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
- tbl->items[item_idx].sent_seq = sent_seq;
- tbl->items[item_idx].ip_id = ip_id;
- tbl->items[item_idx].nb_merged = 1;
- tbl->items[item_idx].is_atomic = is_atomic;
- tbl->item_num++;
-
- /* if the previous packet exists, chain them together. */
- if (prev_idx != INVALID_ARRAY_INDEX) {
- tbl->items[item_idx].next_pkt_idx =
- tbl->items[prev_idx].next_pkt_idx;
- tbl->items[prev_idx].next_pkt_idx = item_idx;
- }
-
- return item_idx;
-}
-
-static inline uint32_t
-delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
- uint32_t prev_item_idx)
-{
- uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
-
- /* NULL indicates an empty item */
- tbl->items[item_idx].firstseg = NULL;
- tbl->item_num--;
- if (prev_item_idx != INVALID_ARRAY_INDEX)
- tbl->items[prev_item_idx].next_pkt_idx = next_idx;
-
- return next_idx;
-}
-
static inline uint32_t
insert_new_flow(struct gro_tcp4_tbl *tbl,
struct tcp4_flow_key *src,
@@ -159,13 +98,10 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
- rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
+ ASSIGN_TCP_FLOW_KEY((&src->tcp_flow), (&dst->tcp_flow));
+
dst->ip_src_addr = src->ip_src_addr;
dst->ip_dst_addr = src->ip_dst_addr;
- dst->recv_ack = src->recv_ack;
- dst->src_port = src->src_port;
- dst->dst_port = src->dst_port;
tbl->flows[flow_idx].start_index = item_idx;
tbl->flow_num++;
@@ -173,21 +109,6 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
return flow_idx;
}
-/*
- * update the packet length for the flushed packet.
- */
-static inline void
-update_header(struct gro_tcp4_item *item)
-{
- struct rte_ipv4_hdr *ipv4_hdr;
- struct rte_mbuf *pkt = item->firstseg;
-
- ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
- pkt->l2_len);
- ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
- pkt->l2_len);
-}
-
int32_t
gro_tcp4_reassemble(struct rte_mbuf *pkt,
struct gro_tcp4_tbl *tbl,
@@ -195,16 +116,15 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
{
struct rte_ether_hdr *eth_hdr;
struct rte_ipv4_hdr *ipv4_hdr;
- struct rte_tcp_hdr *tcp_hdr;
- uint32_t sent_seq;
int32_t tcp_dl;
+ struct rte_tcp_hdr *tcp_hdr;
uint16_t ip_id, hdr_len, frag_off, ip_tlen;
uint8_t is_atomic;
+ uint32_t sent_seq;
struct tcp4_flow_key key;
- uint32_t cur_idx, prev_idx, item_idx;
+ uint32_t item_idx;
uint32_t i, max_flow_num, remaining_flow_num;
- int cmp;
uint8_t find;
/*
@@ -216,7 +136,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
- tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
+ tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt->l2_len + pkt->l3_len);
hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
/*
@@ -230,7 +150,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
ip_tlen = rte_be_to_cpu_16(ipv4_hdr->total_length);
if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len))
rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len);
-
/*
* Don't process the packet whose payload length is less than or
* equal to 0.
@@ -239,6 +158,13 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
if (tcp_dl <= 0)
return -1;
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.tcp_flow.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.tcp_flow.eth_daddr));
+ key.ip_src_addr = ipv4_hdr->src_addr;
+ key.ip_dst_addr = ipv4_hdr->dst_addr;
+ key.tcp_flow.src_port = tcp_hdr->src_port;
+ key.tcp_flow.dst_port = tcp_hdr->dst_port;
+ key.tcp_flow.recv_ack = tcp_hdr->recv_ack;
/*
* Save IPv4 ID for the packet whose DF bit is 0. For the packet
* whose DF bit is 1, IPv4 ID is ignored.
@@ -246,15 +172,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) == RTE_IPV4_HDR_DF_FLAG;
ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
- sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
-
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
- key.ip_src_addr = ipv4_hdr->src_addr;
- key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.src_port = tcp_hdr->src_port;
- key.dst_port = tcp_hdr->dst_port;
- key.recv_ack = tcp_hdr->recv_ack;
/* Search for a matched flow. */
max_flow_num = tbl->max_flow_num;
@@ -270,63 +187,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
}
}
- /*
- * Fail to find a matched flow. Insert a new flow and store the
- * packet into the flow.
- */
if (find == 0) {
- item_idx = insert_new_item(tbl, pkt, start_time,
- INVALID_ARRAY_INDEX, sent_seq, ip_id,
- is_atomic);
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num, tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, ip_id,
+ is_atomic);
if (item_idx == INVALID_ARRAY_INDEX)
return -1;
if (insert_new_flow(tbl, &key, item_idx) ==
- INVALID_ARRAY_INDEX) {
+ INVALID_ARRAY_INDEX) {
/*
* Fail to insert a new flow, so delete the
* stored packet.
- */
- delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
return -1;
}
return 0;
}
+ item_idx = tbl->flows[i].start_index;
- /*
- * Check all packets in the flow and try to find a neighbor for
- * the input packet.
- */
- cur_idx = tbl->flows[i].start_index;
- prev_idx = cur_idx;
- do {
- cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
- sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
- is_atomic);
- if (cmp) {
- if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
- pkt, cmp, sent_seq, ip_id, 0))
- return 1;
- /*
- * Fail to merge the two packets, as the packet
- * length is greater than the max value. Store
- * the packet into the flow.
- */
- if (insert_new_item(tbl, pkt, start_time, cur_idx,
- sent_seq, ip_id, is_atomic) ==
- INVALID_ARRAY_INDEX)
- return -1;
- return 0;
- }
- prev_idx = cur_idx;
- cur_idx = tbl->items[cur_idx].next_pkt_idx;
- } while (cur_idx != INVALID_ARRAY_INDEX);
+ return gro_process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, item_idx,
+ &tbl->item_num, tbl->max_item_num,
+ ip_id, is_atomic, start_time);
+}
- /* Fail to find a neighbor, so store the packet into the flow. */
- if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
- ip_id, is_atomic) == INVALID_ARRAY_INDEX)
- return -1;
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
- return 0;
+ ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len);
}
uint16_t
@@ -353,7 +251,7 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
* Delete the packet and get the next
* packet in the flow.
*/
- j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
+ j = delete_tcp_item(tbl->items, j, &tbl->item_num, INVALID_ARRAY_INDEX);
tbl->flows[i].start_index = j;
if (j == INVALID_ARRAY_INDEX)
tbl->flow_num--;
diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h
index 212f97a042..8ff1eed096 100644
--- a/lib/gro/gro_tcp4.h
+++ b/lib/gro/gro_tcp4.h
@@ -5,32 +5,15 @@
#ifndef _GRO_TCP4_H_
#define _GRO_TCP4_H_
-#include <rte_tcp.h>
+#include <gro_tcp.h>
-#define INVALID_ARRAY_INDEX 0xffffffffUL
#define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
-/*
- * The max length of a IPv4 packet, which includes the length of the L3
- * header, the L4 header and the data payload.
- */
-#define MAX_IPV4_PKT_LENGTH UINT16_MAX
-
-/* The maximum TCP header length */
-#define MAX_TCP_HLEN 60
-#define INVALID_TCP_HDRLEN(len) \
- (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
-
/* Header fields representing a TCP/IPv4 flow */
struct tcp4_flow_key {
- struct rte_ether_addr eth_saddr;
- struct rte_ether_addr eth_daddr;
+ struct gro_tcp_flow tcp_flow;
uint32_t ip_src_addr;
uint32_t ip_dst_addr;
-
- uint32_t recv_ack;
- uint16_t src_port;
- uint16_t dst_port;
};
struct gro_tcp4_flow {
@@ -42,42 +25,12 @@ struct gro_tcp4_flow {
uint32_t start_index;
};
-struct gro_tcp4_item {
- /*
- * The first MBUF segment of the packet. If the value
- * is NULL, it means the item is empty.
- */
- struct rte_mbuf *firstseg;
- /* The last MBUF segment of the packet */
- struct rte_mbuf *lastseg;
- /*
- * The time when the first packet is inserted into the table.
- * This value won't be updated, even if the packet is merged
- * with other packets.
- */
- uint64_t start_time;
- /*
- * next_pkt_idx is used to chain the packets that
- * are in the same flow but can't be merged together
- * (e.g. caused by packet reordering).
- */
- uint32_t next_pkt_idx;
- /* TCP sequence number of the packet */
- uint32_t sent_seq;
- /* IPv4 ID of the packet */
- uint16_t ip_id;
- /* the number of merged packets */
- uint16_t nb_merged;
- /* Indicate if IPv4 ID can be ignored */
- uint8_t is_atomic;
-};
-
/*
* TCP/IPv4 reassembly table structure.
*/
struct gro_tcp4_tbl {
/* item array */
- struct gro_tcp4_item *items;
+ struct gro_tcp_item *items;
/* flow array */
struct gro_tcp4_flow *flows;
/* current item number */
@@ -186,120 +139,9 @@ uint32_t gro_tcp4_tbl_pkt_count(void *tbl);
static inline int
is_same_tcp4_flow(struct tcp4_flow_key k1, struct tcp4_flow_key k2)
{
- return (rte_is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) &&
- rte_is_same_ether_addr(&k1.eth_daddr, &k2.eth_daddr) &&
- (k1.ip_src_addr == k2.ip_src_addr) &&
+ return ((k1.ip_src_addr == k2.ip_src_addr) &&
(k1.ip_dst_addr == k2.ip_dst_addr) &&
- (k1.recv_ack == k2.recv_ack) &&
- (k1.src_port == k2.src_port) &&
- (k1.dst_port == k2.dst_port));
+ is_same_tcp_flow(&k1.tcp_flow, &k2.tcp_flow));
}
-/*
- * Merge two TCP/IPv4 packets without updating checksums.
- * If cmp is larger than 0, append the new packet to the
- * original packet. Otherwise, pre-pend the new packet to
- * the original packet.
- */
-static inline int
-merge_two_tcp4_packets(struct gro_tcp4_item *item,
- struct rte_mbuf *pkt,
- int cmp,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t l2_offset)
-{
- struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
- uint16_t hdr_len, l2_len;
-
- if (cmp > 0) {
- pkt_head = item->firstseg;
- pkt_tail = pkt;
- } else {
- pkt_head = pkt;
- pkt_tail = item->firstseg;
- }
-
- /* check if the IPv4 packet length is greater than the max value */
- hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
- pkt_head->l4_len;
- l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
- if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
- hdr_len > MAX_IPV4_PKT_LENGTH))
- return 0;
-
- /* remove the packet header for the tail packet */
- rte_pktmbuf_adj(pkt_tail, hdr_len);
-
- /* chain two packets together */
- if (cmp > 0) {
- item->lastseg->next = pkt;
- item->lastseg = rte_pktmbuf_lastseg(pkt);
- /* update IP ID to the larger value */
- item->ip_id = ip_id;
- } else {
- lastseg = rte_pktmbuf_lastseg(pkt);
- lastseg->next = item->firstseg;
- item->firstseg = pkt;
- /* update sent_seq to the smaller value */
- item->sent_seq = sent_seq;
- item->ip_id = ip_id;
- }
- item->nb_merged++;
-
- /* update MBUF metadata for the merged packet */
- pkt_head->nb_segs += pkt_tail->nb_segs;
- pkt_head->pkt_len += pkt_tail->pkt_len;
-
- return 1;
-}
-
-/*
- * Check if two TCP/IPv4 packets are neighbors.
- */
-static inline int
-check_seq_option(struct gro_tcp4_item *item,
- struct rte_tcp_hdr *tcph,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t tcp_hl,
- uint16_t tcp_dl,
- uint16_t l2_offset,
- uint8_t is_atomic)
-{
- struct rte_mbuf *pkt_orig = item->firstseg;
- struct rte_ipv4_hdr *iph_orig;
- struct rte_tcp_hdr *tcph_orig;
- uint16_t len, tcp_hl_orig;
-
- iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char *) +
- l2_offset + pkt_orig->l2_len);
- tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig + pkt_orig->l3_len);
- tcp_hl_orig = pkt_orig->l4_len;
-
- /* Check if TCP option fields equal */
- len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
- if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
- (memcmp(tcph + 1, tcph_orig + 1,
- len) != 0)))
- return 0;
-
- /* Don't merge packets whose DF bits are different */
- if (unlikely(item->is_atomic ^ is_atomic))
- return 0;
-
- /* check if the two packets are neighbors */
- len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
- pkt_orig->l3_len - tcp_hl_orig;
- if ((sent_seq == item->sent_seq + len) && (is_atomic ||
- (ip_id == item->ip_id + 1)))
- /* append the new packet */
- return 1;
- else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
- (ip_id + item->nb_merged == item->ip_id)))
- /* pre-pend the new packet */
- return -1;
-
- return 0;
-}
#endif
diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c
new file mode 100644
index 0000000000..66b94b4123
--- /dev/null
+++ b/lib/gro/gro_tcp6.c
@@ -0,0 +1,266 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp6.h"
+#include "gro_tcp.h"
+
+void *
+gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow)
+{
+ struct gro_tcp6_tbl *tbl;
+ size_t size;
+ uint32_t entries_num, i;
+
+ entries_num = max_flow_num * max_item_per_flow;
+ entries_num = RTE_MIN(entries_num, GRO_TCP6_TBL_MAX_ITEM_NUM);
+
+ if (entries_num == 0)
+ return NULL;
+
+ tbl = rte_zmalloc_socket(__func__,
+ sizeof(struct gro_tcp6_tbl),
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl == NULL)
+ return NULL;
+
+ size = sizeof(struct gro_tcp_item) * entries_num;
+ tbl->items = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->items == NULL) {
+ rte_free(tbl);
+ return NULL;
+ }
+ tbl->max_item_num = entries_num;
+
+ size = sizeof(struct gro_tcp6_flow) * entries_num;
+ tbl->flows = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->flows == NULL) {
+ rte_free(tbl->items);
+ rte_free(tbl);
+ return NULL;
+ }
+ /* INVALID_ARRAY_INDEX indicates an empty flow */
+ for (i = 0; i < entries_num; i++)
+ tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
+ tbl->max_flow_num = entries_num;
+
+ return tbl;
+}
+
+void
+gro_tcp6_tbl_destroy(void *tbl)
+{
+ struct gro_tcp6_tbl *tcp_tbl = tbl;
+
+ if (tcp_tbl) {
+ rte_free(tcp_tbl->items);
+ rte_free(tcp_tbl->flows);
+ }
+ rte_free(tcp_tbl);
+}
+
+static inline uint32_t
+find_an_empty_flow(struct gro_tcp6_tbl *tbl)
+{
+ uint32_t i;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++)
+ if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+insert_new_flow(struct gro_tcp6_tbl *tbl,
+ struct tcp6_flow_key *src,
+ uint32_t item_idx)
+{
+ struct tcp6_flow_key *dst;
+ uint32_t flow_idx;
+
+ flow_idx = find_an_empty_flow(tbl);
+ if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
+ return INVALID_ARRAY_INDEX;
+
+ dst = &(tbl->flows[flow_idx].key);
+
+ ASSIGN_TCP_FLOW_KEY((&src->tcp_flow), (&dst->tcp_flow));
+ memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst->src_addr));
+ memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst->dst_addr));
+ dst->vtc_flow = src->vtc_flow;
+
+ tbl->flows[flow_idx].start_index = item_idx;
+ tbl->flow_num++;
+
+ return flow_idx;
+}
+
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
+
+ ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len - pkt->l3_len);
+}
+
+int32_t
+gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time)
+{
+ struct rte_ether_hdr *eth_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ int32_t tcp_dl;
+ uint16_t ip_tlen;
+ struct tcp6_flow_key key;
+ uint32_t i, max_flow_num, remaining_flow_num;
+ uint32_t sent_seq;
+ struct rte_tcp_hdr *tcp_hdr;
+ uint8_t find;
+ uint32_t item_idx;
+ /*
+ * Don't process the packet whose TCP header length is greater
+ * than 60 bytes or less than 20 bytes.
+ */
+ if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
+ return -1;
+
+ eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
+ ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
+ tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt->l2_len + pkt->l3_len);
+
+ /*
+ * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
+ * or CWR set.
+ */
+ if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
+ return -1;
+
+ ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
+ /*
+ * Don't process the packet whose payload length is less than or
+ * equal to 0.
+ */
+ tcp_dl = ip_tlen - pkt->l4_len;
+ if (tcp_dl <= 0)
+ return -1;
+
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.tcp_flow.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.tcp_flow.eth_daddr));
+ memcpy(&key.src_addr[0], &ipv6_hdr->src_addr, sizeof(key.src_addr));
+ memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr, sizeof(key.dst_addr));
+ key.tcp_flow.src_port = tcp_hdr->src_port;
+ key.tcp_flow.dst_port = tcp_hdr->dst_port;
+ key.tcp_flow.recv_ack = tcp_hdr->recv_ack;
+ key.vtc_flow = ipv6_hdr->vtc_flow;
+
+ /* Search for a matched flow. */
+ max_flow_num = tbl->max_flow_num;
+ remaining_flow_num = tbl->flow_num;
+ find = 0;
+ for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
+ if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
+ if (is_same_tcp6_flow(&tbl->flows[i].key, &key)) {
+ find = 1;
+ break;
+ }
+ remaining_flow_num--;
+ }
+ }
+
+ if (find == 0) {
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num, tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, 0, true);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return -1;
+ if (insert_new_flow(tbl, &key, item_idx) ==
+ INVALID_ARRAY_INDEX) {
+ /*
+ * Fail to insert a new flow, so delete the
+ * stored packet.
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
+ return -1;
+ }
+ return 0;
+ }
+ item_idx = tbl->flows[i].start_index;
+ return gro_process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, item_idx,
+ &tbl->item_num, tbl->max_item_num,
+ 0, true, start_time);
+}
+
+uint16_t
+gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out)
+{
+ uint16_t k = 0;
+ uint32_t i, j;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++) {
+ if (unlikely(tbl->flow_num == 0))
+ return k;
+
+ j = tbl->flows[i].start_index;
+ while (j != INVALID_ARRAY_INDEX) {
+ if (tbl->items[j].start_time <= flush_timestamp) {
+ out[k++] = tbl->items[j].firstseg;
+ if (tbl->items[j].nb_merged > 1)
+ update_header(&(tbl->items[j]));
+ /*
+ * Delete the packet and get the next
+ * packet in the flow.
+ */
+ j = delete_tcp_item(tbl->items, j, &tbl->item_num, INVALID_ARRAY_INDEX);
+ tbl->flows[i].start_index = j;
+ if (j == INVALID_ARRAY_INDEX)
+ tbl->flow_num--;
+
+ if (unlikely(k == nb_out))
+ return k;
+ } else
+ /*
+ * The left packets in this flow won't be
+ * timeout. Go to check other flows.
+ */
+ break;
+ }
+ }
+ return k;
+}
+
+uint32_t
+gro_tcp6_tbl_pkt_count(void *tbl)
+{
+ struct gro_tcp6_tbl *gro_tbl = tbl;
+
+ if (gro_tbl)
+ return gro_tbl->item_num;
+
+ return 0;
+}
diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h
new file mode 100644
index 0000000000..9eb6be2dfe
--- /dev/null
+++ b/lib/gro/gro_tcp6.h
@@ -0,0 +1,163 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#ifndef _GRO_TCP6_H_
+#define _GRO_TCP6_H_
+
+#include <gro_tcp.h>
+
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+#define GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
+
+/* Header fields representing a TCP/IPv6 flow */
+struct tcp6_flow_key {
+ struct gro_tcp_flow tcp_flow;
+ uint8_t src_addr[16];
+ uint8_t dst_addr[16];
+ rte_be32_t vtc_flow;
+};
+
+struct gro_tcp6_flow {
+ struct tcp6_flow_key key;
+ /*
+ * The index of the first packet in the flow.
+ * INVALID_ARRAY_INDEX indicates an empty flow.
+ */
+ uint32_t start_index;
+};
+
+/*
+ * TCP/IPv6 reassembly table structure.
+ */
+struct gro_tcp6_tbl {
+ /* item array */
+ struct gro_tcp_item *items;
+ /* flow array */
+ struct gro_tcp6_flow *flows;
+ /* current item number */
+ uint32_t item_num;
+ /* current flow num */
+ uint32_t flow_num;
+ /* item array size */
+ uint32_t max_item_num;
+ /* flow array size */
+ uint32_t max_flow_num;
+};
+
+/**
+ * This function creates a TCP/IPv6 reassembly table.
+ *
+ * @param socket_id
+ * Socket index for allocating the TCP/IPv6 reassemble table
+ * @param max_flow_num
+ * The maximum number of flows in the TCP/IPv6 GRO table
+ * @param max_item_per_flow
+ * The maximum number of packets per flow
+ *
+ * @return
+ * - Return the table pointer on success.
+ * - Return NULL on failure.
+ */
+void *gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow);
+
+/**
+ * This function destroys a TCP/IPv6 reassembly table.
+ *
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table.
+ */
+void gro_tcp6_tbl_destroy(void *tbl);
+
+/**
+ * This function merges a TCP/IPv6 packet. It doesn't process the packet,
+ * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't have
+ * payload.
+ *
+ * This function doesn't check if the packet has correct checksums and
+ * doesn't re-calculate checksums for the merged packet. Additionally,
+ * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
+ * when IP fragmentation is possible (i.e., DF==0). It returns the
+ * packet, if the packet has invalid parameters (e.g. SYN bit is set)
+ * or there is no available space in the table.
+ *
+ * @param pkt
+ * Packet to reassemble
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table
+ * @start_time
+ * The time when the packet is inserted into the table
+ *
+ * @return
+ * - Return a positive value if the packet is merged.
+ * - Return zero if the packet isn't merged but stored in the table.
+ * - Return a negative value for invalid parameters or no available
+ * space in the table.
+ */
+int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time);
+
+/**
+ * This function flushes timeout packets in a TCP/IPv6 reassembly table,
+ * and without updating checksums.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ * @param flush_timestamp
+ * Flush packets which are inserted into the table before or at the
+ * flush_timestamp.
+ * @param out
+ * Pointer array used to keep flushed packets
+ * @param nb_out
+ * The element number in 'out'. It also determines the maximum number of
+ * packets that can be flushed finally.
+ *
+ * @return
+ * The number of flushed packets
+ */
+uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out);
+
+/**
+ * This function returns the number of the packets in a TCP/IPv6
+ * reassembly table.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ *
+ * @return
+ * The number of packets in the table
+ */
+uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
+
+/*
+ * Check if two TCP/IPv6 packets belong to the same flow.
+ */
+static inline int
+is_same_tcp6_flow(struct tcp6_flow_key *k1, struct tcp6_flow_key *k2)
+{
+ rte_be32_t vtc_flow_diff;
+
+ if (memcmp(&k1->src_addr, &k2->src_addr, 16)) {
+ return 0;
+ }
+ if (memcmp(&k1->dst_addr, &k2->dst_addr, 16)) {
+ return 0;
+ }
+ /*
+ * IP version (4) Traffic Class (8) Flow Label (20)
+ * All fields except Traffic class should be same
+ */
+ vtc_flow_diff = (k1->vtc_flow ^ k2->vtc_flow);
+ if (vtc_flow_diff & htonl(0xF00FFFFF))
+ return 0;
+
+ return is_same_tcp_flow(&k1->tcp_flow, &k2->tcp_flow);
+}
+
+#endif
diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
index 3be4deb7c7..16aea7049f 100644
--- a/lib/gro/gro_vxlan_tcp4.c
+++ b/lib/gro/gro_vxlan_tcp4.c
@@ -7,6 +7,7 @@
#include <rte_ethdev.h>
#include <rte_udp.h>
+#include "gro_tcp.h"
#include "gro_vxlan_tcp4.h"
void *
@@ -163,15 +164,9 @@ insert_new_flow(struct gro_vxlan_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->inner_key.eth_saddr),
- &(dst->inner_key.eth_saddr));
- rte_ether_addr_copy(&(src->inner_key.eth_daddr),
- &(dst->inner_key.eth_daddr));
+ ASSIGN_TCP_FLOW_KEY((&(src->inner_key.tcp_flow)), (&(dst->inner_key.tcp_flow)));
dst->inner_key.ip_src_addr = src->inner_key.ip_src_addr;
dst->inner_key.ip_dst_addr = src->inner_key.ip_dst_addr;
- dst->inner_key.recv_ack = src->inner_key.recv_ack;
- dst->inner_key.src_port = src->inner_key.src_port;
- dst->inner_key.dst_port = src->inner_key.dst_port;
dst->vxlan_hdr.vx_flags = src->vxlan_hdr.vx_flags;
dst->vxlan_hdr.vx_vni = src->vxlan_hdr.vx_vni;
@@ -248,7 +243,7 @@ merge_two_vxlan_tcp4_packets(struct gro_vxlan_tcp4_item *item,
uint16_t outer_ip_id,
uint16_t ip_id)
{
- if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp, sent_seq,
+ if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
ip_id, pkt->outer_l2_len +
pkt->outer_l3_len)) {
/* Update the outer IPv4 ID to the large value. */
@@ -357,13 +352,13 @@ gro_vxlan_tcp4_reassemble(struct rte_mbuf *pkt,
sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.eth_daddr));
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.tcp_flow.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.tcp_flow.eth_daddr));
key.inner_key.ip_src_addr = ipv4_hdr->src_addr;
key.inner_key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.inner_key.recv_ack = tcp_hdr->recv_ack;
- key.inner_key.src_port = tcp_hdr->src_port;
- key.inner_key.dst_port = tcp_hdr->dst_port;
+ key.inner_key.tcp_flow.recv_ack = tcp_hdr->recv_ack;
+ key.inner_key.tcp_flow.src_port = tcp_hdr->src_port;
+ key.inner_key.tcp_flow.dst_port = tcp_hdr->dst_port;
key.vxlan_hdr.vx_flags = vxlan_hdr->vx_flags;
key.vxlan_hdr.vx_vni = vxlan_hdr->vx_vni;
diff --git a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h
index 7832942a68..d68d5fcd5b 100644
--- a/lib/gro/gro_vxlan_tcp4.h
+++ b/lib/gro/gro_vxlan_tcp4.h
@@ -5,6 +5,7 @@
#ifndef _GRO_VXLAN_TCP4_H_
#define _GRO_VXLAN_TCP4_H_
+#include "gro_tcp.h"
#include "gro_tcp4.h"
#define GRO_VXLAN_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
@@ -36,7 +37,7 @@ struct gro_vxlan_tcp4_flow {
};
struct gro_vxlan_tcp4_item {
- struct gro_tcp4_item inner_item;
+ struct gro_tcp_item inner_item;
/* IPv4 ID in the outer IPv4 header */
uint16_t outer_ip_id;
/* Indicate if outer IPv4 ID can be ignored */
diff --git a/lib/gro/meson.build b/lib/gro/meson.build
index e4fa2958bd..a7d73f21ec 100644
--- a/lib/gro/meson.build
+++ b/lib/gro/meson.build
@@ -3,7 +3,9 @@
sources = files(
'rte_gro.c',
+ 'gro_tcp.c',
'gro_tcp4.c',
+ 'gro_tcp6.c',
'gro_udp4.c',
'gro_vxlan_tcp4.c',
'gro_vxlan_udp4.c',
diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c
index e35399fd42..d824eebd93 100644
--- a/lib/gro/rte_gro.c
+++ b/lib/gro/rte_gro.c
@@ -8,6 +8,7 @@
#include "rte_gro.h"
#include "gro_tcp4.h"
+#include "gro_tcp6.h"
#include "gro_udp4.h"
#include "gro_vxlan_tcp4.h"
#include "gro_vxlan_udp4.h"
@@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
- gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
+ gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, gro_tcp6_tbl_create, NULL};
static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
+ gro_tcp6_tbl_destroy,
NULL};
static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count,
+ gro_tcp6_tbl_pkt_count,
NULL};
#define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
@@ -35,6 +38,12 @@ static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+/* GRO with extension headers is not supported */
+#define IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
+ ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
+ ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
+ (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+
#define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
@@ -147,7 +156,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
/* allocate a reassembly table for TCP/IPv4 GRO */
struct gro_tcp4_tbl tcp_tbl;
struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
- struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+ struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+
+ struct gro_tcp6_tbl tcp6_tbl;
+ struct gro_tcp6_flow tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
+ struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
/* allocate a reassembly table for UDP/IPv4 GRO */
struct gro_udp4_tbl udp_tbl;
@@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
int32_t ret;
uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
- do_vxlan_udp_gro = 0;
+ do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
do_udp4_gro = 1;
}
+ if (param->gro_types & RTE_GRO_TCP_IPV6) {
+ for (i = 0; i < item_num; i++)
+ tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+ tcp6_tbl.flows = tcp6_flows;
+ tcp6_tbl.items = tcp6_items;
+ tcp6_tbl.flow_num = 0;
+ tcp6_tbl.item_num = 0;
+ tcp6_tbl.max_flow_num = item_num;
+ tcp6_tbl.max_item_num = item_num;
+ do_tcp6_gro = 1;
+ }
for (i = 0; i < nb_pkts; i++) {
/*
@@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
nb_after_gro--;
else if (ret < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
+ if (ret > 0)
+ /* merge successfully */
+ nb_after_gro--;
+ else if (ret < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
if ((nb_after_gro < nb_pkts)
|| (unprocess_num < nb_pkts)) {
i = 0;
+ /* Copy unprocessed packets */
+ if (unprocess_num > 0) {
+ memcpy(&pkts[i], unprocess_pkts,
+ sizeof(struct rte_mbuf *) *
+ unprocess_num);
+ i = unprocess_num;
+ }
+
/* Flush all packets from the tables */
if (do_vxlan_tcp_gro) {
- i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
+ i += gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
0, pkts, nb_pkts);
}
@@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
&pkts[i], nb_pkts - i);
}
- /* Copy unprocessed packets */
- if (unprocess_num > 0) {
- memcpy(&pkts[i], unprocess_pkts,
- sizeof(struct rte_mbuf *) *
- unprocess_num);
+
+ if (do_tcp6_gro) {
+ i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
+ &pkts[i], nb_pkts - i);
}
- nb_after_gro = i + unprocess_num;
}
return nb_after_gro;
@@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
{
struct rte_mbuf *unprocess_pkts[nb_pkts];
struct gro_ctx *gro_ctx = ctx;
- void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
+ void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
uint64_t current_time;
uint16_t i, unprocess_num = 0;
- uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro;
+ uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro, do_tcp6_gro;
if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
+ tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
RTE_GRO_TCP_IPV4;
@@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
RTE_GRO_UDP_IPV4;
do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
RTE_GRO_IPV4_VXLAN_UDP_IPV4;
+ do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) == RTE_GRO_TCP_IPV6;
current_time = rte_rdtsc();
@@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
if (gro_udp4_reassemble(pkts[i], udp_tbl,
current_time) < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
+ current_time) < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
flush_timestamp,
&out[num], left_nb_out);
+ left_nb_out = max_nb_out - num;
+ }
+
+ if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
+ num += gro_tcp6_tbl_timeout_flush(
+ gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
+ flush_timestamp,
+ &out[num], left_nb_out);
+
}
return num;
diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h
index 9f9ed4935a..c83dfd9ad1 100644
--- a/lib/gro/rte_gro.h
+++ b/lib/gro/rte_gro.h
@@ -38,6 +38,9 @@ extern "C" {
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL << RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
/**< VxLAN UDP/IPv4 GRO flag. */
+#define RTE_GRO_TCP_IPV6_INDEX 4
+#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX)
+/**< TCP/IPv6 GRO flag. */
/**
* Structure used to create GRO context objects or used to pass
--
2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* RE: [PATCH v4] gro : ipv6 changes to support GRO for TCP/ipv6
2023-06-06 14:58 ` [PATCH v4] " Kumara Parameshwaran
@ 2023-06-08 4:05 ` Hu, Jiayu
2023-06-08 16:52 ` kumaraparameshwaran rathinavel
0 siblings, 1 reply; 32+ messages in thread
From: Hu, Jiayu @ 2023-06-08 4:05 UTC (permalink / raw)
To: Kumara Parameshwaran; +Cc: dev
Hi Kumara,
Please see replies inline.
In addition, you need to update the programmer guide in generic_receive_offload_lib.rst,
and release note release_23_07.rst.
Thanks,
Jiayu
> -----Original Message-----
> From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> Sent: Tuesday, June 6, 2023 10:58 PM
> To: Hu, Jiayu <jiayu.hu@intel.com>
> Cc: dev@dpdk.org; Kumara Parameshwaran
> <kumaraparamesh92@gmail.com>
> Subject: [PATCH v4] gro : ipv6 changes to support GRO for TCP/ipv6
>
> The patch adds GRO support for TCP/ipv6 packets. This does not include the
> support for vxlan, udp ipv6 packets.
>
> Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>>
> diff --git a/lib/gro/gro_tcp.c b/lib/gro/gro_tcp.c new file mode 100644 index
> 0000000000..02a7d0f8c5
> --- /dev/null
> +++ b/lib/gro/gro_tcp.c
For gro_tcp.c and gro_tcp.h, it's better to add "_internal" in the file name.
> @@ -0,0 +1,128 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
> +#include <rte_malloc.h>
> +#include <rte_mbuf.h>
> +#include <rte_ethdev.h>
> +
> +#include "gro_tcp.h"
> +
> +static inline uint32_t
> +find_an_empty_item(struct gro_tcp_item *items,
> + uint32_t max_item_num)
> +{
> + uint32_t i;
> +
> + for (i = 0; i < max_item_num; i++)
> + if (items[i].firstseg == NULL)
> + return i;
> + return INVALID_ARRAY_INDEX;
> +}
> +
> +uint32_t
> +insert_new_tcp_item(struct rte_mbuf *pkt,
> + struct gro_tcp_item *items,
> + uint32_t *item_num,
> + uint32_t max_item_num,
> + uint64_t start_time,
> + uint32_t prev_idx,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint8_t is_atomic)
This function can be inline.
> +{
> + uint32_t item_idx;
> +
> + item_idx = find_an_empty_item(items, max_item_num);
> + if (item_idx == INVALID_ARRAY_INDEX)
> + return INVALID_ARRAY_INDEX;
> +
> + items[item_idx].firstseg = pkt;
> + items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> + items[item_idx].start_time = start_time;
> + items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> + items[item_idx].sent_seq = sent_seq;
> + items[item_idx].ip_id = ip_id;
> + items[item_idx].nb_merged = 1;
> + items[item_idx].is_atomic = is_atomic;
> + (*item_num) += 1;
> +
> + /* if the previous packet exists, chain them together. */
> + if (prev_idx != INVALID_ARRAY_INDEX) {
> + items[item_idx].next_pkt_idx =
> + items[prev_idx].next_pkt_idx;
> + items[prev_idx].next_pkt_idx = item_idx;
> + }
> +
> + return item_idx;
> +}
> +
> +uint32_t
> +delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t prev_item_idx)
> +{
> + uint32_t next_idx = items[item_idx].next_pkt_idx;
> +
> + /* NULL indicates an empty item */
> + items[item_idx].firstseg = NULL;
> + (*item_num) -= 1;
> + if (prev_item_idx != INVALID_ARRAY_INDEX)
> + items[prev_item_idx].next_pkt_idx = next_idx;
> +
> + return next_idx;
> +}
This function can be inline.
> +
> +int32_t
> +gro_process_tcp_item(struct rte_mbuf *pkt,
> + struct rte_tcp_hdr *tcp_hdr,
> + int32_t tcp_dl,
> + struct gro_tcp_item *items,
> + uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t max_item_num,
> + uint16_t ip_id,
> + uint8_t is_atomic,
> + uint64_t start_time)
It is for internal use, so it's better to remove "gro_" from the function name.
> +{
> + uint32_t cur_idx;
> + uint32_t prev_idx;
> + int cmp;
> + uint32_t sent_seq;
> +
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> + /*
> + * Check all packets in the flow and try to find a neighbor for
> + * the input packet.
> + */
> + cur_idx = item_idx;
> + prev_idx = cur_idx;
> + do {
> + cmp = check_seq_option(&items[cur_idx], tcp_hdr,
> + sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> + is_atomic);
> + if (cmp) {
> + if (merge_two_tcp_packets(&items[cur_idx],
> + pkt, cmp, sent_seq, ip_id, 0))
> + return 1;
> + /*
> + * Fail to merge the two packets, as the packet
> + * length is greater than the max value. Store
> + * the packet into the flow.
> + */
> + if (insert_new_tcp_item(pkt, items, item_num,
> max_item_num, start_time, cur_idx,
> + sent_seq, ip_id, is_atomic) ==
> + INVALID_ARRAY_INDEX)
> + return -1;
> + return 0;
> + }
> + prev_idx = cur_idx;
> + cur_idx = items[cur_idx].next_pkt_idx;
> + } while (cur_idx != INVALID_ARRAY_INDEX);
> +
> + /* Fail to find a neighbor, so store the packet into the flow. */
> + if (insert_new_tcp_item(pkt, items, item_num, max_item_num,
> start_time, prev_idx, sent_seq,
> + ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> + return -1;
> +
> + return 0;
> +}
> diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h new file mode 100644 index
> 0000000000..4b5b4eda9c
> --- /dev/null
> +++ b/lib/gro/gro_tcp.h
> @@ -0,0 +1,209 @@
> +#ifndef _GRO_TCP_H_
> +#define _GRO_TCP_H_
> +
> +#define INVALID_ARRAY_INDEX 0xffffffffUL
> +
> +#include <rte_tcp.h>
> +
> +/*
> + * The max length of a IPv4 packet, which includes the length of the L3
> + * header, the L4 header and the data payload.
> + */
> +#define MAX_IP_PKT_LENGTH UINT16_MAX
> +
> +/* The maximum TCP header length */
> +#define MAX_TCP_HLEN 60
> +#define INVALID_TCP_HDRLEN(len) \
> + (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> +
> +struct gro_tcp_flow {
This structure name is confusing. In the upper layer, tcp4 and tcp6 have gro_tcp4_flow
and gro_tcp6_flow, which represent a flow. Inside gro_tcp4/6_flow, there are keys,
represented by struct tcp4/6_flow_key. But inside struct tcp4/6_flow_key, there is
struct gro_tcp_flow. Need to rename struct gro_tcp_flow, like common_tcp_flow_key.
> + struct rte_ether_addr eth_saddr;
> + struct rte_ether_addr eth_daddr;
> + uint32_t recv_ack;
> + uint16_t src_port;
> + uint16_t dst_port;
> +};
> +
> +#define ASSIGN_TCP_FLOW_KEY(k1, k2) \
Ditto. The macro needs rename, like ASSIGN_COMMON_TCP_FLOW_KEY.
> + do {\
> + rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
> + rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr));
> \
> + k2->recv_ack = k1->recv_ack; \
> + k2->src_port = k1->src_port; \
> + k2->dst_port = k1->dst_port; \
> + } while (0)
> +
> +struct gro_tcp_item {
> + /*
> + * The first MBUF segment of the packet. If the value
> + * is NULL, it means the item is empty.
> + */
> + struct rte_mbuf *firstseg;
> + /* The last MBUF segment of the packet */
> + struct rte_mbuf *lastseg;
> + /*
> + * The time when the first packet is inserted into the table.
> + * This value won't be updated, even if the packet is merged
> + * with other packets.
> + */
> + uint64_t start_time;
> + /*
> + * next_pkt_idx is used to chain the packets that
> + * are in the same flow but can't be merged together
> + * (e.g. caused by packet reordering).
> + */
> + uint32_t next_pkt_idx;
> + /* TCP sequence number of the packet */
> + uint32_t sent_seq;
> + /* IPv4 ID of the packet */
> + uint16_t ip_id;
The ip_id field is not used by tcp6. It's better to use an union to include ip_id for IPv4 and
an useless member for IPv6 with some comments to avoid confusing.
> + /* the number of merged packets */
> + uint16_t nb_merged;
> + /* Indicate if IPv4 ID can be ignored */
> + uint8_t is_atomic;
> +};
> +
> +uint32_t
> +insert_new_tcp_item(struct rte_mbuf *pkt,
> + struct gro_tcp_item *items,
> + uint32_t *item_num,
> + uint32_t table_size,
> + uint64_t start_time,
> + uint32_t prev_idx,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint8_t is_atomic);
> +
> +uint32_t
> +delete_tcp_item(struct gro_tcp_item *items,
> + uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t prev_item_idx);
> +
> +int32_t
> +gro_process_tcp_item(struct rte_mbuf *pkt,
> + struct rte_tcp_hdr *tcp_hdr,
> + int32_t tcp_dl,
> + struct gro_tcp_item *items,
> + uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t table_size,
> + uint16_t ip_id,
> + uint8_t is_atomic,
> + uint64_t start_time);
> +
> +/*
> + * Merge two TCP packets without updating checksums.
> + * If cmp is larger than 0, append the new packet to the
> + * original packet. Otherwise, pre-pend the new packet to
> + * the original packet.
> + */
> +static inline int
> +merge_two_tcp_packets(struct gro_tcp_item *item,
> + struct rte_mbuf *pkt,
> + int cmp,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint16_t l2_offset)
> +{
> + struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
> + uint16_t hdr_len, l2_len;
> +
> + if (cmp > 0) {
> + pkt_head = item->firstseg;
> + pkt_tail = pkt;
> + } else {
> + pkt_head = pkt;
> + pkt_tail = item->firstseg;
> + }
> +
> + /* check if the IPv4 packet length is greater than the max value */
> + hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
> + pkt_head->l4_len;
> + l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
> + if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
> + hdr_len > MAX_IP_PKT_LENGTH))
> + return 0;
> +
> + /* remove the packet header for the tail packet */
> + rte_pktmbuf_adj(pkt_tail, hdr_len);
> +
> + /* chain two packets together */
> + if (cmp > 0) {
> + item->lastseg->next = pkt;
> + item->lastseg = rte_pktmbuf_lastseg(pkt);
> + /* update IP ID to the larger value */
> + item->ip_id = ip_id;
> + } else {
> + lastseg = rte_pktmbuf_lastseg(pkt);
> + lastseg->next = item->firstseg;
> + item->firstseg = pkt;
> + /* update sent_seq to the smaller value */
> + item->sent_seq = sent_seq;
> + item->ip_id = ip_id;
> + }
> + item->nb_merged++;
> +
> + /* update MBUF metadata for the merged packet */
> + pkt_head->nb_segs += pkt_tail->nb_segs;
> + pkt_head->pkt_len += pkt_tail->pkt_len;
> +
> + return 1;
> +}
> +
> +/*
> + * Check if two TCP/IPv4 packets are neighbors.
> + */
> +static inline int
> +check_seq_option(struct gro_tcp_item *item,
> + struct rte_tcp_hdr *tcph,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint16_t tcp_hl,
> + uint16_t tcp_dl,
> + uint16_t l2_offset,
> + uint8_t is_atomic)
> +{
> + struct rte_mbuf *pkt_orig = item->firstseg;
> + char *iph_orig;
> + struct rte_tcp_hdr *tcph_orig;
> + uint16_t len, tcp_hl_orig;
> +
> + iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
> + l2_offset + pkt_orig->l2_len);
> + tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
> + tcp_hl_orig = pkt_orig->l4_len;
> +
> + /* Check if TCP option fields equal */
> + len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> + if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> + (memcmp(tcph + 1, tcph_orig + 1,
> + len) != 0)))
> + return 0;
> +
> + /* Don't merge packets whose DF bits are different */
> + if (unlikely(item->is_atomic ^ is_atomic))
> + return 0;
> +
> + /* check if the two packets are neighbors */
> + len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> + pkt_orig->l3_len - tcp_hl_orig;
> + if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> + (ip_id == item->ip_id + 1)))
> + /* append the new packet */
> + return 1;
> + else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> + (ip_id + item->nb_merged == item->ip_id)))
> + /* pre-pend the new packet */
> + return -1;
> +
> + return 0;
> +}
> +
> +static inline int
> +is_same_tcp_flow(struct gro_tcp_flow *k1, struct gro_tcp_flow *k2) {
> + return (!memcmp(k1, k2, sizeof(struct gro_tcp_flow))); }
I think this function needs rename, as the result of this function cannot identify if they are
same TCP flow.
> +
> +#endif
> diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c index
> 0014096e63..ffc33747c4 100644
> --- a/lib/gro/gro_tcp4.c
> +++ b/lib/gro/gro_tcp4.c
> @@ -7,6 +7,7 @@
> #include <rte_ethdev.h>
>
> #include "gro_tcp4.h"
> +#include "gro_tcp.h"
>
> void *
> gro_tcp4_tbl_create(uint16_t socket_id, @@ -30,7 +31,7 @@
> gro_tcp4_tbl_create(uint16_t socket_id,
> if (tbl == NULL)
> return NULL;
>
> - size = sizeof(struct gro_tcp4_item) * entries_num;
> + size = sizeof(struct gro_tcp_item) * entries_num;
> tbl->items = rte_zmalloc_socket(__func__,
> size,
> RTE_CACHE_LINE_SIZE,
> @@ -71,18 +72,6 @@ gro_tcp4_tbl_destroy(void *tbl)
> rte_free(tcp_tbl);
> }
>
> -static inline uint32_t
> -find_an_empty_item(struct gro_tcp4_tbl *tbl) -{
> - uint32_t i;
> - uint32_t max_item_num = tbl->max_item_num;
> -
> - for (i = 0; i < max_item_num; i++)
> - if (tbl->items[i].firstseg == NULL)
> - return i;
> - return INVALID_ARRAY_INDEX;
> -}
> -
> static inline uint32_t
> find_an_empty_flow(struct gro_tcp4_tbl *tbl) { @@ -95,56 +84,6 @@
> find_an_empty_flow(struct gro_tcp4_tbl *tbl)
> return INVALID_ARRAY_INDEX;
> }
>
> -static inline uint32_t
> -insert_new_item(struct gro_tcp4_tbl *tbl,
> - struct rte_mbuf *pkt,
> - uint64_t start_time,
> - uint32_t prev_idx,
> - uint32_t sent_seq,
> - uint16_t ip_id,
> - uint8_t is_atomic)
> -{
> - uint32_t item_idx;
> -
> - item_idx = find_an_empty_item(tbl);
> - if (item_idx == INVALID_ARRAY_INDEX)
> - return INVALID_ARRAY_INDEX;
> -
> - tbl->items[item_idx].firstseg = pkt;
> - tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> - tbl->items[item_idx].start_time = start_time;
> - tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> - tbl->items[item_idx].sent_seq = sent_seq;
> - tbl->items[item_idx].ip_id = ip_id;
> - tbl->items[item_idx].nb_merged = 1;
> - tbl->items[item_idx].is_atomic = is_atomic;
> - tbl->item_num++;
> -
> - /* if the previous packet exists, chain them together. */
> - if (prev_idx != INVALID_ARRAY_INDEX) {
> - tbl->items[item_idx].next_pkt_idx =
> - tbl->items[prev_idx].next_pkt_idx;
> - tbl->items[prev_idx].next_pkt_idx = item_idx;
> - }
> -
> - return item_idx;
> -}
> -
> -static inline uint32_t
> -delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
> - uint32_t prev_item_idx)
> -{
> - uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
> -
> - /* NULL indicates an empty item */
> - tbl->items[item_idx].firstseg = NULL;
> - tbl->item_num--;
> - if (prev_item_idx != INVALID_ARRAY_INDEX)
> - tbl->items[prev_item_idx].next_pkt_idx = next_idx;
> -
> - return next_idx;
> -}
> -
> static inline uint32_t
> insert_new_flow(struct gro_tcp4_tbl *tbl,
> struct tcp4_flow_key *src,
> @@ -159,13 +98,10 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
>
> dst = &(tbl->flows[flow_idx].key);
>
> - rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
> - rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
> + ASSIGN_TCP_FLOW_KEY((&src->tcp_flow), (&dst->tcp_flow));
> +
> dst->ip_src_addr = src->ip_src_addr;
> dst->ip_dst_addr = src->ip_dst_addr;
> - dst->recv_ack = src->recv_ack;
> - dst->src_port = src->src_port;
> - dst->dst_port = src->dst_port;
>
> tbl->flows[flow_idx].start_index = item_idx;
> tbl->flow_num++;
> @@ -173,21 +109,6 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
> return flow_idx;
> }
>
> -/*
> - * update the packet length for the flushed packet.
> - */
> -static inline void
> -update_header(struct gro_tcp4_item *item) -{
> - struct rte_ipv4_hdr *ipv4_hdr;
> - struct rte_mbuf *pkt = item->firstseg;
> -
> - ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> - pkt->l2_len);
> - ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
> - pkt->l2_len);
> -}
> -
> int32_t
> gro_tcp4_reassemble(struct rte_mbuf *pkt,
> struct gro_tcp4_tbl *tbl,
> @@ -195,16 +116,15 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt, {
> struct rte_ether_hdr *eth_hdr;
> struct rte_ipv4_hdr *ipv4_hdr;
> - struct rte_tcp_hdr *tcp_hdr;
> - uint32_t sent_seq;
> int32_t tcp_dl;
> + struct rte_tcp_hdr *tcp_hdr;
> uint16_t ip_id, hdr_len, frag_off, ip_tlen;
> uint8_t is_atomic;
> + uint32_t sent_seq;
No need to change tcp_hdr and sent_seq here.
>
> struct tcp4_flow_key key;
> - uint32_t cur_idx, prev_idx, item_idx;
> + uint32_t item_idx;
> uint32_t i, max_flow_num, remaining_flow_num;
> - int cmp;
> uint8_t find;
>
> /*
> @@ -216,7 +136,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
>
> eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
> ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
> - tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
> + tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *,
> +pkt->l2_len + pkt->l3_len);
> hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
>
> /*
> @@ -230,7 +150,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> ip_tlen = rte_be_to_cpu_16(ipv4_hdr->total_length);
> if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len))
> rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len);
> -
> /*
> * Don't process the packet whose payload length is less than or
> * equal to 0.
> @@ -239,6 +158,13 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> if (tcp_dl <= 0)
> return -1;
>
> + rte_ether_addr_copy(&(eth_hdr->src_addr),
> &(key.tcp_flow.eth_saddr));
> + rte_ether_addr_copy(&(eth_hdr->dst_addr),
> &(key.tcp_flow.eth_daddr));
> + key.ip_src_addr = ipv4_hdr->src_addr;
> + key.ip_dst_addr = ipv4_hdr->dst_addr;
> + key.tcp_flow.src_port = tcp_hdr->src_port;
> + key.tcp_flow.dst_port = tcp_hdr->dst_port;
> + key.tcp_flow.recv_ack = tcp_hdr->recv_ack;
> /*
> * Save IPv4 ID for the packet whose DF bit is 0. For the packet
> * whose DF bit is 1, IPv4 ID is ignored.
> @@ -246,15 +172,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
> is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) ==
> RTE_IPV4_HDR_DF_FLAG;
> ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
> - sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> -
> - rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
> - rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
> - key.ip_src_addr = ipv4_hdr->src_addr;
> - key.ip_dst_addr = ipv4_hdr->dst_addr;
> - key.src_port = tcp_hdr->src_port;
> - key.dst_port = tcp_hdr->dst_port;
> - key.recv_ack = tcp_hdr->recv_ack;
>
> /* Search for a matched flow. */
> max_flow_num = tbl->max_flow_num;
> @@ -270,63 +187,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> }
> }
>
> - /*
> - * Fail to find a matched flow. Insert a new flow and store the
> - * packet into the flow.
> - */
> if (find == 0) {
> - item_idx = insert_new_item(tbl, pkt, start_time,
> - INVALID_ARRAY_INDEX, sent_seq, ip_id,
> - is_atomic);
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> + item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl-
> >item_num, tbl->max_item_num, start_time,
> + INVALID_ARRAY_INDEX,
> sent_seq, ip_id,
> + is_atomic);
> if (item_idx == INVALID_ARRAY_INDEX)
> return -1;
> if (insert_new_flow(tbl, &key, item_idx) ==
> - INVALID_ARRAY_INDEX) {
> + INVALID_ARRAY_INDEX) {
> /*
> * Fail to insert a new flow, so delete the
> * stored packet.
> - */
> - delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
> + */
> + delete_tcp_item(tbl->items, item_idx, &tbl-
> >item_num,
> +INVALID_ARRAY_INDEX);
> return -1;
> }
> return 0;
> }
> + item_idx = tbl->flows[i].start_index;
No need to update item_idx, and you can directly pass tbl->flows[i].start_index to
gro_process_tcp_item(). And same in gro_tcp6_reassemble().
>
> - /*
> - * Check all packets in the flow and try to find a neighbor for
> - * the input packet.
> - */
> - cur_idx = tbl->flows[i].start_index;
> - prev_idx = cur_idx;
> - do {
> - cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
> - sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> - is_atomic);
> - if (cmp) {
> - if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
> - pkt, cmp, sent_seq, ip_id, 0))
> - return 1;
> - /*
> - * Fail to merge the two packets, as the packet
> - * length is greater than the max value. Store
> - * the packet into the flow.
> - */
> - if (insert_new_item(tbl, pkt, start_time, cur_idx,
> - sent_seq, ip_id, is_atomic) ==
> - INVALID_ARRAY_INDEX)
> - return -1;
> - return 0;
> - }
> - prev_idx = cur_idx;
> - cur_idx = tbl->items[cur_idx].next_pkt_idx;
> - } while (cur_idx != INVALID_ARRAY_INDEX);
> + return gro_process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items,
> item_idx,
> + &tbl->item_num, tbl-
> >max_item_num,
> + ip_id, is_atomic, start_time);
> +}
>
> - /* Fail to find a neighbor, so store the packet into the flow. */
> - if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
> - ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> - return -1;
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [PATCH v4] gro : ipv6 changes to support GRO for TCP/ipv6
2023-06-08 4:05 ` Hu, Jiayu
@ 2023-06-08 16:52 ` kumaraparameshwaran rathinavel
2023-06-09 1:04 ` Hu, Jiayu
0 siblings, 1 reply; 32+ messages in thread
From: kumaraparameshwaran rathinavel @ 2023-06-08 16:52 UTC (permalink / raw)
To: Hu, Jiayu; +Cc: dev
[-- Attachment #1: Type: text/plain, Size: 26737 bytes --]
Hi Jiyau,
Thanks for the quick review comments. Will address the review comments.
Require clarification in one of the comments. Please find it inline.
On Thu, Jun 8, 2023 at 9:35 AM Hu, Jiayu <jiayu.hu@intel.com> wrote:
> Hi Kumara,
>
> Please see replies inline.
>
> In addition, you need to update the programmer guide in
> generic_receive_offload_lib.rst,
> and release note release_23_07.rst.
>
> Thanks,
> Jiayu
>
> > -----Original Message-----
> > From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> > Sent: Tuesday, June 6, 2023 10:58 PM
> > To: Hu, Jiayu <jiayu.hu@intel.com>
> > Cc: dev@dpdk.org; Kumara Parameshwaran
> > <kumaraparamesh92@gmail.com>
> > Subject: [PATCH v4] gro : ipv6 changes to support GRO for TCP/ipv6
> >
> > The patch adds GRO support for TCP/ipv6 packets. This does not include
> the
> > support for vxlan, udp ipv6 packets.
> >
> > Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>>
> > diff --git a/lib/gro/gro_tcp.c b/lib/gro/gro_tcp.c new file mode 100644
> index
> > 0000000000..02a7d0f8c5
> > --- /dev/null
> > +++ b/lib/gro/gro_tcp.c
>
> For gro_tcp.c and gro_tcp.h, it's better to add "_internal" in the file
> name.
>
> > @@ -0,0 +1,128 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2017 Intel Corporation
> > + */
> > +#include <rte_malloc.h>
> > +#include <rte_mbuf.h>
> > +#include <rte_ethdev.h>
> > +
> > +#include "gro_tcp.h"
> > +
> > +static inline uint32_t
> > +find_an_empty_item(struct gro_tcp_item *items,
> > + uint32_t max_item_num)
> > +{
> > + uint32_t i;
> > +
> > + for (i = 0; i < max_item_num; i++)
> > + if (items[i].firstseg == NULL)
> > + return i;
> > + return INVALID_ARRAY_INDEX;
> > +}
> > +
> > +uint32_t
> > +insert_new_tcp_item(struct rte_mbuf *pkt,
> > + struct gro_tcp_item *items,
> > + uint32_t *item_num,
> > + uint32_t max_item_num,
> > + uint64_t start_time,
> > + uint32_t prev_idx,
> > + uint32_t sent_seq,
> > + uint16_t ip_id,
> > + uint8_t is_atomic)
>
> This function can be inline.
>
> > +{
> > + uint32_t item_idx;
> > +
> > + item_idx = find_an_empty_item(items, max_item_num);
> > + if (item_idx == INVALID_ARRAY_INDEX)
> > + return INVALID_ARRAY_INDEX;
> > +
> > + items[item_idx].firstseg = pkt;
> > + items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> > + items[item_idx].start_time = start_time;
> > + items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> > + items[item_idx].sent_seq = sent_seq;
> > + items[item_idx].ip_id = ip_id;
> > + items[item_idx].nb_merged = 1;
> > + items[item_idx].is_atomic = is_atomic;
> > + (*item_num) += 1;
> > +
> > + /* if the previous packet exists, chain them together. */
> > + if (prev_idx != INVALID_ARRAY_INDEX) {
> > + items[item_idx].next_pkt_idx =
> > + items[prev_idx].next_pkt_idx;
> > + items[prev_idx].next_pkt_idx = item_idx;
> > + }
> > +
> > + return item_idx;
> > +}
> > +
> > +uint32_t
> > +delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
> > + uint32_t *item_num,
> > + uint32_t prev_item_idx)
> > +{
> > + uint32_t next_idx = items[item_idx].next_pkt_idx;
> > +
> > + /* NULL indicates an empty item */
> > + items[item_idx].firstseg = NULL;
> > + (*item_num) -= 1;
> > + if (prev_item_idx != INVALID_ARRAY_INDEX)
> > + items[prev_item_idx].next_pkt_idx = next_idx;
> > +
> > + return next_idx;
> > +}
>
> This function can be inline.
>
> > +
> > +int32_t
> > +gro_process_tcp_item(struct rte_mbuf *pkt,
> > + struct rte_tcp_hdr *tcp_hdr,
> > + int32_t tcp_dl,
> > + struct gro_tcp_item *items,
> > + uint32_t item_idx,
> > + uint32_t *item_num,
> > + uint32_t max_item_num,
> > + uint16_t ip_id,
> > + uint8_t is_atomic,
> > + uint64_t start_time)
>
> It is for internal use, so it's better to remove "gro_" from the function
> name.
>
> > +{
> > + uint32_t cur_idx;
> > + uint32_t prev_idx;
> > + int cmp;
> > + uint32_t sent_seq;
> > +
> > + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> > + /*
> > + * Check all packets in the flow and try to find a neighbor for
> > + * the input packet.
> > + */
> > + cur_idx = item_idx;
> > + prev_idx = cur_idx;
> > + do {
> > + cmp = check_seq_option(&items[cur_idx], tcp_hdr,
> > + sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> > + is_atomic);
> > + if (cmp) {
> > + if (merge_two_tcp_packets(&items[cur_idx],
> > + pkt, cmp, sent_seq, ip_id,
> 0))
> > + return 1;
> > + /*
> > + * Fail to merge the two packets, as the packet
> > + * length is greater than the max value. Store
> > + * the packet into the flow.
> > + */
> > + if (insert_new_tcp_item(pkt, items, item_num,
> > max_item_num, start_time, cur_idx,
> > + sent_seq, ip_id,
> is_atomic) ==
> > + INVALID_ARRAY_INDEX)
> > + return -1;
> > + return 0;
> > + }
> > + prev_idx = cur_idx;
> > + cur_idx = items[cur_idx].next_pkt_idx;
> > + } while (cur_idx != INVALID_ARRAY_INDEX);
> > +
> > + /* Fail to find a neighbor, so store the packet into the flow. */
> > + if (insert_new_tcp_item(pkt, items, item_num, max_item_num,
> > start_time, prev_idx, sent_seq,
> > + ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> > + return -1;
> > +
> > + return 0;
> > +}
> > diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h new file mode 100644
> index
> > 0000000000..4b5b4eda9c
> > --- /dev/null
> > +++ b/lib/gro/gro_tcp.h
> > @@ -0,0 +1,209 @@
> > +#ifndef _GRO_TCP_H_
> > +#define _GRO_TCP_H_
> > +
> > +#define INVALID_ARRAY_INDEX 0xffffffffUL
> > +
> > +#include <rte_tcp.h>
> > +
> > +/*
> > + * The max length of a IPv4 packet, which includes the length of the L3
> > + * header, the L4 header and the data payload.
> > + */
> > +#define MAX_IP_PKT_LENGTH UINT16_MAX
> > +
> > +/* The maximum TCP header length */
> > +#define MAX_TCP_HLEN 60
> > +#define INVALID_TCP_HDRLEN(len) \
> > + (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> > +
> > +struct gro_tcp_flow {
>
> This structure name is confusing. In the upper layer, tcp4 and tcp6 have
> gro_tcp4_flow
> and gro_tcp6_flow, which represent a flow. Inside gro_tcp4/6_flow, there
> are keys,
> represented by struct tcp4/6_flow_key. But inside struct tcp4/6_flow_key,
> there is
> struct gro_tcp_flow. Need to rename struct gro_tcp_flow, like
> common_tcp_flow_key.
>
> > + struct rte_ether_addr eth_saddr;
> > + struct rte_ether_addr eth_daddr;
> > + uint32_t recv_ack;
> > + uint16_t src_port;
> > + uint16_t dst_port;
> > +};
> > +
> > +#define ASSIGN_TCP_FLOW_KEY(k1, k2) \
>
> Ditto. The macro needs rename, like ASSIGN_COMMON_TCP_FLOW_KEY.
>
> > + do {\
> > + rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
> > + rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr));
> > \
> > + k2->recv_ack = k1->recv_ack; \
> > + k2->src_port = k1->src_port; \
> > + k2->dst_port = k1->dst_port; \
> > + } while (0)
> > +
> > +struct gro_tcp_item {
> > + /*
> > + * The first MBUF segment of the packet. If the value
> > + * is NULL, it means the item is empty.
> > + */
> > + struct rte_mbuf *firstseg;
> > + /* The last MBUF segment of the packet */
> > + struct rte_mbuf *lastseg;
> > + /*
> > + * The time when the first packet is inserted into the table.
> > + * This value won't be updated, even if the packet is merged
> > + * with other packets.
> > + */
> > + uint64_t start_time;
> > + /*
> > + * next_pkt_idx is used to chain the packets that
> > + * are in the same flow but can't be merged together
> > + * (e.g. caused by packet reordering).
> > + */
> > + uint32_t next_pkt_idx;
> > + /* TCP sequence number of the packet */
> > + uint32_t sent_seq;
> > + /* IPv4 ID of the packet */
> > + uint16_t ip_id;
>
> The ip_id field is not used by tcp6. It's better to use an union to
> include ip_id for IPv4 and
> an useless member for IPv6 with some comments to avoid confusing.
>
> > + /* the number of merged packets */
> > + uint16_t nb_merged;
> > + /* Indicate if IPv4 ID can be ignored */
> > + uint8_t is_atomic;
> > +};
> > +
> > +uint32_t
> > +insert_new_tcp_item(struct rte_mbuf *pkt,
> > + struct gro_tcp_item *items,
> > + uint32_t *item_num,
> > + uint32_t table_size,
> > + uint64_t start_time,
> > + uint32_t prev_idx,
> > + uint32_t sent_seq,
> > + uint16_t ip_id,
> > + uint8_t is_atomic);
> > +
> > +uint32_t
> > +delete_tcp_item(struct gro_tcp_item *items,
> > + uint32_t item_idx,
> > + uint32_t *item_num,
> > + uint32_t prev_item_idx);
> > +
> > +int32_t
> > +gro_process_tcp_item(struct rte_mbuf *pkt,
> > + struct rte_tcp_hdr *tcp_hdr,
> > + int32_t tcp_dl,
> > + struct gro_tcp_item *items,
> > + uint32_t item_idx,
> > + uint32_t *item_num,
> > + uint32_t table_size,
> > + uint16_t ip_id,
> > + uint8_t is_atomic,
> > + uint64_t start_time);
> > +
> > +/*
> > + * Merge two TCP packets without updating checksums.
> > + * If cmp is larger than 0, append the new packet to the
> > + * original packet. Otherwise, pre-pend the new packet to
> > + * the original packet.
> > + */
> > +static inline int
> > +merge_two_tcp_packets(struct gro_tcp_item *item,
> > + struct rte_mbuf *pkt,
> > + int cmp,
> > + uint32_t sent_seq,
> > + uint16_t ip_id,
> > + uint16_t l2_offset)
> > +{
> > + struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
> > + uint16_t hdr_len, l2_len;
> > +
> > + if (cmp > 0) {
> > + pkt_head = item->firstseg;
> > + pkt_tail = pkt;
> > + } else {
> > + pkt_head = pkt;
> > + pkt_tail = item->firstseg;
> > + }
> > +
> > + /* check if the IPv4 packet length is greater than the max value */
> > + hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
> > + pkt_head->l4_len;
> > + l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
> > + if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
> > + hdr_len > MAX_IP_PKT_LENGTH))
> > + return 0;
> > +
> > + /* remove the packet header for the tail packet */
> > + rte_pktmbuf_adj(pkt_tail, hdr_len);
> > +
> > + /* chain two packets together */
> > + if (cmp > 0) {
> > + item->lastseg->next = pkt;
> > + item->lastseg = rte_pktmbuf_lastseg(pkt);
> > + /* update IP ID to the larger value */
> > + item->ip_id = ip_id;
> > + } else {
> > + lastseg = rte_pktmbuf_lastseg(pkt);
> > + lastseg->next = item->firstseg;
> > + item->firstseg = pkt;
> > + /* update sent_seq to the smaller value */
> > + item->sent_seq = sent_seq;
> > + item->ip_id = ip_id;
> > + }
> > + item->nb_merged++;
> > +
> > + /* update MBUF metadata for the merged packet */
> > + pkt_head->nb_segs += pkt_tail->nb_segs;
> > + pkt_head->pkt_len += pkt_tail->pkt_len;
> > +
> > + return 1;
> > +}
> > +
> > +/*
> > + * Check if two TCP/IPv4 packets are neighbors.
> > + */
> > +static inline int
> > +check_seq_option(struct gro_tcp_item *item,
> > + struct rte_tcp_hdr *tcph,
> > + uint32_t sent_seq,
> > + uint16_t ip_id,
> > + uint16_t tcp_hl,
> > + uint16_t tcp_dl,
> > + uint16_t l2_offset,
> > + uint8_t is_atomic)
> > +{
> > + struct rte_mbuf *pkt_orig = item->firstseg;
> > + char *iph_orig;
> > + struct rte_tcp_hdr *tcph_orig;
> > + uint16_t len, tcp_hl_orig;
> > +
> > + iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
> > + l2_offset + pkt_orig->l2_len);
> > + tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
> > + tcp_hl_orig = pkt_orig->l4_len;
> > +
> > + /* Check if TCP option fields equal */
> > + len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> > + if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> > + (memcmp(tcph + 1, tcph_orig + 1,
> > + len) != 0)))
> > + return 0;
> > +
> > + /* Don't merge packets whose DF bits are different */
> > + if (unlikely(item->is_atomic ^ is_atomic))
> > + return 0;
> > +
> > + /* check if the two packets are neighbors */
> > + len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> > + pkt_orig->l3_len - tcp_hl_orig;
> > + if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> > + (ip_id == item->ip_id + 1)))
> > + /* append the new packet */
> > + return 1;
> > + else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> > + (ip_id + item->nb_merged == item->ip_id)))
> > + /* pre-pend the new packet */
> > + return -1;
> > +
> > + return 0;
> > +}
> > +
> > +static inline int
> > +is_same_tcp_flow(struct gro_tcp_flow *k1, struct gro_tcp_flow *k2) {
> > + return (!memcmp(k1, k2, sizeof(struct gro_tcp_flow))); }
>
> I think this function needs rename, as the result of this function cannot
> identify if they are
> same TCP flow.
>
> > +
> > +#endif
> > diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c index
> > 0014096e63..ffc33747c4 100644
> > --- a/lib/gro/gro_tcp4.c
> > +++ b/lib/gro/gro_tcp4.c
> > @@ -7,6 +7,7 @@
> > #include <rte_ethdev.h>
> >
> > #include "gro_tcp4.h"
> > +#include "gro_tcp.h"
> >
> > void *
> > gro_tcp4_tbl_create(uint16_t socket_id, @@ -30,7 +31,7 @@
> > gro_tcp4_tbl_create(uint16_t socket_id,
> > if (tbl == NULL)
> > return NULL;
> >
> > - size = sizeof(struct gro_tcp4_item) * entries_num;
> > + size = sizeof(struct gro_tcp_item) * entries_num;
> > tbl->items = rte_zmalloc_socket(__func__,
> > size,
> > RTE_CACHE_LINE_SIZE,
> > @@ -71,18 +72,6 @@ gro_tcp4_tbl_destroy(void *tbl)
> > rte_free(tcp_tbl);
> > }
> >
> > -static inline uint32_t
> > -find_an_empty_item(struct gro_tcp4_tbl *tbl) -{
> > - uint32_t i;
> > - uint32_t max_item_num = tbl->max_item_num;
> > -
> > - for (i = 0; i < max_item_num; i++)
> > - if (tbl->items[i].firstseg == NULL)
> > - return i;
> > - return INVALID_ARRAY_INDEX;
> > -}
> > -
> > static inline uint32_t
> > find_an_empty_flow(struct gro_tcp4_tbl *tbl) { @@ -95,56 +84,6 @@
> > find_an_empty_flow(struct gro_tcp4_tbl *tbl)
> > return INVALID_ARRAY_INDEX;
> > }
> >
> > -static inline uint32_t
> > -insert_new_item(struct gro_tcp4_tbl *tbl,
> > - struct rte_mbuf *pkt,
> > - uint64_t start_time,
> > - uint32_t prev_idx,
> > - uint32_t sent_seq,
> > - uint16_t ip_id,
> > - uint8_t is_atomic)
> > -{
> > - uint32_t item_idx;
> > -
> > - item_idx = find_an_empty_item(tbl);
> > - if (item_idx == INVALID_ARRAY_INDEX)
> > - return INVALID_ARRAY_INDEX;
> > -
> > - tbl->items[item_idx].firstseg = pkt;
> > - tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> > - tbl->items[item_idx].start_time = start_time;
> > - tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> > - tbl->items[item_idx].sent_seq = sent_seq;
> > - tbl->items[item_idx].ip_id = ip_id;
> > - tbl->items[item_idx].nb_merged = 1;
> > - tbl->items[item_idx].is_atomic = is_atomic;
> > - tbl->item_num++;
> > -
> > - /* if the previous packet exists, chain them together. */
> > - if (prev_idx != INVALID_ARRAY_INDEX) {
> > - tbl->items[item_idx].next_pkt_idx =
> > - tbl->items[prev_idx].next_pkt_idx;
> > - tbl->items[prev_idx].next_pkt_idx = item_idx;
> > - }
> > -
> > - return item_idx;
> > -}
> > -
> > -static inline uint32_t
> > -delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
> > - uint32_t prev_item_idx)
> > -{
> > - uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
> > -
> > - /* NULL indicates an empty item */
> > - tbl->items[item_idx].firstseg = NULL;
> > - tbl->item_num--;
> > - if (prev_item_idx != INVALID_ARRAY_INDEX)
> > - tbl->items[prev_item_idx].next_pkt_idx = next_idx;
> > -
> > - return next_idx;
> > -}
> > -
> > static inline uint32_t
> > insert_new_flow(struct gro_tcp4_tbl *tbl,
> > struct tcp4_flow_key *src,
> > @@ -159,13 +98,10 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
> >
> > dst = &(tbl->flows[flow_idx].key);
> >
> > - rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
> > - rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
> > + ASSIGN_TCP_FLOW_KEY((&src->tcp_flow), (&dst->tcp_flow));
> > +
> > dst->ip_src_addr = src->ip_src_addr;
> > dst->ip_dst_addr = src->ip_dst_addr;
> > - dst->recv_ack = src->recv_ack;
> > - dst->src_port = src->src_port;
> > - dst->dst_port = src->dst_port;
> >
> > tbl->flows[flow_idx].start_index = item_idx;
> > tbl->flow_num++;
> > @@ -173,21 +109,6 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
> > return flow_idx;
> > }
> >
> > -/*
> > - * update the packet length for the flushed packet.
> > - */
> > -static inline void
> > -update_header(struct gro_tcp4_item *item) -{
> > - struct rte_ipv4_hdr *ipv4_hdr;
> > - struct rte_mbuf *pkt = item->firstseg;
> > -
> > - ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> > - pkt->l2_len);
> > - ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
> > - pkt->l2_len);
> > -}
> > -
> > int32_t
> > gro_tcp4_reassemble(struct rte_mbuf *pkt,
> > struct gro_tcp4_tbl *tbl,
> > @@ -195,16 +116,15 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt, {
> > struct rte_ether_hdr *eth_hdr;
> > struct rte_ipv4_hdr *ipv4_hdr;
> > - struct rte_tcp_hdr *tcp_hdr;
> > - uint32_t sent_seq;
> > int32_t tcp_dl;
> > + struct rte_tcp_hdr *tcp_hdr;
> > uint16_t ip_id, hdr_len, frag_off, ip_tlen;
> > uint8_t is_atomic;
> > + uint32_t sent_seq;
>
> No need to change tcp_hdr and sent_seq here.
>
>> The flow matching is done in the function and if the flow is not found
>> insert_new_tcp_item is invoked from this function itself. Did you mean to
>> move that to the process_tcp_item as well? If that is the case we should
>> pass the start_idx as INVALID_ARRAY_INDEX and in process_tcp_item check if
>> INVALID_ARRAY_INDEX do a insert_new_tcp_item and return, do not do the
>> sequnce number checks etc.
>>
> >
> > struct tcp4_flow_key key;
> > - uint32_t cur_idx, prev_idx, item_idx;
> > + uint32_t item_idx;
> > uint32_t i, max_flow_num, remaining_flow_num;
> > - int cmp;
> > uint8_t find;
> >
> > /*
> > @@ -216,7 +136,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> >
> > eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
> > ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
> > - tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
> > + tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *,
> > +pkt->l2_len + pkt->l3_len);
> > hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
> >
> > /*
> > @@ -230,7 +150,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> > ip_tlen = rte_be_to_cpu_16(ipv4_hdr->total_length);
> > if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len))
> > rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen -
> pkt->l2_len);
> > -
> > /*
> > * Don't process the packet whose payload length is less than or
> > * equal to 0.
> > @@ -239,6 +158,13 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> > if (tcp_dl <= 0)
> > return -1;
> >
> > + rte_ether_addr_copy(&(eth_hdr->src_addr),
> > &(key.tcp_flow.eth_saddr));
> > + rte_ether_addr_copy(&(eth_hdr->dst_addr),
> > &(key.tcp_flow.eth_daddr));
> > + key.ip_src_addr = ipv4_hdr->src_addr;
> > + key.ip_dst_addr = ipv4_hdr->dst_addr;
> > + key.tcp_flow.src_port = tcp_hdr->src_port;
> > + key.tcp_flow.dst_port = tcp_hdr->dst_port;
> > + key.tcp_flow.recv_ack = tcp_hdr->recv_ack;
> > /*
> > * Save IPv4 ID for the packet whose DF bit is 0. For the packet
> > * whose DF bit is 1, IPv4 ID is ignored.
> > @@ -246,15 +172,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> > frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
> > is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) ==
> > RTE_IPV4_HDR_DF_FLAG;
> > ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
> > - sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> > -
> > - rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
> > - rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
> > - key.ip_src_addr = ipv4_hdr->src_addr;
> > - key.ip_dst_addr = ipv4_hdr->dst_addr;
> > - key.src_port = tcp_hdr->src_port;
> > - key.dst_port = tcp_hdr->dst_port;
> > - key.recv_ack = tcp_hdr->recv_ack;
> >
> > /* Search for a matched flow. */
> > max_flow_num = tbl->max_flow_num;
> > @@ -270,63 +187,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> > }
> > }
> >
> > - /*
> > - * Fail to find a matched flow. Insert a new flow and store the
> > - * packet into the flow.
> > - */
> > if (find == 0) {
> > - item_idx = insert_new_item(tbl, pkt, start_time,
> > - INVALID_ARRAY_INDEX, sent_seq, ip_id,
> > - is_atomic);
> > + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> > + item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl-
> > >item_num, tbl->max_item_num, start_time,
> > + INVALID_ARRAY_INDEX,
> > sent_seq, ip_id,
> > + is_atomic);
> > if (item_idx == INVALID_ARRAY_INDEX)
> > return -1;
> > if (insert_new_flow(tbl, &key, item_idx) ==
> > - INVALID_ARRAY_INDEX) {
> > + INVALID_ARRAY_INDEX) {
> > /*
> > * Fail to insert a new flow, so delete the
> > * stored packet.
> > - */
> > - delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
> > + */
> > + delete_tcp_item(tbl->items, item_idx, &tbl-
> > >item_num,
> > +INVALID_ARRAY_INDEX);
> > return -1;
> > }
> > return 0;
> > }
> > + item_idx = tbl->flows[i].start_index;
>
> No need to update item_idx, and you can directly pass
> tbl->flows[i].start_index to
> gro_process_tcp_item(). And same in gro_tcp6_reassemble().
>
> >
> > - /*
> > - * Check all packets in the flow and try to find a neighbor for
> > - * the input packet.
> > - */
> > - cur_idx = tbl->flows[i].start_index;
> > - prev_idx = cur_idx;
> > - do {
> > - cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
> > - sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> > - is_atomic);
> > - if (cmp) {
> > - if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
> > - pkt, cmp, sent_seq, ip_id,
> 0))
> > - return 1;
> > - /*
> > - * Fail to merge the two packets, as the packet
> > - * length is greater than the max value. Store
> > - * the packet into the flow.
> > - */
> > - if (insert_new_item(tbl, pkt, start_time, cur_idx,
> > - sent_seq, ip_id,
> is_atomic) ==
> > - INVALID_ARRAY_INDEX)
> > - return -1;
> > - return 0;
> > - }
> > - prev_idx = cur_idx;
> > - cur_idx = tbl->items[cur_idx].next_pkt_idx;
> > - } while (cur_idx != INVALID_ARRAY_INDEX);
> > + return gro_process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items,
> > item_idx,
> > + &tbl->item_num, tbl-
> > >max_item_num,
> > + ip_id, is_atomic,
> start_time);
> > +}
> >
> > - /* Fail to find a neighbor, so store the packet into the flow. */
> > - if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
> > - ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> > - return -1;
>
[-- Attachment #2: Type: text/html, Size: 33888 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread
* RE: [PATCH v4] gro : ipv6 changes to support GRO for TCP/ipv6
2023-06-08 16:52 ` kumaraparameshwaran rathinavel
@ 2023-06-09 1:04 ` Hu, Jiayu
0 siblings, 0 replies; 32+ messages in thread
From: Hu, Jiayu @ 2023-06-09 1:04 UTC (permalink / raw)
To: kumaraparameshwaran rathinavel; +Cc: dev
[-- Attachment #1: Type: text/plain, Size: 26029 bytes --]
Hi Kumara,
In your replied mail, the line doesn’t preface with “>” and it’s hard to find your replies.
So I reply here. In gro_tcp4_reassemble(), I mean there is no need to switch the sequences
of tcp_hdr and sent_seq definition statements in the code, rather than not updating them.
Thanks,
Jiayu
From: kumaraparameshwaran rathinavel <kumaraparamesh92@gmail.com>
Sent: Friday, June 9, 2023 12:52 AM
To: Hu, Jiayu <jiayu.hu@intel.com>
Cc: dev@dpdk.org
Subject: Re: [PATCH v4] gro : ipv6 changes to support GRO for TCP/ipv6
Hi Jiyau,
Thanks for the quick review comments. Will address the review comments. Require clarification in one of the comments. Please find it inline.
On Thu, Jun 8, 2023 at 9:35 AM Hu, Jiayu <jiayu.hu@intel.com<mailto:jiayu.hu@intel.com>> wrote:
Hi Kumara,
Please see replies inline.
In addition, you need to update the programmer guide in generic_receive_offload_lib.rst,
and release note release_23_07.rst.
Thanks,
Jiayu
> -----Original Message-----
> From: Kumara Parameshwaran <kumaraparamesh92@gmail.com<mailto:kumaraparamesh92@gmail.com>>
> Sent: Tuesday, June 6, 2023 10:58 PM
> To: Hu, Jiayu <jiayu.hu@intel.com<mailto:jiayu.hu@intel.com>>
> Cc: dev@dpdk.org<mailto:dev@dpdk.org>; Kumara Parameshwaran
> <kumaraparamesh92@gmail.com<mailto:kumaraparamesh92@gmail.com>>
> Subject: [PATCH v4] gro : ipv6 changes to support GRO for TCP/ipv6
>
> The patch adds GRO support for TCP/ipv6 packets. This does not include the
> support for vxlan, udp ipv6 packets.
>
> Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com<mailto:kumaraparamesh92@gmail.com>>>
> diff --git a/lib/gro/gro_tcp.c b/lib/gro/gro_tcp.c new file mode 100644 index
> 0000000000..02a7d0f8c5
> --- /dev/null
> +++ b/lib/gro/gro_tcp.c
For gro_tcp.c and gro_tcp.h, it's better to add "_internal" in the file name.
> @@ -0,0 +1,128 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
> +#include <rte_malloc.h>
> +#include <rte_mbuf.h>
> +#include <rte_ethdev.h>
> +
> +#include "gro_tcp.h"
> +
> +static inline uint32_t
> +find_an_empty_item(struct gro_tcp_item *items,
> + uint32_t max_item_num)
> +{
> + uint32_t i;
> +
> + for (i = 0; i < max_item_num; i++)
> + if (items[i].firstseg == NULL)
> + return i;
> + return INVALID_ARRAY_INDEX;
> +}
> +
> +uint32_t
> +insert_new_tcp_item(struct rte_mbuf *pkt,
> + struct gro_tcp_item *items,
> + uint32_t *item_num,
> + uint32_t max_item_num,
> + uint64_t start_time,
> + uint32_t prev_idx,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint8_t is_atomic)
This function can be inline.
> +{
> + uint32_t item_idx;
> +
> + item_idx = find_an_empty_item(items, max_item_num);
> + if (item_idx == INVALID_ARRAY_INDEX)
> + return INVALID_ARRAY_INDEX;
> +
> + items[item_idx].firstseg = pkt;
> + items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> + items[item_idx].start_time = start_time;
> + items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> + items[item_idx].sent_seq = sent_seq;
> + items[item_idx].ip_id = ip_id;
> + items[item_idx].nb_merged = 1;
> + items[item_idx].is_atomic = is_atomic;
> + (*item_num) += 1;
> +
> + /* if the previous packet exists, chain them together. */
> + if (prev_idx != INVALID_ARRAY_INDEX) {
> + items[item_idx].next_pkt_idx =
> + items[prev_idx].next_pkt_idx;
> + items[prev_idx].next_pkt_idx = item_idx;
> + }
> +
> + return item_idx;
> +}
> +
> +uint32_t
> +delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t prev_item_idx)
> +{
> + uint32_t next_idx = items[item_idx].next_pkt_idx;
> +
> + /* NULL indicates an empty item */
> + items[item_idx].firstseg = NULL;
> + (*item_num) -= 1;
> + if (prev_item_idx != INVALID_ARRAY_INDEX)
> + items[prev_item_idx].next_pkt_idx = next_idx;
> +
> + return next_idx;
> +}
This function can be inline.
> +
> +int32_t
> +gro_process_tcp_item(struct rte_mbuf *pkt,
> + struct rte_tcp_hdr *tcp_hdr,
> + int32_t tcp_dl,
> + struct gro_tcp_item *items,
> + uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t max_item_num,
> + uint16_t ip_id,
> + uint8_t is_atomic,
> + uint64_t start_time)
It is for internal use, so it's better to remove "gro_" from the function name.
> +{
> + uint32_t cur_idx;
> + uint32_t prev_idx;
> + int cmp;
> + uint32_t sent_seq;
> +
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> + /*
> + * Check all packets in the flow and try to find a neighbor for
> + * the input packet.
> + */
> + cur_idx = item_idx;
> + prev_idx = cur_idx;
> + do {
> + cmp = check_seq_option(&items[cur_idx], tcp_hdr,
> + sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> + is_atomic);
> + if (cmp) {
> + if (merge_two_tcp_packets(&items[cur_idx],
> + pkt, cmp, sent_seq, ip_id, 0))
> + return 1;
> + /*
> + * Fail to merge the two packets, as the packet
> + * length is greater than the max value. Store
> + * the packet into the flow.
> + */
> + if (insert_new_tcp_item(pkt, items, item_num,
> max_item_num, start_time, cur_idx,
> + sent_seq, ip_id, is_atomic) ==
> + INVALID_ARRAY_INDEX)
> + return -1;
> + return 0;
> + }
> + prev_idx = cur_idx;
> + cur_idx = items[cur_idx].next_pkt_idx;
> + } while (cur_idx != INVALID_ARRAY_INDEX);
> +
> + /* Fail to find a neighbor, so store the packet into the flow. */
> + if (insert_new_tcp_item(pkt, items, item_num, max_item_num,
> start_time, prev_idx, sent_seq,
> + ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> + return -1;
> +
> + return 0;
> +}
> diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h new file mode 100644 index
> 0000000000..4b5b4eda9c
> --- /dev/null
> +++ b/lib/gro/gro_tcp.h
> @@ -0,0 +1,209 @@
> +#ifndef _GRO_TCP_H_
> +#define _GRO_TCP_H_
> +
> +#define INVALID_ARRAY_INDEX 0xffffffffUL
> +
> +#include <rte_tcp.h>
> +
> +/*
> + * The max length of a IPv4 packet, which includes the length of the L3
> + * header, the L4 header and the data payload.
> + */
> +#define MAX_IP_PKT_LENGTH UINT16_MAX
> +
> +/* The maximum TCP header length */
> +#define MAX_TCP_HLEN 60
> +#define INVALID_TCP_HDRLEN(len) \
> + (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> +
> +struct gro_tcp_flow {
This structure name is confusing. In the upper layer, tcp4 and tcp6 have gro_tcp4_flow
and gro_tcp6_flow, which represent a flow. Inside gro_tcp4/6_flow, there are keys,
represented by struct tcp4/6_flow_key. But inside struct tcp4/6_flow_key, there is
struct gro_tcp_flow. Need to rename struct gro_tcp_flow, like common_tcp_flow_key.
> + struct rte_ether_addr eth_saddr;
> + struct rte_ether_addr eth_daddr;
> + uint32_t recv_ack;
> + uint16_t src_port;
> + uint16_t dst_port;
> +};
> +
> +#define ASSIGN_TCP_FLOW_KEY(k1, k2) \
Ditto. The macro needs rename, like ASSIGN_COMMON_TCP_FLOW_KEY.
> + do {\
> + rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
> + rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr));
> \
> + k2->recv_ack = k1->recv_ack; \
> + k2->src_port = k1->src_port; \
> + k2->dst_port = k1->dst_port; \
> + } while (0)
> +
> +struct gro_tcp_item {
> + /*
> + * The first MBUF segment of the packet. If the value
> + * is NULL, it means the item is empty.
> + */
> + struct rte_mbuf *firstseg;
> + /* The last MBUF segment of the packet */
> + struct rte_mbuf *lastseg;
> + /*
> + * The time when the first packet is inserted into the table.
> + * This value won't be updated, even if the packet is merged
> + * with other packets.
> + */
> + uint64_t start_time;
> + /*
> + * next_pkt_idx is used to chain the packets that
> + * are in the same flow but can't be merged together
> + * (e.g. caused by packet reordering).
> + */
> + uint32_t next_pkt_idx;
> + /* TCP sequence number of the packet */
> + uint32_t sent_seq;
> + /* IPv4 ID of the packet */
> + uint16_t ip_id;
The ip_id field is not used by tcp6. It's better to use an union to include ip_id for IPv4 and
an useless member for IPv6 with some comments to avoid confusing.
> + /* the number of merged packets */
> + uint16_t nb_merged;
> + /* Indicate if IPv4 ID can be ignored */
> + uint8_t is_atomic;
> +};
> +
> +uint32_t
> +insert_new_tcp_item(struct rte_mbuf *pkt,
> + struct gro_tcp_item *items,
> + uint32_t *item_num,
> + uint32_t table_size,
> + uint64_t start_time,
> + uint32_t prev_idx,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint8_t is_atomic);
> +
> +uint32_t
> +delete_tcp_item(struct gro_tcp_item *items,
> + uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t prev_item_idx);
> +
> +int32_t
> +gro_process_tcp_item(struct rte_mbuf *pkt,
> + struct rte_tcp_hdr *tcp_hdr,
> + int32_t tcp_dl,
> + struct gro_tcp_item *items,
> + uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t table_size,
> + uint16_t ip_id,
> + uint8_t is_atomic,
> + uint64_t start_time);
> +
> +/*
> + * Merge two TCP packets without updating checksums.
> + * If cmp is larger than 0, append the new packet to the
> + * original packet. Otherwise, pre-pend the new packet to
> + * the original packet.
> + */
> +static inline int
> +merge_two_tcp_packets(struct gro_tcp_item *item,
> + struct rte_mbuf *pkt,
> + int cmp,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint16_t l2_offset)
> +{
> + struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
> + uint16_t hdr_len, l2_len;
> +
> + if (cmp > 0) {
> + pkt_head = item->firstseg;
> + pkt_tail = pkt;
> + } else {
> + pkt_head = pkt;
> + pkt_tail = item->firstseg;
> + }
> +
> + /* check if the IPv4 packet length is greater than the max value */
> + hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
> + pkt_head->l4_len;
> + l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
> + if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
> + hdr_len > MAX_IP_PKT_LENGTH))
> + return 0;
> +
> + /* remove the packet header for the tail packet */
> + rte_pktmbuf_adj(pkt_tail, hdr_len);
> +
> + /* chain two packets together */
> + if (cmp > 0) {
> + item->lastseg->next = pkt;
> + item->lastseg = rte_pktmbuf_lastseg(pkt);
> + /* update IP ID to the larger value */
> + item->ip_id = ip_id;
> + } else {
> + lastseg = rte_pktmbuf_lastseg(pkt);
> + lastseg->next = item->firstseg;
> + item->firstseg = pkt;
> + /* update sent_seq to the smaller value */
> + item->sent_seq = sent_seq;
> + item->ip_id = ip_id;
> + }
> + item->nb_merged++;
> +
> + /* update MBUF metadata for the merged packet */
> + pkt_head->nb_segs += pkt_tail->nb_segs;
> + pkt_head->pkt_len += pkt_tail->pkt_len;
> +
> + return 1;
> +}
> +
> +/*
> + * Check if two TCP/IPv4 packets are neighbors.
> + */
> +static inline int
> +check_seq_option(struct gro_tcp_item *item,
> + struct rte_tcp_hdr *tcph,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint16_t tcp_hl,
> + uint16_t tcp_dl,
> + uint16_t l2_offset,
> + uint8_t is_atomic)
> +{
> + struct rte_mbuf *pkt_orig = item->firstseg;
> + char *iph_orig;
> + struct rte_tcp_hdr *tcph_orig;
> + uint16_t len, tcp_hl_orig;
> +
> + iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
> + l2_offset + pkt_orig->l2_len);
> + tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
> + tcp_hl_orig = pkt_orig->l4_len;
> +
> + /* Check if TCP option fields equal */
> + len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> + if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> + (memcmp(tcph + 1, tcph_orig + 1,
> + len) != 0)))
> + return 0;
> +
> + /* Don't merge packets whose DF bits are different */
> + if (unlikely(item->is_atomic ^ is_atomic))
> + return 0;
> +
> + /* check if the two packets are neighbors */
> + len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> + pkt_orig->l3_len - tcp_hl_orig;
> + if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> + (ip_id == item->ip_id + 1)))
> + /* append the new packet */
> + return 1;
> + else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> + (ip_id + item->nb_merged == item->ip_id)))
> + /* pre-pend the new packet */
> + return -1;
> +
> + return 0;
> +}
> +
> +static inline int
> +is_same_tcp_flow(struct gro_tcp_flow *k1, struct gro_tcp_flow *k2) {
> + return (!memcmp(k1, k2, sizeof(struct gro_tcp_flow))); }
I think this function needs rename, as the result of this function cannot identify if they are
same TCP flow.
> +
> +#endif
> diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c index
> 0014096e63..ffc33747c4 100644
> --- a/lib/gro/gro_tcp4.c
> +++ b/lib/gro/gro_tcp4.c
> @@ -7,6 +7,7 @@
> #include <rte_ethdev.h>
>
> #include "gro_tcp4.h"
> +#include "gro_tcp.h"
>
> void *
> gro_tcp4_tbl_create(uint16_t socket_id, @@ -30,7 +31,7 @@
> gro_tcp4_tbl_create(uint16_t socket_id,
> if (tbl == NULL)
> return NULL;
>
> - size = sizeof(struct gro_tcp4_item) * entries_num;
> + size = sizeof(struct gro_tcp_item) * entries_num;
> tbl->items = rte_zmalloc_socket(__func__,
> size,
> RTE_CACHE_LINE_SIZE,
> @@ -71,18 +72,6 @@ gro_tcp4_tbl_destroy(void *tbl)
> rte_free(tcp_tbl);
> }
>
> -static inline uint32_t
> -find_an_empty_item(struct gro_tcp4_tbl *tbl) -{
> - uint32_t i;
> - uint32_t max_item_num = tbl->max_item_num;
> -
> - for (i = 0; i < max_item_num; i++)
> - if (tbl->items[i].firstseg == NULL)
> - return i;
> - return INVALID_ARRAY_INDEX;
> -}
> -
> static inline uint32_t
> find_an_empty_flow(struct gro_tcp4_tbl *tbl) { @@ -95,56 +84,6 @@
> find_an_empty_flow(struct gro_tcp4_tbl *tbl)
> return INVALID_ARRAY_INDEX;
> }
>
> -static inline uint32_t
> -insert_new_item(struct gro_tcp4_tbl *tbl,
> - struct rte_mbuf *pkt,
> - uint64_t start_time,
> - uint32_t prev_idx,
> - uint32_t sent_seq,
> - uint16_t ip_id,
> - uint8_t is_atomic)
> -{
> - uint32_t item_idx;
> -
> - item_idx = find_an_empty_item(tbl);
> - if (item_idx == INVALID_ARRAY_INDEX)
> - return INVALID_ARRAY_INDEX;
> -
> - tbl->items[item_idx].firstseg = pkt;
> - tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> - tbl->items[item_idx].start_time = start_time;
> - tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> - tbl->items[item_idx].sent_seq = sent_seq;
> - tbl->items[item_idx].ip_id = ip_id;
> - tbl->items[item_idx].nb_merged = 1;
> - tbl->items[item_idx].is_atomic = is_atomic;
> - tbl->item_num++;
> -
> - /* if the previous packet exists, chain them together. */
> - if (prev_idx != INVALID_ARRAY_INDEX) {
> - tbl->items[item_idx].next_pkt_idx =
> - tbl->items[prev_idx].next_pkt_idx;
> - tbl->items[prev_idx].next_pkt_idx = item_idx;
> - }
> -
> - return item_idx;
> -}
> -
> -static inline uint32_t
> -delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
> - uint32_t prev_item_idx)
> -{
> - uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
> -
> - /* NULL indicates an empty item */
> - tbl->items[item_idx].firstseg = NULL;
> - tbl->item_num--;
> - if (prev_item_idx != INVALID_ARRAY_INDEX)
> - tbl->items[prev_item_idx].next_pkt_idx = next_idx;
> -
> - return next_idx;
> -}
> -
> static inline uint32_t
> insert_new_flow(struct gro_tcp4_tbl *tbl,
> struct tcp4_flow_key *src,
> @@ -159,13 +98,10 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
>
> dst = &(tbl->flows[flow_idx].key);
>
> - rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
> - rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
> + ASSIGN_TCP_FLOW_KEY((&src->tcp_flow), (&dst->tcp_flow));
> +
> dst->ip_src_addr = src->ip_src_addr;
> dst->ip_dst_addr = src->ip_dst_addr;
> - dst->recv_ack = src->recv_ack;
> - dst->src_port = src->src_port;
> - dst->dst_port = src->dst_port;
>
> tbl->flows[flow_idx].start_index = item_idx;
> tbl->flow_num++;
> @@ -173,21 +109,6 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
> return flow_idx;
> }
>
> -/*
> - * update the packet length for the flushed packet.
> - */
> -static inline void
> -update_header(struct gro_tcp4_item *item) -{
> - struct rte_ipv4_hdr *ipv4_hdr;
> - struct rte_mbuf *pkt = item->firstseg;
> -
> - ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> - pkt->l2_len);
> - ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
> - pkt->l2_len);
> -}
> -
> int32_t
> gro_tcp4_reassemble(struct rte_mbuf *pkt,
> struct gro_tcp4_tbl *tbl,
> @@ -195,16 +116,15 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt, {
> struct rte_ether_hdr *eth_hdr;
> struct rte_ipv4_hdr *ipv4_hdr;
> - struct rte_tcp_hdr *tcp_hdr;
> - uint32_t sent_seq;
> int32_t tcp_dl;
> + struct rte_tcp_hdr *tcp_hdr;
> uint16_t ip_id, hdr_len, frag_off, ip_tlen;
> uint8_t is_atomic;
> + uint32_t sent_seq;
No need to change tcp_hdr and sent_seq here.
The flow matching is done in the function and if the flow is not found insert_new_tcp_item is invoked from this function itself. Did you mean to move that to the process_tcp_item as well? If that is the case we should pass the start_idx as INVALID_ARRAY_INDEX and in process_tcp_item check if INVALID_ARRAY_INDEX do a insert_new_tcp_item and return, do not do the sequnce number checks etc.
>
> struct tcp4_flow_key key;
> - uint32_t cur_idx, prev_idx, item_idx;
> + uint32_t item_idx;
> uint32_t i, max_flow_num, remaining_flow_num;
> - int cmp;
> uint8_t find;
>
> /*
> @@ -216,7 +136,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
>
> eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
> ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
> - tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
> + tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *,
> +pkt->l2_len + pkt->l3_len);
> hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
>
> /*
> @@ -230,7 +150,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> ip_tlen = rte_be_to_cpu_16(ipv4_hdr->total_length);
> if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len))
> rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len);
> -
> /*
> * Don't process the packet whose payload length is less than or
> * equal to 0.
> @@ -239,6 +158,13 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> if (tcp_dl <= 0)
> return -1;
>
> + rte_ether_addr_copy(&(eth_hdr->src_addr),
> &(key.tcp_flow.eth_saddr));
> + rte_ether_addr_copy(&(eth_hdr->dst_addr),
> &(key.tcp_flow.eth_daddr));
> + key.ip_src_addr = ipv4_hdr->src_addr;
> + key.ip_dst_addr = ipv4_hdr->dst_addr;
> + key.tcp_flow.src_port = tcp_hdr->src_port;
> + key.tcp_flow.dst_port = tcp_hdr->dst_port;
> + key.tcp_flow.recv_ack = tcp_hdr->recv_ack;
> /*
> * Save IPv4 ID for the packet whose DF bit is 0. For the packet
> * whose DF bit is 1, IPv4 ID is ignored.
> @@ -246,15 +172,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
> is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) ==
> RTE_IPV4_HDR_DF_FLAG;
> ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
> - sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> -
> - rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
> - rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
> - key.ip_src_addr = ipv4_hdr->src_addr;
> - key.ip_dst_addr = ipv4_hdr->dst_addr;
> - key.src_port = tcp_hdr->src_port;
> - key.dst_port = tcp_hdr->dst_port;
> - key.recv_ack = tcp_hdr->recv_ack;
>
> /* Search for a matched flow. */
> max_flow_num = tbl->max_flow_num;
> @@ -270,63 +187,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> }
> }
>
> - /*
> - * Fail to find a matched flow. Insert a new flow and store the
> - * packet into the flow.
> - */
> if (find == 0) {
> - item_idx = insert_new_item(tbl, pkt, start_time,
> - INVALID_ARRAY_INDEX, sent_seq, ip_id,
> - is_atomic);
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> + item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl-
> >item_num, tbl->max_item_num, start_time,
> + INVALID_ARRAY_INDEX,
> sent_seq, ip_id,
> + is_atomic);
> if (item_idx == INVALID_ARRAY_INDEX)
> return -1;
> if (insert_new_flow(tbl, &key, item_idx) ==
> - INVALID_ARRAY_INDEX) {
> + INVALID_ARRAY_INDEX) {
> /*
> * Fail to insert a new flow, so delete the
> * stored packet.
> - */
> - delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
> + */
> + delete_tcp_item(tbl->items, item_idx, &tbl-
> >item_num,
> +INVALID_ARRAY_INDEX);
> return -1;
> }
> return 0;
> }
> + item_idx = tbl->flows[i].start_index;
No need to update item_idx, and you can directly pass tbl->flows[i].start_index to
gro_process_tcp_item(). And same in gro_tcp6_reassemble().
>
> - /*
> - * Check all packets in the flow and try to find a neighbor for
> - * the input packet.
> - */
> - cur_idx = tbl->flows[i].start_index;
> - prev_idx = cur_idx;
> - do {
> - cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
> - sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> - is_atomic);
> - if (cmp) {
> - if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
> - pkt, cmp, sent_seq, ip_id, 0))
> - return 1;
> - /*
> - * Fail to merge the two packets, as the packet
> - * length is greater than the max value. Store
> - * the packet into the flow.
> - */
> - if (insert_new_item(tbl, pkt, start_time, cur_idx,
> - sent_seq, ip_id, is_atomic) ==
> - INVALID_ARRAY_INDEX)
> - return -1;
> - return 0;
> - }
> - prev_idx = cur_idx;
> - cur_idx = tbl->items[cur_idx].next_pkt_idx;
> - } while (cur_idx != INVALID_ARRAY_INDEX);
> + return gro_process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items,
> item_idx,
> + &tbl->item_num, tbl-
> >max_item_num,
> + ip_id, is_atomic, start_time);
> +}
>
> - /* Fail to find a neighbor, so store the packet into the flow. */
> - if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
> - ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> - return -1;
[-- Attachment #2: Type: text/html, Size: 46703 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread
* [PATCH v5] gro : ipv6 changes to support GRO for TCP/ipv6
2022-10-20 18:14 ` [PATCH v2] " Kumara Parameshwaran
` (3 preceding siblings ...)
2023-06-06 14:58 ` [PATCH v4] " Kumara Parameshwaran
@ 2023-06-12 11:05 ` Kumara Parameshwaran
2023-06-12 11:23 ` [PATCH v6] " Kumara Parameshwaran
` (5 subsequent siblings)
10 siblings, 0 replies; 32+ messages in thread
From: Kumara Parameshwaran @ 2023-06-12 11:05 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev, Kumara Parameshwaran
The patch adds GRO support for TCP/ipv6 packets. This does not
include the support for vxlan, udp ipv6 packets.
Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
---
v1:
* Changes to support GRO for TCP/ipv6 packets. This does not include
vxlan changes.
* The GRO is performed only for ipv6 packets that does not contain
extension headers.
* The logic for the TCP coalescing remains the same, in ipv6 header
the source address, destination address, flow label, version fields
are expected to be the same.
* Re-organised the code to reuse certain tcp functions for both ipv4 and
ipv6 flows.
v2:
* Fix comments in gro_tcp6.h header file.
v3:
* Adderess review comments to fix code duplication for v4 and v6
v4:
* Addresses review comments for v3, do not use callbacks
v5:
* Address review comments
lib/gro/gro_tcp4.c | 176 +++++-------------------
lib/gro/gro_tcp4.h | 170 +-----------------------
lib/gro/gro_tcp6.c | 265 +++++++++++++++++++++++++++++++++++++
lib/gro/gro_tcp6.h | 163 +++++++++++++++++++++++
lib/gro/gro_tcp_internal.c | 128 ++++++++++++++++++
lib/gro/gro_tcp_internal.h | 212 +++++++++++++++++++++++++++++
lib/gro/gro_vxlan_tcp4.c | 23 ++--
lib/gro/gro_vxlan_tcp4.h | 3 +-
lib/gro/meson.build | 2 +
lib/gro/rte_gro.c | 83 ++++++++++--
lib/gro/rte_gro.h | 3 +
11 files changed, 895 insertions(+), 333 deletions(-)
create mode 100644 lib/gro/gro_tcp6.c
create mode 100644 lib/gro/gro_tcp6.h
create mode 100644 lib/gro/gro_tcp_internal.c
create mode 100644 lib/gro/gro_tcp_internal.h
diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c
index 0014096e63..8feea90c49 100644
--- a/lib/gro/gro_tcp4.c
+++ b/lib/gro/gro_tcp4.c
@@ -30,7 +30,7 @@ gro_tcp4_tbl_create(uint16_t socket_id,
if (tbl == NULL)
return NULL;
- size = sizeof(struct gro_tcp4_item) * entries_num;
+ size = sizeof(struct gro_tcp_item) * entries_num;
tbl->items = rte_zmalloc_socket(__func__,
size,
RTE_CACHE_LINE_SIZE,
@@ -71,18 +71,6 @@ gro_tcp4_tbl_destroy(void *tbl)
rte_free(tcp_tbl);
}
-static inline uint32_t
-find_an_empty_item(struct gro_tcp4_tbl *tbl)
-{
- uint32_t i;
- uint32_t max_item_num = tbl->max_item_num;
-
- for (i = 0; i < max_item_num; i++)
- if (tbl->items[i].firstseg == NULL)
- return i;
- return INVALID_ARRAY_INDEX;
-}
-
static inline uint32_t
find_an_empty_flow(struct gro_tcp4_tbl *tbl)
{
@@ -95,56 +83,6 @@ find_an_empty_flow(struct gro_tcp4_tbl *tbl)
return INVALID_ARRAY_INDEX;
}
-static inline uint32_t
-insert_new_item(struct gro_tcp4_tbl *tbl,
- struct rte_mbuf *pkt,
- uint64_t start_time,
- uint32_t prev_idx,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint8_t is_atomic)
-{
- uint32_t item_idx;
-
- item_idx = find_an_empty_item(tbl);
- if (item_idx == INVALID_ARRAY_INDEX)
- return INVALID_ARRAY_INDEX;
-
- tbl->items[item_idx].firstseg = pkt;
- tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
- tbl->items[item_idx].start_time = start_time;
- tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
- tbl->items[item_idx].sent_seq = sent_seq;
- tbl->items[item_idx].ip_id = ip_id;
- tbl->items[item_idx].nb_merged = 1;
- tbl->items[item_idx].is_atomic = is_atomic;
- tbl->item_num++;
-
- /* if the previous packet exists, chain them together. */
- if (prev_idx != INVALID_ARRAY_INDEX) {
- tbl->items[item_idx].next_pkt_idx =
- tbl->items[prev_idx].next_pkt_idx;
- tbl->items[prev_idx].next_pkt_idx = item_idx;
- }
-
- return item_idx;
-}
-
-static inline uint32_t
-delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
- uint32_t prev_item_idx)
-{
- uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
-
- /* NULL indicates an empty item */
- tbl->items[item_idx].firstseg = NULL;
- tbl->item_num--;
- if (prev_item_idx != INVALID_ARRAY_INDEX)
- tbl->items[prev_item_idx].next_pkt_idx = next_idx;
-
- return next_idx;
-}
-
static inline uint32_t
insert_new_flow(struct gro_tcp4_tbl *tbl,
struct tcp4_flow_key *src,
@@ -159,13 +97,10 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
- rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
+ ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
+
dst->ip_src_addr = src->ip_src_addr;
dst->ip_dst_addr = src->ip_dst_addr;
- dst->recv_ack = src->recv_ack;
- dst->src_port = src->src_port;
- dst->dst_port = src->dst_port;
tbl->flows[flow_idx].start_index = item_idx;
tbl->flow_num++;
@@ -173,21 +108,6 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
return flow_idx;
}
-/*
- * update the packet length for the flushed packet.
- */
-static inline void
-update_header(struct gro_tcp4_item *item)
-{
- struct rte_ipv4_hdr *ipv4_hdr;
- struct rte_mbuf *pkt = item->firstseg;
-
- ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
- pkt->l2_len);
- ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
- pkt->l2_len);
-}
-
int32_t
gro_tcp4_reassemble(struct rte_mbuf *pkt,
struct gro_tcp4_tbl *tbl,
@@ -202,9 +122,8 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
uint8_t is_atomic;
struct tcp4_flow_key key;
- uint32_t cur_idx, prev_idx, item_idx;
+ uint32_t item_idx;
uint32_t i, max_flow_num, remaining_flow_num;
- int cmp;
uint8_t find;
/*
@@ -216,7 +135,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
- tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
+ tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt->l2_len + pkt->l3_len);
hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
/*
@@ -230,7 +149,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
ip_tlen = rte_be_to_cpu_16(ipv4_hdr->total_length);
if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len))
rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len);
-
/*
* Don't process the packet whose payload length is less than or
* equal to 0.
@@ -239,6 +157,13 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
if (tcp_dl <= 0)
return -1;
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.cmn_key.eth_daddr));
+ key.ip_src_addr = ipv4_hdr->src_addr;
+ key.ip_dst_addr = ipv4_hdr->dst_addr;
+ key.cmn_key.src_port = tcp_hdr->src_port;
+ key.cmn_key.dst_port = tcp_hdr->dst_port;
+ key.cmn_key.recv_ack = tcp_hdr->recv_ack;
/*
* Save IPv4 ID for the packet whose DF bit is 0. For the packet
* whose DF bit is 1, IPv4 ID is ignored.
@@ -246,15 +171,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) == RTE_IPV4_HDR_DF_FLAG;
ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
- sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
-
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
- key.ip_src_addr = ipv4_hdr->src_addr;
- key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.src_port = tcp_hdr->src_port;
- key.dst_port = tcp_hdr->dst_port;
- key.recv_ack = tcp_hdr->recv_ack;
/* Search for a matched flow. */
max_flow_num = tbl->max_flow_num;
@@ -270,63 +186,43 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
}
}
- /*
- * Fail to find a matched flow. Insert a new flow and store the
- * packet into the flow.
- */
if (find == 0) {
- item_idx = insert_new_item(tbl, pkt, start_time,
- INVALID_ARRAY_INDEX, sent_seq, ip_id,
- is_atomic);
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num, tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, ip_id,
+ is_atomic);
if (item_idx == INVALID_ARRAY_INDEX)
return -1;
if (insert_new_flow(tbl, &key, item_idx) ==
- INVALID_ARRAY_INDEX) {
+ INVALID_ARRAY_INDEX) {
/*
* Fail to insert a new flow, so delete the
* stored packet.
- */
- delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
return -1;
}
return 0;
}
- /*
- * Check all packets in the flow and try to find a neighbor for
- * the input packet.
- */
- cur_idx = tbl->flows[i].start_index;
- prev_idx = cur_idx;
- do {
- cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
- sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
- is_atomic);
- if (cmp) {
- if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
- pkt, cmp, sent_seq, ip_id, 0))
- return 1;
- /*
- * Fail to merge the two packets, as the packet
- * length is greater than the max value. Store
- * the packet into the flow.
- */
- if (insert_new_item(tbl, pkt, start_time, cur_idx,
- sent_seq, ip_id, is_atomic) ==
- INVALID_ARRAY_INDEX)
- return -1;
- return 0;
- }
- prev_idx = cur_idx;
- cur_idx = tbl->items[cur_idx].next_pkt_idx;
- } while (cur_idx != INVALID_ARRAY_INDEX);
+ return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl->flows[i].start_index,
+ &tbl->item_num, tbl->max_item_num,
+ ip_id, is_atomic, start_time);
+}
- /* Fail to find a neighbor, so store the packet into the flow. */
- if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
- ip_id, is_atomic) == INVALID_ARRAY_INDEX)
- return -1;
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
- return 0;
+ ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len);
}
uint16_t
@@ -353,7 +249,7 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
* Delete the packet and get the next
* packet in the flow.
*/
- j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
+ j = delete_tcp_item(tbl->items, j, &tbl->item_num, INVALID_ARRAY_INDEX);
tbl->flows[i].start_index = j;
if (j == INVALID_ARRAY_INDEX)
tbl->flow_num--;
diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h
index 212f97a042..c0154afa24 100644
--- a/lib/gro/gro_tcp4.h
+++ b/lib/gro/gro_tcp4.h
@@ -5,32 +5,15 @@
#ifndef _GRO_TCP4_H_
#define _GRO_TCP4_H_
-#include <rte_tcp.h>
+#include <gro_tcp_internal.h>
-#define INVALID_ARRAY_INDEX 0xffffffffUL
#define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
-/*
- * The max length of a IPv4 packet, which includes the length of the L3
- * header, the L4 header and the data payload.
- */
-#define MAX_IPV4_PKT_LENGTH UINT16_MAX
-
-/* The maximum TCP header length */
-#define MAX_TCP_HLEN 60
-#define INVALID_TCP_HDRLEN(len) \
- (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
-
-/* Header fields representing a TCP/IPv4 flow */
+/* Header fields representing common fields in TCP flow */
struct tcp4_flow_key {
- struct rte_ether_addr eth_saddr;
- struct rte_ether_addr eth_daddr;
+ struct cmn_tcp_key cmn_key;
uint32_t ip_src_addr;
uint32_t ip_dst_addr;
-
- uint32_t recv_ack;
- uint16_t src_port;
- uint16_t dst_port;
};
struct gro_tcp4_flow {
@@ -42,42 +25,12 @@ struct gro_tcp4_flow {
uint32_t start_index;
};
-struct gro_tcp4_item {
- /*
- * The first MBUF segment of the packet. If the value
- * is NULL, it means the item is empty.
- */
- struct rte_mbuf *firstseg;
- /* The last MBUF segment of the packet */
- struct rte_mbuf *lastseg;
- /*
- * The time when the first packet is inserted into the table.
- * This value won't be updated, even if the packet is merged
- * with other packets.
- */
- uint64_t start_time;
- /*
- * next_pkt_idx is used to chain the packets that
- * are in the same flow but can't be merged together
- * (e.g. caused by packet reordering).
- */
- uint32_t next_pkt_idx;
- /* TCP sequence number of the packet */
- uint32_t sent_seq;
- /* IPv4 ID of the packet */
- uint16_t ip_id;
- /* the number of merged packets */
- uint16_t nb_merged;
- /* Indicate if IPv4 ID can be ignored */
- uint8_t is_atomic;
-};
-
/*
* TCP/IPv4 reassembly table structure.
*/
struct gro_tcp4_tbl {
/* item array */
- struct gro_tcp4_item *items;
+ struct gro_tcp_item *items;
/* flow array */
struct gro_tcp4_flow *flows;
/* current item number */
@@ -186,120 +139,9 @@ uint32_t gro_tcp4_tbl_pkt_count(void *tbl);
static inline int
is_same_tcp4_flow(struct tcp4_flow_key k1, struct tcp4_flow_key k2)
{
- return (rte_is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) &&
- rte_is_same_ether_addr(&k1.eth_daddr, &k2.eth_daddr) &&
- (k1.ip_src_addr == k2.ip_src_addr) &&
+ return ((k1.ip_src_addr == k2.ip_src_addr) &&
(k1.ip_dst_addr == k2.ip_dst_addr) &&
- (k1.recv_ack == k2.recv_ack) &&
- (k1.src_port == k2.src_port) &&
- (k1.dst_port == k2.dst_port));
+ is_common_tcp_key(&k1.cmn_key, &k2.cmn_key));
}
-/*
- * Merge two TCP/IPv4 packets without updating checksums.
- * If cmp is larger than 0, append the new packet to the
- * original packet. Otherwise, pre-pend the new packet to
- * the original packet.
- */
-static inline int
-merge_two_tcp4_packets(struct gro_tcp4_item *item,
- struct rte_mbuf *pkt,
- int cmp,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t l2_offset)
-{
- struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
- uint16_t hdr_len, l2_len;
-
- if (cmp > 0) {
- pkt_head = item->firstseg;
- pkt_tail = pkt;
- } else {
- pkt_head = pkt;
- pkt_tail = item->firstseg;
- }
-
- /* check if the IPv4 packet length is greater than the max value */
- hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
- pkt_head->l4_len;
- l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
- if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
- hdr_len > MAX_IPV4_PKT_LENGTH))
- return 0;
-
- /* remove the packet header for the tail packet */
- rte_pktmbuf_adj(pkt_tail, hdr_len);
-
- /* chain two packets together */
- if (cmp > 0) {
- item->lastseg->next = pkt;
- item->lastseg = rte_pktmbuf_lastseg(pkt);
- /* update IP ID to the larger value */
- item->ip_id = ip_id;
- } else {
- lastseg = rte_pktmbuf_lastseg(pkt);
- lastseg->next = item->firstseg;
- item->firstseg = pkt;
- /* update sent_seq to the smaller value */
- item->sent_seq = sent_seq;
- item->ip_id = ip_id;
- }
- item->nb_merged++;
-
- /* update MBUF metadata for the merged packet */
- pkt_head->nb_segs += pkt_tail->nb_segs;
- pkt_head->pkt_len += pkt_tail->pkt_len;
-
- return 1;
-}
-
-/*
- * Check if two TCP/IPv4 packets are neighbors.
- */
-static inline int
-check_seq_option(struct gro_tcp4_item *item,
- struct rte_tcp_hdr *tcph,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t tcp_hl,
- uint16_t tcp_dl,
- uint16_t l2_offset,
- uint8_t is_atomic)
-{
- struct rte_mbuf *pkt_orig = item->firstseg;
- struct rte_ipv4_hdr *iph_orig;
- struct rte_tcp_hdr *tcph_orig;
- uint16_t len, tcp_hl_orig;
-
- iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char *) +
- l2_offset + pkt_orig->l2_len);
- tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig + pkt_orig->l3_len);
- tcp_hl_orig = pkt_orig->l4_len;
-
- /* Check if TCP option fields equal */
- len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
- if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
- (memcmp(tcph + 1, tcph_orig + 1,
- len) != 0)))
- return 0;
-
- /* Don't merge packets whose DF bits are different */
- if (unlikely(item->is_atomic ^ is_atomic))
- return 0;
-
- /* check if the two packets are neighbors */
- len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
- pkt_orig->l3_len - tcp_hl_orig;
- if ((sent_seq == item->sent_seq + len) && (is_atomic ||
- (ip_id == item->ip_id + 1)))
- /* append the new packet */
- return 1;
- else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
- (ip_id + item->nb_merged == item->ip_id)))
- /* pre-pend the new packet */
- return -1;
-
- return 0;
-}
#endif
diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c
new file mode 100644
index 0000000000..8e27fba2e4
--- /dev/null
+++ b/lib/gro/gro_tcp6.c
@@ -0,0 +1,265 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp6.h"
+
+void *
+gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow)
+{
+ struct gro_tcp6_tbl *tbl;
+ size_t size;
+ uint32_t entries_num, i;
+
+ entries_num = max_flow_num * max_item_per_flow;
+ entries_num = RTE_MIN(entries_num, GRO_TCP6_TBL_MAX_ITEM_NUM);
+
+ if (entries_num == 0)
+ return NULL;
+
+ tbl = rte_zmalloc_socket(__func__,
+ sizeof(struct gro_tcp6_tbl),
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl == NULL)
+ return NULL;
+
+ size = sizeof(struct gro_tcp_item) * entries_num;
+ tbl->items = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->items == NULL) {
+ rte_free(tbl);
+ return NULL;
+ }
+ tbl->max_item_num = entries_num;
+
+ size = sizeof(struct gro_tcp6_flow) * entries_num;
+ tbl->flows = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->flows == NULL) {
+ rte_free(tbl->items);
+ rte_free(tbl);
+ return NULL;
+ }
+ /* INVALID_ARRAY_INDEX indicates an empty flow */
+ for (i = 0; i < entries_num; i++)
+ tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
+ tbl->max_flow_num = entries_num;
+
+ return tbl;
+}
+
+void
+gro_tcp6_tbl_destroy(void *tbl)
+{
+ struct gro_tcp6_tbl *tcp_tbl = tbl;
+
+ if (tcp_tbl) {
+ rte_free(tcp_tbl->items);
+ rte_free(tcp_tbl->flows);
+ }
+ rte_free(tcp_tbl);
+}
+
+static inline uint32_t
+find_an_empty_flow(struct gro_tcp6_tbl *tbl)
+{
+ uint32_t i;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++)
+ if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+insert_new_flow(struct gro_tcp6_tbl *tbl,
+ struct tcp6_flow_key *src,
+ uint32_t item_idx)
+{
+ struct tcp6_flow_key *dst;
+ uint32_t flow_idx;
+
+ flow_idx = find_an_empty_flow(tbl);
+ if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
+ return INVALID_ARRAY_INDEX;
+
+ dst = &(tbl->flows[flow_idx].key);
+
+ ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
+ memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst->src_addr));
+ memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst->dst_addr));
+ dst->vtc_flow = src->vtc_flow;
+
+ tbl->flows[flow_idx].start_index = item_idx;
+ tbl->flow_num++;
+
+ return flow_idx;
+}
+
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
+
+ ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len - pkt->l3_len);
+}
+
+int32_t
+gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time)
+{
+ struct rte_ether_hdr *eth_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ int32_t tcp_dl;
+ uint16_t ip_tlen;
+ struct tcp6_flow_key key;
+ uint32_t i, max_flow_num, remaining_flow_num;
+ uint32_t sent_seq;
+ struct rte_tcp_hdr *tcp_hdr;
+ uint8_t find;
+ uint32_t item_idx;
+ /*
+ * Don't process the packet whose TCP header length is greater
+ * than 60 bytes or less than 20 bytes.
+ */
+ if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
+ return -1;
+
+ eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
+ ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
+ tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt->l2_len + pkt->l3_len);
+
+ /*
+ * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
+ * or CWR set.
+ */
+ if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
+ return -1;
+
+ ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
+ /*
+ * Don't process the packet whose payload length is less than or
+ * equal to 0.
+ */
+ tcp_dl = ip_tlen - pkt->l4_len;
+ if (tcp_dl <= 0)
+ return -1;
+
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.cmn_key.eth_daddr));
+ memcpy(&key.src_addr[0], &ipv6_hdr->src_addr, sizeof(key.src_addr));
+ memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr, sizeof(key.dst_addr));
+ key.cmn_key.src_port = tcp_hdr->src_port;
+ key.cmn_key.dst_port = tcp_hdr->dst_port;
+ key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+ key.vtc_flow = ipv6_hdr->vtc_flow;
+
+ /* Search for a matched flow. */
+ max_flow_num = tbl->max_flow_num;
+ remaining_flow_num = tbl->flow_num;
+ find = 0;
+ for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
+ if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
+ if (is_same_tcp6_flow(&tbl->flows[i].key, &key)) {
+ find = 1;
+ break;
+ }
+ remaining_flow_num--;
+ }
+ }
+
+ if (find == 0) {
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num, tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, 0, true);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return -1;
+ if (insert_new_flow(tbl, &key, item_idx) ==
+ INVALID_ARRAY_INDEX) {
+ /*
+ * Fail to insert a new flow, so delete the
+ * stored packet.
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
+ return -1;
+ }
+ return 0;
+ }
+
+ return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl->flows[i].start_index,
+ &tbl->item_num, tbl->max_item_num,
+ 0, true, start_time);
+}
+
+uint16_t
+gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out)
+{
+ uint16_t k = 0;
+ uint32_t i, j;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++) {
+ if (unlikely(tbl->flow_num == 0))
+ return k;
+
+ j = tbl->flows[i].start_index;
+ while (j != INVALID_ARRAY_INDEX) {
+ if (tbl->items[j].start_time <= flush_timestamp) {
+ out[k++] = tbl->items[j].firstseg;
+ if (tbl->items[j].nb_merged > 1)
+ update_header(&(tbl->items[j]));
+ /*
+ * Delete the packet and get the next
+ * packet in the flow.
+ */
+ j = delete_tcp_item(tbl->items, j, &tbl->item_num, INVALID_ARRAY_INDEX);
+ tbl->flows[i].start_index = j;
+ if (j == INVALID_ARRAY_INDEX)
+ tbl->flow_num--;
+
+ if (unlikely(k == nb_out))
+ return k;
+ } else
+ /*
+ * The left packets in this flow won't be
+ * timeout. Go to check other flows.
+ */
+ break;
+ }
+ }
+ return k;
+}
+
+uint32_t
+gro_tcp6_tbl_pkt_count(void *tbl)
+{
+ struct gro_tcp6_tbl *gro_tbl = tbl;
+
+ if (gro_tbl)
+ return gro_tbl->item_num;
+
+ return 0;
+}
diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h
new file mode 100644
index 0000000000..46b547b39f
--- /dev/null
+++ b/lib/gro/gro_tcp6.h
@@ -0,0 +1,163 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#ifndef _GRO_TCP6_H_
+#define _GRO_TCP6_H_
+
+#include <gro_tcp_internal.h>
+
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+#define GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
+
+/* Header fields representing a TCP/IPv6 flow */
+struct tcp6_flow_key {
+ struct cmn_tcp_key cmn_key;
+ uint8_t src_addr[16];
+ uint8_t dst_addr[16];
+ rte_be32_t vtc_flow;
+};
+
+struct gro_tcp6_flow {
+ struct tcp6_flow_key key;
+ /*
+ * The index of the first packet in the flow.
+ * INVALID_ARRAY_INDEX indicates an empty flow.
+ */
+ uint32_t start_index;
+};
+
+/*
+ * TCP/IPv6 reassembly table structure.
+ */
+struct gro_tcp6_tbl {
+ /* item array */
+ struct gro_tcp_item *items;
+ /* flow array */
+ struct gro_tcp6_flow *flows;
+ /* current item number */
+ uint32_t item_num;
+ /* current flow num */
+ uint32_t flow_num;
+ /* item array size */
+ uint32_t max_item_num;
+ /* flow array size */
+ uint32_t max_flow_num;
+};
+
+/**
+ * This function creates a TCP/IPv6 reassembly table.
+ *
+ * @param socket_id
+ * Socket index for allocating the TCP/IPv6 reassemble table
+ * @param max_flow_num
+ * The maximum number of flows in the TCP/IPv6 GRO table
+ * @param max_item_per_flow
+ * The maximum number of packets per flow
+ *
+ * @return
+ * - Return the table pointer on success.
+ * - Return NULL on failure.
+ */
+void *gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow);
+
+/**
+ * This function destroys a TCP/IPv6 reassembly table.
+ *
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table.
+ */
+void gro_tcp6_tbl_destroy(void *tbl);
+
+/**
+ * This function merges a TCP/IPv6 packet. It doesn't process the packet,
+ * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't have
+ * payload.
+ *
+ * This function doesn't check if the packet has correct checksums and
+ * doesn't re-calculate checksums for the merged packet. Additionally,
+ * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
+ * when IP fragmentation is possible (i.e., DF==0). It returns the
+ * packet, if the packet has invalid parameters (e.g. SYN bit is set)
+ * or there is no available space in the table.
+ *
+ * @param pkt
+ * Packet to reassemble
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table
+ * @start_time
+ * The time when the packet is inserted into the table
+ *
+ * @return
+ * - Return a positive value if the packet is merged.
+ * - Return zero if the packet isn't merged but stored in the table.
+ * - Return a negative value for invalid parameters or no available
+ * space in the table.
+ */
+int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time);
+
+/**
+ * This function flushes timeout packets in a TCP/IPv6 reassembly table,
+ * and without updating checksums.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ * @param flush_timestamp
+ * Flush packets which are inserted into the table before or at the
+ * flush_timestamp.
+ * @param out
+ * Pointer array used to keep flushed packets
+ * @param nb_out
+ * The element number in 'out'. It also determines the maximum number of
+ * packets that can be flushed finally.
+ *
+ * @return
+ * The number of flushed packets
+ */
+uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out);
+
+/**
+ * This function returns the number of the packets in a TCP/IPv6
+ * reassembly table.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ *
+ * @return
+ * The number of packets in the table
+ */
+uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
+
+/*
+ * Check if two TCP/IPv6 packets belong to the same flow.
+ */
+static inline int
+is_same_tcp6_flow(struct tcp6_flow_key *k1, struct tcp6_flow_key *k2)
+{
+ rte_be32_t vtc_flow_diff;
+
+ if (memcmp(&k1->src_addr, &k2->src_addr, 16)) {
+ return 0;
+ }
+ if (memcmp(&k1->dst_addr, &k2->dst_addr, 16)) {
+ return 0;
+ }
+ /*
+ * IP version (4) Traffic Class (8) Flow Label (20)
+ * All fields except Traffic class should be same
+ */
+ vtc_flow_diff = (k1->vtc_flow ^ k2->vtc_flow);
+ if (vtc_flow_diff & htonl(0xF00FFFFF))
+ return 0;
+
+ return is_common_tcp_key(&k1->cmn_key, &k2->cmn_key);
+}
+
+#endif
diff --git a/lib/gro/gro_tcp_internal.c b/lib/gro/gro_tcp_internal.c
new file mode 100644
index 0000000000..4b37121899
--- /dev/null
+++ b/lib/gro/gro_tcp_internal.c
@@ -0,0 +1,128 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp_internal.h"
+
+static inline uint32_t
+find_an_empty_item(struct gro_tcp_item *items,
+ uint32_t max_item_num)
+{
+ uint32_t i;
+
+ for (i = 0; i < max_item_num; i++)
+ if (items[i].firstseg == NULL)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+inline uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic)
+{
+ uint32_t item_idx;
+
+ item_idx = find_an_empty_item(items, max_item_num);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return INVALID_ARRAY_INDEX;
+
+ items[item_idx].firstseg = pkt;
+ items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
+ items[item_idx].start_time = start_time;
+ items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
+ items[item_idx].sent_seq = sent_seq;
+ items[item_idx].l3.ip_id = ip_id;
+ items[item_idx].nb_merged = 1;
+ items[item_idx].is_atomic = is_atomic;
+ (*item_num) += 1;
+
+ /* if the previous packet exists, chain them together. */
+ if (prev_idx != INVALID_ARRAY_INDEX) {
+ items[item_idx].next_pkt_idx =
+ items[prev_idx].next_pkt_idx;
+ items[prev_idx].next_pkt_idx = item_idx;
+ }
+
+ return item_idx;
+}
+
+inline uint32_t
+delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx)
+{
+ uint32_t next_idx = items[item_idx].next_pkt_idx;
+
+ /* NULL indicates an empty item */
+ items[item_idx].firstseg = NULL;
+ (*item_num) -= 1;
+ if (prev_item_idx != INVALID_ARRAY_INDEX)
+ items[prev_item_idx].next_pkt_idx = next_idx;
+
+ return next_idx;
+}
+
+int32_t
+process_tcp_item(struct rte_mbuf *pkt,
+ struct rte_tcp_hdr *tcp_hdr,
+ int32_t tcp_dl,
+ struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time)
+{
+ uint32_t cur_idx;
+ uint32_t prev_idx;
+ int cmp;
+ uint32_t sent_seq;
+
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ /*
+ * Check all packets in the flow and try to find a neighbor for
+ * the input packet.
+ */
+ cur_idx = item_idx;
+ prev_idx = cur_idx;
+ do {
+ cmp = check_seq_option(&items[cur_idx], tcp_hdr,
+ sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
+ is_atomic);
+ if (cmp) {
+ if (merge_two_tcp_packets(&items[cur_idx],
+ pkt, cmp, sent_seq, ip_id, 0))
+ return 1;
+ /*
+ * Fail to merge the two packets, as the packet
+ * length is greater than the max value. Store
+ * the packet into the flow.
+ */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num, start_time, cur_idx,
+ sent_seq, ip_id, is_atomic) ==
+ INVALID_ARRAY_INDEX)
+ return -1;
+ return 0;
+ }
+ prev_idx = cur_idx;
+ cur_idx = items[cur_idx].next_pkt_idx;
+ } while (cur_idx != INVALID_ARRAY_INDEX);
+
+ /* Fail to find a neighbor, so store the packet into the flow. */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num, start_time, prev_idx, sent_seq,
+ ip_id, is_atomic) == INVALID_ARRAY_INDEX)
+ return -1;
+
+ return 0;
+}
diff --git a/lib/gro/gro_tcp_internal.h b/lib/gro/gro_tcp_internal.h
new file mode 100644
index 0000000000..072b7aea13
--- /dev/null
+++ b/lib/gro/gro_tcp_internal.h
@@ -0,0 +1,212 @@
+#ifndef _GRO_TCP_H_
+#define _GRO_TCP_H_
+
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+
+#include <rte_tcp.h>
+
+/*
+ * The max length of a IPv4 packet, which includes the length of the L3
+ * header, the L4 header and the data payload.
+ */
+#define MAX_IP_PKT_LENGTH UINT16_MAX
+
+/* The maximum TCP header length */
+#define MAX_TCP_HLEN 60
+#define INVALID_TCP_HDRLEN(len) \
+ (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
+
+struct cmn_tcp_key {
+ struct rte_ether_addr eth_saddr;
+ struct rte_ether_addr eth_daddr;
+ uint32_t recv_ack;
+ uint16_t src_port;
+ uint16_t dst_port;
+};
+
+#define ASSIGN_COMMON_TCP_KEY(k1, k2) \
+ do {\
+ rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
+ rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr)); \
+ k2->recv_ack = k1->recv_ack; \
+ k2->src_port = k1->src_port; \
+ k2->dst_port = k1->dst_port; \
+ } while (0)
+
+struct gro_tcp_item {
+ /*
+ * The first MBUF segment of the packet. If the value
+ * is NULL, it means the item is empty.
+ */
+ struct rte_mbuf *firstseg;
+ /* The last MBUF segment of the packet */
+ struct rte_mbuf *lastseg;
+ /*
+ * The time when the first packet is inserted into the table.
+ * This value won't be updated, even if the packet is merged
+ * with other packets.
+ */
+ uint64_t start_time;
+ /*
+ * next_pkt_idx is used to chain the packets that
+ * are in the same flow but can't be merged together
+ * (e.g. caused by packet reordering).
+ */
+ uint32_t next_pkt_idx;
+ /* TCP sequence number of the packet */
+ uint32_t sent_seq;
+ union {
+ /* IPv4 ID of the packet */
+ uint16_t ip_id;
+ /* Unused field for IPv6 */
+ uint16_t unused;
+ } l3;
+ /* the number of merged packets */
+ uint16_t nb_merged;
+ /* Indicate if IPv4 ID can be ignored */
+ uint8_t is_atomic;
+};
+
+uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic);
+
+uint32_t
+delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx);
+
+int32_t
+process_tcp_item(struct rte_mbuf *pkt,
+ struct rte_tcp_hdr *tcp_hdr,
+ int32_t tcp_dl,
+ struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time);
+
+/*
+ * Merge two TCP packets without updating checksums.
+ * If cmp is larger than 0, append the new packet to the
+ * original packet. Otherwise, pre-pend the new packet to
+ * the original packet.
+ */
+static inline int
+merge_two_tcp_packets(struct gro_tcp_item *item,
+ struct rte_mbuf *pkt,
+ int cmp,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t l2_offset)
+{
+ struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
+ uint16_t hdr_len, l2_len;
+
+ if (cmp > 0) {
+ pkt_head = item->firstseg;
+ pkt_tail = pkt;
+ } else {
+ pkt_head = pkt;
+ pkt_tail = item->firstseg;
+ }
+
+ /* check if the IPv4 packet length is greater than the max value */
+ hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
+ pkt_head->l4_len;
+ l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
+ if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
+ hdr_len > MAX_IP_PKT_LENGTH))
+ return 0;
+
+ /* remove the packet header for the tail packet */
+ rte_pktmbuf_adj(pkt_tail, hdr_len);
+
+ /* chain two packets together */
+ if (cmp > 0) {
+ item->lastseg->next = pkt;
+ item->lastseg = rte_pktmbuf_lastseg(pkt);
+ /* update IP ID to the larger value */
+ item->l3.ip_id = ip_id;
+ } else {
+ lastseg = rte_pktmbuf_lastseg(pkt);
+ lastseg->next = item->firstseg;
+ item->firstseg = pkt;
+ /* update sent_seq to the smaller value */
+ item->sent_seq = sent_seq;
+ item->l3.ip_id = ip_id;
+ }
+ item->nb_merged++;
+
+ /* update MBUF metadata for the merged packet */
+ pkt_head->nb_segs += pkt_tail->nb_segs;
+ pkt_head->pkt_len += pkt_tail->pkt_len;
+
+ return 1;
+}
+
+/*
+ * Check if two TCP/IPv4 packets are neighbors.
+ */
+static inline int
+check_seq_option(struct gro_tcp_item *item,
+ struct rte_tcp_hdr *tcph,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t tcp_hl,
+ uint16_t tcp_dl,
+ uint16_t l2_offset,
+ uint8_t is_atomic)
+{
+ struct rte_mbuf *pkt_orig = item->firstseg;
+ char *iph_orig;
+ struct rte_tcp_hdr *tcph_orig;
+ uint16_t len, tcp_hl_orig;
+
+ iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
+ l2_offset + pkt_orig->l2_len);
+ tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
+ tcp_hl_orig = pkt_orig->l4_len;
+
+ /* Check if TCP option fields equal */
+ len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
+ if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
+ (memcmp(tcph + 1, tcph_orig + 1,
+ len) != 0)))
+ return 0;
+
+ /* Don't merge packets whose DF bits are different */
+ if (unlikely(item->is_atomic ^ is_atomic))
+ return 0;
+
+ /* check if the two packets are neighbors */
+ len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
+ pkt_orig->l3_len - tcp_hl_orig;
+ if ((sent_seq == item->sent_seq + len) && (is_atomic ||
+ (ip_id == item->l3.ip_id + 1)))
+ /* append the new packet */
+ return 1;
+ else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
+ (ip_id + item->nb_merged == item->l3.ip_id)))
+ /* pre-pend the new packet */
+ return -1;
+
+ return 0;
+}
+
+static inline int
+is_common_tcp_key(struct cmn_tcp_key *k1, struct cmn_tcp_key *k2)
+{
+ return (!memcmp(k1, k2, sizeof(struct cmn_tcp_key)));
+}
+
+#endif
diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
index 3be4deb7c7..81eebf0d2d 100644
--- a/lib/gro/gro_vxlan_tcp4.c
+++ b/lib/gro/gro_vxlan_tcp4.c
@@ -7,6 +7,7 @@
#include <rte_ethdev.h>
#include <rte_udp.h>
+#include "gro_tcp_internal.h"
#include "gro_vxlan_tcp4.h"
void *
@@ -116,7 +117,7 @@ insert_new_item(struct gro_vxlan_tcp4_tbl *tbl,
tbl->items[item_idx].inner_item.start_time = start_time;
tbl->items[item_idx].inner_item.next_pkt_idx = INVALID_ARRAY_INDEX;
tbl->items[item_idx].inner_item.sent_seq = sent_seq;
- tbl->items[item_idx].inner_item.ip_id = ip_id;
+ tbl->items[item_idx].inner_item.l3.ip_id = ip_id;
tbl->items[item_idx].inner_item.nb_merged = 1;
tbl->items[item_idx].inner_item.is_atomic = is_atomic;
tbl->items[item_idx].outer_ip_id = outer_ip_id;
@@ -163,15 +164,9 @@ insert_new_flow(struct gro_vxlan_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->inner_key.eth_saddr),
- &(dst->inner_key.eth_saddr));
- rte_ether_addr_copy(&(src->inner_key.eth_daddr),
- &(dst->inner_key.eth_daddr));
+ ASSIGN_COMMON_TCP_KEY((&(src->inner_key.cmn_key)), (&(dst->inner_key.cmn_key)));
dst->inner_key.ip_src_addr = src->inner_key.ip_src_addr;
dst->inner_key.ip_dst_addr = src->inner_key.ip_dst_addr;
- dst->inner_key.recv_ack = src->inner_key.recv_ack;
- dst->inner_key.src_port = src->inner_key.src_port;
- dst->inner_key.dst_port = src->inner_key.dst_port;
dst->vxlan_hdr.vx_flags = src->vxlan_hdr.vx_flags;
dst->vxlan_hdr.vx_vni = src->vxlan_hdr.vx_vni;
@@ -248,7 +243,7 @@ merge_two_vxlan_tcp4_packets(struct gro_vxlan_tcp4_item *item,
uint16_t outer_ip_id,
uint16_t ip_id)
{
- if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp, sent_seq,
+ if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
ip_id, pkt->outer_l2_len +
pkt->outer_l3_len)) {
/* Update the outer IPv4 ID to the large value. */
@@ -357,13 +352,13 @@ gro_vxlan_tcp4_reassemble(struct rte_mbuf *pkt,
sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.eth_daddr));
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.cmn_key.eth_daddr));
key.inner_key.ip_src_addr = ipv4_hdr->src_addr;
key.inner_key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.inner_key.recv_ack = tcp_hdr->recv_ack;
- key.inner_key.src_port = tcp_hdr->src_port;
- key.inner_key.dst_port = tcp_hdr->dst_port;
+ key.inner_key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+ key.inner_key.cmn_key.src_port = tcp_hdr->src_port;
+ key.inner_key.cmn_key.dst_port = tcp_hdr->dst_port;
key.vxlan_hdr.vx_flags = vxlan_hdr->vx_flags;
key.vxlan_hdr.vx_vni = vxlan_hdr->vx_vni;
diff --git a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h
index 7832942a68..82eaaee11e 100644
--- a/lib/gro/gro_vxlan_tcp4.h
+++ b/lib/gro/gro_vxlan_tcp4.h
@@ -5,6 +5,7 @@
#ifndef _GRO_VXLAN_TCP4_H_
#define _GRO_VXLAN_TCP4_H_
+#include "gro_tcp_internal.h"
#include "gro_tcp4.h"
#define GRO_VXLAN_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
@@ -36,7 +37,7 @@ struct gro_vxlan_tcp4_flow {
};
struct gro_vxlan_tcp4_item {
- struct gro_tcp4_item inner_item;
+ struct gro_tcp_item inner_item;
/* IPv4 ID in the outer IPv4 header */
uint16_t outer_ip_id;
/* Indicate if outer IPv4 ID can be ignored */
diff --git a/lib/gro/meson.build b/lib/gro/meson.build
index e4fa2958bd..1640317890 100644
--- a/lib/gro/meson.build
+++ b/lib/gro/meson.build
@@ -3,7 +3,9 @@
sources = files(
'rte_gro.c',
+ 'gro_tcp_internal.c',
'gro_tcp4.c',
+ 'gro_tcp6.c',
'gro_udp4.c',
'gro_vxlan_tcp4.c',
'gro_vxlan_udp4.c',
diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c
index e35399fd42..d824eebd93 100644
--- a/lib/gro/rte_gro.c
+++ b/lib/gro/rte_gro.c
@@ -8,6 +8,7 @@
#include "rte_gro.h"
#include "gro_tcp4.h"
+#include "gro_tcp6.h"
#include "gro_udp4.h"
#include "gro_vxlan_tcp4.h"
#include "gro_vxlan_udp4.h"
@@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
- gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
+ gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, gro_tcp6_tbl_create, NULL};
static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
+ gro_tcp6_tbl_destroy,
NULL};
static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count,
+ gro_tcp6_tbl_pkt_count,
NULL};
#define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
@@ -35,6 +38,12 @@ static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+/* GRO with extension headers is not supported */
+#define IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
+ ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
+ ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
+ (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+
#define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
@@ -147,7 +156,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
/* allocate a reassembly table for TCP/IPv4 GRO */
struct gro_tcp4_tbl tcp_tbl;
struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
- struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+ struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+
+ struct gro_tcp6_tbl tcp6_tbl;
+ struct gro_tcp6_flow tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
+ struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
/* allocate a reassembly table for UDP/IPv4 GRO */
struct gro_udp4_tbl udp_tbl;
@@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
int32_t ret;
uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
- do_vxlan_udp_gro = 0;
+ do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
do_udp4_gro = 1;
}
+ if (param->gro_types & RTE_GRO_TCP_IPV6) {
+ for (i = 0; i < item_num; i++)
+ tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+ tcp6_tbl.flows = tcp6_flows;
+ tcp6_tbl.items = tcp6_items;
+ tcp6_tbl.flow_num = 0;
+ tcp6_tbl.item_num = 0;
+ tcp6_tbl.max_flow_num = item_num;
+ tcp6_tbl.max_item_num = item_num;
+ do_tcp6_gro = 1;
+ }
for (i = 0; i < nb_pkts; i++) {
/*
@@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
nb_after_gro--;
else if (ret < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
+ if (ret > 0)
+ /* merge successfully */
+ nb_after_gro--;
+ else if (ret < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
if ((nb_after_gro < nb_pkts)
|| (unprocess_num < nb_pkts)) {
i = 0;
+ /* Copy unprocessed packets */
+ if (unprocess_num > 0) {
+ memcpy(&pkts[i], unprocess_pkts,
+ sizeof(struct rte_mbuf *) *
+ unprocess_num);
+ i = unprocess_num;
+ }
+
/* Flush all packets from the tables */
if (do_vxlan_tcp_gro) {
- i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
+ i += gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
0, pkts, nb_pkts);
}
@@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
&pkts[i], nb_pkts - i);
}
- /* Copy unprocessed packets */
- if (unprocess_num > 0) {
- memcpy(&pkts[i], unprocess_pkts,
- sizeof(struct rte_mbuf *) *
- unprocess_num);
+
+ if (do_tcp6_gro) {
+ i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
+ &pkts[i], nb_pkts - i);
}
- nb_after_gro = i + unprocess_num;
}
return nb_after_gro;
@@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
{
struct rte_mbuf *unprocess_pkts[nb_pkts];
struct gro_ctx *gro_ctx = ctx;
- void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
+ void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
uint64_t current_time;
uint16_t i, unprocess_num = 0;
- uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro;
+ uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro, do_tcp6_gro;
if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
+ tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
RTE_GRO_TCP_IPV4;
@@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
RTE_GRO_UDP_IPV4;
do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
RTE_GRO_IPV4_VXLAN_UDP_IPV4;
+ do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) == RTE_GRO_TCP_IPV6;
current_time = rte_rdtsc();
@@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
if (gro_udp4_reassemble(pkts[i], udp_tbl,
current_time) < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
+ current_time) < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
flush_timestamp,
&out[num], left_nb_out);
+ left_nb_out = max_nb_out - num;
+ }
+
+ if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
+ num += gro_tcp6_tbl_timeout_flush(
+ gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
+ flush_timestamp,
+ &out[num], left_nb_out);
+
}
return num;
diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h
index 9f9ed4935a..c83dfd9ad1 100644
--- a/lib/gro/rte_gro.h
+++ b/lib/gro/rte_gro.h
@@ -38,6 +38,9 @@ extern "C" {
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL << RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
/**< VxLAN UDP/IPv4 GRO flag. */
+#define RTE_GRO_TCP_IPV6_INDEX 4
+#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX)
+/**< TCP/IPv6 GRO flag. */
/**
* Structure used to create GRO context objects or used to pass
--
2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* [PATCH v6] gro : ipv6 changes to support GRO for TCP/ipv6
2022-10-20 18:14 ` [PATCH v2] " Kumara Parameshwaran
` (4 preceding siblings ...)
2023-06-12 11:05 ` [PATCH v5] " Kumara Parameshwaran
@ 2023-06-12 11:23 ` Kumara Parameshwaran
2023-06-12 11:31 ` [PATCH v7] " Kumara Parameshwaran
` (4 subsequent siblings)
10 siblings, 0 replies; 32+ messages in thread
From: Kumara Parameshwaran @ 2023-06-12 11:23 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev, Kumara Parameshwaran
The patch adds GRO support for TCP/ipv6 packets. This does not
include the support for vxlan, udp ipv6 packets.
Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
---
v1:
* Changes to support GRO for TCP/ipv6 packets. This does not include
vxlan changes.
* The GRO is performed only for ipv6 packets that does not contain
extension headers.
* The logic for the TCP coalescing remains the same, in ipv6 header
the source address, destination address, flow label, version fields
are expected to be the same.
* Re-organised the code to reuse certain tcp functions for both ipv4 and
ipv6 flows.
v2:
* Fix comments in gro_tcp6.h header file.
v3:
* Adderess review comments to fix code duplication for v4 and v6
v4:
* Addresses review comments for v3, do not use callbacks
v5:
* Address review comments
v6:
* Fix warning and coding style issues
lib/gro/gro_tcp4.c | 178 ++++++-------------------
lib/gro/gro_tcp4.h | 170 +-----------------------
lib/gro/gro_tcp6.c | 266 +++++++++++++++++++++++++++++++++++++
lib/gro/gro_tcp6.h | 161 ++++++++++++++++++++++
lib/gro/gro_tcp_internal.c | 128 ++++++++++++++++++
lib/gro/gro_tcp_internal.h | 212 +++++++++++++++++++++++++++++
lib/gro/gro_vxlan_tcp4.c | 23 ++--
lib/gro/gro_vxlan_tcp4.h | 3 +-
lib/gro/meson.build | 2 +
lib/gro/rte_gro.c | 83 ++++++++++--
lib/gro/rte_gro.h | 3 +
11 files changed, 896 insertions(+), 333 deletions(-)
create mode 100644 lib/gro/gro_tcp6.c
create mode 100644 lib/gro/gro_tcp6.h
create mode 100644 lib/gro/gro_tcp_internal.c
create mode 100644 lib/gro/gro_tcp_internal.h
diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c
index 0014096e63..42fee78f30 100644
--- a/lib/gro/gro_tcp4.c
+++ b/lib/gro/gro_tcp4.c
@@ -30,7 +30,7 @@ gro_tcp4_tbl_create(uint16_t socket_id,
if (tbl == NULL)
return NULL;
- size = sizeof(struct gro_tcp4_item) * entries_num;
+ size = sizeof(struct gro_tcp_item) * entries_num;
tbl->items = rte_zmalloc_socket(__func__,
size,
RTE_CACHE_LINE_SIZE,
@@ -71,18 +71,6 @@ gro_tcp4_tbl_destroy(void *tbl)
rte_free(tcp_tbl);
}
-static inline uint32_t
-find_an_empty_item(struct gro_tcp4_tbl *tbl)
-{
- uint32_t i;
- uint32_t max_item_num = tbl->max_item_num;
-
- for (i = 0; i < max_item_num; i++)
- if (tbl->items[i].firstseg == NULL)
- return i;
- return INVALID_ARRAY_INDEX;
-}
-
static inline uint32_t
find_an_empty_flow(struct gro_tcp4_tbl *tbl)
{
@@ -95,56 +83,6 @@ find_an_empty_flow(struct gro_tcp4_tbl *tbl)
return INVALID_ARRAY_INDEX;
}
-static inline uint32_t
-insert_new_item(struct gro_tcp4_tbl *tbl,
- struct rte_mbuf *pkt,
- uint64_t start_time,
- uint32_t prev_idx,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint8_t is_atomic)
-{
- uint32_t item_idx;
-
- item_idx = find_an_empty_item(tbl);
- if (item_idx == INVALID_ARRAY_INDEX)
- return INVALID_ARRAY_INDEX;
-
- tbl->items[item_idx].firstseg = pkt;
- tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
- tbl->items[item_idx].start_time = start_time;
- tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
- tbl->items[item_idx].sent_seq = sent_seq;
- tbl->items[item_idx].ip_id = ip_id;
- tbl->items[item_idx].nb_merged = 1;
- tbl->items[item_idx].is_atomic = is_atomic;
- tbl->item_num++;
-
- /* if the previous packet exists, chain them together. */
- if (prev_idx != INVALID_ARRAY_INDEX) {
- tbl->items[item_idx].next_pkt_idx =
- tbl->items[prev_idx].next_pkt_idx;
- tbl->items[prev_idx].next_pkt_idx = item_idx;
- }
-
- return item_idx;
-}
-
-static inline uint32_t
-delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
- uint32_t prev_item_idx)
-{
- uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
-
- /* NULL indicates an empty item */
- tbl->items[item_idx].firstseg = NULL;
- tbl->item_num--;
- if (prev_item_idx != INVALID_ARRAY_INDEX)
- tbl->items[prev_item_idx].next_pkt_idx = next_idx;
-
- return next_idx;
-}
-
static inline uint32_t
insert_new_flow(struct gro_tcp4_tbl *tbl,
struct tcp4_flow_key *src,
@@ -159,13 +97,10 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
- rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
+ ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
+
dst->ip_src_addr = src->ip_src_addr;
dst->ip_dst_addr = src->ip_dst_addr;
- dst->recv_ack = src->recv_ack;
- dst->src_port = src->src_port;
- dst->dst_port = src->dst_port;
tbl->flows[flow_idx].start_index = item_idx;
tbl->flow_num++;
@@ -173,21 +108,6 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
return flow_idx;
}
-/*
- * update the packet length for the flushed packet.
- */
-static inline void
-update_header(struct gro_tcp4_item *item)
-{
- struct rte_ipv4_hdr *ipv4_hdr;
- struct rte_mbuf *pkt = item->firstseg;
-
- ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
- pkt->l2_len);
- ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
- pkt->l2_len);
-}
-
int32_t
gro_tcp4_reassemble(struct rte_mbuf *pkt,
struct gro_tcp4_tbl *tbl,
@@ -202,9 +122,8 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
uint8_t is_atomic;
struct tcp4_flow_key key;
- uint32_t cur_idx, prev_idx, item_idx;
+ uint32_t item_idx;
uint32_t i, max_flow_num, remaining_flow_num;
- int cmp;
uint8_t find;
/*
@@ -216,7 +135,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
- tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
+ tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt->l2_len + pkt->l3_len);
hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
/*
@@ -230,7 +149,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
ip_tlen = rte_be_to_cpu_16(ipv4_hdr->total_length);
if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len))
rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len);
-
/*
* Don't process the packet whose payload length is less than or
* equal to 0.
@@ -239,6 +157,13 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
if (tcp_dl <= 0)
return -1;
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.cmn_key.eth_daddr));
+ key.ip_src_addr = ipv4_hdr->src_addr;
+ key.ip_dst_addr = ipv4_hdr->dst_addr;
+ key.cmn_key.src_port = tcp_hdr->src_port;
+ key.cmn_key.dst_port = tcp_hdr->dst_port;
+ key.cmn_key.recv_ack = tcp_hdr->recv_ack;
/*
* Save IPv4 ID for the packet whose DF bit is 0. For the packet
* whose DF bit is 1, IPv4 ID is ignored.
@@ -246,15 +171,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) == RTE_IPV4_HDR_DF_FLAG;
ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
- sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
-
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
- key.ip_src_addr = ipv4_hdr->src_addr;
- key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.src_port = tcp_hdr->src_port;
- key.dst_port = tcp_hdr->dst_port;
- key.recv_ack = tcp_hdr->recv_ack;
/* Search for a matched flow. */
max_flow_num = tbl->max_flow_num;
@@ -270,63 +186,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
}
}
- /*
- * Fail to find a matched flow. Insert a new flow and store the
- * packet into the flow.
- */
if (find == 0) {
- item_idx = insert_new_item(tbl, pkt, start_time,
- INVALID_ARRAY_INDEX, sent_seq, ip_id,
- is_atomic);
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num,
+ tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, ip_id,
+ is_atomic);
if (item_idx == INVALID_ARRAY_INDEX)
return -1;
if (insert_new_flow(tbl, &key, item_idx) ==
- INVALID_ARRAY_INDEX) {
+ INVALID_ARRAY_INDEX) {
/*
* Fail to insert a new flow, so delete the
* stored packet.
- */
- delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
return -1;
}
return 0;
}
- /*
- * Check all packets in the flow and try to find a neighbor for
- * the input packet.
- */
- cur_idx = tbl->flows[i].start_index;
- prev_idx = cur_idx;
- do {
- cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
- sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
- is_atomic);
- if (cmp) {
- if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
- pkt, cmp, sent_seq, ip_id, 0))
- return 1;
- /*
- * Fail to merge the two packets, as the packet
- * length is greater than the max value. Store
- * the packet into the flow.
- */
- if (insert_new_item(tbl, pkt, start_time, cur_idx,
- sent_seq, ip_id, is_atomic) ==
- INVALID_ARRAY_INDEX)
- return -1;
- return 0;
- }
- prev_idx = cur_idx;
- cur_idx = tbl->items[cur_idx].next_pkt_idx;
- } while (cur_idx != INVALID_ARRAY_INDEX);
+ return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl->flows[i].start_index,
+ &tbl->item_num, tbl->max_item_num,
+ ip_id, is_atomic, start_time);
+}
- /* Fail to find a neighbor, so store the packet into the flow. */
- if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
- ip_id, is_atomic) == INVALID_ARRAY_INDEX)
- return -1;
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
- return 0;
+ ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len);
}
uint16_t
@@ -353,7 +250,8 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
* Delete the packet and get the next
* packet in the flow.
*/
- j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
+ j = delete_tcp_item(tbl->items, j,
+ &tbl->item_num, INVALID_ARRAY_INDEX);
tbl->flows[i].start_index = j;
if (j == INVALID_ARRAY_INDEX)
tbl->flow_num--;
diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h
index 212f97a042..c0154afa24 100644
--- a/lib/gro/gro_tcp4.h
+++ b/lib/gro/gro_tcp4.h
@@ -5,32 +5,15 @@
#ifndef _GRO_TCP4_H_
#define _GRO_TCP4_H_
-#include <rte_tcp.h>
+#include <gro_tcp_internal.h>
-#define INVALID_ARRAY_INDEX 0xffffffffUL
#define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
-/*
- * The max length of a IPv4 packet, which includes the length of the L3
- * header, the L4 header and the data payload.
- */
-#define MAX_IPV4_PKT_LENGTH UINT16_MAX
-
-/* The maximum TCP header length */
-#define MAX_TCP_HLEN 60
-#define INVALID_TCP_HDRLEN(len) \
- (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
-
-/* Header fields representing a TCP/IPv4 flow */
+/* Header fields representing common fields in TCP flow */
struct tcp4_flow_key {
- struct rte_ether_addr eth_saddr;
- struct rte_ether_addr eth_daddr;
+ struct cmn_tcp_key cmn_key;
uint32_t ip_src_addr;
uint32_t ip_dst_addr;
-
- uint32_t recv_ack;
- uint16_t src_port;
- uint16_t dst_port;
};
struct gro_tcp4_flow {
@@ -42,42 +25,12 @@ struct gro_tcp4_flow {
uint32_t start_index;
};
-struct gro_tcp4_item {
- /*
- * The first MBUF segment of the packet. If the value
- * is NULL, it means the item is empty.
- */
- struct rte_mbuf *firstseg;
- /* The last MBUF segment of the packet */
- struct rte_mbuf *lastseg;
- /*
- * The time when the first packet is inserted into the table.
- * This value won't be updated, even if the packet is merged
- * with other packets.
- */
- uint64_t start_time;
- /*
- * next_pkt_idx is used to chain the packets that
- * are in the same flow but can't be merged together
- * (e.g. caused by packet reordering).
- */
- uint32_t next_pkt_idx;
- /* TCP sequence number of the packet */
- uint32_t sent_seq;
- /* IPv4 ID of the packet */
- uint16_t ip_id;
- /* the number of merged packets */
- uint16_t nb_merged;
- /* Indicate if IPv4 ID can be ignored */
- uint8_t is_atomic;
-};
-
/*
* TCP/IPv4 reassembly table structure.
*/
struct gro_tcp4_tbl {
/* item array */
- struct gro_tcp4_item *items;
+ struct gro_tcp_item *items;
/* flow array */
struct gro_tcp4_flow *flows;
/* current item number */
@@ -186,120 +139,9 @@ uint32_t gro_tcp4_tbl_pkt_count(void *tbl);
static inline int
is_same_tcp4_flow(struct tcp4_flow_key k1, struct tcp4_flow_key k2)
{
- return (rte_is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) &&
- rte_is_same_ether_addr(&k1.eth_daddr, &k2.eth_daddr) &&
- (k1.ip_src_addr == k2.ip_src_addr) &&
+ return ((k1.ip_src_addr == k2.ip_src_addr) &&
(k1.ip_dst_addr == k2.ip_dst_addr) &&
- (k1.recv_ack == k2.recv_ack) &&
- (k1.src_port == k2.src_port) &&
- (k1.dst_port == k2.dst_port));
+ is_common_tcp_key(&k1.cmn_key, &k2.cmn_key));
}
-/*
- * Merge two TCP/IPv4 packets without updating checksums.
- * If cmp is larger than 0, append the new packet to the
- * original packet. Otherwise, pre-pend the new packet to
- * the original packet.
- */
-static inline int
-merge_two_tcp4_packets(struct gro_tcp4_item *item,
- struct rte_mbuf *pkt,
- int cmp,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t l2_offset)
-{
- struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
- uint16_t hdr_len, l2_len;
-
- if (cmp > 0) {
- pkt_head = item->firstseg;
- pkt_tail = pkt;
- } else {
- pkt_head = pkt;
- pkt_tail = item->firstseg;
- }
-
- /* check if the IPv4 packet length is greater than the max value */
- hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
- pkt_head->l4_len;
- l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
- if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
- hdr_len > MAX_IPV4_PKT_LENGTH))
- return 0;
-
- /* remove the packet header for the tail packet */
- rte_pktmbuf_adj(pkt_tail, hdr_len);
-
- /* chain two packets together */
- if (cmp > 0) {
- item->lastseg->next = pkt;
- item->lastseg = rte_pktmbuf_lastseg(pkt);
- /* update IP ID to the larger value */
- item->ip_id = ip_id;
- } else {
- lastseg = rte_pktmbuf_lastseg(pkt);
- lastseg->next = item->firstseg;
- item->firstseg = pkt;
- /* update sent_seq to the smaller value */
- item->sent_seq = sent_seq;
- item->ip_id = ip_id;
- }
- item->nb_merged++;
-
- /* update MBUF metadata for the merged packet */
- pkt_head->nb_segs += pkt_tail->nb_segs;
- pkt_head->pkt_len += pkt_tail->pkt_len;
-
- return 1;
-}
-
-/*
- * Check if two TCP/IPv4 packets are neighbors.
- */
-static inline int
-check_seq_option(struct gro_tcp4_item *item,
- struct rte_tcp_hdr *tcph,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t tcp_hl,
- uint16_t tcp_dl,
- uint16_t l2_offset,
- uint8_t is_atomic)
-{
- struct rte_mbuf *pkt_orig = item->firstseg;
- struct rte_ipv4_hdr *iph_orig;
- struct rte_tcp_hdr *tcph_orig;
- uint16_t len, tcp_hl_orig;
-
- iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char *) +
- l2_offset + pkt_orig->l2_len);
- tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig + pkt_orig->l3_len);
- tcp_hl_orig = pkt_orig->l4_len;
-
- /* Check if TCP option fields equal */
- len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
- if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
- (memcmp(tcph + 1, tcph_orig + 1,
- len) != 0)))
- return 0;
-
- /* Don't merge packets whose DF bits are different */
- if (unlikely(item->is_atomic ^ is_atomic))
- return 0;
-
- /* check if the two packets are neighbors */
- len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
- pkt_orig->l3_len - tcp_hl_orig;
- if ((sent_seq == item->sent_seq + len) && (is_atomic ||
- (ip_id == item->ip_id + 1)))
- /* append the new packet */
- return 1;
- else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
- (ip_id + item->nb_merged == item->ip_id)))
- /* pre-pend the new packet */
- return -1;
-
- return 0;
-}
#endif
diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c
new file mode 100644
index 0000000000..e0b21c33c9
--- /dev/null
+++ b/lib/gro/gro_tcp6.c
@@ -0,0 +1,266 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp6.h"
+
+void *
+gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow)
+{
+ struct gro_tcp6_tbl *tbl;
+ size_t size;
+ uint32_t entries_num, i;
+
+ entries_num = max_flow_num * max_item_per_flow;
+ entries_num = RTE_MIN(entries_num, GRO_TCP6_TBL_MAX_ITEM_NUM);
+
+ if (entries_num == 0)
+ return NULL;
+
+ tbl = rte_zmalloc_socket(__func__,
+ sizeof(struct gro_tcp6_tbl),
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl == NULL)
+ return NULL;
+
+ size = sizeof(struct gro_tcp_item) * entries_num;
+ tbl->items = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->items == NULL) {
+ rte_free(tbl);
+ return NULL;
+ }
+ tbl->max_item_num = entries_num;
+
+ size = sizeof(struct gro_tcp6_flow) * entries_num;
+ tbl->flows = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->flows == NULL) {
+ rte_free(tbl->items);
+ rte_free(tbl);
+ return NULL;
+ }
+ /* INVALID_ARRAY_INDEX indicates an empty flow */
+ for (i = 0; i < entries_num; i++)
+ tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
+ tbl->max_flow_num = entries_num;
+
+ return tbl;
+}
+
+void
+gro_tcp6_tbl_destroy(void *tbl)
+{
+ struct gro_tcp6_tbl *tcp_tbl = tbl;
+
+ if (tcp_tbl) {
+ rte_free(tcp_tbl->items);
+ rte_free(tcp_tbl->flows);
+ }
+ rte_free(tcp_tbl);
+}
+
+static inline uint32_t
+find_an_empty_flow(struct gro_tcp6_tbl *tbl)
+{
+ uint32_t i;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++)
+ if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+insert_new_flow(struct gro_tcp6_tbl *tbl,
+ struct tcp6_flow_key *src,
+ uint32_t item_idx)
+{
+ struct tcp6_flow_key *dst;
+ uint32_t flow_idx;
+
+ flow_idx = find_an_empty_flow(tbl);
+ if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
+ return INVALID_ARRAY_INDEX;
+
+ dst = &(tbl->flows[flow_idx].key);
+
+ ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
+ memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst->src_addr));
+ memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst->dst_addr));
+ dst->vtc_flow = src->vtc_flow;
+
+ tbl->flows[flow_idx].start_index = item_idx;
+ tbl->flow_num++;
+
+ return flow_idx;
+}
+
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
+
+ ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len - pkt->l3_len);
+}
+
+int32_t
+gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time)
+{
+ struct rte_ether_hdr *eth_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ int32_t tcp_dl;
+ uint16_t ip_tlen;
+ struct tcp6_flow_key key;
+ uint32_t i, max_flow_num, remaining_flow_num;
+ uint32_t sent_seq;
+ struct rte_tcp_hdr *tcp_hdr;
+ uint8_t find;
+ uint32_t item_idx;
+ /*
+ * Don't process the packet whose TCP header length is greater
+ * than 60 bytes or less than 20 bytes.
+ */
+ if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
+ return -1;
+
+ eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
+ ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
+ tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt->l2_len + pkt->l3_len);
+
+ /*
+ * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
+ * or CWR set.
+ */
+ if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
+ return -1;
+
+ ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
+ /*
+ * Don't process the packet whose payload length is less than or
+ * equal to 0.
+ */
+ tcp_dl = ip_tlen - pkt->l4_len;
+ if (tcp_dl <= 0)
+ return -1;
+
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.cmn_key.eth_daddr));
+ memcpy(&key.src_addr[0], &ipv6_hdr->src_addr, sizeof(key.src_addr));
+ memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr, sizeof(key.dst_addr));
+ key.cmn_key.src_port = tcp_hdr->src_port;
+ key.cmn_key.dst_port = tcp_hdr->dst_port;
+ key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+ key.vtc_flow = ipv6_hdr->vtc_flow;
+
+ /* Search for a matched flow. */
+ max_flow_num = tbl->max_flow_num;
+ remaining_flow_num = tbl->flow_num;
+ find = 0;
+ for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
+ if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
+ if (is_same_tcp6_flow(&tbl->flows[i].key, &key)) {
+ find = 1;
+ break;
+ }
+ remaining_flow_num--;
+ }
+ }
+
+ if (find == 0) {
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num, tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, 0, true);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return -1;
+ if (insert_new_flow(tbl, &key, item_idx) ==
+ INVALID_ARRAY_INDEX) {
+ /*
+ * Fail to insert a new flow, so delete the
+ * stored packet.
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
+ return -1;
+ }
+ return 0;
+ }
+
+ return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl->flows[i].start_index,
+ &tbl->item_num, tbl->max_item_num,
+ 0, true, start_time);
+}
+
+uint16_t
+gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out)
+{
+ uint16_t k = 0;
+ uint32_t i, j;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++) {
+ if (unlikely(tbl->flow_num == 0))
+ return k;
+
+ j = tbl->flows[i].start_index;
+ while (j != INVALID_ARRAY_INDEX) {
+ if (tbl->items[j].start_time <= flush_timestamp) {
+ out[k++] = tbl->items[j].firstseg;
+ if (tbl->items[j].nb_merged > 1)
+ update_header(&(tbl->items[j]));
+ /*
+ * Delete the packet and get the next
+ * packet in the flow.
+ */
+ j = delete_tcp_item(tbl->items, j,
+ &tbl->item_num, INVALID_ARRAY_INDEX);
+ tbl->flows[i].start_index = j;
+ if (j == INVALID_ARRAY_INDEX)
+ tbl->flow_num--;
+
+ if (unlikely(k == nb_out))
+ return k;
+ } else
+ /*
+ * The left packets in this flow won't be
+ * timeout. Go to check other flows.
+ */
+ break;
+ }
+ }
+ return k;
+}
+
+uint32_t
+gro_tcp6_tbl_pkt_count(void *tbl)
+{
+ struct gro_tcp6_tbl *gro_tbl = tbl;
+
+ if (gro_tbl)
+ return gro_tbl->item_num;
+
+ return 0;
+}
diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h
new file mode 100644
index 0000000000..2de465ecd1
--- /dev/null
+++ b/lib/gro/gro_tcp6.h
@@ -0,0 +1,161 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#ifndef _GRO_TCP6_H_
+#define _GRO_TCP6_H_
+
+#include <gro_tcp_internal.h>
+
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+#define GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
+
+/* Header fields representing a TCP/IPv6 flow */
+struct tcp6_flow_key {
+ struct cmn_tcp_key cmn_key;
+ uint8_t src_addr[16];
+ uint8_t dst_addr[16];
+ rte_be32_t vtc_flow;
+};
+
+struct gro_tcp6_flow {
+ struct tcp6_flow_key key;
+ /*
+ * The index of the first packet in the flow.
+ * INVALID_ARRAY_INDEX indicates an empty flow.
+ */
+ uint32_t start_index;
+};
+
+/*
+ * TCP/IPv6 reassembly table structure.
+ */
+struct gro_tcp6_tbl {
+ /* item array */
+ struct gro_tcp_item *items;
+ /* flow array */
+ struct gro_tcp6_flow *flows;
+ /* current item number */
+ uint32_t item_num;
+ /* current flow num */
+ uint32_t flow_num;
+ /* item array size */
+ uint32_t max_item_num;
+ /* flow array size */
+ uint32_t max_flow_num;
+};
+
+/**
+ * This function creates a TCP/IPv6 reassembly table.
+ *
+ * @param socket_id
+ * Socket index for allocating the TCP/IPv6 reassemble table
+ * @param max_flow_num
+ * The maximum number of flows in the TCP/IPv6 GRO table
+ * @param max_item_per_flow
+ * The maximum number of packets per flow
+ *
+ * @return
+ * - Return the table pointer on success.
+ * - Return NULL on failure.
+ */
+void *gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow);
+
+/**
+ * This function destroys a TCP/IPv6 reassembly table.
+ *
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table.
+ */
+void gro_tcp6_tbl_destroy(void *tbl);
+
+/**
+ * This function merges a TCP/IPv6 packet. It doesn't process the packet,
+ * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't have
+ * payload.
+ *
+ * This function doesn't check if the packet has correct checksums and
+ * doesn't re-calculate checksums for the merged packet. Additionally,
+ * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
+ * when IP fragmentation is possible (i.e., DF==0). It returns the
+ * packet, if the packet has invalid parameters (e.g. SYN bit is set)
+ * or there is no available space in the table.
+ *
+ * @param pkt
+ * Packet to reassemble
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table
+ * @start_time
+ * The time when the packet is inserted into the table
+ *
+ * @return
+ * - Return a positive value if the packet is merged.
+ * - Return zero if the packet isn't merged but stored in the table.
+ * - Return a negative value for invalid parameters or no available
+ * space in the table.
+ */
+int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time);
+
+/**
+ * This function flushes timeout packets in a TCP/IPv6 reassembly table,
+ * and without updating checksums.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ * @param flush_timestamp
+ * Flush packets which are inserted into the table before or at the
+ * flush_timestamp.
+ * @param out
+ * Pointer array used to keep flushed packets
+ * @param nb_out
+ * The element number in 'out'. It also determines the maximum number of
+ * packets that can be flushed finally.
+ *
+ * @return
+ * The number of flushed packets
+ */
+uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out);
+
+/**
+ * This function returns the number of the packets in a TCP/IPv6
+ * reassembly table.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ *
+ * @return
+ * The number of packets in the table
+ */
+uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
+
+/*
+ * Check if two TCP/IPv6 packets belong to the same flow.
+ */
+static inline int
+is_same_tcp6_flow(struct tcp6_flow_key *k1, struct tcp6_flow_key *k2)
+{
+ rte_be32_t vtc_flow_diff;
+
+ if (memcmp(&k1->src_addr, &k2->src_addr, 16)
+ return 0;
+ if (memcmp(&k1->dst_addr, &k2->dst_addr, 16)
+ return 0;
+ /*
+ * IP version (4) Traffic Class (8) Flow Label (20)
+ * All fields except Traffic class should be same
+ */
+ vtc_flow_diff = (k1->vtc_flow ^ k2->vtc_flow);
+ if (vtc_flow_diff & htonl(0xF00FFFFF))
+ return 0;
+
+ return is_common_tcp_key(&k1->cmn_key, &k2->cmn_key);
+}
+
+#endif
diff --git a/lib/gro/gro_tcp_internal.c b/lib/gro/gro_tcp_internal.c
new file mode 100644
index 0000000000..5a21bca7f8
--- /dev/null
+++ b/lib/gro/gro_tcp_internal.c
@@ -0,0 +1,128 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp_internal.h"
+
+static inline uint32_t
+find_an_empty_item(struct gro_tcp_item *items,
+ uint32_t max_item_num)
+{
+ uint32_t i;
+
+ for (i = 0; i < max_item_num; i++)
+ if (items[i].firstseg == NULL)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+inline uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic)
+{
+ uint32_t item_idx;
+
+ item_idx = find_an_empty_item(items, max_item_num);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return INVALID_ARRAY_INDEX;
+
+ items[item_idx].firstseg = pkt;
+ items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
+ items[item_idx].start_time = start_time;
+ items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
+ items[item_idx].sent_seq = sent_seq;
+ items[item_idx].l3.ip_id = ip_id;
+ items[item_idx].nb_merged = 1;
+ items[item_idx].is_atomic = is_atomic;
+ (*item_num) += 1;
+
+ /* if the previous packet exists, chain them together. */
+ if (prev_idx != INVALID_ARRAY_INDEX) {
+ items[item_idx].next_pkt_idx =
+ items[prev_idx].next_pkt_idx;
+ items[prev_idx].next_pkt_idx = item_idx;
+ }
+
+ return item_idx;
+}
+
+inline uint32_t
+delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx)
+{
+ uint32_t next_idx = items[item_idx].next_pkt_idx;
+
+ /* NULL indicates an empty item */
+ items[item_idx].firstseg = NULL;
+ (*item_num) -= 1;
+ if (prev_item_idx != INVALID_ARRAY_INDEX)
+ items[prev_item_idx].next_pkt_idx = next_idx;
+
+ return next_idx;
+}
+
+int32_t
+process_tcp_item(struct rte_mbuf *pkt,
+ struct rte_tcp_hdr *tcp_hdr,
+ int32_t tcp_dl,
+ struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time)
+{
+ uint32_t cur_idx;
+ uint32_t prev_idx;
+ int cmp;
+ uint32_t sent_seq;
+
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ /*
+ * Check all packets in the flow and try to find a neighbor for
+ * the input packet.
+ */
+ cur_idx = item_idx;
+ prev_idx = cur_idx;
+ do {
+ cmp = check_seq_option(&items[cur_idx], tcp_hdr,
+ sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
+ is_atomic);
+ if (cmp) {
+ if (merge_two_tcp_packets(&items[cur_idx],
+ pkt, cmp, sent_seq, ip_id, 0))
+ return 1;
+ /*
+ * Fail to merge the two packets, as the packet
+ * length is greater than the max value. Store
+ * the packet into the flow.
+ */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num,
+ start_time, cur_idx, sent_seq, ip_id, is_atomic) ==
+ INVALID_ARRAY_INDEX)
+ return -1;
+ return 0;
+ }
+ prev_idx = cur_idx;
+ cur_idx = items[cur_idx].next_pkt_idx;
+ } while (cur_idx != INVALID_ARRAY_INDEX);
+
+ /* Fail to find a neighbor, so store the packet into the flow. */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num, start_time, prev_idx, sent_seq,
+ ip_id, is_atomic) == INVALID_ARRAY_INDEX)
+ return -1;
+
+ return 0;
+}
diff --git a/lib/gro/gro_tcp_internal.h b/lib/gro/gro_tcp_internal.h
new file mode 100644
index 0000000000..072b7aea13
--- /dev/null
+++ b/lib/gro/gro_tcp_internal.h
@@ -0,0 +1,212 @@
+#ifndef _GRO_TCP_H_
+#define _GRO_TCP_H_
+
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+
+#include <rte_tcp.h>
+
+/*
+ * The max length of a IPv4 packet, which includes the length of the L3
+ * header, the L4 header and the data payload.
+ */
+#define MAX_IP_PKT_LENGTH UINT16_MAX
+
+/* The maximum TCP header length */
+#define MAX_TCP_HLEN 60
+#define INVALID_TCP_HDRLEN(len) \
+ (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
+
+struct cmn_tcp_key {
+ struct rte_ether_addr eth_saddr;
+ struct rte_ether_addr eth_daddr;
+ uint32_t recv_ack;
+ uint16_t src_port;
+ uint16_t dst_port;
+};
+
+#define ASSIGN_COMMON_TCP_KEY(k1, k2) \
+ do {\
+ rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
+ rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr)); \
+ k2->recv_ack = k1->recv_ack; \
+ k2->src_port = k1->src_port; \
+ k2->dst_port = k1->dst_port; \
+ } while (0)
+
+struct gro_tcp_item {
+ /*
+ * The first MBUF segment of the packet. If the value
+ * is NULL, it means the item is empty.
+ */
+ struct rte_mbuf *firstseg;
+ /* The last MBUF segment of the packet */
+ struct rte_mbuf *lastseg;
+ /*
+ * The time when the first packet is inserted into the table.
+ * This value won't be updated, even if the packet is merged
+ * with other packets.
+ */
+ uint64_t start_time;
+ /*
+ * next_pkt_idx is used to chain the packets that
+ * are in the same flow but can't be merged together
+ * (e.g. caused by packet reordering).
+ */
+ uint32_t next_pkt_idx;
+ /* TCP sequence number of the packet */
+ uint32_t sent_seq;
+ union {
+ /* IPv4 ID of the packet */
+ uint16_t ip_id;
+ /* Unused field for IPv6 */
+ uint16_t unused;
+ } l3;
+ /* the number of merged packets */
+ uint16_t nb_merged;
+ /* Indicate if IPv4 ID can be ignored */
+ uint8_t is_atomic;
+};
+
+uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic);
+
+uint32_t
+delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx);
+
+int32_t
+process_tcp_item(struct rte_mbuf *pkt,
+ struct rte_tcp_hdr *tcp_hdr,
+ int32_t tcp_dl,
+ struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time);
+
+/*
+ * Merge two TCP packets without updating checksums.
+ * If cmp is larger than 0, append the new packet to the
+ * original packet. Otherwise, pre-pend the new packet to
+ * the original packet.
+ */
+static inline int
+merge_two_tcp_packets(struct gro_tcp_item *item,
+ struct rte_mbuf *pkt,
+ int cmp,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t l2_offset)
+{
+ struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
+ uint16_t hdr_len, l2_len;
+
+ if (cmp > 0) {
+ pkt_head = item->firstseg;
+ pkt_tail = pkt;
+ } else {
+ pkt_head = pkt;
+ pkt_tail = item->firstseg;
+ }
+
+ /* check if the IPv4 packet length is greater than the max value */
+ hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
+ pkt_head->l4_len;
+ l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
+ if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
+ hdr_len > MAX_IP_PKT_LENGTH))
+ return 0;
+
+ /* remove the packet header for the tail packet */
+ rte_pktmbuf_adj(pkt_tail, hdr_len);
+
+ /* chain two packets together */
+ if (cmp > 0) {
+ item->lastseg->next = pkt;
+ item->lastseg = rte_pktmbuf_lastseg(pkt);
+ /* update IP ID to the larger value */
+ item->l3.ip_id = ip_id;
+ } else {
+ lastseg = rte_pktmbuf_lastseg(pkt);
+ lastseg->next = item->firstseg;
+ item->firstseg = pkt;
+ /* update sent_seq to the smaller value */
+ item->sent_seq = sent_seq;
+ item->l3.ip_id = ip_id;
+ }
+ item->nb_merged++;
+
+ /* update MBUF metadata for the merged packet */
+ pkt_head->nb_segs += pkt_tail->nb_segs;
+ pkt_head->pkt_len += pkt_tail->pkt_len;
+
+ return 1;
+}
+
+/*
+ * Check if two TCP/IPv4 packets are neighbors.
+ */
+static inline int
+check_seq_option(struct gro_tcp_item *item,
+ struct rte_tcp_hdr *tcph,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t tcp_hl,
+ uint16_t tcp_dl,
+ uint16_t l2_offset,
+ uint8_t is_atomic)
+{
+ struct rte_mbuf *pkt_orig = item->firstseg;
+ char *iph_orig;
+ struct rte_tcp_hdr *tcph_orig;
+ uint16_t len, tcp_hl_orig;
+
+ iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
+ l2_offset + pkt_orig->l2_len);
+ tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
+ tcp_hl_orig = pkt_orig->l4_len;
+
+ /* Check if TCP option fields equal */
+ len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
+ if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
+ (memcmp(tcph + 1, tcph_orig + 1,
+ len) != 0)))
+ return 0;
+
+ /* Don't merge packets whose DF bits are different */
+ if (unlikely(item->is_atomic ^ is_atomic))
+ return 0;
+
+ /* check if the two packets are neighbors */
+ len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
+ pkt_orig->l3_len - tcp_hl_orig;
+ if ((sent_seq == item->sent_seq + len) && (is_atomic ||
+ (ip_id == item->l3.ip_id + 1)))
+ /* append the new packet */
+ return 1;
+ else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
+ (ip_id + item->nb_merged == item->l3.ip_id)))
+ /* pre-pend the new packet */
+ return -1;
+
+ return 0;
+}
+
+static inline int
+is_common_tcp_key(struct cmn_tcp_key *k1, struct cmn_tcp_key *k2)
+{
+ return (!memcmp(k1, k2, sizeof(struct cmn_tcp_key)));
+}
+
+#endif
diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
index 3be4deb7c7..81eebf0d2d 100644
--- a/lib/gro/gro_vxlan_tcp4.c
+++ b/lib/gro/gro_vxlan_tcp4.c
@@ -7,6 +7,7 @@
#include <rte_ethdev.h>
#include <rte_udp.h>
+#include "gro_tcp_internal.h"
#include "gro_vxlan_tcp4.h"
void *
@@ -116,7 +117,7 @@ insert_new_item(struct gro_vxlan_tcp4_tbl *tbl,
tbl->items[item_idx].inner_item.start_time = start_time;
tbl->items[item_idx].inner_item.next_pkt_idx = INVALID_ARRAY_INDEX;
tbl->items[item_idx].inner_item.sent_seq = sent_seq;
- tbl->items[item_idx].inner_item.ip_id = ip_id;
+ tbl->items[item_idx].inner_item.l3.ip_id = ip_id;
tbl->items[item_idx].inner_item.nb_merged = 1;
tbl->items[item_idx].inner_item.is_atomic = is_atomic;
tbl->items[item_idx].outer_ip_id = outer_ip_id;
@@ -163,15 +164,9 @@ insert_new_flow(struct gro_vxlan_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->inner_key.eth_saddr),
- &(dst->inner_key.eth_saddr));
- rte_ether_addr_copy(&(src->inner_key.eth_daddr),
- &(dst->inner_key.eth_daddr));
+ ASSIGN_COMMON_TCP_KEY((&(src->inner_key.cmn_key)), (&(dst->inner_key.cmn_key)));
dst->inner_key.ip_src_addr = src->inner_key.ip_src_addr;
dst->inner_key.ip_dst_addr = src->inner_key.ip_dst_addr;
- dst->inner_key.recv_ack = src->inner_key.recv_ack;
- dst->inner_key.src_port = src->inner_key.src_port;
- dst->inner_key.dst_port = src->inner_key.dst_port;
dst->vxlan_hdr.vx_flags = src->vxlan_hdr.vx_flags;
dst->vxlan_hdr.vx_vni = src->vxlan_hdr.vx_vni;
@@ -248,7 +243,7 @@ merge_two_vxlan_tcp4_packets(struct gro_vxlan_tcp4_item *item,
uint16_t outer_ip_id,
uint16_t ip_id)
{
- if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp, sent_seq,
+ if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
ip_id, pkt->outer_l2_len +
pkt->outer_l3_len)) {
/* Update the outer IPv4 ID to the large value. */
@@ -357,13 +352,13 @@ gro_vxlan_tcp4_reassemble(struct rte_mbuf *pkt,
sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.eth_daddr));
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.cmn_key.eth_daddr));
key.inner_key.ip_src_addr = ipv4_hdr->src_addr;
key.inner_key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.inner_key.recv_ack = tcp_hdr->recv_ack;
- key.inner_key.src_port = tcp_hdr->src_port;
- key.inner_key.dst_port = tcp_hdr->dst_port;
+ key.inner_key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+ key.inner_key.cmn_key.src_port = tcp_hdr->src_port;
+ key.inner_key.cmn_key.dst_port = tcp_hdr->dst_port;
key.vxlan_hdr.vx_flags = vxlan_hdr->vx_flags;
key.vxlan_hdr.vx_vni = vxlan_hdr->vx_vni;
diff --git a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h
index 7832942a68..82eaaee11e 100644
--- a/lib/gro/gro_vxlan_tcp4.h
+++ b/lib/gro/gro_vxlan_tcp4.h
@@ -5,6 +5,7 @@
#ifndef _GRO_VXLAN_TCP4_H_
#define _GRO_VXLAN_TCP4_H_
+#include "gro_tcp_internal.h"
#include "gro_tcp4.h"
#define GRO_VXLAN_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
@@ -36,7 +37,7 @@ struct gro_vxlan_tcp4_flow {
};
struct gro_vxlan_tcp4_item {
- struct gro_tcp4_item inner_item;
+ struct gro_tcp_item inner_item;
/* IPv4 ID in the outer IPv4 header */
uint16_t outer_ip_id;
/* Indicate if outer IPv4 ID can be ignored */
diff --git a/lib/gro/meson.build b/lib/gro/meson.build
index e4fa2958bd..1640317890 100644
--- a/lib/gro/meson.build
+++ b/lib/gro/meson.build
@@ -3,7 +3,9 @@
sources = files(
'rte_gro.c',
+ 'gro_tcp_internal.c',
'gro_tcp4.c',
+ 'gro_tcp6.c',
'gro_udp4.c',
'gro_vxlan_tcp4.c',
'gro_vxlan_udp4.c',
diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c
index e35399fd42..d824eebd93 100644
--- a/lib/gro/rte_gro.c
+++ b/lib/gro/rte_gro.c
@@ -8,6 +8,7 @@
#include "rte_gro.h"
#include "gro_tcp4.h"
+#include "gro_tcp6.h"
#include "gro_udp4.h"
#include "gro_vxlan_tcp4.h"
#include "gro_vxlan_udp4.h"
@@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
- gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
+ gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, gro_tcp6_tbl_create, NULL};
static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
+ gro_tcp6_tbl_destroy,
NULL};
static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count,
+ gro_tcp6_tbl_pkt_count,
NULL};
#define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
@@ -35,6 +38,12 @@ static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+/* GRO with extension headers is not supported */
+#define IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
+ ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
+ ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
+ (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+
#define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
@@ -147,7 +156,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
/* allocate a reassembly table for TCP/IPv4 GRO */
struct gro_tcp4_tbl tcp_tbl;
struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
- struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+ struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+
+ struct gro_tcp6_tbl tcp6_tbl;
+ struct gro_tcp6_flow tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
+ struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
/* allocate a reassembly table for UDP/IPv4 GRO */
struct gro_udp4_tbl udp_tbl;
@@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
int32_t ret;
uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
- do_vxlan_udp_gro = 0;
+ do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
do_udp4_gro = 1;
}
+ if (param->gro_types & RTE_GRO_TCP_IPV6) {
+ for (i = 0; i < item_num; i++)
+ tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+ tcp6_tbl.flows = tcp6_flows;
+ tcp6_tbl.items = tcp6_items;
+ tcp6_tbl.flow_num = 0;
+ tcp6_tbl.item_num = 0;
+ tcp6_tbl.max_flow_num = item_num;
+ tcp6_tbl.max_item_num = item_num;
+ do_tcp6_gro = 1;
+ }
for (i = 0; i < nb_pkts; i++) {
/*
@@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
nb_after_gro--;
else if (ret < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
+ if (ret > 0)
+ /* merge successfully */
+ nb_after_gro--;
+ else if (ret < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
if ((nb_after_gro < nb_pkts)
|| (unprocess_num < nb_pkts)) {
i = 0;
+ /* Copy unprocessed packets */
+ if (unprocess_num > 0) {
+ memcpy(&pkts[i], unprocess_pkts,
+ sizeof(struct rte_mbuf *) *
+ unprocess_num);
+ i = unprocess_num;
+ }
+
/* Flush all packets from the tables */
if (do_vxlan_tcp_gro) {
- i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
+ i += gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
0, pkts, nb_pkts);
}
@@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
&pkts[i], nb_pkts - i);
}
- /* Copy unprocessed packets */
- if (unprocess_num > 0) {
- memcpy(&pkts[i], unprocess_pkts,
- sizeof(struct rte_mbuf *) *
- unprocess_num);
+
+ if (do_tcp6_gro) {
+ i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
+ &pkts[i], nb_pkts - i);
}
- nb_after_gro = i + unprocess_num;
}
return nb_after_gro;
@@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
{
struct rte_mbuf *unprocess_pkts[nb_pkts];
struct gro_ctx *gro_ctx = ctx;
- void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
+ void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
uint64_t current_time;
uint16_t i, unprocess_num = 0;
- uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro;
+ uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro, do_tcp6_gro;
if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
+ tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
RTE_GRO_TCP_IPV4;
@@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
RTE_GRO_UDP_IPV4;
do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
RTE_GRO_IPV4_VXLAN_UDP_IPV4;
+ do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) == RTE_GRO_TCP_IPV6;
current_time = rte_rdtsc();
@@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
if (gro_udp4_reassemble(pkts[i], udp_tbl,
current_time) < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
+ current_time) < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
flush_timestamp,
&out[num], left_nb_out);
+ left_nb_out = max_nb_out - num;
+ }
+
+ if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
+ num += gro_tcp6_tbl_timeout_flush(
+ gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
+ flush_timestamp,
+ &out[num], left_nb_out);
+
}
return num;
diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h
index 9f9ed4935a..c83dfd9ad1 100644
--- a/lib/gro/rte_gro.h
+++ b/lib/gro/rte_gro.h
@@ -38,6 +38,9 @@ extern "C" {
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL << RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
/**< VxLAN UDP/IPv4 GRO flag. */
+#define RTE_GRO_TCP_IPV6_INDEX 4
+#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX)
+/**< TCP/IPv6 GRO flag. */
/**
* Structure used to create GRO context objects or used to pass
--
2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* [PATCH v7] gro : ipv6 changes to support GRO for TCP/ipv6
2022-10-20 18:14 ` [PATCH v2] " Kumara Parameshwaran
` (5 preceding siblings ...)
2023-06-12 11:23 ` [PATCH v6] " Kumara Parameshwaran
@ 2023-06-12 11:31 ` Kumara Parameshwaran
2023-06-12 12:04 ` kumaraparameshwaran rathinavel
2023-06-13 2:25 ` Hu, Jiayu
2023-06-15 5:40 ` [PATCH v8] " Kumara Parameshwaran
` (3 subsequent siblings)
10 siblings, 2 replies; 32+ messages in thread
From: Kumara Parameshwaran @ 2023-06-12 11:31 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev, Kumara Parameshwaran
The patch adds GRO support for TCP/ipv6 packets. This does not
include the support for vxlan, udp ipv6 packets.
Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
---
v1:
* Changes to support GRO for TCP/ipv6 packets. This does not include
vxlan changes.
* The GRO is performed only for ipv6 packets that does not contain
extension headers.
* The logic for the TCP coalescing remains the same, in ipv6 header
the source address, destination address, flow label, version fields
are expected to be the same.
* Re-organised the code to reuse certain tcp functions for both ipv4 and
ipv6 flows.
v2:
* Fix comments in gro_tcp6.h header file.
v3:
* Adderess review comments to fix code duplication for v4 and v6
v4:
* Addresses review comments for v3, do not use callbacks
v5:
* Address review comments
v6:
* Fix warning and coding style issues
v7:
* Fix build compilation issue
lib/gro/gro_tcp4.c | 178 ++++++-------------------
lib/gro/gro_tcp4.h | 170 +----------------------
lib/gro/gro_tcp6.c | 267 +++++++++++++++++++++++++++++++++++++
lib/gro/gro_tcp6.h | 161 ++++++++++++++++++++++
lib/gro/gro_tcp_internal.c | 128 ++++++++++++++++++
lib/gro/gro_tcp_internal.h | 212 +++++++++++++++++++++++++++++
lib/gro/gro_vxlan_tcp4.c | 23 ++--
lib/gro/gro_vxlan_tcp4.h | 3 +-
lib/gro/meson.build | 2 +
lib/gro/rte_gro.c | 83 ++++++++++--
lib/gro/rte_gro.h | 3 +
11 files changed, 897 insertions(+), 333 deletions(-)
create mode 100644 lib/gro/gro_tcp6.c
create mode 100644 lib/gro/gro_tcp6.h
create mode 100644 lib/gro/gro_tcp_internal.c
create mode 100644 lib/gro/gro_tcp_internal.h
diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c
index 0014096e63..42fee78f30 100644
--- a/lib/gro/gro_tcp4.c
+++ b/lib/gro/gro_tcp4.c
@@ -30,7 +30,7 @@ gro_tcp4_tbl_create(uint16_t socket_id,
if (tbl == NULL)
return NULL;
- size = sizeof(struct gro_tcp4_item) * entries_num;
+ size = sizeof(struct gro_tcp_item) * entries_num;
tbl->items = rte_zmalloc_socket(__func__,
size,
RTE_CACHE_LINE_SIZE,
@@ -71,18 +71,6 @@ gro_tcp4_tbl_destroy(void *tbl)
rte_free(tcp_tbl);
}
-static inline uint32_t
-find_an_empty_item(struct gro_tcp4_tbl *tbl)
-{
- uint32_t i;
- uint32_t max_item_num = tbl->max_item_num;
-
- for (i = 0; i < max_item_num; i++)
- if (tbl->items[i].firstseg == NULL)
- return i;
- return INVALID_ARRAY_INDEX;
-}
-
static inline uint32_t
find_an_empty_flow(struct gro_tcp4_tbl *tbl)
{
@@ -95,56 +83,6 @@ find_an_empty_flow(struct gro_tcp4_tbl *tbl)
return INVALID_ARRAY_INDEX;
}
-static inline uint32_t
-insert_new_item(struct gro_tcp4_tbl *tbl,
- struct rte_mbuf *pkt,
- uint64_t start_time,
- uint32_t prev_idx,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint8_t is_atomic)
-{
- uint32_t item_idx;
-
- item_idx = find_an_empty_item(tbl);
- if (item_idx == INVALID_ARRAY_INDEX)
- return INVALID_ARRAY_INDEX;
-
- tbl->items[item_idx].firstseg = pkt;
- tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
- tbl->items[item_idx].start_time = start_time;
- tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
- tbl->items[item_idx].sent_seq = sent_seq;
- tbl->items[item_idx].ip_id = ip_id;
- tbl->items[item_idx].nb_merged = 1;
- tbl->items[item_idx].is_atomic = is_atomic;
- tbl->item_num++;
-
- /* if the previous packet exists, chain them together. */
- if (prev_idx != INVALID_ARRAY_INDEX) {
- tbl->items[item_idx].next_pkt_idx =
- tbl->items[prev_idx].next_pkt_idx;
- tbl->items[prev_idx].next_pkt_idx = item_idx;
- }
-
- return item_idx;
-}
-
-static inline uint32_t
-delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
- uint32_t prev_item_idx)
-{
- uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
-
- /* NULL indicates an empty item */
- tbl->items[item_idx].firstseg = NULL;
- tbl->item_num--;
- if (prev_item_idx != INVALID_ARRAY_INDEX)
- tbl->items[prev_item_idx].next_pkt_idx = next_idx;
-
- return next_idx;
-}
-
static inline uint32_t
insert_new_flow(struct gro_tcp4_tbl *tbl,
struct tcp4_flow_key *src,
@@ -159,13 +97,10 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
- rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
+ ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
+
dst->ip_src_addr = src->ip_src_addr;
dst->ip_dst_addr = src->ip_dst_addr;
- dst->recv_ack = src->recv_ack;
- dst->src_port = src->src_port;
- dst->dst_port = src->dst_port;
tbl->flows[flow_idx].start_index = item_idx;
tbl->flow_num++;
@@ -173,21 +108,6 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
return flow_idx;
}
-/*
- * update the packet length for the flushed packet.
- */
-static inline void
-update_header(struct gro_tcp4_item *item)
-{
- struct rte_ipv4_hdr *ipv4_hdr;
- struct rte_mbuf *pkt = item->firstseg;
-
- ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
- pkt->l2_len);
- ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
- pkt->l2_len);
-}
-
int32_t
gro_tcp4_reassemble(struct rte_mbuf *pkt,
struct gro_tcp4_tbl *tbl,
@@ -202,9 +122,8 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
uint8_t is_atomic;
struct tcp4_flow_key key;
- uint32_t cur_idx, prev_idx, item_idx;
+ uint32_t item_idx;
uint32_t i, max_flow_num, remaining_flow_num;
- int cmp;
uint8_t find;
/*
@@ -216,7 +135,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
- tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
+ tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt->l2_len + pkt->l3_len);
hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
/*
@@ -230,7 +149,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
ip_tlen = rte_be_to_cpu_16(ipv4_hdr->total_length);
if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len))
rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len);
-
/*
* Don't process the packet whose payload length is less than or
* equal to 0.
@@ -239,6 +157,13 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
if (tcp_dl <= 0)
return -1;
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.cmn_key.eth_daddr));
+ key.ip_src_addr = ipv4_hdr->src_addr;
+ key.ip_dst_addr = ipv4_hdr->dst_addr;
+ key.cmn_key.src_port = tcp_hdr->src_port;
+ key.cmn_key.dst_port = tcp_hdr->dst_port;
+ key.cmn_key.recv_ack = tcp_hdr->recv_ack;
/*
* Save IPv4 ID for the packet whose DF bit is 0. For the packet
* whose DF bit is 1, IPv4 ID is ignored.
@@ -246,15 +171,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) == RTE_IPV4_HDR_DF_FLAG;
ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
- sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
-
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
- key.ip_src_addr = ipv4_hdr->src_addr;
- key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.src_port = tcp_hdr->src_port;
- key.dst_port = tcp_hdr->dst_port;
- key.recv_ack = tcp_hdr->recv_ack;
/* Search for a matched flow. */
max_flow_num = tbl->max_flow_num;
@@ -270,63 +186,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
}
}
- /*
- * Fail to find a matched flow. Insert a new flow and store the
- * packet into the flow.
- */
if (find == 0) {
- item_idx = insert_new_item(tbl, pkt, start_time,
- INVALID_ARRAY_INDEX, sent_seq, ip_id,
- is_atomic);
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num,
+ tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, ip_id,
+ is_atomic);
if (item_idx == INVALID_ARRAY_INDEX)
return -1;
if (insert_new_flow(tbl, &key, item_idx) ==
- INVALID_ARRAY_INDEX) {
+ INVALID_ARRAY_INDEX) {
/*
* Fail to insert a new flow, so delete the
* stored packet.
- */
- delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
return -1;
}
return 0;
}
- /*
- * Check all packets in the flow and try to find a neighbor for
- * the input packet.
- */
- cur_idx = tbl->flows[i].start_index;
- prev_idx = cur_idx;
- do {
- cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
- sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
- is_atomic);
- if (cmp) {
- if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
- pkt, cmp, sent_seq, ip_id, 0))
- return 1;
- /*
- * Fail to merge the two packets, as the packet
- * length is greater than the max value. Store
- * the packet into the flow.
- */
- if (insert_new_item(tbl, pkt, start_time, cur_idx,
- sent_seq, ip_id, is_atomic) ==
- INVALID_ARRAY_INDEX)
- return -1;
- return 0;
- }
- prev_idx = cur_idx;
- cur_idx = tbl->items[cur_idx].next_pkt_idx;
- } while (cur_idx != INVALID_ARRAY_INDEX);
+ return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl->flows[i].start_index,
+ &tbl->item_num, tbl->max_item_num,
+ ip_id, is_atomic, start_time);
+}
- /* Fail to find a neighbor, so store the packet into the flow. */
- if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
- ip_id, is_atomic) == INVALID_ARRAY_INDEX)
- return -1;
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
- return 0;
+ ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len);
}
uint16_t
@@ -353,7 +250,8 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
* Delete the packet and get the next
* packet in the flow.
*/
- j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
+ j = delete_tcp_item(tbl->items, j,
+ &tbl->item_num, INVALID_ARRAY_INDEX);
tbl->flows[i].start_index = j;
if (j == INVALID_ARRAY_INDEX)
tbl->flow_num--;
diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h
index 212f97a042..c0154afa24 100644
--- a/lib/gro/gro_tcp4.h
+++ b/lib/gro/gro_tcp4.h
@@ -5,32 +5,15 @@
#ifndef _GRO_TCP4_H_
#define _GRO_TCP4_H_
-#include <rte_tcp.h>
+#include <gro_tcp_internal.h>
-#define INVALID_ARRAY_INDEX 0xffffffffUL
#define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
-/*
- * The max length of a IPv4 packet, which includes the length of the L3
- * header, the L4 header and the data payload.
- */
-#define MAX_IPV4_PKT_LENGTH UINT16_MAX
-
-/* The maximum TCP header length */
-#define MAX_TCP_HLEN 60
-#define INVALID_TCP_HDRLEN(len) \
- (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
-
-/* Header fields representing a TCP/IPv4 flow */
+/* Header fields representing common fields in TCP flow */
struct tcp4_flow_key {
- struct rte_ether_addr eth_saddr;
- struct rte_ether_addr eth_daddr;
+ struct cmn_tcp_key cmn_key;
uint32_t ip_src_addr;
uint32_t ip_dst_addr;
-
- uint32_t recv_ack;
- uint16_t src_port;
- uint16_t dst_port;
};
struct gro_tcp4_flow {
@@ -42,42 +25,12 @@ struct gro_tcp4_flow {
uint32_t start_index;
};
-struct gro_tcp4_item {
- /*
- * The first MBUF segment of the packet. If the value
- * is NULL, it means the item is empty.
- */
- struct rte_mbuf *firstseg;
- /* The last MBUF segment of the packet */
- struct rte_mbuf *lastseg;
- /*
- * The time when the first packet is inserted into the table.
- * This value won't be updated, even if the packet is merged
- * with other packets.
- */
- uint64_t start_time;
- /*
- * next_pkt_idx is used to chain the packets that
- * are in the same flow but can't be merged together
- * (e.g. caused by packet reordering).
- */
- uint32_t next_pkt_idx;
- /* TCP sequence number of the packet */
- uint32_t sent_seq;
- /* IPv4 ID of the packet */
- uint16_t ip_id;
- /* the number of merged packets */
- uint16_t nb_merged;
- /* Indicate if IPv4 ID can be ignored */
- uint8_t is_atomic;
-};
-
/*
* TCP/IPv4 reassembly table structure.
*/
struct gro_tcp4_tbl {
/* item array */
- struct gro_tcp4_item *items;
+ struct gro_tcp_item *items;
/* flow array */
struct gro_tcp4_flow *flows;
/* current item number */
@@ -186,120 +139,9 @@ uint32_t gro_tcp4_tbl_pkt_count(void *tbl);
static inline int
is_same_tcp4_flow(struct tcp4_flow_key k1, struct tcp4_flow_key k2)
{
- return (rte_is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) &&
- rte_is_same_ether_addr(&k1.eth_daddr, &k2.eth_daddr) &&
- (k1.ip_src_addr == k2.ip_src_addr) &&
+ return ((k1.ip_src_addr == k2.ip_src_addr) &&
(k1.ip_dst_addr == k2.ip_dst_addr) &&
- (k1.recv_ack == k2.recv_ack) &&
- (k1.src_port == k2.src_port) &&
- (k1.dst_port == k2.dst_port));
+ is_common_tcp_key(&k1.cmn_key, &k2.cmn_key));
}
-/*
- * Merge two TCP/IPv4 packets without updating checksums.
- * If cmp is larger than 0, append the new packet to the
- * original packet. Otherwise, pre-pend the new packet to
- * the original packet.
- */
-static inline int
-merge_two_tcp4_packets(struct gro_tcp4_item *item,
- struct rte_mbuf *pkt,
- int cmp,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t l2_offset)
-{
- struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
- uint16_t hdr_len, l2_len;
-
- if (cmp > 0) {
- pkt_head = item->firstseg;
- pkt_tail = pkt;
- } else {
- pkt_head = pkt;
- pkt_tail = item->firstseg;
- }
-
- /* check if the IPv4 packet length is greater than the max value */
- hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
- pkt_head->l4_len;
- l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
- if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
- hdr_len > MAX_IPV4_PKT_LENGTH))
- return 0;
-
- /* remove the packet header for the tail packet */
- rte_pktmbuf_adj(pkt_tail, hdr_len);
-
- /* chain two packets together */
- if (cmp > 0) {
- item->lastseg->next = pkt;
- item->lastseg = rte_pktmbuf_lastseg(pkt);
- /* update IP ID to the larger value */
- item->ip_id = ip_id;
- } else {
- lastseg = rte_pktmbuf_lastseg(pkt);
- lastseg->next = item->firstseg;
- item->firstseg = pkt;
- /* update sent_seq to the smaller value */
- item->sent_seq = sent_seq;
- item->ip_id = ip_id;
- }
- item->nb_merged++;
-
- /* update MBUF metadata for the merged packet */
- pkt_head->nb_segs += pkt_tail->nb_segs;
- pkt_head->pkt_len += pkt_tail->pkt_len;
-
- return 1;
-}
-
-/*
- * Check if two TCP/IPv4 packets are neighbors.
- */
-static inline int
-check_seq_option(struct gro_tcp4_item *item,
- struct rte_tcp_hdr *tcph,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t tcp_hl,
- uint16_t tcp_dl,
- uint16_t l2_offset,
- uint8_t is_atomic)
-{
- struct rte_mbuf *pkt_orig = item->firstseg;
- struct rte_ipv4_hdr *iph_orig;
- struct rte_tcp_hdr *tcph_orig;
- uint16_t len, tcp_hl_orig;
-
- iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char *) +
- l2_offset + pkt_orig->l2_len);
- tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig + pkt_orig->l3_len);
- tcp_hl_orig = pkt_orig->l4_len;
-
- /* Check if TCP option fields equal */
- len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
- if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
- (memcmp(tcph + 1, tcph_orig + 1,
- len) != 0)))
- return 0;
-
- /* Don't merge packets whose DF bits are different */
- if (unlikely(item->is_atomic ^ is_atomic))
- return 0;
-
- /* check if the two packets are neighbors */
- len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
- pkt_orig->l3_len - tcp_hl_orig;
- if ((sent_seq == item->sent_seq + len) && (is_atomic ||
- (ip_id == item->ip_id + 1)))
- /* append the new packet */
- return 1;
- else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
- (ip_id + item->nb_merged == item->ip_id)))
- /* pre-pend the new packet */
- return -1;
-
- return 0;
-}
#endif
diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c
new file mode 100644
index 0000000000..0ea73741c1
--- /dev/null
+++ b/lib/gro/gro_tcp6.c
@@ -0,0 +1,267 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp6.h"
+
+void *
+gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow)
+{
+ struct gro_tcp6_tbl *tbl;
+ size_t size;
+ uint32_t entries_num, i;
+
+ entries_num = max_flow_num * max_item_per_flow;
+ entries_num = RTE_MIN(entries_num, GRO_TCP6_TBL_MAX_ITEM_NUM);
+
+ if (entries_num == 0)
+ return NULL;
+
+ tbl = rte_zmalloc_socket(__func__,
+ sizeof(struct gro_tcp6_tbl),
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl == NULL)
+ return NULL;
+
+ size = sizeof(struct gro_tcp_item) * entries_num;
+ tbl->items = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->items == NULL) {
+ rte_free(tbl);
+ return NULL;
+ }
+ tbl->max_item_num = entries_num;
+
+ size = sizeof(struct gro_tcp6_flow) * entries_num;
+ tbl->flows = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->flows == NULL) {
+ rte_free(tbl->items);
+ rte_free(tbl);
+ return NULL;
+ }
+ /* INVALID_ARRAY_INDEX indicates an empty flow */
+ for (i = 0; i < entries_num; i++)
+ tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
+ tbl->max_flow_num = entries_num;
+
+ return tbl;
+}
+
+void
+gro_tcp6_tbl_destroy(void *tbl)
+{
+ struct gro_tcp6_tbl *tcp_tbl = tbl;
+
+ if (tcp_tbl) {
+ rte_free(tcp_tbl->items);
+ rte_free(tcp_tbl->flows);
+ }
+ rte_free(tcp_tbl);
+}
+
+static inline uint32_t
+find_an_empty_flow(struct gro_tcp6_tbl *tbl)
+{
+ uint32_t i;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++)
+ if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+insert_new_flow(struct gro_tcp6_tbl *tbl,
+ struct tcp6_flow_key *src,
+ uint32_t item_idx)
+{
+ struct tcp6_flow_key *dst;
+ uint32_t flow_idx;
+
+ flow_idx = find_an_empty_flow(tbl);
+ if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
+ return INVALID_ARRAY_INDEX;
+
+ dst = &(tbl->flows[flow_idx].key);
+
+ ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
+ memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst->src_addr));
+ memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst->dst_addr));
+ dst->vtc_flow = src->vtc_flow;
+
+ tbl->flows[flow_idx].start_index = item_idx;
+ tbl->flow_num++;
+
+ return flow_idx;
+}
+
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
+
+ ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len - pkt->l3_len);
+}
+
+int32_t
+gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time)
+{
+ struct rte_ether_hdr *eth_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ int32_t tcp_dl;
+ uint16_t ip_tlen;
+ struct tcp6_flow_key key;
+ uint32_t i, max_flow_num, remaining_flow_num;
+ uint32_t sent_seq;
+ struct rte_tcp_hdr *tcp_hdr;
+ uint8_t find;
+ uint32_t item_idx;
+ /*
+ * Don't process the packet whose TCP header length is greater
+ * than 60 bytes or less than 20 bytes.
+ */
+ if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
+ return -1;
+
+ eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
+ ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
+ tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt->l2_len + pkt->l3_len);
+
+ /*
+ * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
+ * or CWR set.
+ */
+ if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
+ return -1;
+
+ ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
+ /*
+ * Don't process the packet whose payload length is less than or
+ * equal to 0.
+ */
+ tcp_dl = ip_tlen - pkt->l4_len;
+ if (tcp_dl <= 0)
+ return -1;
+
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.cmn_key.eth_daddr));
+ memcpy(&key.src_addr[0], &ipv6_hdr->src_addr, sizeof(key.src_addr));
+ memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr, sizeof(key.dst_addr));
+ key.cmn_key.src_port = tcp_hdr->src_port;
+ key.cmn_key.dst_port = tcp_hdr->dst_port;
+ key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+ key.vtc_flow = ipv6_hdr->vtc_flow;
+
+ /* Search for a matched flow. */
+ max_flow_num = tbl->max_flow_num;
+ remaining_flow_num = tbl->flow_num;
+ find = 0;
+ for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
+ if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
+ if (is_same_tcp6_flow(&tbl->flows[i].key, &key)) {
+ find = 1;
+ break;
+ }
+ remaining_flow_num--;
+ }
+ }
+
+ if (find == 0) {
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num,
+ tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, 0, true);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return -1;
+ if (insert_new_flow(tbl, &key, item_idx) ==
+ INVALID_ARRAY_INDEX) {
+ /*
+ * Fail to insert a new flow, so delete the
+ * stored packet.
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
+ return -1;
+ }
+ return 0;
+ }
+
+ return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl->flows[i].start_index,
+ &tbl->item_num, tbl->max_item_num,
+ 0, true, start_time);
+}
+
+uint16_t
+gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out)
+{
+ uint16_t k = 0;
+ uint32_t i, j;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++) {
+ if (unlikely(tbl->flow_num == 0))
+ return k;
+
+ j = tbl->flows[i].start_index;
+ while (j != INVALID_ARRAY_INDEX) {
+ if (tbl->items[j].start_time <= flush_timestamp) {
+ out[k++] = tbl->items[j].firstseg;
+ if (tbl->items[j].nb_merged > 1)
+ update_header(&(tbl->items[j]));
+ /*
+ * Delete the packet and get the next
+ * packet in the flow.
+ */
+ j = delete_tcp_item(tbl->items, j,
+ &tbl->item_num, INVALID_ARRAY_INDEX);
+ tbl->flows[i].start_index = j;
+ if (j == INVALID_ARRAY_INDEX)
+ tbl->flow_num--;
+
+ if (unlikely(k == nb_out))
+ return k;
+ } else
+ /*
+ * The left packets in this flow won't be
+ * timeout. Go to check other flows.
+ */
+ break;
+ }
+ }
+ return k;
+}
+
+uint32_t
+gro_tcp6_tbl_pkt_count(void *tbl)
+{
+ struct gro_tcp6_tbl *gro_tbl = tbl;
+
+ if (gro_tbl)
+ return gro_tbl->item_num;
+
+ return 0;
+}
diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h
new file mode 100644
index 0000000000..cc02b0004a
--- /dev/null
+++ b/lib/gro/gro_tcp6.h
@@ -0,0 +1,161 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+
+#ifndef _GRO_TCP6_H_
+#define _GRO_TCP6_H_
+
+#include <gro_tcp_internal.h>
+
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+#define GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
+
+/* Header fields representing a TCP/IPv6 flow */
+struct tcp6_flow_key {
+ struct cmn_tcp_key cmn_key;
+ uint8_t src_addr[16];
+ uint8_t dst_addr[16];
+ rte_be32_t vtc_flow;
+};
+
+struct gro_tcp6_flow {
+ struct tcp6_flow_key key;
+ /*
+ * The index of the first packet in the flow.
+ * INVALID_ARRAY_INDEX indicates an empty flow.
+ */
+ uint32_t start_index;
+};
+
+/*
+ * TCP/IPv6 reassembly table structure.
+ */
+struct gro_tcp6_tbl {
+ /* item array */
+ struct gro_tcp_item *items;
+ /* flow array */
+ struct gro_tcp6_flow *flows;
+ /* current item number */
+ uint32_t item_num;
+ /* current flow num */
+ uint32_t flow_num;
+ /* item array size */
+ uint32_t max_item_num;
+ /* flow array size */
+ uint32_t max_flow_num;
+};
+
+/**
+ * This function creates a TCP/IPv6 reassembly table.
+ *
+ * @param socket_id
+ * Socket index for allocating the TCP/IPv6 reassemble table
+ * @param max_flow_num
+ * The maximum number of flows in the TCP/IPv6 GRO table
+ * @param max_item_per_flow
+ * The maximum number of packets per flow
+ *
+ * @return
+ * - Return the table pointer on success.
+ * - Return NULL on failure.
+ */
+void *gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow);
+
+/**
+ * This function destroys a TCP/IPv6 reassembly table.
+ *
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table.
+ */
+void gro_tcp6_tbl_destroy(void *tbl);
+
+/**
+ * This function merges a TCP/IPv6 packet. It doesn't process the packet,
+ * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't have
+ * payload.
+ *
+ * This function doesn't check if the packet has correct checksums and
+ * doesn't re-calculate checksums for the merged packet. Additionally,
+ * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
+ * when IP fragmentation is possible (i.e., DF==0). It returns the
+ * packet, if the packet has invalid parameters (e.g. SYN bit is set)
+ * or there is no available space in the table.
+ *
+ * @param pkt
+ * Packet to reassemble
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table
+ * @start_time
+ * The time when the packet is inserted into the table
+ *
+ * @return
+ * - Return a positive value if the packet is merged.
+ * - Return zero if the packet isn't merged but stored in the table.
+ * - Return a negative value for invalid parameters or no available
+ * space in the table.
+ */
+int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time);
+
+/**
+ * This function flushes timeout packets in a TCP/IPv6 reassembly table,
+ * and without updating checksums.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ * @param flush_timestamp
+ * Flush packets which are inserted into the table before or at the
+ * flush_timestamp.
+ * @param out
+ * Pointer array used to keep flushed packets
+ * @param nb_out
+ * The element number in 'out'. It also determines the maximum number of
+ * packets that can be flushed finally.
+ *
+ * @return
+ * The number of flushed packets
+ */
+uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out);
+
+/**
+ * This function returns the number of the packets in a TCP/IPv6
+ * reassembly table.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ *
+ * @return
+ * The number of packets in the table
+ */
+uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
+
+/*
+ * Check if two TCP/IPv6 packets belong to the same flow.
+ */
+static inline int
+is_same_tcp6_flow(struct tcp6_flow_key *k1, struct tcp6_flow_key *k2)
+{
+ rte_be32_t vtc_flow_diff;
+
+ if (memcmp(&k1->src_addr, &k2->src_addr, 16))
+ return 0;
+ if (memcmp(&k1->dst_addr, &k2->dst_addr, 16))
+ return 0;
+ /*
+ * IP version (4) Traffic Class (8) Flow Label (20)
+ * All fields except Traffic class should be same
+ */
+ vtc_flow_diff = (k1->vtc_flow ^ k2->vtc_flow);
+ if (vtc_flow_diff & htonl(0xF00FFFFF))
+ return 0;
+
+ return is_common_tcp_key(&k1->cmn_key, &k2->cmn_key);
+}
+
+#endif
diff --git a/lib/gro/gro_tcp_internal.c b/lib/gro/gro_tcp_internal.c
new file mode 100644
index 0000000000..5a21bca7f8
--- /dev/null
+++ b/lib/gro/gro_tcp_internal.c
@@ -0,0 +1,128 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
+ */
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp_internal.h"
+
+static inline uint32_t
+find_an_empty_item(struct gro_tcp_item *items,
+ uint32_t max_item_num)
+{
+ uint32_t i;
+
+ for (i = 0; i < max_item_num; i++)
+ if (items[i].firstseg == NULL)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+inline uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic)
+{
+ uint32_t item_idx;
+
+ item_idx = find_an_empty_item(items, max_item_num);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return INVALID_ARRAY_INDEX;
+
+ items[item_idx].firstseg = pkt;
+ items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
+ items[item_idx].start_time = start_time;
+ items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
+ items[item_idx].sent_seq = sent_seq;
+ items[item_idx].l3.ip_id = ip_id;
+ items[item_idx].nb_merged = 1;
+ items[item_idx].is_atomic = is_atomic;
+ (*item_num) += 1;
+
+ /* if the previous packet exists, chain them together. */
+ if (prev_idx != INVALID_ARRAY_INDEX) {
+ items[item_idx].next_pkt_idx =
+ items[prev_idx].next_pkt_idx;
+ items[prev_idx].next_pkt_idx = item_idx;
+ }
+
+ return item_idx;
+}
+
+inline uint32_t
+delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx)
+{
+ uint32_t next_idx = items[item_idx].next_pkt_idx;
+
+ /* NULL indicates an empty item */
+ items[item_idx].firstseg = NULL;
+ (*item_num) -= 1;
+ if (prev_item_idx != INVALID_ARRAY_INDEX)
+ items[prev_item_idx].next_pkt_idx = next_idx;
+
+ return next_idx;
+}
+
+int32_t
+process_tcp_item(struct rte_mbuf *pkt,
+ struct rte_tcp_hdr *tcp_hdr,
+ int32_t tcp_dl,
+ struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time)
+{
+ uint32_t cur_idx;
+ uint32_t prev_idx;
+ int cmp;
+ uint32_t sent_seq;
+
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ /*
+ * Check all packets in the flow and try to find a neighbor for
+ * the input packet.
+ */
+ cur_idx = item_idx;
+ prev_idx = cur_idx;
+ do {
+ cmp = check_seq_option(&items[cur_idx], tcp_hdr,
+ sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
+ is_atomic);
+ if (cmp) {
+ if (merge_two_tcp_packets(&items[cur_idx],
+ pkt, cmp, sent_seq, ip_id, 0))
+ return 1;
+ /*
+ * Fail to merge the two packets, as the packet
+ * length is greater than the max value. Store
+ * the packet into the flow.
+ */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num,
+ start_time, cur_idx, sent_seq, ip_id, is_atomic) ==
+ INVALID_ARRAY_INDEX)
+ return -1;
+ return 0;
+ }
+ prev_idx = cur_idx;
+ cur_idx = items[cur_idx].next_pkt_idx;
+ } while (cur_idx != INVALID_ARRAY_INDEX);
+
+ /* Fail to find a neighbor, so store the packet into the flow. */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num, start_time, prev_idx, sent_seq,
+ ip_id, is_atomic) == INVALID_ARRAY_INDEX)
+ return -1;
+
+ return 0;
+}
diff --git a/lib/gro/gro_tcp_internal.h b/lib/gro/gro_tcp_internal.h
new file mode 100644
index 0000000000..072b7aea13
--- /dev/null
+++ b/lib/gro/gro_tcp_internal.h
@@ -0,0 +1,212 @@
+#ifndef _GRO_TCP_H_
+#define _GRO_TCP_H_
+
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+
+#include <rte_tcp.h>
+
+/*
+ * The max length of a IPv4 packet, which includes the length of the L3
+ * header, the L4 header and the data payload.
+ */
+#define MAX_IP_PKT_LENGTH UINT16_MAX
+
+/* The maximum TCP header length */
+#define MAX_TCP_HLEN 60
+#define INVALID_TCP_HDRLEN(len) \
+ (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
+
+struct cmn_tcp_key {
+ struct rte_ether_addr eth_saddr;
+ struct rte_ether_addr eth_daddr;
+ uint32_t recv_ack;
+ uint16_t src_port;
+ uint16_t dst_port;
+};
+
+#define ASSIGN_COMMON_TCP_KEY(k1, k2) \
+ do {\
+ rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
+ rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr)); \
+ k2->recv_ack = k1->recv_ack; \
+ k2->src_port = k1->src_port; \
+ k2->dst_port = k1->dst_port; \
+ } while (0)
+
+struct gro_tcp_item {
+ /*
+ * The first MBUF segment of the packet. If the value
+ * is NULL, it means the item is empty.
+ */
+ struct rte_mbuf *firstseg;
+ /* The last MBUF segment of the packet */
+ struct rte_mbuf *lastseg;
+ /*
+ * The time when the first packet is inserted into the table.
+ * This value won't be updated, even if the packet is merged
+ * with other packets.
+ */
+ uint64_t start_time;
+ /*
+ * next_pkt_idx is used to chain the packets that
+ * are in the same flow but can't be merged together
+ * (e.g. caused by packet reordering).
+ */
+ uint32_t next_pkt_idx;
+ /* TCP sequence number of the packet */
+ uint32_t sent_seq;
+ union {
+ /* IPv4 ID of the packet */
+ uint16_t ip_id;
+ /* Unused field for IPv6 */
+ uint16_t unused;
+ } l3;
+ /* the number of merged packets */
+ uint16_t nb_merged;
+ /* Indicate if IPv4 ID can be ignored */
+ uint8_t is_atomic;
+};
+
+uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic);
+
+uint32_t
+delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx);
+
+int32_t
+process_tcp_item(struct rte_mbuf *pkt,
+ struct rte_tcp_hdr *tcp_hdr,
+ int32_t tcp_dl,
+ struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time);
+
+/*
+ * Merge two TCP packets without updating checksums.
+ * If cmp is larger than 0, append the new packet to the
+ * original packet. Otherwise, pre-pend the new packet to
+ * the original packet.
+ */
+static inline int
+merge_two_tcp_packets(struct gro_tcp_item *item,
+ struct rte_mbuf *pkt,
+ int cmp,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t l2_offset)
+{
+ struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
+ uint16_t hdr_len, l2_len;
+
+ if (cmp > 0) {
+ pkt_head = item->firstseg;
+ pkt_tail = pkt;
+ } else {
+ pkt_head = pkt;
+ pkt_tail = item->firstseg;
+ }
+
+ /* check if the IPv4 packet length is greater than the max value */
+ hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
+ pkt_head->l4_len;
+ l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
+ if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
+ hdr_len > MAX_IP_PKT_LENGTH))
+ return 0;
+
+ /* remove the packet header for the tail packet */
+ rte_pktmbuf_adj(pkt_tail, hdr_len);
+
+ /* chain two packets together */
+ if (cmp > 0) {
+ item->lastseg->next = pkt;
+ item->lastseg = rte_pktmbuf_lastseg(pkt);
+ /* update IP ID to the larger value */
+ item->l3.ip_id = ip_id;
+ } else {
+ lastseg = rte_pktmbuf_lastseg(pkt);
+ lastseg->next = item->firstseg;
+ item->firstseg = pkt;
+ /* update sent_seq to the smaller value */
+ item->sent_seq = sent_seq;
+ item->l3.ip_id = ip_id;
+ }
+ item->nb_merged++;
+
+ /* update MBUF metadata for the merged packet */
+ pkt_head->nb_segs += pkt_tail->nb_segs;
+ pkt_head->pkt_len += pkt_tail->pkt_len;
+
+ return 1;
+}
+
+/*
+ * Check if two TCP/IPv4 packets are neighbors.
+ */
+static inline int
+check_seq_option(struct gro_tcp_item *item,
+ struct rte_tcp_hdr *tcph,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t tcp_hl,
+ uint16_t tcp_dl,
+ uint16_t l2_offset,
+ uint8_t is_atomic)
+{
+ struct rte_mbuf *pkt_orig = item->firstseg;
+ char *iph_orig;
+ struct rte_tcp_hdr *tcph_orig;
+ uint16_t len, tcp_hl_orig;
+
+ iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
+ l2_offset + pkt_orig->l2_len);
+ tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
+ tcp_hl_orig = pkt_orig->l4_len;
+
+ /* Check if TCP option fields equal */
+ len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
+ if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
+ (memcmp(tcph + 1, tcph_orig + 1,
+ len) != 0)))
+ return 0;
+
+ /* Don't merge packets whose DF bits are different */
+ if (unlikely(item->is_atomic ^ is_atomic))
+ return 0;
+
+ /* check if the two packets are neighbors */
+ len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
+ pkt_orig->l3_len - tcp_hl_orig;
+ if ((sent_seq == item->sent_seq + len) && (is_atomic ||
+ (ip_id == item->l3.ip_id + 1)))
+ /* append the new packet */
+ return 1;
+ else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
+ (ip_id + item->nb_merged == item->l3.ip_id)))
+ /* pre-pend the new packet */
+ return -1;
+
+ return 0;
+}
+
+static inline int
+is_common_tcp_key(struct cmn_tcp_key *k1, struct cmn_tcp_key *k2)
+{
+ return (!memcmp(k1, k2, sizeof(struct cmn_tcp_key)));
+}
+
+#endif
diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
index 3be4deb7c7..81eebf0d2d 100644
--- a/lib/gro/gro_vxlan_tcp4.c
+++ b/lib/gro/gro_vxlan_tcp4.c
@@ -7,6 +7,7 @@
#include <rte_ethdev.h>
#include <rte_udp.h>
+#include "gro_tcp_internal.h"
#include "gro_vxlan_tcp4.h"
void *
@@ -116,7 +117,7 @@ insert_new_item(struct gro_vxlan_tcp4_tbl *tbl,
tbl->items[item_idx].inner_item.start_time = start_time;
tbl->items[item_idx].inner_item.next_pkt_idx = INVALID_ARRAY_INDEX;
tbl->items[item_idx].inner_item.sent_seq = sent_seq;
- tbl->items[item_idx].inner_item.ip_id = ip_id;
+ tbl->items[item_idx].inner_item.l3.ip_id = ip_id;
tbl->items[item_idx].inner_item.nb_merged = 1;
tbl->items[item_idx].inner_item.is_atomic = is_atomic;
tbl->items[item_idx].outer_ip_id = outer_ip_id;
@@ -163,15 +164,9 @@ insert_new_flow(struct gro_vxlan_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->inner_key.eth_saddr),
- &(dst->inner_key.eth_saddr));
- rte_ether_addr_copy(&(src->inner_key.eth_daddr),
- &(dst->inner_key.eth_daddr));
+ ASSIGN_COMMON_TCP_KEY((&(src->inner_key.cmn_key)), (&(dst->inner_key.cmn_key)));
dst->inner_key.ip_src_addr = src->inner_key.ip_src_addr;
dst->inner_key.ip_dst_addr = src->inner_key.ip_dst_addr;
- dst->inner_key.recv_ack = src->inner_key.recv_ack;
- dst->inner_key.src_port = src->inner_key.src_port;
- dst->inner_key.dst_port = src->inner_key.dst_port;
dst->vxlan_hdr.vx_flags = src->vxlan_hdr.vx_flags;
dst->vxlan_hdr.vx_vni = src->vxlan_hdr.vx_vni;
@@ -248,7 +243,7 @@ merge_two_vxlan_tcp4_packets(struct gro_vxlan_tcp4_item *item,
uint16_t outer_ip_id,
uint16_t ip_id)
{
- if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp, sent_seq,
+ if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
ip_id, pkt->outer_l2_len +
pkt->outer_l3_len)) {
/* Update the outer IPv4 ID to the large value. */
@@ -357,13 +352,13 @@ gro_vxlan_tcp4_reassemble(struct rte_mbuf *pkt,
sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.eth_daddr));
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.cmn_key.eth_daddr));
key.inner_key.ip_src_addr = ipv4_hdr->src_addr;
key.inner_key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.inner_key.recv_ack = tcp_hdr->recv_ack;
- key.inner_key.src_port = tcp_hdr->src_port;
- key.inner_key.dst_port = tcp_hdr->dst_port;
+ key.inner_key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+ key.inner_key.cmn_key.src_port = tcp_hdr->src_port;
+ key.inner_key.cmn_key.dst_port = tcp_hdr->dst_port;
key.vxlan_hdr.vx_flags = vxlan_hdr->vx_flags;
key.vxlan_hdr.vx_vni = vxlan_hdr->vx_vni;
diff --git a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h
index 7832942a68..82eaaee11e 100644
--- a/lib/gro/gro_vxlan_tcp4.h
+++ b/lib/gro/gro_vxlan_tcp4.h
@@ -5,6 +5,7 @@
#ifndef _GRO_VXLAN_TCP4_H_
#define _GRO_VXLAN_TCP4_H_
+#include "gro_tcp_internal.h"
#include "gro_tcp4.h"
#define GRO_VXLAN_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
@@ -36,7 +37,7 @@ struct gro_vxlan_tcp4_flow {
};
struct gro_vxlan_tcp4_item {
- struct gro_tcp4_item inner_item;
+ struct gro_tcp_item inner_item;
/* IPv4 ID in the outer IPv4 header */
uint16_t outer_ip_id;
/* Indicate if outer IPv4 ID can be ignored */
diff --git a/lib/gro/meson.build b/lib/gro/meson.build
index e4fa2958bd..1640317890 100644
--- a/lib/gro/meson.build
+++ b/lib/gro/meson.build
@@ -3,7 +3,9 @@
sources = files(
'rte_gro.c',
+ 'gro_tcp_internal.c',
'gro_tcp4.c',
+ 'gro_tcp6.c',
'gro_udp4.c',
'gro_vxlan_tcp4.c',
'gro_vxlan_udp4.c',
diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c
index e35399fd42..d824eebd93 100644
--- a/lib/gro/rte_gro.c
+++ b/lib/gro/rte_gro.c
@@ -8,6 +8,7 @@
#include "rte_gro.h"
#include "gro_tcp4.h"
+#include "gro_tcp6.h"
#include "gro_udp4.h"
#include "gro_vxlan_tcp4.h"
#include "gro_vxlan_udp4.h"
@@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
- gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
+ gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, gro_tcp6_tbl_create, NULL};
static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
+ gro_tcp6_tbl_destroy,
NULL};
static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count,
+ gro_tcp6_tbl_pkt_count,
NULL};
#define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
@@ -35,6 +38,12 @@ static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+/* GRO with extension headers is not supported */
+#define IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
+ ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
+ ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
+ (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+
#define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
@@ -147,7 +156,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
/* allocate a reassembly table for TCP/IPv4 GRO */
struct gro_tcp4_tbl tcp_tbl;
struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
- struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+ struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+
+ struct gro_tcp6_tbl tcp6_tbl;
+ struct gro_tcp6_flow tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
+ struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
/* allocate a reassembly table for UDP/IPv4 GRO */
struct gro_udp4_tbl udp_tbl;
@@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
int32_t ret;
uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
- do_vxlan_udp_gro = 0;
+ do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
do_udp4_gro = 1;
}
+ if (param->gro_types & RTE_GRO_TCP_IPV6) {
+ for (i = 0; i < item_num; i++)
+ tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+ tcp6_tbl.flows = tcp6_flows;
+ tcp6_tbl.items = tcp6_items;
+ tcp6_tbl.flow_num = 0;
+ tcp6_tbl.item_num = 0;
+ tcp6_tbl.max_flow_num = item_num;
+ tcp6_tbl.max_item_num = item_num;
+ do_tcp6_gro = 1;
+ }
for (i = 0; i < nb_pkts; i++) {
/*
@@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
nb_after_gro--;
else if (ret < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
+ if (ret > 0)
+ /* merge successfully */
+ nb_after_gro--;
+ else if (ret < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
if ((nb_after_gro < nb_pkts)
|| (unprocess_num < nb_pkts)) {
i = 0;
+ /* Copy unprocessed packets */
+ if (unprocess_num > 0) {
+ memcpy(&pkts[i], unprocess_pkts,
+ sizeof(struct rte_mbuf *) *
+ unprocess_num);
+ i = unprocess_num;
+ }
+
/* Flush all packets from the tables */
if (do_vxlan_tcp_gro) {
- i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
+ i += gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
0, pkts, nb_pkts);
}
@@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
&pkts[i], nb_pkts - i);
}
- /* Copy unprocessed packets */
- if (unprocess_num > 0) {
- memcpy(&pkts[i], unprocess_pkts,
- sizeof(struct rte_mbuf *) *
- unprocess_num);
+
+ if (do_tcp6_gro) {
+ i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
+ &pkts[i], nb_pkts - i);
}
- nb_after_gro = i + unprocess_num;
}
return nb_after_gro;
@@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
{
struct rte_mbuf *unprocess_pkts[nb_pkts];
struct gro_ctx *gro_ctx = ctx;
- void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
+ void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
uint64_t current_time;
uint16_t i, unprocess_num = 0;
- uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro;
+ uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro, do_tcp6_gro;
if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
+ tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
RTE_GRO_TCP_IPV4;
@@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
RTE_GRO_UDP_IPV4;
do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
RTE_GRO_IPV4_VXLAN_UDP_IPV4;
+ do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) == RTE_GRO_TCP_IPV6;
current_time = rte_rdtsc();
@@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
if (gro_udp4_reassemble(pkts[i], udp_tbl,
current_time) < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
+ current_time) < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
flush_timestamp,
&out[num], left_nb_out);
+ left_nb_out = max_nb_out - num;
+ }
+
+ if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
+ num += gro_tcp6_tbl_timeout_flush(
+ gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
+ flush_timestamp,
+ &out[num], left_nb_out);
+
}
return num;
diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h
index 9f9ed4935a..c83dfd9ad1 100644
--- a/lib/gro/rte_gro.h
+++ b/lib/gro/rte_gro.h
@@ -38,6 +38,9 @@ extern "C" {
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL << RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
/**< VxLAN UDP/IPv4 GRO flag. */
+#define RTE_GRO_TCP_IPV6_INDEX 4
+#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX)
+/**< TCP/IPv6 GRO flag. */
/**
* Structure used to create GRO context objects or used to pass
--
2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [PATCH v7] gro : ipv6 changes to support GRO for TCP/ipv6
2023-06-12 11:31 ` [PATCH v7] " Kumara Parameshwaran
@ 2023-06-12 12:04 ` kumaraparameshwaran rathinavel
2023-06-13 2:25 ` Hu, Jiayu
1 sibling, 0 replies; 32+ messages in thread
From: kumaraparameshwaran rathinavel @ 2023-06-12 12:04 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev
[-- Attachment #1: Type: text/plain, Size: 65378 bytes --]
On Mon, Jun 12, 2023 at 5:01 PM Kumara Parameshwaran <
kumaraparamesh92@gmail.com> wrote:
> The patch adds GRO support for TCP/ipv6 packets. This does not
> include the support for vxlan, udp ipv6 packets.
>
> Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> ---
> v1:
> * Changes to support GRO for TCP/ipv6 packets. This does not
> include
> vxlan changes.
> * The GRO is performed only for ipv6 packets that does not contain
> extension headers.
> * The logic for the TCP coalescing remains the same, in ipv6
> header
> the source address, destination address, flow label, version
> fields
> are expected to be the same.
> * Re-organised the code to reuse certain tcp functions for both
> ipv4 and
> ipv6 flows.
> v2:
> * Fix comments in gro_tcp6.h header file.
>
> v3:
> * Adderess review comments to fix code duplication for v4 and v6
>
> v4:
> * Addresses review comments for v3, do not use callbacks
>
> v5:
> * Address review comments
>
> v6:
> * Fix warning and coding style issues
>
> v7:
> * Fix build compilation issue
> lib/gro/gro_tcp4.c | 178 ++++++-------------------
> lib/gro/gro_tcp4.h | 170 +----------------------
> lib/gro/gro_tcp6.c | 267 +++++++++++++++++++++++++++++++++++++
> lib/gro/gro_tcp6.h | 161 ++++++++++++++++++++++
> lib/gro/gro_tcp_internal.c | 128 ++++++++++++++++++
> lib/gro/gro_tcp_internal.h | 212 +++++++++++++++++++++++++++++
> lib/gro/gro_vxlan_tcp4.c | 23 ++--
> lib/gro/gro_vxlan_tcp4.h | 3 +-
> lib/gro/meson.build | 2 +
> lib/gro/rte_gro.c | 83 ++++++++++--
> lib/gro/rte_gro.h | 3 +
> 11 files changed, 897 insertions(+), 333 deletions(-)
> create mode 100644 lib/gro/gro_tcp6.c
> create mode 100644 lib/gro/gro_tcp6.h
> create mode 100644 lib/gro/gro_tcp_internal.c
> create mode 100644 lib/gro/gro_tcp_internal.h
>
> diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c
> index 0014096e63..42fee78f30 100644
> --- a/lib/gro/gro_tcp4.c
> +++ b/lib/gro/gro_tcp4.c
> @@ -30,7 +30,7 @@ gro_tcp4_tbl_create(uint16_t socket_id,
> if (tbl == NULL)
> return NULL;
>
> - size = sizeof(struct gro_tcp4_item) * entries_num;
> + size = sizeof(struct gro_tcp_item) * entries_num;
> tbl->items = rte_zmalloc_socket(__func__,
> size,
> RTE_CACHE_LINE_SIZE,
> @@ -71,18 +71,6 @@ gro_tcp4_tbl_destroy(void *tbl)
> rte_free(tcp_tbl);
> }
>
> -static inline uint32_t
> -find_an_empty_item(struct gro_tcp4_tbl *tbl)
> -{
> - uint32_t i;
> - uint32_t max_item_num = tbl->max_item_num;
> -
> - for (i = 0; i < max_item_num; i++)
> - if (tbl->items[i].firstseg == NULL)
> - return i;
> - return INVALID_ARRAY_INDEX;
> -}
> -
> static inline uint32_t
> find_an_empty_flow(struct gro_tcp4_tbl *tbl)
> {
> @@ -95,56 +83,6 @@ find_an_empty_flow(struct gro_tcp4_tbl *tbl)
> return INVALID_ARRAY_INDEX;
> }
>
> -static inline uint32_t
> -insert_new_item(struct gro_tcp4_tbl *tbl,
> - struct rte_mbuf *pkt,
> - uint64_t start_time,
> - uint32_t prev_idx,
> - uint32_t sent_seq,
> - uint16_t ip_id,
> - uint8_t is_atomic)
> -{
> - uint32_t item_idx;
> -
> - item_idx = find_an_empty_item(tbl);
> - if (item_idx == INVALID_ARRAY_INDEX)
> - return INVALID_ARRAY_INDEX;
> -
> - tbl->items[item_idx].firstseg = pkt;
> - tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> - tbl->items[item_idx].start_time = start_time;
> - tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> - tbl->items[item_idx].sent_seq = sent_seq;
> - tbl->items[item_idx].ip_id = ip_id;
> - tbl->items[item_idx].nb_merged = 1;
> - tbl->items[item_idx].is_atomic = is_atomic;
> - tbl->item_num++;
> -
> - /* if the previous packet exists, chain them together. */
> - if (prev_idx != INVALID_ARRAY_INDEX) {
> - tbl->items[item_idx].next_pkt_idx =
> - tbl->items[prev_idx].next_pkt_idx;
> - tbl->items[prev_idx].next_pkt_idx = item_idx;
> - }
> -
> - return item_idx;
> -}
> -
> -static inline uint32_t
> -delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
> - uint32_t prev_item_idx)
> -{
> - uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
> -
> - /* NULL indicates an empty item */
> - tbl->items[item_idx].firstseg = NULL;
> - tbl->item_num--;
> - if (prev_item_idx != INVALID_ARRAY_INDEX)
> - tbl->items[prev_item_idx].next_pkt_idx = next_idx;
> -
> - return next_idx;
> -}
> -
> static inline uint32_t
> insert_new_flow(struct gro_tcp4_tbl *tbl,
> struct tcp4_flow_key *src,
> @@ -159,13 +97,10 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
>
> dst = &(tbl->flows[flow_idx].key);
>
> - rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
> - rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
> + ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
> +
> dst->ip_src_addr = src->ip_src_addr;
> dst->ip_dst_addr = src->ip_dst_addr;
> - dst->recv_ack = src->recv_ack;
> - dst->src_port = src->src_port;
> - dst->dst_port = src->dst_port;
>
> tbl->flows[flow_idx].start_index = item_idx;
> tbl->flow_num++;
> @@ -173,21 +108,6 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
> return flow_idx;
> }
>
> -/*
> - * update the packet length for the flushed packet.
> - */
> -static inline void
> -update_header(struct gro_tcp4_item *item)
> -{
> - struct rte_ipv4_hdr *ipv4_hdr;
> - struct rte_mbuf *pkt = item->firstseg;
> -
> - ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> - pkt->l2_len);
> - ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
> - pkt->l2_len);
> -}
> -
> int32_t
> gro_tcp4_reassemble(struct rte_mbuf *pkt,
> struct gro_tcp4_tbl *tbl,
> @@ -202,9 +122,8 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> uint8_t is_atomic;
>
> struct tcp4_flow_key key;
> - uint32_t cur_idx, prev_idx, item_idx;
> + uint32_t item_idx;
> uint32_t i, max_flow_num, remaining_flow_num;
> - int cmp;
> uint8_t find;
>
> /*
> @@ -216,7 +135,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
>
> eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
> ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
> - tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
> + tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *,
> pkt->l2_len + pkt->l3_len);
> hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
>
> /*
> @@ -230,7 +149,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> ip_tlen = rte_be_to_cpu_16(ipv4_hdr->total_length);
> if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len))
> rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen -
> pkt->l2_len);
> -
> /*
> * Don't process the packet whose payload length is less than or
> * equal to 0.
> @@ -239,6 +157,13 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> if (tcp_dl <= 0)
> return -1;
>
> + rte_ether_addr_copy(&(eth_hdr->src_addr),
> &(key.cmn_key.eth_saddr));
> + rte_ether_addr_copy(&(eth_hdr->dst_addr),
> &(key.cmn_key.eth_daddr));
> + key.ip_src_addr = ipv4_hdr->src_addr;
> + key.ip_dst_addr = ipv4_hdr->dst_addr;
> + key.cmn_key.src_port = tcp_hdr->src_port;
> + key.cmn_key.dst_port = tcp_hdr->dst_port;
> + key.cmn_key.recv_ack = tcp_hdr->recv_ack;
> /*
> * Save IPv4 ID for the packet whose DF bit is 0. For the packet
> * whose DF bit is 1, IPv4 ID is ignored.
> @@ -246,15 +171,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
> is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) ==
> RTE_IPV4_HDR_DF_FLAG;
> ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
> - sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> -
> - rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
> - rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
> - key.ip_src_addr = ipv4_hdr->src_addr;
> - key.ip_dst_addr = ipv4_hdr->dst_addr;
> - key.src_port = tcp_hdr->src_port;
> - key.dst_port = tcp_hdr->dst_port;
> - key.recv_ack = tcp_hdr->recv_ack;
>
> /* Search for a matched flow. */
> max_flow_num = tbl->max_flow_num;
> @@ -270,63 +186,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> }
> }
>
> - /*
> - * Fail to find a matched flow. Insert a new flow and store the
> - * packet into the flow.
> - */
> if (find == 0) {
> - item_idx = insert_new_item(tbl, pkt, start_time,
> - INVALID_ARRAY_INDEX, sent_seq, ip_id,
> - is_atomic);
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> + item_idx = insert_new_tcp_item(pkt, tbl->items,
> &tbl->item_num,
> + tbl->max_item_num,
> start_time,
> + INVALID_ARRAY_INDEX,
> sent_seq, ip_id,
> + is_atomic);
> if (item_idx == INVALID_ARRAY_INDEX)
> return -1;
> if (insert_new_flow(tbl, &key, item_idx) ==
> - INVALID_ARRAY_INDEX) {
> + INVALID_ARRAY_INDEX) {
> /*
> * Fail to insert a new flow, so delete the
> * stored packet.
> - */
> - delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
> + */
> + delete_tcp_item(tbl->items, item_idx,
> &tbl->item_num, INVALID_ARRAY_INDEX);
> return -1;
> }
> return 0;
> }
>
> - /*
> - * Check all packets in the flow and try to find a neighbor for
> - * the input packet.
> - */
> - cur_idx = tbl->flows[i].start_index;
> - prev_idx = cur_idx;
> - do {
> - cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
> - sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> - is_atomic);
> - if (cmp) {
> - if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
> - pkt, cmp, sent_seq, ip_id,
> 0))
> - return 1;
> - /*
> - * Fail to merge the two packets, as the packet
> - * length is greater than the max value. Store
> - * the packet into the flow.
> - */
> - if (insert_new_item(tbl, pkt, start_time, cur_idx,
> - sent_seq, ip_id,
> is_atomic) ==
> - INVALID_ARRAY_INDEX)
> - return -1;
> - return 0;
> - }
> - prev_idx = cur_idx;
> - cur_idx = tbl->items[cur_idx].next_pkt_idx;
> - } while (cur_idx != INVALID_ARRAY_INDEX);
> + return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items,
> tbl->flows[i].start_index,
> + &tbl->item_num,
> tbl->max_item_num,
> + ip_id, is_atomic,
> start_time);
> +}
>
> - /* Fail to find a neighbor, so store the packet into the flow. */
> - if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
> - ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> - return -1;
> +/*
> + * update the packet length for the flushed packet.
> + */
> +static inline void
> +update_header(struct gro_tcp_item *item)
> +{
> + struct rte_ipv4_hdr *ipv4_hdr;
> + struct rte_mbuf *pkt = item->firstseg;
>
> - return 0;
> + ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> + pkt->l2_len);
> + ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
> + pkt->l2_len);
> }
>
> uint16_t
> @@ -353,7 +250,8 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
> * Delete the packet and get the next
> * packet in the flow.
> */
> - j = delete_item(tbl, j,
> INVALID_ARRAY_INDEX);
> + j = delete_tcp_item(tbl->items, j,
> + &tbl->item_num,
> INVALID_ARRAY_INDEX);
> tbl->flows[i].start_index = j;
> if (j == INVALID_ARRAY_INDEX)
> tbl->flow_num--;
> diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h
> index 212f97a042..c0154afa24 100644
> --- a/lib/gro/gro_tcp4.h
> +++ b/lib/gro/gro_tcp4.h
> @@ -5,32 +5,15 @@
> #ifndef _GRO_TCP4_H_
> #define _GRO_TCP4_H_
>
> -#include <rte_tcp.h>
> +#include <gro_tcp_internal.h>
>
> -#define INVALID_ARRAY_INDEX 0xffffffffUL
> #define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
>
> -/*
> - * The max length of a IPv4 packet, which includes the length of the L3
> - * header, the L4 header and the data payload.
> - */
> -#define MAX_IPV4_PKT_LENGTH UINT16_MAX
> -
> -/* The maximum TCP header length */
> -#define MAX_TCP_HLEN 60
> -#define INVALID_TCP_HDRLEN(len) \
> - (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> -
> -/* Header fields representing a TCP/IPv4 flow */
> +/* Header fields representing common fields in TCP flow */
> struct tcp4_flow_key {
> - struct rte_ether_addr eth_saddr;
> - struct rte_ether_addr eth_daddr;
> + struct cmn_tcp_key cmn_key;
> uint32_t ip_src_addr;
> uint32_t ip_dst_addr;
> -
> - uint32_t recv_ack;
> - uint16_t src_port;
> - uint16_t dst_port;
> };
>
> struct gro_tcp4_flow {
> @@ -42,42 +25,12 @@ struct gro_tcp4_flow {
> uint32_t start_index;
> };
>
> -struct gro_tcp4_item {
> - /*
> - * The first MBUF segment of the packet. If the value
> - * is NULL, it means the item is empty.
> - */
> - struct rte_mbuf *firstseg;
> - /* The last MBUF segment of the packet */
> - struct rte_mbuf *lastseg;
> - /*
> - * The time when the first packet is inserted into the table.
> - * This value won't be updated, even if the packet is merged
> - * with other packets.
> - */
> - uint64_t start_time;
> - /*
> - * next_pkt_idx is used to chain the packets that
> - * are in the same flow but can't be merged together
> - * (e.g. caused by packet reordering).
> - */
> - uint32_t next_pkt_idx;
> - /* TCP sequence number of the packet */
> - uint32_t sent_seq;
> - /* IPv4 ID of the packet */
> - uint16_t ip_id;
> - /* the number of merged packets */
> - uint16_t nb_merged;
> - /* Indicate if IPv4 ID can be ignored */
> - uint8_t is_atomic;
> -};
> -
> /*
> * TCP/IPv4 reassembly table structure.
> */
> struct gro_tcp4_tbl {
> /* item array */
> - struct gro_tcp4_item *items;
> + struct gro_tcp_item *items;
> /* flow array */
> struct gro_tcp4_flow *flows;
> /* current item number */
> @@ -186,120 +139,9 @@ uint32_t gro_tcp4_tbl_pkt_count(void *tbl);
> static inline int
> is_same_tcp4_flow(struct tcp4_flow_key k1, struct tcp4_flow_key k2)
> {
> - return (rte_is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) &&
> - rte_is_same_ether_addr(&k1.eth_daddr,
> &k2.eth_daddr) &&
> - (k1.ip_src_addr == k2.ip_src_addr) &&
> + return ((k1.ip_src_addr == k2.ip_src_addr) &&
> (k1.ip_dst_addr == k2.ip_dst_addr) &&
> - (k1.recv_ack == k2.recv_ack) &&
> - (k1.src_port == k2.src_port) &&
> - (k1.dst_port == k2.dst_port));
> + is_common_tcp_key(&k1.cmn_key, &k2.cmn_key));
> }
>
> -/*
> - * Merge two TCP/IPv4 packets without updating checksums.
> - * If cmp is larger than 0, append the new packet to the
> - * original packet. Otherwise, pre-pend the new packet to
> - * the original packet.
> - */
> -static inline int
> -merge_two_tcp4_packets(struct gro_tcp4_item *item,
> - struct rte_mbuf *pkt,
> - int cmp,
> - uint32_t sent_seq,
> - uint16_t ip_id,
> - uint16_t l2_offset)
> -{
> - struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
> - uint16_t hdr_len, l2_len;
> -
> - if (cmp > 0) {
> - pkt_head = item->firstseg;
> - pkt_tail = pkt;
> - } else {
> - pkt_head = pkt;
> - pkt_tail = item->firstseg;
> - }
> -
> - /* check if the IPv4 packet length is greater than the max value */
> - hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
> - pkt_head->l4_len;
> - l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
> - if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
> - hdr_len > MAX_IPV4_PKT_LENGTH))
> - return 0;
> -
> - /* remove the packet header for the tail packet */
> - rte_pktmbuf_adj(pkt_tail, hdr_len);
> -
> - /* chain two packets together */
> - if (cmp > 0) {
> - item->lastseg->next = pkt;
> - item->lastseg = rte_pktmbuf_lastseg(pkt);
> - /* update IP ID to the larger value */
> - item->ip_id = ip_id;
> - } else {
> - lastseg = rte_pktmbuf_lastseg(pkt);
> - lastseg->next = item->firstseg;
> - item->firstseg = pkt;
> - /* update sent_seq to the smaller value */
> - item->sent_seq = sent_seq;
> - item->ip_id = ip_id;
> - }
> - item->nb_merged++;
> -
> - /* update MBUF metadata for the merged packet */
> - pkt_head->nb_segs += pkt_tail->nb_segs;
> - pkt_head->pkt_len += pkt_tail->pkt_len;
> -
> - return 1;
> -}
> -
> -/*
> - * Check if two TCP/IPv4 packets are neighbors.
> - */
> -static inline int
> -check_seq_option(struct gro_tcp4_item *item,
> - struct rte_tcp_hdr *tcph,
> - uint32_t sent_seq,
> - uint16_t ip_id,
> - uint16_t tcp_hl,
> - uint16_t tcp_dl,
> - uint16_t l2_offset,
> - uint8_t is_atomic)
> -{
> - struct rte_mbuf *pkt_orig = item->firstseg;
> - struct rte_ipv4_hdr *iph_orig;
> - struct rte_tcp_hdr *tcph_orig;
> - uint16_t len, tcp_hl_orig;
> -
> - iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char
> *) +
> - l2_offset + pkt_orig->l2_len);
> - tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig +
> pkt_orig->l3_len);
> - tcp_hl_orig = pkt_orig->l4_len;
> -
> - /* Check if TCP option fields equal */
> - len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> - if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> - (memcmp(tcph + 1, tcph_orig + 1,
> - len) != 0)))
> - return 0;
> -
> - /* Don't merge packets whose DF bits are different */
> - if (unlikely(item->is_atomic ^ is_atomic))
> - return 0;
> -
> - /* check if the two packets are neighbors */
> - len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> - pkt_orig->l3_len - tcp_hl_orig;
> - if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> - (ip_id == item->ip_id + 1)))
> - /* append the new packet */
> - return 1;
> - else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> - (ip_id + item->nb_merged == item->ip_id)))
> - /* pre-pend the new packet */
> - return -1;
> -
> - return 0;
> -}
> #endif
> diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c
> new file mode 100644
> index 0000000000..0ea73741c1
> --- /dev/null
> +++ b/lib/gro/gro_tcp6.c
> @@ -0,0 +1,267 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
> +
> +#include <rte_malloc.h>
> +#include <rte_mbuf.h>
> +#include <rte_ethdev.h>
> +
> +#include "gro_tcp6.h"
> +
> +void *
> +gro_tcp6_tbl_create(uint16_t socket_id,
> + uint16_t max_flow_num,
> + uint16_t max_item_per_flow)
> +{
> + struct gro_tcp6_tbl *tbl;
> + size_t size;
> + uint32_t entries_num, i;
> +
> + entries_num = max_flow_num * max_item_per_flow;
> + entries_num = RTE_MIN(entries_num, GRO_TCP6_TBL_MAX_ITEM_NUM);
> +
> + if (entries_num == 0)
> + return NULL;
> +
> + tbl = rte_zmalloc_socket(__func__,
> + sizeof(struct gro_tcp6_tbl),
> + RTE_CACHE_LINE_SIZE,
> + socket_id);
> + if (tbl == NULL)
> + return NULL;
> +
> + size = sizeof(struct gro_tcp_item) * entries_num;
> + tbl->items = rte_zmalloc_socket(__func__,
> + size,
> + RTE_CACHE_LINE_SIZE,
> + socket_id);
> + if (tbl->items == NULL) {
> + rte_free(tbl);
> + return NULL;
> + }
> + tbl->max_item_num = entries_num;
> +
> + size = sizeof(struct gro_tcp6_flow) * entries_num;
> + tbl->flows = rte_zmalloc_socket(__func__,
> + size,
> + RTE_CACHE_LINE_SIZE,
> + socket_id);
> + if (tbl->flows == NULL) {
> + rte_free(tbl->items);
> + rte_free(tbl);
> + return NULL;
> + }
> + /* INVALID_ARRAY_INDEX indicates an empty flow */
> + for (i = 0; i < entries_num; i++)
> + tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
> + tbl->max_flow_num = entries_num;
> +
> + return tbl;
> +}
> +
> +void
> +gro_tcp6_tbl_destroy(void *tbl)
> +{
> + struct gro_tcp6_tbl *tcp_tbl = tbl;
> +
> + if (tcp_tbl) {
> + rte_free(tcp_tbl->items);
> + rte_free(tcp_tbl->flows);
> + }
> + rte_free(tcp_tbl);
> +}
> +
> +static inline uint32_t
> +find_an_empty_flow(struct gro_tcp6_tbl *tbl)
> +{
> + uint32_t i;
> + uint32_t max_flow_num = tbl->max_flow_num;
> +
> + for (i = 0; i < max_flow_num; i++)
> + if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
> + return i;
> + return INVALID_ARRAY_INDEX;
> +}
> +
> +static inline uint32_t
> +insert_new_flow(struct gro_tcp6_tbl *tbl,
> + struct tcp6_flow_key *src,
> + uint32_t item_idx)
> +{
> + struct tcp6_flow_key *dst;
> + uint32_t flow_idx;
> +
> + flow_idx = find_an_empty_flow(tbl);
> + if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
> + return INVALID_ARRAY_INDEX;
> +
> + dst = &(tbl->flows[flow_idx].key);
> +
> + ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
> + memcpy(&dst->src_addr[0], &src->src_addr[0],
> sizeof(dst->src_addr));
> + memcpy(&dst->dst_addr[0], &src->dst_addr[0],
> sizeof(dst->dst_addr));
> + dst->vtc_flow = src->vtc_flow;
> +
> + tbl->flows[flow_idx].start_index = item_idx;
> + tbl->flow_num++;
> +
> + return flow_idx;
> +}
> +
> +/*
> + * update the packet length for the flushed packet.
> + */
> +static inline void
> +update_header(struct gro_tcp_item *item)
> +{
> + struct rte_ipv6_hdr *ipv6_hdr;
> + struct rte_mbuf *pkt = item->firstseg;
> +
> + ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> + pkt->l2_len);
> + ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
> + pkt->l2_len - pkt->l3_len);
> +}
> +
> +int32_t
> +gro_tcp6_reassemble(struct rte_mbuf *pkt,
> + struct gro_tcp6_tbl *tbl,
> + uint64_t start_time)
> +{
> + struct rte_ether_hdr *eth_hdr;
> + struct rte_ipv6_hdr *ipv6_hdr;
> + int32_t tcp_dl;
> + uint16_t ip_tlen;
> + struct tcp6_flow_key key;
> + uint32_t i, max_flow_num, remaining_flow_num;
> + uint32_t sent_seq;
> + struct rte_tcp_hdr *tcp_hdr;
> + uint8_t find;
> + uint32_t item_idx;
> + /*
> + * Don't process the packet whose TCP header length is greater
> + * than 60 bytes or less than 20 bytes.
> + */
> + if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
> + return -1;
> +
> + eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
> + ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
> + tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *,
> pkt->l2_len + pkt->l3_len);
> +
> + /*
> + * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
> + * or CWR set.
> + */
> + if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
> + return -1;
> +
> + ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
> + /*
> + * Don't process the packet whose payload length is less than or
> + * equal to 0.
> + */
> + tcp_dl = ip_tlen - pkt->l4_len;
> + if (tcp_dl <= 0)
> + return -1;
> +
> + rte_ether_addr_copy(&(eth_hdr->src_addr),
> &(key.cmn_key.eth_saddr));
> + rte_ether_addr_copy(&(eth_hdr->dst_addr),
> &(key.cmn_key.eth_daddr));
> + memcpy(&key.src_addr[0], &ipv6_hdr->src_addr,
> sizeof(key.src_addr));
> + memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr,
> sizeof(key.dst_addr));
> + key.cmn_key.src_port = tcp_hdr->src_port;
> + key.cmn_key.dst_port = tcp_hdr->dst_port;
> + key.cmn_key.recv_ack = tcp_hdr->recv_ack;
> + key.vtc_flow = ipv6_hdr->vtc_flow;
> +
> + /* Search for a matched flow. */
> + max_flow_num = tbl->max_flow_num;
> + remaining_flow_num = tbl->flow_num;
> + find = 0;
> + for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
> + if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
> + if (is_same_tcp6_flow(&tbl->flows[i].key, &key)) {
> + find = 1;
> + break;
> + }
> + remaining_flow_num--;
> + }
> + }
> +
> + if (find == 0) {
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> + item_idx = insert_new_tcp_item(pkt, tbl->items,
> &tbl->item_num,
> + tbl->max_item_num,
> start_time,
> + INVALID_ARRAY_INDEX,
> sent_seq, 0, true);
> + if (item_idx == INVALID_ARRAY_INDEX)
> + return -1;
> + if (insert_new_flow(tbl, &key, item_idx) ==
> + INVALID_ARRAY_INDEX) {
> + /*
> + * Fail to insert a new flow, so delete the
> + * stored packet.
> + */
> + delete_tcp_item(tbl->items, item_idx,
> &tbl->item_num, INVALID_ARRAY_INDEX);
> + return -1;
> + }
> + return 0;
> + }
> +
> + return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items,
> tbl->flows[i].start_index,
> + &tbl->item_num,
> tbl->max_item_num,
> + 0, true, start_time);
> +}
> +
> +uint16_t
> +gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
> + uint64_t flush_timestamp,
> + struct rte_mbuf **out,
> + uint16_t nb_out)
> +{
> + uint16_t k = 0;
> + uint32_t i, j;
> + uint32_t max_flow_num = tbl->max_flow_num;
> +
> + for (i = 0; i < max_flow_num; i++) {
> + if (unlikely(tbl->flow_num == 0))
> + return k;
> +
> + j = tbl->flows[i].start_index;
> + while (j != INVALID_ARRAY_INDEX) {
> + if (tbl->items[j].start_time <= flush_timestamp) {
> + out[k++] = tbl->items[j].firstseg;
> + if (tbl->items[j].nb_merged > 1)
> + update_header(&(tbl->items[j]));
> + /*
> + * Delete the packet and get the next
> + * packet in the flow.
> + */
> + j = delete_tcp_item(tbl->items, j,
> + &tbl->item_num,
> INVALID_ARRAY_INDEX);
> + tbl->flows[i].start_index = j;
> + if (j == INVALID_ARRAY_INDEX)
> + tbl->flow_num--;
> +
> + if (unlikely(k == nb_out))
> + return k;
> + } else
> + /*
> + * The left packets in this flow won't be
> + * timeout. Go to check other flows.
> + */
> + break;
> + }
> + }
> + return k;
> +}
> +
> +uint32_t
> +gro_tcp6_tbl_pkt_count(void *tbl)
> +{
> + struct gro_tcp6_tbl *gro_tbl = tbl;
> +
> + if (gro_tbl)
> + return gro_tbl->item_num;
> +
> + return 0;
> +}
> diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h
> new file mode 100644
> index 0000000000..cc02b0004a
> --- /dev/null
> +++ b/lib/gro/gro_tcp6.h
> @@ -0,0 +1,161 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
> +
> +#ifndef _GRO_TCP6_H_
> +#define _GRO_TCP6_H_
> +
> +#include <gro_tcp_internal.h>
> +
> +#define INVALID_ARRAY_INDEX 0xffffffffUL
> +#define GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
> +
> +/* Header fields representing a TCP/IPv6 flow */
> +struct tcp6_flow_key {
> + struct cmn_tcp_key cmn_key;
> + uint8_t src_addr[16];
> + uint8_t dst_addr[16];
> + rte_be32_t vtc_flow;
> +};
> +
> +struct gro_tcp6_flow {
> + struct tcp6_flow_key key;
> + /*
> + * The index of the first packet in the flow.
> + * INVALID_ARRAY_INDEX indicates an empty flow.
> + */
> + uint32_t start_index;
> +};
> +
> +/*
> + * TCP/IPv6 reassembly table structure.
> + */
> +struct gro_tcp6_tbl {
> + /* item array */
> + struct gro_tcp_item *items;
> + /* flow array */
> + struct gro_tcp6_flow *flows;
> + /* current item number */
> + uint32_t item_num;
> + /* current flow num */
> + uint32_t flow_num;
> + /* item array size */
> + uint32_t max_item_num;
> + /* flow array size */
> + uint32_t max_flow_num;
> +};
> +
> +/**
> + * This function creates a TCP/IPv6 reassembly table.
> + *
> + * @param socket_id
> + * Socket index for allocating the TCP/IPv6 reassemble table
> + * @param max_flow_num
> + * The maximum number of flows in the TCP/IPv6 GRO table
> + * @param max_item_per_flow
> + * The maximum number of packets per flow
> + *
> + * @return
> + * - Return the table pointer on success.
> + * - Return NULL on failure.
> + */
> +void *gro_tcp6_tbl_create(uint16_t socket_id,
> + uint16_t max_flow_num,
> + uint16_t max_item_per_flow);
> +
> +/**
> + * This function destroys a TCP/IPv6 reassembly table.
> + *
> + * @param tbl
> + * Pointer pointing to the TCP/IPv6 reassembly table.
> + */
> +void gro_tcp6_tbl_destroy(void *tbl);
> +
> +/**
> + * This function merges a TCP/IPv6 packet. It doesn't process the packet,
> + * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't have
> + * payload.
> + *
> + * This function doesn't check if the packet has correct checksums and
> + * doesn't re-calculate checksums for the merged packet. Additionally,
> + * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
> + * when IP fragmentation is possible (i.e., DF==0). It returns the
> + * packet, if the packet has invalid parameters (e.g. SYN bit is set)
> + * or there is no available space in the table.
> + *
> + * @param pkt
> + * Packet to reassemble
> + * @param tbl
> + * Pointer pointing to the TCP/IPv6 reassembly table
> + * @start_time
> + * The time when the packet is inserted into the table
> + *
> + * @return
> + * - Return a positive value if the packet is merged.
> + * - Return zero if the packet isn't merged but stored in the table.
> + * - Return a negative value for invalid parameters or no available
> + * space in the table.
> + */
> +int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
> + struct gro_tcp6_tbl *tbl,
> + uint64_t start_time);
> +
> +/**
> + * This function flushes timeout packets in a TCP/IPv6 reassembly table,
> + * and without updating checksums.
> + *
> + * @param tbl
> + * TCP/IPv6 reassembly table pointer
> + * @param flush_timestamp
> + * Flush packets which are inserted into the table before or at the
> + * flush_timestamp.
> + * @param out
> + * Pointer array used to keep flushed packets
> + * @param nb_out
> + * The element number in 'out'. It also determines the maximum number of
> + * packets that can be flushed finally.
> + *
> + * @return
> + * The number of flushed packets
> + */
> +uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
> + uint64_t flush_timestamp,
> + struct rte_mbuf **out,
> + uint16_t nb_out);
> +
> +/**
> + * This function returns the number of the packets in a TCP/IPv6
> + * reassembly table.
> + *
> + * @param tbl
> + * TCP/IPv6 reassembly table pointer
> + *
> + * @return
> + * The number of packets in the table
> + */
> +uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
> +
> +/*
> + * Check if two TCP/IPv6 packets belong to the same flow.
> + */
> +static inline int
> +is_same_tcp6_flow(struct tcp6_flow_key *k1, struct tcp6_flow_key *k2)
> +{
> + rte_be32_t vtc_flow_diff;
> +
> + if (memcmp(&k1->src_addr, &k2->src_addr, 16))
> + return 0;
> + if (memcmp(&k1->dst_addr, &k2->dst_addr, 16))
> + return 0;
> + /*
> + * IP version (4) Traffic Class (8) Flow Label (20)
> + * All fields except Traffic class should be same
> + */
> + vtc_flow_diff = (k1->vtc_flow ^ k2->vtc_flow);
> + if (vtc_flow_diff & htonl(0xF00FFFFF))
> + return 0;
> +
> + return is_common_tcp_key(&k1->cmn_key, &k2->cmn_key);
> +}
> +
> +#endif
> diff --git a/lib/gro/gro_tcp_internal.c b/lib/gro/gro_tcp_internal.c
> new file mode 100644
> index 0000000000..5a21bca7f8
> --- /dev/null
> +++ b/lib/gro/gro_tcp_internal.c
> @@ -0,0 +1,128 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
> +#include <rte_malloc.h>
> +#include <rte_mbuf.h>
> +#include <rte_ethdev.h>
> +
> +#include "gro_tcp_internal.h"
> +
> +static inline uint32_t
> +find_an_empty_item(struct gro_tcp_item *items,
> + uint32_t max_item_num)
> +{
> + uint32_t i;
> +
> + for (i = 0; i < max_item_num; i++)
> + if (items[i].firstseg == NULL)
> + return i;
> + return INVALID_ARRAY_INDEX;
> +}
> +
> +inline uint32_t
> +insert_new_tcp_item(struct rte_mbuf *pkt,
> + struct gro_tcp_item *items,
> + uint32_t *item_num,
> + uint32_t max_item_num,
> + uint64_t start_time,
> + uint32_t prev_idx,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint8_t is_atomic)
> +{
> + uint32_t item_idx;
> +
> + item_idx = find_an_empty_item(items, max_item_num);
> + if (item_idx == INVALID_ARRAY_INDEX)
> + return INVALID_ARRAY_INDEX;
> +
> + items[item_idx].firstseg = pkt;
> + items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> + items[item_idx].start_time = start_time;
> + items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> + items[item_idx].sent_seq = sent_seq;
> + items[item_idx].l3.ip_id = ip_id;
> + items[item_idx].nb_merged = 1;
> + items[item_idx].is_atomic = is_atomic;
> + (*item_num) += 1;
> +
> + /* if the previous packet exists, chain them together. */
> + if (prev_idx != INVALID_ARRAY_INDEX) {
> + items[item_idx].next_pkt_idx =
> + items[prev_idx].next_pkt_idx;
> + items[prev_idx].next_pkt_idx = item_idx;
> + }
> +
> + return item_idx;
> +}
> +
> +inline uint32_t
> +delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t prev_item_idx)
> +{
> + uint32_t next_idx = items[item_idx].next_pkt_idx;
> +
> + /* NULL indicates an empty item */
> + items[item_idx].firstseg = NULL;
> + (*item_num) -= 1;
> + if (prev_item_idx != INVALID_ARRAY_INDEX)
> + items[prev_item_idx].next_pkt_idx = next_idx;
> +
> + return next_idx;
> +}
> +
> +int32_t
> +process_tcp_item(struct rte_mbuf *pkt,
> + struct rte_tcp_hdr *tcp_hdr,
> + int32_t tcp_dl,
> + struct gro_tcp_item *items,
> + uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t max_item_num,
> + uint16_t ip_id,
> + uint8_t is_atomic,
> + uint64_t start_time)
> +{
> + uint32_t cur_idx;
> + uint32_t prev_idx;
> + int cmp;
> + uint32_t sent_seq;
> +
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> + /*
> + * Check all packets in the flow and try to find a neighbor for
> + * the input packet.
> + */
> + cur_idx = item_idx;
> + prev_idx = cur_idx;
> + do {
> + cmp = check_seq_option(&items[cur_idx], tcp_hdr,
> + sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> + is_atomic);
> + if (cmp) {
> + if (merge_two_tcp_packets(&items[cur_idx],
> + pkt, cmp, sent_seq, ip_id,
> 0))
> + return 1;
> + /*
> + * Fail to merge the two packets, as the packet
> + * length is greater than the max value. Store
> + * the packet into the flow.
> + */
> + if (insert_new_tcp_item(pkt, items, item_num,
> max_item_num,
> + start_time, cur_idx,
> sent_seq, ip_id, is_atomic) ==
> + INVALID_ARRAY_INDEX)
> + return -1;
> + return 0;
> + }
> + prev_idx = cur_idx;
> + cur_idx = items[cur_idx].next_pkt_idx;
> + } while (cur_idx != INVALID_ARRAY_INDEX);
> +
> + /* Fail to find a neighbor, so store the packet into the flow. */
> + if (insert_new_tcp_item(pkt, items, item_num, max_item_num,
> start_time, prev_idx, sent_seq,
> + ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> + return -1;
> +
> + return 0;
> +}
> diff --git a/lib/gro/gro_tcp_internal.h b/lib/gro/gro_tcp_internal.h
> new file mode 100644
> index 0000000000..072b7aea13
> --- /dev/null
> +++ b/lib/gro/gro_tcp_internal.h
> @@ -0,0 +1,212 @@
> +#ifndef _GRO_TCP_H_
> +#define _GRO_TCP_H_
> +
> +#define INVALID_ARRAY_INDEX 0xffffffffUL
> +
> +#include <rte_tcp.h>
> +
> +/*
> + * The max length of a IPv4 packet, which includes the length of the L3
> + * header, the L4 header and the data payload.
> + */
> +#define MAX_IP_PKT_LENGTH UINT16_MAX
> +
> +/* The maximum TCP header length */
> +#define MAX_TCP_HLEN 60
> +#define INVALID_TCP_HDRLEN(len) \
> + (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> +
> +struct cmn_tcp_key {
> + struct rte_ether_addr eth_saddr;
> + struct rte_ether_addr eth_daddr;
> + uint32_t recv_ack;
> + uint16_t src_port;
> + uint16_t dst_port;
> +};
> +
> +#define ASSIGN_COMMON_TCP_KEY(k1, k2) \
> + do {\
> + rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
> + rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr)); \
> + k2->recv_ack = k1->recv_ack; \
> + k2->src_port = k1->src_port; \
> + k2->dst_port = k1->dst_port; \
> + } while (0)
> +
> +struct gro_tcp_item {
> + /*
> + * The first MBUF segment of the packet. If the value
> + * is NULL, it means the item is empty.
> + */
> + struct rte_mbuf *firstseg;
> + /* The last MBUF segment of the packet */
> + struct rte_mbuf *lastseg;
> + /*
> + * The time when the first packet is inserted into the table.
> + * This value won't be updated, even if the packet is merged
> + * with other packets.
> + */
> + uint64_t start_time;
> + /*
> + * next_pkt_idx is used to chain the packets that
> + * are in the same flow but can't be merged together
> + * (e.g. caused by packet reordering).
> + */
> + uint32_t next_pkt_idx;
> + /* TCP sequence number of the packet */
> + uint32_t sent_seq;
> + union {
> + /* IPv4 ID of the packet */
> + uint16_t ip_id;
> + /* Unused field for IPv6 */
> + uint16_t unused;
> + } l3;
> + /* the number of merged packets */
> + uint16_t nb_merged;
> + /* Indicate if IPv4 ID can be ignored */
> + uint8_t is_atomic;
> +};
> +
> +uint32_t
> +insert_new_tcp_item(struct rte_mbuf *pkt,
> + struct gro_tcp_item *items,
> + uint32_t *item_num,
> + uint32_t max_item_num,
> + uint64_t start_time,
> + uint32_t prev_idx,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint8_t is_atomic);
> +
> +uint32_t
> +delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t prev_item_idx);
> +
> +int32_t
> +process_tcp_item(struct rte_mbuf *pkt,
> + struct rte_tcp_hdr *tcp_hdr,
> + int32_t tcp_dl,
> + struct gro_tcp_item *items,
> + uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t max_item_num,
> + uint16_t ip_id,
> + uint8_t is_atomic,
> + uint64_t start_time);
> +
> +/*
> + * Merge two TCP packets without updating checksums.
> + * If cmp is larger than 0, append the new packet to the
> + * original packet. Otherwise, pre-pend the new packet to
> + * the original packet.
> + */
> +static inline int
> +merge_two_tcp_packets(struct gro_tcp_item *item,
> + struct rte_mbuf *pkt,
> + int cmp,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint16_t l2_offset)
> +{
> + struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
> + uint16_t hdr_len, l2_len;
> +
> + if (cmp > 0) {
> + pkt_head = item->firstseg;
> + pkt_tail = pkt;
> + } else {
> + pkt_head = pkt;
> + pkt_tail = item->firstseg;
> + }
> +
> + /* check if the IPv4 packet length is greater than the max value */
> + hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
> + pkt_head->l4_len;
> + l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
> + if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
> + hdr_len > MAX_IP_PKT_LENGTH))
> + return 0;
> +
> + /* remove the packet header for the tail packet */
> + rte_pktmbuf_adj(pkt_tail, hdr_len);
> +
> + /* chain two packets together */
> + if (cmp > 0) {
> + item->lastseg->next = pkt;
> + item->lastseg = rte_pktmbuf_lastseg(pkt);
> + /* update IP ID to the larger value */
> + item->l3.ip_id = ip_id;
> + } else {
> + lastseg = rte_pktmbuf_lastseg(pkt);
> + lastseg->next = item->firstseg;
> + item->firstseg = pkt;
> + /* update sent_seq to the smaller value */
> + item->sent_seq = sent_seq;
> + item->l3.ip_id = ip_id;
> + }
> + item->nb_merged++;
> +
> + /* update MBUF metadata for the merged packet */
> + pkt_head->nb_segs += pkt_tail->nb_segs;
> + pkt_head->pkt_len += pkt_tail->pkt_len;
> +
> + return 1;
> +}
> +
> +/*
> + * Check if two TCP/IPv4 packets are neighbors.
> + */
> +static inline int
> +check_seq_option(struct gro_tcp_item *item,
> + struct rte_tcp_hdr *tcph,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint16_t tcp_hl,
> + uint16_t tcp_dl,
> + uint16_t l2_offset,
> + uint8_t is_atomic)
> +{
> + struct rte_mbuf *pkt_orig = item->firstseg;
> + char *iph_orig;
> + struct rte_tcp_hdr *tcph_orig;
> + uint16_t len, tcp_hl_orig;
> +
> + iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
> + l2_offset + pkt_orig->l2_len);
> + tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
> + tcp_hl_orig = pkt_orig->l4_len;
> +
> + /* Check if TCP option fields equal */
> + len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> + if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> + (memcmp(tcph + 1, tcph_orig + 1,
> + len) != 0)))
> + return 0;
> +
> + /* Don't merge packets whose DF bits are different */
> + if (unlikely(item->is_atomic ^ is_atomic))
> + return 0;
> +
> + /* check if the two packets are neighbors */
> + len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> + pkt_orig->l3_len - tcp_hl_orig;
> + if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> + (ip_id == item->l3.ip_id + 1)))
> + /* append the new packet */
> + return 1;
> + else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> + (ip_id + item->nb_merged ==
> item->l3.ip_id)))
> + /* pre-pend the new packet */
> + return -1;
> +
> + return 0;
> +}
> +
> +static inline int
> +is_common_tcp_key(struct cmn_tcp_key *k1, struct cmn_tcp_key *k2)
> +{
> + return (!memcmp(k1, k2, sizeof(struct cmn_tcp_key)));
> +}
> +
> +#endif
> diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
> index 3be4deb7c7..81eebf0d2d 100644
> --- a/lib/gro/gro_vxlan_tcp4.c
> +++ b/lib/gro/gro_vxlan_tcp4.c
> @@ -7,6 +7,7 @@
> #include <rte_ethdev.h>
> #include <rte_udp.h>
>
> +#include "gro_tcp_internal.h"
> #include "gro_vxlan_tcp4.h"
>
> void *
> @@ -116,7 +117,7 @@ insert_new_item(struct gro_vxlan_tcp4_tbl *tbl,
> tbl->items[item_idx].inner_item.start_time = start_time;
> tbl->items[item_idx].inner_item.next_pkt_idx = INVALID_ARRAY_INDEX;
> tbl->items[item_idx].inner_item.sent_seq = sent_seq;
> - tbl->items[item_idx].inner_item.ip_id = ip_id;
> + tbl->items[item_idx].inner_item.l3.ip_id = ip_id;
> tbl->items[item_idx].inner_item.nb_merged = 1;
> tbl->items[item_idx].inner_item.is_atomic = is_atomic;
> tbl->items[item_idx].outer_ip_id = outer_ip_id;
> @@ -163,15 +164,9 @@ insert_new_flow(struct gro_vxlan_tcp4_tbl *tbl,
>
> dst = &(tbl->flows[flow_idx].key);
>
> - rte_ether_addr_copy(&(src->inner_key.eth_saddr),
> - &(dst->inner_key.eth_saddr));
> - rte_ether_addr_copy(&(src->inner_key.eth_daddr),
> - &(dst->inner_key.eth_daddr));
> + ASSIGN_COMMON_TCP_KEY((&(src->inner_key.cmn_key)),
> (&(dst->inner_key.cmn_key)));
> dst->inner_key.ip_src_addr = src->inner_key.ip_src_addr;
> dst->inner_key.ip_dst_addr = src->inner_key.ip_dst_addr;
> - dst->inner_key.recv_ack = src->inner_key.recv_ack;
> - dst->inner_key.src_port = src->inner_key.src_port;
> - dst->inner_key.dst_port = src->inner_key.dst_port;
>
> dst->vxlan_hdr.vx_flags = src->vxlan_hdr.vx_flags;
> dst->vxlan_hdr.vx_vni = src->vxlan_hdr.vx_vni;
> @@ -248,7 +243,7 @@ merge_two_vxlan_tcp4_packets(struct
> gro_vxlan_tcp4_item *item,
> uint16_t outer_ip_id,
> uint16_t ip_id)
> {
> - if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp, sent_seq,
> + if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
> ip_id, pkt->outer_l2_len +
> pkt->outer_l3_len)) {
> /* Update the outer IPv4 ID to the large value. */
> @@ -357,13 +352,13 @@ gro_vxlan_tcp4_reassemble(struct rte_mbuf *pkt,
>
> sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
>
> - rte_ether_addr_copy(&(eth_hdr->src_addr),
> &(key.inner_key.eth_saddr));
> - rte_ether_addr_copy(&(eth_hdr->dst_addr),
> &(key.inner_key.eth_daddr));
> + rte_ether_addr_copy(&(eth_hdr->src_addr),
> &(key.inner_key.cmn_key.eth_saddr));
> + rte_ether_addr_copy(&(eth_hdr->dst_addr),
> &(key.inner_key.cmn_key.eth_daddr));
> key.inner_key.ip_src_addr = ipv4_hdr->src_addr;
> key.inner_key.ip_dst_addr = ipv4_hdr->dst_addr;
> - key.inner_key.recv_ack = tcp_hdr->recv_ack;
> - key.inner_key.src_port = tcp_hdr->src_port;
> - key.inner_key.dst_port = tcp_hdr->dst_port;
> + key.inner_key.cmn_key.recv_ack = tcp_hdr->recv_ack;
> + key.inner_key.cmn_key.src_port = tcp_hdr->src_port;
> + key.inner_key.cmn_key.dst_port = tcp_hdr->dst_port;
>
> key.vxlan_hdr.vx_flags = vxlan_hdr->vx_flags;
> key.vxlan_hdr.vx_vni = vxlan_hdr->vx_vni;
> diff --git a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h
> index 7832942a68..82eaaee11e 100644
> --- a/lib/gro/gro_vxlan_tcp4.h
> +++ b/lib/gro/gro_vxlan_tcp4.h
> @@ -5,6 +5,7 @@
> #ifndef _GRO_VXLAN_TCP4_H_
> #define _GRO_VXLAN_TCP4_H_
>
> +#include "gro_tcp_internal.h"
> #include "gro_tcp4.h"
>
> #define GRO_VXLAN_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
> @@ -36,7 +37,7 @@ struct gro_vxlan_tcp4_flow {
> };
>
> struct gro_vxlan_tcp4_item {
> - struct gro_tcp4_item inner_item;
> + struct gro_tcp_item inner_item;
> /* IPv4 ID in the outer IPv4 header */
> uint16_t outer_ip_id;
> /* Indicate if outer IPv4 ID can be ignored */
> diff --git a/lib/gro/meson.build b/lib/gro/meson.build
> index e4fa2958bd..1640317890 100644
> --- a/lib/gro/meson.build
> +++ b/lib/gro/meson.build
> @@ -3,7 +3,9 @@
>
> sources = files(
> 'rte_gro.c',
> + 'gro_tcp_internal.c',
> 'gro_tcp4.c',
> + 'gro_tcp6.c',
> 'gro_udp4.c',
> 'gro_vxlan_tcp4.c',
> 'gro_vxlan_udp4.c',
> diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c
> index e35399fd42..d824eebd93 100644
> --- a/lib/gro/rte_gro.c
> +++ b/lib/gro/rte_gro.c
> @@ -8,6 +8,7 @@
>
> #include "rte_gro.h"
> #include "gro_tcp4.h"
> +#include "gro_tcp6.h"
> #include "gro_udp4.h"
> #include "gro_vxlan_tcp4.h"
> #include "gro_vxlan_udp4.h"
> @@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
>
> static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
> gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
> - gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
> + gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create,
> gro_tcp6_tbl_create, NULL};
> static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
> gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
> gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
> + gro_tcp6_tbl_destroy,
> NULL};
> static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
> gro_tcp4_tbl_pkt_count,
> gro_vxlan_tcp4_tbl_pkt_count,
> gro_udp4_tbl_pkt_count,
> gro_vxlan_udp4_tbl_pkt_count,
> + gro_tcp6_tbl_pkt_count,
> NULL};
>
> #define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
> @@ -35,6 +38,12 @@ static gro_tbl_pkt_count_fn
> tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
> ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
> (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
>
> +/* GRO with extension headers is not supported */
> +#define IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
> + ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
> + ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
> + (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
> +
> #define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
> ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
> (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
> @@ -147,7 +156,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> /* allocate a reassembly table for TCP/IPv4 GRO */
> struct gro_tcp4_tbl tcp_tbl;
> struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
> - struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0}
> };
> + struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
> +
> + struct gro_tcp6_tbl tcp6_tbl;
> + struct gro_tcp6_flow tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
> + struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0}
> };
>
> /* allocate a reassembly table for UDP/IPv4 GRO */
> struct gro_udp4_tbl udp_tbl;
> @@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> int32_t ret;
> uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
> uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
> - do_vxlan_udp_gro = 0;
> + do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
>
> if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
> - RTE_GRO_TCP_IPV4 |
> + RTE_GRO_TCP_IPV4 |
> RTE_GRO_TCP_IPV6 |
> RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
> RTE_GRO_UDP_IPV4)) == 0))
> return nb_pkts;
> @@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> do_udp4_gro = 1;
> }
>
> + if (param->gro_types & RTE_GRO_TCP_IPV6) {
> + for (i = 0; i < item_num; i++)
> + tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
> +
> + tcp6_tbl.flows = tcp6_flows;
> + tcp6_tbl.items = tcp6_items;
> + tcp6_tbl.flow_num = 0;
> + tcp6_tbl.item_num = 0;
> + tcp6_tbl.max_flow_num = item_num;
> + tcp6_tbl.max_item_num = item_num;
> + do_tcp6_gro = 1;
> + }
>
> for (i = 0; i < nb_pkts; i++) {
> /*
> @@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> nb_after_gro--;
> else if (ret < 0)
> unprocess_pkts[unprocess_num++] = pkts[i];
> + } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
> + do_tcp6_gro) {
> + ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
> + if (ret > 0)
> + /* merge successfully */
> + nb_after_gro--;
> + else if (ret < 0)
> + unprocess_pkts[unprocess_num++] = pkts[i];
> } else
> unprocess_pkts[unprocess_num++] = pkts[i];
> }
> @@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> if ((nb_after_gro < nb_pkts)
> || (unprocess_num < nb_pkts)) {
> i = 0;
> + /* Copy unprocessed packets */
> + if (unprocess_num > 0) {
> + memcpy(&pkts[i], unprocess_pkts,
> + sizeof(struct rte_mbuf *) *
> + unprocess_num);
> + i = unprocess_num;
> + }
> +
> /* Flush all packets from the tables */
> if (do_vxlan_tcp_gro) {
> - i =
> gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
> + i +=
> gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
> 0, pkts, nb_pkts);
> }
>
> @@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
> &pkts[i], nb_pkts - i);
> }
> - /* Copy unprocessed packets */
> - if (unprocess_num > 0) {
> - memcpy(&pkts[i], unprocess_pkts,
> - sizeof(struct rte_mbuf *) *
> - unprocess_num);
> +
> + if (do_tcp6_gro) {
> + i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
> + &pkts[i], nb_pkts - i);
> }
> - nb_after_gro = i + unprocess_num;
> }
>
> return nb_after_gro;
> @@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> {
> struct rte_mbuf *unprocess_pkts[nb_pkts];
> struct gro_ctx *gro_ctx = ctx;
> - void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
> + void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
> uint64_t current_time;
> uint16_t i, unprocess_num = 0;
> - uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro,
> do_vxlan_udp_gro;
> + uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro,
> do_vxlan_udp_gro, do_tcp6_gro;
>
> if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
> - RTE_GRO_TCP_IPV4 |
> + RTE_GRO_TCP_IPV4 |
> RTE_GRO_TCP_IPV6 |
> RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
> RTE_GRO_UDP_IPV4)) == 0))
> return nb_pkts;
> @@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
> udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
> vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
> + tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
>
> do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
> RTE_GRO_TCP_IPV4;
> @@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> RTE_GRO_UDP_IPV4;
> do_vxlan_udp_gro = (gro_ctx->gro_types &
> RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
> RTE_GRO_IPV4_VXLAN_UDP_IPV4;
> + do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) ==
> RTE_GRO_TCP_IPV6;
>
> current_time = rte_rdtsc();
>
> @@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> if (gro_udp4_reassemble(pkts[i], udp_tbl,
> current_time) < 0)
> unprocess_pkts[unprocess_num++] = pkts[i];
> + } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
> + do_tcp6_gro) {
> + if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
> + current_time) < 0)
> + unprocess_pkts[unprocess_num++] = pkts[i];
> } else
> unprocess_pkts[unprocess_num++] = pkts[i];
> }
> @@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
> gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
> flush_timestamp,
> &out[num], left_nb_out);
> + left_nb_out = max_nb_out - num;
> + }
> +
> + if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
> + num += gro_tcp6_tbl_timeout_flush(
> + gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
> + flush_timestamp,
> + &out[num], left_nb_out);
> +
> }
>
> return num;
> diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h
> index 9f9ed4935a..c83dfd9ad1 100644
> --- a/lib/gro/rte_gro.h
> +++ b/lib/gro/rte_gro.h
> @@ -38,6 +38,9 @@ extern "C" {
> #define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3
> #define RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL <<
> RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
> /**< VxLAN UDP/IPv4 GRO flag. */
> +#define RTE_GRO_TCP_IPV6_INDEX 4
> +#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX)
> +/**< TCP/IPv6 GRO flag. */
>
> /**
> * Structure used to create GRO context objects or used to pass
> --
> 2.25.1
>
Hi Jiyau, I have updated the review comments. Tested GRO for both v4 and
>> v6. I will update the generic_receive_offload_lib.rst and release note in
>> subsequent change. Would be great if you can review the current changes by
>> then.
>>
>
[-- Attachment #2: Type: text/html, Size: 78605 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread
* RE: [PATCH v7] gro : ipv6 changes to support GRO for TCP/ipv6
2023-06-12 11:31 ` [PATCH v7] " Kumara Parameshwaran
2023-06-12 12:04 ` kumaraparameshwaran rathinavel
@ 2023-06-13 2:25 ` Hu, Jiayu
2023-06-14 3:43 ` kumaraparameshwaran rathinavel
1 sibling, 1 reply; 32+ messages in thread
From: Hu, Jiayu @ 2023-06-13 2:25 UTC (permalink / raw)
To: Kumara Parameshwaran; +Cc: dev
Hi Kumara,
Overall, the patch looks good to me. But you need to update the doc and there
are some minor changes needed to be made. Please see replies inline.
After all comments are addressed, you can add "Reviewed-by: Jiayu Hu <jiayu.hu@intel.com>".
Thanks,
Jiayu
> -----Original Message-----
> From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> Sent: Monday, June 12, 2023 7:31 PM
> To: Hu, Jiayu <jiayu.hu@intel.com>
> Cc: dev@dpdk.org; Kumara Parameshwaran
> <kumaraparamesh92@gmail.com>
> Subject: [PATCH v7] gro : ipv6 changes to support GRO for TCP/ipv6
>
> The patch adds GRO support for TCP/ipv6 packets. This does not include the
> support for vxlan, udp ipv6 packets.
>
> Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> ---
> v1:
> * Changes to support GRO for TCP/ipv6 packets. This does not
> include
> vxlan changes.
> * The GRO is performed only for ipv6 packets that does not contain
> extension headers.
> * The logic for the TCP coalescing remains the same, in ipv6 header
> the source address, destination address, flow label, version fields
> are expected to be the same.
> * Re-organised the code to reuse certain tcp functions for both ipv4
> and
> ipv6 flows.
> v2:
> * Fix comments in gro_tcp6.h header file.
>
> v3:
> * Adderess review comments to fix code duplication for v4 and v6
>
> v4:
> * Addresses review comments for v3, do not use callbacks
>
> v5:
> * Address review comments
>
> v6:
> * Fix warning and coding style issues
>
> v7:
> * Fix build compilation issue
> lib/gro/gro_tcp4.c | 178 ++++++-------------------
> lib/gro/gro_tcp4.h | 170 +----------------------
> lib/gro/gro_tcp6.c | 267 +++++++++++++++++++++++++++++++++++++
> lib/gro/gro_tcp6.h | 161 ++++++++++++++++++++++
> lib/gro/gro_tcp_internal.c | 128 ++++++++++++++++++
> lib/gro/gro_tcp_internal.h | 212 +++++++++++++++++++++++++++++
> lib/gro/gro_vxlan_tcp4.c | 23 ++--
> lib/gro/gro_vxlan_tcp4.h | 3 +-
> lib/gro/meson.build | 2 +
> lib/gro/rte_gro.c | 83 ++++++++++--
> lib/gro/rte_gro.h | 3 +
> 11 files changed, 897 insertions(+), 333 deletions(-) create mode 100644
> lib/gro/gro_tcp6.c create mode 100644 lib/gro/gro_tcp6.h create mode
> 100644 lib/gro/gro_tcp_internal.c create mode 100644
> lib/gro/gro_tcp_internal.h
>
> diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c index
> 0014096e63..42fee78f30 100644
> --- a/lib/gro/gro_tcp4.c
> +++ b/lib/gro/gro_tcp4.c
> -
> int32_t
> gro_tcp4_reassemble(struct rte_mbuf *pkt,
> struct gro_tcp4_tbl *tbl,
> @@ -202,9 +122,8 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> uint8_t is_atomic;
>
> struct tcp4_flow_key key;
> - uint32_t cur_idx, prev_idx, item_idx;
> + uint32_t item_idx;
> uint32_t i, max_flow_num, remaining_flow_num;
> - int cmp;
> uint8_t find;
>
> /*
> @@ -216,7 +135,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
>
> eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
> ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
> - tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
> + tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *,
> +pkt->l2_len + pkt->l3_len);
No need to change the method to obtain tcp_hdr.
> hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
>
> /*
> @@ -230,7 +149,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> ip_tlen = rte_be_to_cpu_16(ipv4_hdr->total_length);
> if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len))
> rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len);
> -
No need to delete the blank line.
> /*
> * Don't process the packet whose payload length is less than or
> * equal to 0.
> @@ -239,6 +157,13 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> if (tcp_dl <= 0)
> return -1;
>
> + rte_ether_addr_copy(&(eth_hdr->src_addr),
> &(key.cmn_key.eth_saddr));
> + rte_ether_addr_copy(&(eth_hdr->dst_addr),
> &(key.cmn_key.eth_daddr));
> + key.ip_src_addr = ipv4_hdr->src_addr;
> + key.ip_dst_addr = ipv4_hdr->dst_addr;
> + key.cmn_key.src_port = tcp_hdr->src_port;
> + key.cmn_key.dst_port = tcp_hdr->dst_port;
> + key.cmn_key.recv_ack = tcp_hdr->recv_ack;
Add a blank line between key.cmn_key.recv_ack assignment and annotation.
> /*
> * Save IPv4 ID for the packet whose DF bit is 0. For the packet
> * whose DF bit is 1, IPv4 ID is ignored.
> @@ -246,15 +171,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
> is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) ==
> RTE_IPV4_HDR_DF_FLAG;
> ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
> - sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> -
> - rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
> - rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
> - key.ip_src_addr = ipv4_hdr->src_addr;
> - key.ip_dst_addr = ipv4_hdr->dst_addr;
> - key.src_port = tcp_hdr->src_port;
> - key.dst_port = tcp_hdr->dst_port;
> - key.recv_ack = tcp_hdr->recv_ack;
>
> /* Search for a matched flow. */
> max_flow_num = tbl->max_flow_num;
> @@ -270,63 +186,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> }
> }
>
> - /*
> - * Fail to find a matched flow. Insert a new flow and store the
> - * packet into the flow.
> - */
> if (find == 0) {
> - item_idx = insert_new_item(tbl, pkt, start_time,
> - INVALID_ARRAY_INDEX, sent_seq, ip_id,
> - is_atomic);
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> + item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl-
> >item_num,
> + tbl->max_item_num,
> start_time,
> + INVALID_ARRAY_INDEX,
> sent_seq, ip_id,
> + is_atomic);
> if (item_idx == INVALID_ARRAY_INDEX)
> return -1;
> if (insert_new_flow(tbl, &key, item_idx) ==
> - INVALID_ARRAY_INDEX) {
> + INVALID_ARRAY_INDEX) {
> /*
> * Fail to insert a new flow, so delete the
> * stored packet.
> - */
> - delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
> + */
> + delete_tcp_item(tbl->items, item_idx, &tbl-
> >item_num,
> +INVALID_ARRAY_INDEX);
> return -1;
> }
> return 0;
> }
>
> - /*
> - * Check all packets in the flow and try to find a neighbor for
> - * the input packet.
> - */
> - cur_idx = tbl->flows[i].start_index;
> - prev_idx = cur_idx;
> - do {
> - cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
> - sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> - is_atomic);
> - if (cmp) {
> - if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
> - pkt, cmp, sent_seq, ip_id, 0))
> - return 1;
> - /*
> - * Fail to merge the two packets, as the packet
> - * length is greater than the max value. Store
> - * the packet into the flow.
> - */
> - if (insert_new_item(tbl, pkt, start_time, cur_idx,
> - sent_seq, ip_id, is_atomic) ==
> - INVALID_ARRAY_INDEX)
> - return -1;
> - return 0;
> - }
> - prev_idx = cur_idx;
> - cur_idx = tbl->items[cur_idx].next_pkt_idx;
> - } while (cur_idx != INVALID_ARRAY_INDEX);
> + return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl-
> >flows[i].start_index,
> + &tbl->item_num, tbl-
> >max_item_num,
> + ip_id, is_atomic, start_time);
> +}
>
> - /* Fail to find a neighbor, so store the packet into the flow. */
> - if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
> - ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> - return -1;
> +/*
> + * update the packet length for the flushed packet.
> + */
> +static inline void
> +update_header(struct gro_tcp_item *item) {
> + struct rte_ipv4_hdr *ipv4_hdr;
> + struct rte_mbuf *pkt = item->firstseg;
>
> - return 0;
> + ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> + pkt->l2_len);
> + ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
> + pkt->l2_len);
> }
>
> uint16_t
> @@ -353,7 +250,8 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
> * Delete the packet and get the next
> * packet in the flow.
> */
> - j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
> + j = delete_tcp_item(tbl->items, j,
> + &tbl->item_num,
> INVALID_ARRAY_INDEX);
> tbl->flows[i].start_index = j;
> if (j == INVALID_ARRAY_INDEX)
> tbl->flow_num--;
> diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h index
> 212f97a042..c0154afa24 100644
> --- a/lib/gro/gro_tcp4.h
> +++ b/lib/gro/gro_tcp4.h
> @@ -5,32 +5,15 @@
> #ifndef _GRO_TCP4_H_
> #define _GRO_TCP4_H_
>
> -#include <rte_tcp.h>
> +#include <gro_tcp_internal.h>
Use "#include "gro_tcp_internal.h"".
>
> -#define INVALID_ARRAY_INDEX 0xffffffffUL #define
> GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
>
> diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c new file mode 100644
> index 0000000000..0ea73741c1
> --- /dev/null
> +++ b/lib/gro/gro_tcp6.c
> @@ -0,0 +1,267 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
The license header is incorrect, and please update. Same and similar issue happens
in the new added files, like gro_tcp_internal.h.
> +
> diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h new file mode 100644
> index 0000000000..cc02b0004a
> --- /dev/null
> +++ b/lib/gro/gro_tcp6.h
> @@ -0,0 +1,161 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
Ditto.
> +
> +#ifndef _GRO_TCP6_H_
> +#define _GRO_TCP6_H_
> +
> +#include <gro_tcp_internal.h>
Use "#include "gro_tcp_internal.h"".
> +
> +#define INVALID_ARRAY_INDEX 0xffffffffUL #define
Duplicate definition for INVALID_ARRAY_INDEX in both gro_tcp6.h
and gro_tcp_internal.h.
> +GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
> +
> +/* Header fields representing a TCP/IPv6 flow */ struct tcp6_flow_key {
> + struct cmn_tcp_key cmn_key;
> + uint8_t src_addr[16];
> + uint8_t dst_addr[16];
> + rte_be32_t vtc_flow;
> +};
> +
> diff --git a/lib/gro/gro_tcp_internal.c b/lib/gro/gro_tcp_internal.c new file
> mode 100644 index 0000000000..5a21bca7f8
> --- /dev/null
> +++ b/lib/gro/gro_tcp_internal.c
> @@ -0,0 +1,128 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
Update license header.
> +#include <rte_malloc.h>
> +#include <rte_mbuf.h>
> +#include <rte_ethdev.h>
> +
> +#include "gro_tcp_internal.h"
> +
> diff --git a/lib/gro/gro_tcp_internal.h b/lib/gro/gro_tcp_internal.h new file
> mode 100644 index 0000000000..072b7aea13
> --- /dev/null
> +++ b/lib/gro/gro_tcp_internal.h
> @@ -0,0 +1,212 @@
> +#ifndef _GRO_TCP_H_
> +#define _GRO_TCP_H_
> +
Add license header in this file.
> +#define INVALID_ARRAY_INDEX 0xffffffffUL
> +
> +#include <rte_tcp.h>
> +
> +/*
> + * The max length of a IPv4 packet, which includes the length of the L3
> + * header, the L4 header and the data payload.
> + */
> +#define MAX_IP_PKT_LENGTH UINT16_MAX
> +
> +/* The maximum TCP header length */
> +#define MAX_TCP_HLEN 60
> +#define INVALID_TCP_HDRLEN(len) \
> + (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> +
> +/*
> + * Check if two TCP/IPv4 packets are neighbors.
> + */
This function is not only for TCP4, so use "Check if two TCP packets are neighbors".
> +static inline int
> +check_seq_option(struct gro_tcp_item *item,
> + struct rte_tcp_hdr *tcph,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint16_t tcp_hl,
> + uint16_t tcp_dl,
> + uint16_t l2_offset,
> + uint8_t is_atomic)
> +{
> + struct rte_mbuf *pkt_orig = item->firstseg;
> + char *iph_orig;
> + struct rte_tcp_hdr *tcph_orig;
> + uint16_t len, tcp_hl_orig;
> +
> + iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
> + l2_offset + pkt_orig->l2_len);
> + tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
> + tcp_hl_orig = pkt_orig->l4_len;
> +
> + /* Check if TCP option fields equal */
> + len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> + if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> + (memcmp(tcph + 1, tcph_orig + 1,
> + len) != 0)))
> + return 0;
> +
> + /* Don't merge packets whose DF bits are different */
> + if (unlikely(item->is_atomic ^ is_atomic))
> + return 0;
> +
> + /* check if the two packets are neighbors */
> + len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> + pkt_orig->l3_len - tcp_hl_orig;
> + if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> + (ip_id == item->l3.ip_id + 1)))
> + /* append the new packet */
> + return 1;
> + else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> + (ip_id + item->nb_merged == item->l3.ip_id)))
> + /* pre-pend the new packet */
> + return -1;
> +
> + return 0;
> +}
> +
> +static inline int
> +is_common_tcp_key(struct cmn_tcp_key *k1, struct cmn_tcp_key *k2) {
Rename it to "is_same_common_tcp_key()"?
> + return (!memcmp(k1, k2, sizeof(struct cmn_tcp_key))); }
> +
> +#endif
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [PATCH v7] gro : ipv6 changes to support GRO for TCP/ipv6
2023-06-13 2:25 ` Hu, Jiayu
@ 2023-06-14 3:43 ` kumaraparameshwaran rathinavel
2023-06-14 4:56 ` Hu, Jiayu
0 siblings, 1 reply; 32+ messages in thread
From: kumaraparameshwaran rathinavel @ 2023-06-14 3:43 UTC (permalink / raw)
To: Hu, Jiayu; +Cc: dev
[-- Attachment #1: Type: text/plain, Size: 16616 bytes --]
On Tue, Jun 13, 2023 at 7:56 AM Hu, Jiayu <jiayu.hu@intel.com> wrote:
> Hi Kumara,
>
> Overall, the patch looks good to me. But you need to update the doc and
> there
> are some minor changes needed to be made. Please see replies inline.
>
>> Thanks Jiayu. I see some linking errors for windows. It was a for
>> inesert_new_tcp_item which is now in *.c file. Earlier this was
>> inert_new_item in *.h file. The error is that the insert_new_tcp_item
>> function is undefined. Was there a reason why it was in a header file
>> earlier ? Just wanted to check with you if you had encountered this before.
>> Will try it on a windows setup as well.
>>
>
> After all comments are addressed, you can add "Reviewed-by: Jiayu Hu <
> jiayu.hu@intel.com>".
>
> Thanks,
> Jiayu
>
> > -----Original Message-----
> > From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> > Sent: Monday, June 12, 2023 7:31 PM
> > To: Hu, Jiayu <jiayu.hu@intel.com>
> > Cc: dev@dpdk.org; Kumara Parameshwaran
> > <kumaraparamesh92@gmail.com>
> > Subject: [PATCH v7] gro : ipv6 changes to support GRO for TCP/ipv6
> >
> > The patch adds GRO support for TCP/ipv6 packets. This does not include
> the
> > support for vxlan, udp ipv6 packets.
> >
> > Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> > ---
> > v1:
> > * Changes to support GRO for TCP/ipv6 packets. This does not
> > include
> > vxlan changes.
> > * The GRO is performed only for ipv6 packets that does not contain
> > extension headers.
> > * The logic for the TCP coalescing remains the same, in ipv6 header
> > the source address, destination address, flow label, version
> fields
> > are expected to be the same.
> > * Re-organised the code to reuse certain tcp functions for both
> ipv4
> > and
> > ipv6 flows.
> > v2:
> > * Fix comments in gro_tcp6.h header file.
> >
> > v3:
> > * Adderess review comments to fix code duplication for v4 and v6
> >
> > v4:
> > * Addresses review comments for v3, do not use callbacks
> >
> > v5:
> > * Address review comments
> >
> > v6:
> > * Fix warning and coding style issues
> >
> > v7:
> > * Fix build compilation issue
> > lib/gro/gro_tcp4.c | 178 ++++++-------------------
> > lib/gro/gro_tcp4.h | 170 +----------------------
> > lib/gro/gro_tcp6.c | 267 +++++++++++++++++++++++++++++++++++++
> > lib/gro/gro_tcp6.h | 161 ++++++++++++++++++++++
> > lib/gro/gro_tcp_internal.c | 128 ++++++++++++++++++
> > lib/gro/gro_tcp_internal.h | 212 +++++++++++++++++++++++++++++
> > lib/gro/gro_vxlan_tcp4.c | 23 ++--
> > lib/gro/gro_vxlan_tcp4.h | 3 +-
> > lib/gro/meson.build | 2 +
> > lib/gro/rte_gro.c | 83 ++++++++++--
> > lib/gro/rte_gro.h | 3 +
> > 11 files changed, 897 insertions(+), 333 deletions(-) create mode
> 100644
> > lib/gro/gro_tcp6.c create mode 100644 lib/gro/gro_tcp6.h create mode
> > 100644 lib/gro/gro_tcp_internal.c create mode 100644
> > lib/gro/gro_tcp_internal.h
> >
> > diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c index
> > 0014096e63..42fee78f30 100644
> > --- a/lib/gro/gro_tcp4.c
> > +++ b/lib/gro/gro_tcp4.c
> > -
> > int32_t
> > gro_tcp4_reassemble(struct rte_mbuf *pkt,
> > struct gro_tcp4_tbl *tbl,
> > @@ -202,9 +122,8 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> > uint8_t is_atomic;
> >
> > struct tcp4_flow_key key;
> > - uint32_t cur_idx, prev_idx, item_idx;
> > + uint32_t item_idx;
> > uint32_t i, max_flow_num, remaining_flow_num;
> > - int cmp;
> > uint8_t find;
> >
> > /*
> > @@ -216,7 +135,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> >
> > eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
> > ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
> > - tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
> > + tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *,
> > +pkt->l2_len + pkt->l3_len);
>
> No need to change the method to obtain tcp_hdr.
>
> > hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
> >
> > /*
> > @@ -230,7 +149,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> > ip_tlen = rte_be_to_cpu_16(ipv4_hdr->total_length);
> > if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len))
> > rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen -
> pkt->l2_len);
> > -
> No need to delete the blank line.
>
> > /*
> > * Don't process the packet whose payload length is less than or
> > * equal to 0.
> > @@ -239,6 +157,13 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> > if (tcp_dl <= 0)
> > return -1;
> >
> > + rte_ether_addr_copy(&(eth_hdr->src_addr),
> > &(key.cmn_key.eth_saddr));
> > + rte_ether_addr_copy(&(eth_hdr->dst_addr),
> > &(key.cmn_key.eth_daddr));
> > + key.ip_src_addr = ipv4_hdr->src_addr;
> > + key.ip_dst_addr = ipv4_hdr->dst_addr;
> > + key.cmn_key.src_port = tcp_hdr->src_port;
> > + key.cmn_key.dst_port = tcp_hdr->dst_port;
> > + key.cmn_key.recv_ack = tcp_hdr->recv_ack;
>
> Add a blank line between key.cmn_key.recv_ack assignment and annotation.
>
> > /*
> > * Save IPv4 ID for the packet whose DF bit is 0. For the packet
> > * whose DF bit is 1, IPv4 ID is ignored.
> > @@ -246,15 +171,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> > frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
> > is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) ==
> > RTE_IPV4_HDR_DF_FLAG;
> > ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
> > - sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> > -
> > - rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
> > - rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
> > - key.ip_src_addr = ipv4_hdr->src_addr;
> > - key.ip_dst_addr = ipv4_hdr->dst_addr;
> > - key.src_port = tcp_hdr->src_port;
> > - key.dst_port = tcp_hdr->dst_port;
> > - key.recv_ack = tcp_hdr->recv_ack;
> >
> > /* Search for a matched flow. */
> > max_flow_num = tbl->max_flow_num;
> > @@ -270,63 +186,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> > }
> > }
> >
> > - /*
> > - * Fail to find a matched flow. Insert a new flow and store the
> > - * packet into the flow.
> > - */
> > if (find == 0) {
> > - item_idx = insert_new_item(tbl, pkt, start_time,
> > - INVALID_ARRAY_INDEX, sent_seq, ip_id,
> > - is_atomic);
> > + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> > + item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl-
> > >item_num,
> > + tbl->max_item_num,
> > start_time,
> > + INVALID_ARRAY_INDEX,
> > sent_seq, ip_id,
> > + is_atomic);
> > if (item_idx == INVALID_ARRAY_INDEX)
> > return -1;
> > if (insert_new_flow(tbl, &key, item_idx) ==
> > - INVALID_ARRAY_INDEX) {
> > + INVALID_ARRAY_INDEX) {
> > /*
> > * Fail to insert a new flow, so delete the
> > * stored packet.
> > - */
> > - delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
> > + */
> > + delete_tcp_item(tbl->items, item_idx, &tbl-
> > >item_num,
> > +INVALID_ARRAY_INDEX);
> > return -1;
> > }
> > return 0;
> > }
> >
> > - /*
> > - * Check all packets in the flow and try to find a neighbor for
> > - * the input packet.
> > - */
> > - cur_idx = tbl->flows[i].start_index;
> > - prev_idx = cur_idx;
> > - do {
> > - cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
> > - sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> > - is_atomic);
> > - if (cmp) {
> > - if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
> > - pkt, cmp, sent_seq, ip_id,
> 0))
> > - return 1;
> > - /*
> > - * Fail to merge the two packets, as the packet
> > - * length is greater than the max value. Store
> > - * the packet into the flow.
> > - */
> > - if (insert_new_item(tbl, pkt, start_time, cur_idx,
> > - sent_seq, ip_id,
> is_atomic) ==
> > - INVALID_ARRAY_INDEX)
> > - return -1;
> > - return 0;
> > - }
> > - prev_idx = cur_idx;
> > - cur_idx = tbl->items[cur_idx].next_pkt_idx;
> > - } while (cur_idx != INVALID_ARRAY_INDEX);
> > + return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl-
> > >flows[i].start_index,
> > + &tbl->item_num, tbl-
> > >max_item_num,
> > + ip_id, is_atomic,
> start_time);
> > +}
> >
> > - /* Fail to find a neighbor, so store the packet into the flow. */
> > - if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
> > - ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> > - return -1;
> > +/*
> > + * update the packet length for the flushed packet.
> > + */
> > +static inline void
> > +update_header(struct gro_tcp_item *item) {
> > + struct rte_ipv4_hdr *ipv4_hdr;
> > + struct rte_mbuf *pkt = item->firstseg;
> >
> > - return 0;
> > + ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> > + pkt->l2_len);
> > + ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
> > + pkt->l2_len);
> > }
> >
> > uint16_t
> > @@ -353,7 +250,8 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
> > * Delete the packet and get the next
> > * packet in the flow.
> > */
> > - j = delete_item(tbl, j,
> INVALID_ARRAY_INDEX);
> > + j = delete_tcp_item(tbl->items, j,
> > + &tbl->item_num,
> > INVALID_ARRAY_INDEX);
> > tbl->flows[i].start_index = j;
> > if (j == INVALID_ARRAY_INDEX)
> > tbl->flow_num--;
> > diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h index
> > 212f97a042..c0154afa24 100644
> > --- a/lib/gro/gro_tcp4.h
> > +++ b/lib/gro/gro_tcp4.h
> > @@ -5,32 +5,15 @@
> > #ifndef _GRO_TCP4_H_
> > #define _GRO_TCP4_H_
> >
> > -#include <rte_tcp.h>
> > +#include <gro_tcp_internal.h>
>
> Use "#include "gro_tcp_internal.h"".
>
> >
> > -#define INVALID_ARRAY_INDEX 0xffffffffUL #define
> > GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
> >
> > diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c new file mode 100644
> > index 0000000000..0ea73741c1
> > --- /dev/null
> > +++ b/lib/gro/gro_tcp6.c
> > @@ -0,0 +1,267 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2017 Intel Corporation
> > + */
>
> The license header is incorrect, and please update. Same and similar issue
> happens
> in the new added files, like gro_tcp_internal.h.
>
> > +
> > diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h new file mode 100644
> > index 0000000000..cc02b0004a
> > --- /dev/null
> > +++ b/lib/gro/gro_tcp6.h
> > @@ -0,0 +1,161 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2017 Intel Corporation
> > + */
>
> Ditto.
>
> > +
> > +#ifndef _GRO_TCP6_H_
> > +#define _GRO_TCP6_H_
> > +
> > +#include <gro_tcp_internal.h>
>
> Use "#include "gro_tcp_internal.h"".
>
> > +
> > +#define INVALID_ARRAY_INDEX 0xffffffffUL #define
>
> Duplicate definition for INVALID_ARRAY_INDEX in both gro_tcp6.h
> and gro_tcp_internal.h.
>
> > +GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
> > +
> > +/* Header fields representing a TCP/IPv6 flow */ struct tcp6_flow_key {
> > + struct cmn_tcp_key cmn_key;
> > + uint8_t src_addr[16];
> > + uint8_t dst_addr[16];
> > + rte_be32_t vtc_flow;
> > +};
> > +
> > diff --git a/lib/gro/gro_tcp_internal.c b/lib/gro/gro_tcp_internal.c new
> file
> > mode 100644 index 0000000000..5a21bca7f8
> > --- /dev/null
> > +++ b/lib/gro/gro_tcp_internal.c
> > @@ -0,0 +1,128 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2017 Intel Corporation
> > + */
>
> Update license header.
>
> > +#include <rte_malloc.h>
> > +#include <rte_mbuf.h>
> > +#include <rte_ethdev.h>
> > +
> > +#include "gro_tcp_internal.h"
> > +
> > diff --git a/lib/gro/gro_tcp_internal.h b/lib/gro/gro_tcp_internal.h new
> file
> > mode 100644 index 0000000000..072b7aea13
> > --- /dev/null
> > +++ b/lib/gro/gro_tcp_internal.h
> > @@ -0,0 +1,212 @@
> > +#ifndef _GRO_TCP_H_
> > +#define _GRO_TCP_H_
> > +
>
> Add license header in this file.
>
> > +#define INVALID_ARRAY_INDEX 0xffffffffUL
> > +
> > +#include <rte_tcp.h>
> > +
> > +/*
> > + * The max length of a IPv4 packet, which includes the length of the L3
> > + * header, the L4 header and the data payload.
> > + */
> > +#define MAX_IP_PKT_LENGTH UINT16_MAX
> > +
> > +/* The maximum TCP header length */
> > +#define MAX_TCP_HLEN 60
> > +#define INVALID_TCP_HDRLEN(len) \
> > + (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> > +
> > +/*
> > + * Check if two TCP/IPv4 packets are neighbors.
> > + */
>
> This function is not only for TCP4, so use "Check if two TCP packets are
> neighbors".
>
> > +static inline int
> > +check_seq_option(struct gro_tcp_item *item,
> > + struct rte_tcp_hdr *tcph,
> > + uint32_t sent_seq,
> > + uint16_t ip_id,
> > + uint16_t tcp_hl,
> > + uint16_t tcp_dl,
> > + uint16_t l2_offset,
> > + uint8_t is_atomic)
> > +{
> > + struct rte_mbuf *pkt_orig = item->firstseg;
> > + char *iph_orig;
> > + struct rte_tcp_hdr *tcph_orig;
> > + uint16_t len, tcp_hl_orig;
> > +
> > + iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
> > + l2_offset + pkt_orig->l2_len);
> > + tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
> > + tcp_hl_orig = pkt_orig->l4_len;
> > +
> > + /* Check if TCP option fields equal */
> > + len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> > + if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> > + (memcmp(tcph + 1, tcph_orig + 1,
> > + len) != 0)))
> > + return 0;
> > +
> > + /* Don't merge packets whose DF bits are different */
> > + if (unlikely(item->is_atomic ^ is_atomic))
> > + return 0;
> > +
> > + /* check if the two packets are neighbors */
> > + len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> > + pkt_orig->l3_len - tcp_hl_orig;
> > + if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> > + (ip_id == item->l3.ip_id + 1)))
> > + /* append the new packet */
> > + return 1;
> > + else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> > + (ip_id + item->nb_merged ==
> item->l3.ip_id)))
> > + /* pre-pend the new packet */
> > + return -1;
> > +
> > + return 0;
> > +}
> > +
> > +static inline int
> > +is_common_tcp_key(struct cmn_tcp_key *k1, struct cmn_tcp_key *k2) {
>
> Rename it to "is_same_common_tcp_key()"?
>
> > + return (!memcmp(k1, k2, sizeof(struct cmn_tcp_key))); }
> > +
> > +#endif
>
[-- Attachment #2: Type: text/html, Size: 21330 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread
* RE: [PATCH v7] gro : ipv6 changes to support GRO for TCP/ipv6
2023-06-14 3:43 ` kumaraparameshwaran rathinavel
@ 2023-06-14 4:56 ` Hu, Jiayu
0 siblings, 0 replies; 32+ messages in thread
From: Hu, Jiayu @ 2023-06-14 4:56 UTC (permalink / raw)
To: kumaraparameshwaran rathinavel; +Cc: dev
[-- Attachment #1: Type: text/plain, Size: 16620 bytes --]
From my experience, inline is used with “static” as a compiler hint.
In case of a raw inline, which is used by both insert_new_tcp_item()
and delete_tcp_item() now, it seems you need to provide an external
one to avoid linking error. But I haven’t used it in this way before.
If inline is the root cause of window compiler errors, defining the two
functions as static inline in gro_tcp_internal.h may be a solution.
Thanks,
Jiayu
From: kumaraparameshwaran rathinavel <kumaraparamesh92@gmail.com>
Sent: Wednesday, June 14, 2023 11:43 AM
To: Hu, Jiayu <jiayu.hu@intel.com>
Cc: dev@dpdk.org
Subject: Re: [PATCH v7] gro : ipv6 changes to support GRO for TCP/ipv6
On Tue, Jun 13, 2023 at 7:56 AM Hu, Jiayu <jiayu.hu@intel.com<mailto:jiayu.hu@intel.com>> wrote:
Hi Kumara,
Overall, the patch looks good to me. But you need to update the doc and there
are some minor changes needed to be made. Please see replies inline.
Thanks Jiayu. I see some linking errors for windows. It was a for inesert_new_tcp_item which is now in *.c file. Earlier this was inert_new_item in *.h file. The error is that the insert_new_tcp_item function is undefined. Was there a reason why it was in a header file earlier ? Just wanted to check with you if you had encountered this before. Will try it on a windows setup as well.
After all comments are addressed, you can add "Reviewed-by: Jiayu Hu <jiayu.hu@intel.com<mailto:jiayu.hu@intel.com>>".
Thanks,
Jiayu
> -----Original Message-----
> From: Kumara Parameshwaran <kumaraparamesh92@gmail.com<mailto:kumaraparamesh92@gmail.com>>
> Sent: Monday, June 12, 2023 7:31 PM
> To: Hu, Jiayu <jiayu.hu@intel.com<mailto:jiayu.hu@intel.com>>
> Cc: dev@dpdk.org<mailto:dev@dpdk.org>; Kumara Parameshwaran
> <kumaraparamesh92@gmail.com<mailto:kumaraparamesh92@gmail.com>>
> Subject: [PATCH v7] gro : ipv6 changes to support GRO for TCP/ipv6
>
> The patch adds GRO support for TCP/ipv6 packets. This does not include the
> support for vxlan, udp ipv6 packets.
>
> Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com<mailto:kumaraparamesh92@gmail.com>>
> ---
> v1:
> * Changes to support GRO for TCP/ipv6 packets. This does not
> include
> vxlan changes.
> * The GRO is performed only for ipv6 packets that does not contain
> extension headers.
> * The logic for the TCP coalescing remains the same, in ipv6 header
> the source address, destination address, flow label, version fields
> are expected to be the same.
> * Re-organised the code to reuse certain tcp functions for both ipv4
> and
> ipv6 flows.
> v2:
> * Fix comments in gro_tcp6.h header file.
>
> v3:
> * Adderess review comments to fix code duplication for v4 and v6
>
> v4:
> * Addresses review comments for v3, do not use callbacks
>
> v5:
> * Address review comments
>
> v6:
> * Fix warning and coding style issues
>
> v7:
> * Fix build compilation issue
> lib/gro/gro_tcp4.c | 178 ++++++-------------------
> lib/gro/gro_tcp4.h | 170 +----------------------
> lib/gro/gro_tcp6.c | 267 +++++++++++++++++++++++++++++++++++++
> lib/gro/gro_tcp6.h | 161 ++++++++++++++++++++++
> lib/gro/gro_tcp_internal.c | 128 ++++++++++++++++++
> lib/gro/gro_tcp_internal.h | 212 +++++++++++++++++++++++++++++
> lib/gro/gro_vxlan_tcp4.c | 23 ++--
> lib/gro/gro_vxlan_tcp4.h | 3 +-
> lib/gro/meson.build | 2 +
> lib/gro/rte_gro.c | 83 ++++++++++--
> lib/gro/rte_gro.h | 3 +
> 11 files changed, 897 insertions(+), 333 deletions(-) create mode 100644
> lib/gro/gro_tcp6.c create mode 100644 lib/gro/gro_tcp6.h create mode
> 100644 lib/gro/gro_tcp_internal.c create mode 100644
> lib/gro/gro_tcp_internal.h
>
> diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c index
> 0014096e63..42fee78f30 100644
> --- a/lib/gro/gro_tcp4.c
> +++ b/lib/gro/gro_tcp4.c
> -
> int32_t
> gro_tcp4_reassemble(struct rte_mbuf *pkt,
> struct gro_tcp4_tbl *tbl,
> @@ -202,9 +122,8 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> uint8_t is_atomic;
>
> struct tcp4_flow_key key;
> - uint32_t cur_idx, prev_idx, item_idx;
> + uint32_t item_idx;
> uint32_t i, max_flow_num, remaining_flow_num;
> - int cmp;
> uint8_t find;
>
> /*
> @@ -216,7 +135,7 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
>
> eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
> ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
> - tcp_hdr = (struct rte_tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len);
> + tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *,
> +pkt->l2_len + pkt->l3_len);
No need to change the method to obtain tcp_hdr.
> hdr_len = pkt->l2_len + pkt->l3_len + pkt->l4_len;
>
> /*
> @@ -230,7 +149,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> ip_tlen = rte_be_to_cpu_16(ipv4_hdr->total_length);
> if (pkt->pkt_len > (uint32_t)(ip_tlen + pkt->l2_len))
> rte_pktmbuf_trim(pkt, pkt->pkt_len - ip_tlen - pkt->l2_len);
> -
No need to delete the blank line.
> /*
> * Don't process the packet whose payload length is less than or
> * equal to 0.
> @@ -239,6 +157,13 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> if (tcp_dl <= 0)
> return -1;
>
> + rte_ether_addr_copy(&(eth_hdr->src_addr),
> &(key.cmn_key.eth_saddr));
> + rte_ether_addr_copy(&(eth_hdr->dst_addr),
> &(key.cmn_key.eth_daddr));
> + key.ip_src_addr = ipv4_hdr->src_addr;
> + key.ip_dst_addr = ipv4_hdr->dst_addr;
> + key.cmn_key.src_port = tcp_hdr->src_port;
> + key.cmn_key.dst_port = tcp_hdr->dst_port;
> + key.cmn_key.recv_ack = tcp_hdr->recv_ack;
Add a blank line between key.cmn_key.recv_ack assignment and annotation.
> /*
> * Save IPv4 ID for the packet whose DF bit is 0. For the packet
> * whose DF bit is 1, IPv4 ID is ignored.
> @@ -246,15 +171,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
> is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) ==
> RTE_IPV4_HDR_DF_FLAG;
> ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
> - sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> -
> - rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
> - rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
> - key.ip_src_addr = ipv4_hdr->src_addr;
> - key.ip_dst_addr = ipv4_hdr->dst_addr;
> - key.src_port = tcp_hdr->src_port;
> - key.dst_port = tcp_hdr->dst_port;
> - key.recv_ack = tcp_hdr->recv_ack;
>
> /* Search for a matched flow. */
> max_flow_num = tbl->max_flow_num;
> @@ -270,63 +186,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> }
> }
>
> - /*
> - * Fail to find a matched flow. Insert a new flow and store the
> - * packet into the flow.
> - */
> if (find == 0) {
> - item_idx = insert_new_item(tbl, pkt, start_time,
> - INVALID_ARRAY_INDEX, sent_seq, ip_id,
> - is_atomic);
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> + item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl-
> >item_num,
> + tbl->max_item_num,
> start_time,
> + INVALID_ARRAY_INDEX,
> sent_seq, ip_id,
> + is_atomic);
> if (item_idx == INVALID_ARRAY_INDEX)
> return -1;
> if (insert_new_flow(tbl, &key, item_idx) ==
> - INVALID_ARRAY_INDEX) {
> + INVALID_ARRAY_INDEX) {
> /*
> * Fail to insert a new flow, so delete the
> * stored packet.
> - */
> - delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
> + */
> + delete_tcp_item(tbl->items, item_idx, &tbl-
> >item_num,
> +INVALID_ARRAY_INDEX);
> return -1;
> }
> return 0;
> }
>
> - /*
> - * Check all packets in the flow and try to find a neighbor for
> - * the input packet.
> - */
> - cur_idx = tbl->flows[i].start_index;
> - prev_idx = cur_idx;
> - do {
> - cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
> - sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> - is_atomic);
> - if (cmp) {
> - if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
> - pkt, cmp, sent_seq, ip_id, 0))
> - return 1;
> - /*
> - * Fail to merge the two packets, as the packet
> - * length is greater than the max value. Store
> - * the packet into the flow.
> - */
> - if (insert_new_item(tbl, pkt, start_time, cur_idx,
> - sent_seq, ip_id, is_atomic) ==
> - INVALID_ARRAY_INDEX)
> - return -1;
> - return 0;
> - }
> - prev_idx = cur_idx;
> - cur_idx = tbl->items[cur_idx].next_pkt_idx;
> - } while (cur_idx != INVALID_ARRAY_INDEX);
> + return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl-
> >flows[i].start_index,
> + &tbl->item_num, tbl-
> >max_item_num,
> + ip_id, is_atomic, start_time);
> +}
>
> - /* Fail to find a neighbor, so store the packet into the flow. */
> - if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
> - ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> - return -1;
> +/*
> + * update the packet length for the flushed packet.
> + */
> +static inline void
> +update_header(struct gro_tcp_item *item) {
> + struct rte_ipv4_hdr *ipv4_hdr;
> + struct rte_mbuf *pkt = item->firstseg;
>
> - return 0;
> + ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> + pkt->l2_len);
> + ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
> + pkt->l2_len);
> }
>
> uint16_t
> @@ -353,7 +250,8 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
> * Delete the packet and get the next
> * packet in the flow.
> */
> - j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
> + j = delete_tcp_item(tbl->items, j,
> + &tbl->item_num,
> INVALID_ARRAY_INDEX);
> tbl->flows[i].start_index = j;
> if (j == INVALID_ARRAY_INDEX)
> tbl->flow_num--;
> diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h index
> 212f97a042..c0154afa24 100644
> --- a/lib/gro/gro_tcp4.h
> +++ b/lib/gro/gro_tcp4.h
> @@ -5,32 +5,15 @@
> #ifndef _GRO_TCP4_H_
> #define _GRO_TCP4_H_
>
> -#include <rte_tcp.h>
> +#include <gro_tcp_internal.h>
Use "#include "gro_tcp_internal.h"".
>
> -#define INVALID_ARRAY_INDEX 0xffffffffUL #define
> GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
>
> diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c new file mode 100644
> index 0000000000..0ea73741c1
> --- /dev/null
> +++ b/lib/gro/gro_tcp6.c
> @@ -0,0 +1,267 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
The license header is incorrect, and please update. Same and similar issue happens
in the new added files, like gro_tcp_internal.h.
> +
> diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h new file mode 100644
> index 0000000000..cc02b0004a
> --- /dev/null
> +++ b/lib/gro/gro_tcp6.h
> @@ -0,0 +1,161 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
Ditto.
> +
> +#ifndef _GRO_TCP6_H_
> +#define _GRO_TCP6_H_
> +
> +#include <gro_tcp_internal.h>
Use "#include "gro_tcp_internal.h"".
> +
> +#define INVALID_ARRAY_INDEX 0xffffffffUL #define
Duplicate definition for INVALID_ARRAY_INDEX in both gro_tcp6.h
and gro_tcp_internal.h.
> +GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
> +
> +/* Header fields representing a TCP/IPv6 flow */ struct tcp6_flow_key {
> + struct cmn_tcp_key cmn_key;
> + uint8_t src_addr[16];
> + uint8_t dst_addr[16];
> + rte_be32_t vtc_flow;
> +};
> +
> diff --git a/lib/gro/gro_tcp_internal.c b/lib/gro/gro_tcp_internal.c new file
> mode 100644 index 0000000000..5a21bca7f8
> --- /dev/null
> +++ b/lib/gro/gro_tcp_internal.c
> @@ -0,0 +1,128 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Intel Corporation
> + */
Update license header.
> +#include <rte_malloc.h>
> +#include <rte_mbuf.h>
> +#include <rte_ethdev.h>
> +
> +#include "gro_tcp_internal.h"
> +
> diff --git a/lib/gro/gro_tcp_internal.h b/lib/gro/gro_tcp_internal.h new file
> mode 100644 index 0000000000..072b7aea13
> --- /dev/null
> +++ b/lib/gro/gro_tcp_internal.h
> @@ -0,0 +1,212 @@
> +#ifndef _GRO_TCP_H_
> +#define _GRO_TCP_H_
> +
Add license header in this file.
> +#define INVALID_ARRAY_INDEX 0xffffffffUL
> +
> +#include <rte_tcp.h>
> +
> +/*
> + * The max length of a IPv4 packet, which includes the length of the L3
> + * header, the L4 header and the data payload.
> + */
> +#define MAX_IP_PKT_LENGTH UINT16_MAX
> +
> +/* The maximum TCP header length */
> +#define MAX_TCP_HLEN 60
> +#define INVALID_TCP_HDRLEN(len) \
> + (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> +
> +/*
> + * Check if two TCP/IPv4 packets are neighbors.
> + */
This function is not only for TCP4, so use "Check if two TCP packets are neighbors".
> +static inline int
> +check_seq_option(struct gro_tcp_item *item,
> + struct rte_tcp_hdr *tcph,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint16_t tcp_hl,
> + uint16_t tcp_dl,
> + uint16_t l2_offset,
> + uint8_t is_atomic)
> +{
> + struct rte_mbuf *pkt_orig = item->firstseg;
> + char *iph_orig;
> + struct rte_tcp_hdr *tcph_orig;
> + uint16_t len, tcp_hl_orig;
> +
> + iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
> + l2_offset + pkt_orig->l2_len);
> + tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
> + tcp_hl_orig = pkt_orig->l4_len;
> +
> + /* Check if TCP option fields equal */
> + len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> + if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> + (memcmp(tcph + 1, tcph_orig + 1,
> + len) != 0)))
> + return 0;
> +
> + /* Don't merge packets whose DF bits are different */
> + if (unlikely(item->is_atomic ^ is_atomic))
> + return 0;
> +
> + /* check if the two packets are neighbors */
> + len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> + pkt_orig->l3_len - tcp_hl_orig;
> + if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> + (ip_id == item->l3.ip_id + 1)))
> + /* append the new packet */
> + return 1;
> + else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> + (ip_id + item->nb_merged == item->l3.ip_id)))
> + /* pre-pend the new packet */
> + return -1;
> +
> + return 0;
> +}
> +
> +static inline int
> +is_common_tcp_key(struct cmn_tcp_key *k1, struct cmn_tcp_key *k2) {
Rename it to "is_same_common_tcp_key()"?
> + return (!memcmp(k1, k2, sizeof(struct cmn_tcp_key))); }
> +
> +#endif
[-- Attachment #2: Type: text/html, Size: 30227 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread
* [PATCH v8] gro : ipv6 changes to support GRO for TCP/ipv6
2022-10-20 18:14 ` [PATCH v2] " Kumara Parameshwaran
` (6 preceding siblings ...)
2023-06-12 11:31 ` [PATCH v7] " Kumara Parameshwaran
@ 2023-06-15 5:40 ` Kumara Parameshwaran
2023-06-15 6:20 ` [PATCH v9] " Kumara Parameshwaran
` (2 subsequent siblings)
10 siblings, 0 replies; 32+ messages in thread
From: Kumara Parameshwaran @ 2023-06-15 5:40 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev, Kumara Parameshwaran
The patch adds GRO support for TCP/ipv6 packets. This does not
include the support for vxlan, udp ipv6 packets.
Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
---
v1:
* Changes to support GRO for TCP/ipv6 packets. This does not include
vxlan changes.
* The GRO is performed only for ipv6 packets that does not contain
extension headers.
* The logic for the TCP coalescing remains the same, in ipv6 header
the source address, destination address, flow label, version fields
are expected to be the same.
* Re-organised the code to reuse certain tcp functions for both ipv4 and
ipv6 flows.
v2:
* Fix comments in gro_tcp6.h header file.
v3:
* Adderess review comments to fix code duplication for v4 and v6
v4:
* Addresses review comments for v3, do not use callbacks
v5:
* Address review comments
v6:
* Fix warning and coding style issues
v7:
* Fix build compilation issue
v8:
* Use gro_tcp_internal.h for functions used for gro_tcp4 and gro_tcp6 and use
gro_tcp.h for data structures and functions used across gro_vxlan_tcp4
.../generic_receive_offload_lib.rst | 21 +-
doc/guides/rel_notes/release_23_07.rst | 2 +
lib/gro/gro_tcp4.c | 177 +++---------
lib/gro/gro_tcp4.h | 170 +----------
lib/gro/gro_tcp6.c | 268 ++++++++++++++++++
lib/gro/gro_tcp6.h | 160 +++++++++++
lib/gro/gro_tcp_internal.h | 129 +++++++++
lib/gro/gro_vxlan_tcp4.c | 22 +-
lib/gro/gro_vxlan_tcp4.h | 2 +-
lib/gro/meson.build | 1 +
lib/gro/rte_gro.c | 83 +++++-
lib/gro/rte_gro.h | 3 +
12 files changed, 697 insertions(+), 341 deletions(-)
create mode 100644 lib/gro/gro_tcp6.c
create mode 100644 lib/gro/gro_tcp6.h
create mode 100644 lib/gro/gro_tcp_internal.h
diff --git a/doc/guides/prog_guide/generic_receive_offload_lib.rst b/doc/guides/prog_guide/generic_receive_offload_lib.rst
index 98a5d29bbc..533cda7f5c 100644
--- a/doc/guides/prog_guide/generic_receive_offload_lib.rst
+++ b/doc/guides/prog_guide/generic_receive_offload_lib.rst
@@ -138,20 +138,21 @@ The key-based algorithm has two characters:
Key-based Reassembly Algorithm
-TCP/IPv4 GRO
-------------
+TCP-IPv4/IPv6 GRO
+-----------------
-The table structure used by TCP/IPv4 GRO contains two arrays: flow array
+The table structure used by TCP-IPv4/IPv6 GRO contains two arrays: flow array
and item array. The flow array keeps flow information, and the item array
-keeps packet information.
+keeps packet information. The flow array is different for IPv4 and IPv6 while
+the item array is the same.
-Header fields used to define a TCP/IPv4 flow include:
+Header fields used to define a TCP-IPv4/IPv6 flow include:
-- source and destination: Ethernet and IP address, TCP port
+- common tcp key fields : Ethernet address, TCP port, TCP acknowledge number
+- version specific IP address
+- IPv6 flow label for IPv6 flow
-- TCP acknowledge number
-
-TCP/IPv4 packets whose FIN, SYN, RST, URG, PSH, ECE or CWR bit is set
+TCP packets whose FIN, SYN, RST, URG, PSH, ECE or CWR bit is set
won't be processed.
Header fields deciding if two packets are neighbors include:
@@ -159,7 +160,7 @@ Header fields deciding if two packets are neighbors include:
- TCP sequence number
- IPv4 ID. The IPv4 ID fields of the packets, whose DF bit is 0, should
- be increased by 1.
+ be increased by 1. This is applicable only for IPv4
VxLAN GRO
---------
diff --git a/doc/guides/rel_notes/release_23_07.rst b/doc/guides/rel_notes/release_23_07.rst
index a9b1293689..3930590d94 100644
--- a/doc/guides/rel_notes/release_23_07.rst
+++ b/doc/guides/rel_notes/release_23_07.rst
@@ -55,6 +55,8 @@ New Features
Also, make sure to start the actual text at the margin.
=======================================================
+* **Added support for TCP/IPv6 GRO.**
+ Enhanced the existing library to support GRO for TCP packets over IPv6 network.
Removed Items
-------------
diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c
index 0014096e63..6645de592b 100644
--- a/lib/gro/gro_tcp4.c
+++ b/lib/gro/gro_tcp4.c
@@ -7,6 +7,7 @@
#include <rte_ethdev.h>
#include "gro_tcp4.h"
+#include "gro_tcp_internal.h"
void *
gro_tcp4_tbl_create(uint16_t socket_id,
@@ -30,7 +31,7 @@ gro_tcp4_tbl_create(uint16_t socket_id,
if (tbl == NULL)
return NULL;
- size = sizeof(struct gro_tcp4_item) * entries_num;
+ size = sizeof(struct gro_tcp_item) * entries_num;
tbl->items = rte_zmalloc_socket(__func__,
size,
RTE_CACHE_LINE_SIZE,
@@ -71,18 +72,6 @@ gro_tcp4_tbl_destroy(void *tbl)
rte_free(tcp_tbl);
}
-static inline uint32_t
-find_an_empty_item(struct gro_tcp4_tbl *tbl)
-{
- uint32_t i;
- uint32_t max_item_num = tbl->max_item_num;
-
- for (i = 0; i < max_item_num; i++)
- if (tbl->items[i].firstseg == NULL)
- return i;
- return INVALID_ARRAY_INDEX;
-}
-
static inline uint32_t
find_an_empty_flow(struct gro_tcp4_tbl *tbl)
{
@@ -95,56 +84,6 @@ find_an_empty_flow(struct gro_tcp4_tbl *tbl)
return INVALID_ARRAY_INDEX;
}
-static inline uint32_t
-insert_new_item(struct gro_tcp4_tbl *tbl,
- struct rte_mbuf *pkt,
- uint64_t start_time,
- uint32_t prev_idx,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint8_t is_atomic)
-{
- uint32_t item_idx;
-
- item_idx = find_an_empty_item(tbl);
- if (item_idx == INVALID_ARRAY_INDEX)
- return INVALID_ARRAY_INDEX;
-
- tbl->items[item_idx].firstseg = pkt;
- tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
- tbl->items[item_idx].start_time = start_time;
- tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
- tbl->items[item_idx].sent_seq = sent_seq;
- tbl->items[item_idx].ip_id = ip_id;
- tbl->items[item_idx].nb_merged = 1;
- tbl->items[item_idx].is_atomic = is_atomic;
- tbl->item_num++;
-
- /* if the previous packet exists, chain them together. */
- if (prev_idx != INVALID_ARRAY_INDEX) {
- tbl->items[item_idx].next_pkt_idx =
- tbl->items[prev_idx].next_pkt_idx;
- tbl->items[prev_idx].next_pkt_idx = item_idx;
- }
-
- return item_idx;
-}
-
-static inline uint32_t
-delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
- uint32_t prev_item_idx)
-{
- uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
-
- /* NULL indicates an empty item */
- tbl->items[item_idx].firstseg = NULL;
- tbl->item_num--;
- if (prev_item_idx != INVALID_ARRAY_INDEX)
- tbl->items[prev_item_idx].next_pkt_idx = next_idx;
-
- return next_idx;
-}
-
static inline uint32_t
insert_new_flow(struct gro_tcp4_tbl *tbl,
struct tcp4_flow_key *src,
@@ -159,13 +98,10 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
- rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
+ ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
+
dst->ip_src_addr = src->ip_src_addr;
dst->ip_dst_addr = src->ip_dst_addr;
- dst->recv_ack = src->recv_ack;
- dst->src_port = src->src_port;
- dst->dst_port = src->dst_port;
tbl->flows[flow_idx].start_index = item_idx;
tbl->flow_num++;
@@ -173,21 +109,6 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
return flow_idx;
}
-/*
- * update the packet length for the flushed packet.
- */
-static inline void
-update_header(struct gro_tcp4_item *item)
-{
- struct rte_ipv4_hdr *ipv4_hdr;
- struct rte_mbuf *pkt = item->firstseg;
-
- ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
- pkt->l2_len);
- ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
- pkt->l2_len);
-}
-
int32_t
gro_tcp4_reassemble(struct rte_mbuf *pkt,
struct gro_tcp4_tbl *tbl,
@@ -202,9 +123,8 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
uint8_t is_atomic;
struct tcp4_flow_key key;
- uint32_t cur_idx, prev_idx, item_idx;
+ uint32_t item_idx;
uint32_t i, max_flow_num, remaining_flow_num;
- int cmp;
uint8_t find;
/*
@@ -239,6 +159,14 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
if (tcp_dl <= 0)
return -1;
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.cmn_key.eth_daddr));
+ key.ip_src_addr = ipv4_hdr->src_addr;
+ key.ip_dst_addr = ipv4_hdr->dst_addr;
+ key.cmn_key.src_port = tcp_hdr->src_port;
+ key.cmn_key.dst_port = tcp_hdr->dst_port;
+ key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+
/*
* Save IPv4 ID for the packet whose DF bit is 0. For the packet
* whose DF bit is 1, IPv4 ID is ignored.
@@ -246,15 +174,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) == RTE_IPV4_HDR_DF_FLAG;
ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
- sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
-
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
- key.ip_src_addr = ipv4_hdr->src_addr;
- key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.src_port = tcp_hdr->src_port;
- key.dst_port = tcp_hdr->dst_port;
- key.recv_ack = tcp_hdr->recv_ack;
/* Search for a matched flow. */
max_flow_num = tbl->max_flow_num;
@@ -270,63 +189,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
}
}
- /*
- * Fail to find a matched flow. Insert a new flow and store the
- * packet into the flow.
- */
if (find == 0) {
- item_idx = insert_new_item(tbl, pkt, start_time,
- INVALID_ARRAY_INDEX, sent_seq, ip_id,
- is_atomic);
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num,
+ tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, ip_id,
+ is_atomic);
if (item_idx == INVALID_ARRAY_INDEX)
return -1;
if (insert_new_flow(tbl, &key, item_idx) ==
- INVALID_ARRAY_INDEX) {
+ INVALID_ARRAY_INDEX) {
/*
* Fail to insert a new flow, so delete the
* stored packet.
- */
- delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
return -1;
}
return 0;
}
- /*
- * Check all packets in the flow and try to find a neighbor for
- * the input packet.
- */
- cur_idx = tbl->flows[i].start_index;
- prev_idx = cur_idx;
- do {
- cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
- sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
- is_atomic);
- if (cmp) {
- if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
- pkt, cmp, sent_seq, ip_id, 0))
- return 1;
- /*
- * Fail to merge the two packets, as the packet
- * length is greater than the max value. Store
- * the packet into the flow.
- */
- if (insert_new_item(tbl, pkt, start_time, cur_idx,
- sent_seq, ip_id, is_atomic) ==
- INVALID_ARRAY_INDEX)
- return -1;
- return 0;
- }
- prev_idx = cur_idx;
- cur_idx = tbl->items[cur_idx].next_pkt_idx;
- } while (cur_idx != INVALID_ARRAY_INDEX);
+ return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl->flows[i].start_index,
+ &tbl->item_num, tbl->max_item_num,
+ ip_id, is_atomic, start_time);
+}
- /* Fail to find a neighbor, so store the packet into the flow. */
- if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
- ip_id, is_atomic) == INVALID_ARRAY_INDEX)
- return -1;
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
- return 0;
+ ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len);
}
uint16_t
@@ -353,7 +253,8 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
* Delete the packet and get the next
* packet in the flow.
*/
- j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
+ j = delete_tcp_item(tbl->items, j,
+ &tbl->item_num, INVALID_ARRAY_INDEX);
tbl->flows[i].start_index = j;
if (j == INVALID_ARRAY_INDEX)
tbl->flow_num--;
diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h
index 212f97a042..245e5da486 100644
--- a/lib/gro/gro_tcp4.h
+++ b/lib/gro/gro_tcp4.h
@@ -5,32 +5,15 @@
#ifndef _GRO_TCP4_H_
#define _GRO_TCP4_H_
-#include <rte_tcp.h>
+#include "gro_tcp.h"
-#define INVALID_ARRAY_INDEX 0xffffffffUL
#define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
-/*
- * The max length of a IPv4 packet, which includes the length of the L3
- * header, the L4 header and the data payload.
- */
-#define MAX_IPV4_PKT_LENGTH UINT16_MAX
-
-/* The maximum TCP header length */
-#define MAX_TCP_HLEN 60
-#define INVALID_TCP_HDRLEN(len) \
- (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
-
-/* Header fields representing a TCP/IPv4 flow */
+/* Header fields representing common fields in TCP flow */
struct tcp4_flow_key {
- struct rte_ether_addr eth_saddr;
- struct rte_ether_addr eth_daddr;
+ struct cmn_tcp_key cmn_key;
uint32_t ip_src_addr;
uint32_t ip_dst_addr;
-
- uint32_t recv_ack;
- uint16_t src_port;
- uint16_t dst_port;
};
struct gro_tcp4_flow {
@@ -42,42 +25,12 @@ struct gro_tcp4_flow {
uint32_t start_index;
};
-struct gro_tcp4_item {
- /*
- * The first MBUF segment of the packet. If the value
- * is NULL, it means the item is empty.
- */
- struct rte_mbuf *firstseg;
- /* The last MBUF segment of the packet */
- struct rte_mbuf *lastseg;
- /*
- * The time when the first packet is inserted into the table.
- * This value won't be updated, even if the packet is merged
- * with other packets.
- */
- uint64_t start_time;
- /*
- * next_pkt_idx is used to chain the packets that
- * are in the same flow but can't be merged together
- * (e.g. caused by packet reordering).
- */
- uint32_t next_pkt_idx;
- /* TCP sequence number of the packet */
- uint32_t sent_seq;
- /* IPv4 ID of the packet */
- uint16_t ip_id;
- /* the number of merged packets */
- uint16_t nb_merged;
- /* Indicate if IPv4 ID can be ignored */
- uint8_t is_atomic;
-};
-
/*
* TCP/IPv4 reassembly table structure.
*/
struct gro_tcp4_tbl {
/* item array */
- struct gro_tcp4_item *items;
+ struct gro_tcp_item *items;
/* flow array */
struct gro_tcp4_flow *flows;
/* current item number */
@@ -186,120 +139,9 @@ uint32_t gro_tcp4_tbl_pkt_count(void *tbl);
static inline int
is_same_tcp4_flow(struct tcp4_flow_key k1, struct tcp4_flow_key k2)
{
- return (rte_is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) &&
- rte_is_same_ether_addr(&k1.eth_daddr, &k2.eth_daddr) &&
- (k1.ip_src_addr == k2.ip_src_addr) &&
+ return ((k1.ip_src_addr == k2.ip_src_addr) &&
(k1.ip_dst_addr == k2.ip_dst_addr) &&
- (k1.recv_ack == k2.recv_ack) &&
- (k1.src_port == k2.src_port) &&
- (k1.dst_port == k2.dst_port));
+ is_same_common_tcp_key(&k1.cmn_key, &k2.cmn_key));
}
-/*
- * Merge two TCP/IPv4 packets without updating checksums.
- * If cmp is larger than 0, append the new packet to the
- * original packet. Otherwise, pre-pend the new packet to
- * the original packet.
- */
-static inline int
-merge_two_tcp4_packets(struct gro_tcp4_item *item,
- struct rte_mbuf *pkt,
- int cmp,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t l2_offset)
-{
- struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
- uint16_t hdr_len, l2_len;
-
- if (cmp > 0) {
- pkt_head = item->firstseg;
- pkt_tail = pkt;
- } else {
- pkt_head = pkt;
- pkt_tail = item->firstseg;
- }
-
- /* check if the IPv4 packet length is greater than the max value */
- hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
- pkt_head->l4_len;
- l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
- if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
- hdr_len > MAX_IPV4_PKT_LENGTH))
- return 0;
-
- /* remove the packet header for the tail packet */
- rte_pktmbuf_adj(pkt_tail, hdr_len);
-
- /* chain two packets together */
- if (cmp > 0) {
- item->lastseg->next = pkt;
- item->lastseg = rte_pktmbuf_lastseg(pkt);
- /* update IP ID to the larger value */
- item->ip_id = ip_id;
- } else {
- lastseg = rte_pktmbuf_lastseg(pkt);
- lastseg->next = item->firstseg;
- item->firstseg = pkt;
- /* update sent_seq to the smaller value */
- item->sent_seq = sent_seq;
- item->ip_id = ip_id;
- }
- item->nb_merged++;
-
- /* update MBUF metadata for the merged packet */
- pkt_head->nb_segs += pkt_tail->nb_segs;
- pkt_head->pkt_len += pkt_tail->pkt_len;
-
- return 1;
-}
-
-/*
- * Check if two TCP/IPv4 packets are neighbors.
- */
-static inline int
-check_seq_option(struct gro_tcp4_item *item,
- struct rte_tcp_hdr *tcph,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t tcp_hl,
- uint16_t tcp_dl,
- uint16_t l2_offset,
- uint8_t is_atomic)
-{
- struct rte_mbuf *pkt_orig = item->firstseg;
- struct rte_ipv4_hdr *iph_orig;
- struct rte_tcp_hdr *tcph_orig;
- uint16_t len, tcp_hl_orig;
-
- iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char *) +
- l2_offset + pkt_orig->l2_len);
- tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig + pkt_orig->l3_len);
- tcp_hl_orig = pkt_orig->l4_len;
-
- /* Check if TCP option fields equal */
- len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
- if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
- (memcmp(tcph + 1, tcph_orig + 1,
- len) != 0)))
- return 0;
-
- /* Don't merge packets whose DF bits are different */
- if (unlikely(item->is_atomic ^ is_atomic))
- return 0;
-
- /* check if the two packets are neighbors */
- len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
- pkt_orig->l3_len - tcp_hl_orig;
- if ((sent_seq == item->sent_seq + len) && (is_atomic ||
- (ip_id == item->ip_id + 1)))
- /* append the new packet */
- return 1;
- else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
- (ip_id + item->nb_merged == item->ip_id)))
- /* pre-pend the new packet */
- return -1;
-
- return 0;
-}
#endif
diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c
new file mode 100644
index 0000000000..5aa39801e1
--- /dev/null
+++ b/lib/gro/gro_tcp6.c
@@ -0,0 +1,268 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp6.h"
+#include "gro_tcp_internal.h"
+
+void *
+gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow)
+{
+ struct gro_tcp6_tbl *tbl;
+ size_t size;
+ uint32_t entries_num, i;
+
+ entries_num = max_flow_num * max_item_per_flow;
+ entries_num = RTE_MIN(entries_num, GRO_TCP6_TBL_MAX_ITEM_NUM);
+
+ if (entries_num == 0)
+ return NULL;
+
+ tbl = rte_zmalloc_socket(__func__,
+ sizeof(struct gro_tcp6_tbl),
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl == NULL)
+ return NULL;
+
+ size = sizeof(struct gro_tcp_item) * entries_num;
+ tbl->items = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->items == NULL) {
+ rte_free(tbl);
+ return NULL;
+ }
+ tbl->max_item_num = entries_num;
+
+ size = sizeof(struct gro_tcp6_flow) * entries_num;
+ tbl->flows = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->flows == NULL) {
+ rte_free(tbl->items);
+ rte_free(tbl);
+ return NULL;
+ }
+ /* INVALID_ARRAY_INDEX indicates an empty flow */
+ for (i = 0; i < entries_num; i++)
+ tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
+ tbl->max_flow_num = entries_num;
+
+ return tbl;
+}
+
+void
+gro_tcp6_tbl_destroy(void *tbl)
+{
+ struct gro_tcp6_tbl *tcp_tbl = tbl;
+
+ if (tcp_tbl) {
+ rte_free(tcp_tbl->items);
+ rte_free(tcp_tbl->flows);
+ }
+ rte_free(tcp_tbl);
+}
+
+static inline uint32_t
+find_an_empty_flow(struct gro_tcp6_tbl *tbl)
+{
+ uint32_t i;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++)
+ if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+insert_new_flow(struct gro_tcp6_tbl *tbl,
+ struct tcp6_flow_key *src,
+ uint32_t item_idx)
+{
+ struct tcp6_flow_key *dst;
+ uint32_t flow_idx;
+
+ flow_idx = find_an_empty_flow(tbl);
+ if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
+ return INVALID_ARRAY_INDEX;
+
+ dst = &(tbl->flows[flow_idx].key);
+
+ ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
+ memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst->src_addr));
+ memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst->dst_addr));
+ dst->vtc_flow = src->vtc_flow;
+
+ tbl->flows[flow_idx].start_index = item_idx;
+ tbl->flow_num++;
+
+ return flow_idx;
+}
+
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
+
+ ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len - pkt->l3_len);
+}
+
+int32_t
+gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time)
+{
+ struct rte_ether_hdr *eth_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ int32_t tcp_dl;
+ uint16_t ip_tlen;
+ struct tcp6_flow_key key;
+ uint32_t i, max_flow_num, remaining_flow_num;
+ uint32_t sent_seq;
+ struct rte_tcp_hdr *tcp_hdr;
+ uint8_t find;
+ uint32_t item_idx;
+ /*
+ * Don't process the packet whose TCP header length is greater
+ * than 60 bytes or less than 20 bytes.
+ */
+ if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
+ return -1;
+
+ eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
+ ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
+ tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt->l2_len + pkt->l3_len);
+
+ /*
+ * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
+ * or CWR set.
+ */
+ if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
+ return -1;
+
+ ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
+ /*
+ * Don't process the packet whose payload length is less than or
+ * equal to 0.
+ */
+ tcp_dl = ip_tlen - pkt->l4_len;
+ if (tcp_dl <= 0)
+ return -1;
+
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.cmn_key.eth_daddr));
+ memcpy(&key.src_addr[0], &ipv6_hdr->src_addr, sizeof(key.src_addr));
+ memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr, sizeof(key.dst_addr));
+ key.cmn_key.src_port = tcp_hdr->src_port;
+ key.cmn_key.dst_port = tcp_hdr->dst_port;
+ key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+ key.vtc_flow = ipv6_hdr->vtc_flow;
+
+ /* Search for a matched flow. */
+ max_flow_num = tbl->max_flow_num;
+ remaining_flow_num = tbl->flow_num;
+ find = 0;
+ for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
+ if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
+ if (is_same_tcp6_flow(&tbl->flows[i].key, &key)) {
+ find = 1;
+ break;
+ }
+ remaining_flow_num--;
+ }
+ }
+
+ if (find == 0) {
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num,
+ tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, 0, true);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return -1;
+ if (insert_new_flow(tbl, &key, item_idx) ==
+ INVALID_ARRAY_INDEX) {
+ /*
+ * Fail to insert a new flow, so delete the
+ * stored packet.
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
+ return -1;
+ }
+ return 0;
+ }
+
+ return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl->flows[i].start_index,
+ &tbl->item_num, tbl->max_item_num,
+ 0, true, start_time);
+}
+
+uint16_t
+gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out)
+{
+ uint16_t k = 0;
+ uint32_t i, j;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++) {
+ if (unlikely(tbl->flow_num == 0))
+ return k;
+
+ j = tbl->flows[i].start_index;
+ while (j != INVALID_ARRAY_INDEX) {
+ if (tbl->items[j].start_time <= flush_timestamp) {
+ out[k++] = tbl->items[j].firstseg;
+ if (tbl->items[j].nb_merged > 1)
+ update_header(&(tbl->items[j]));
+ /*
+ * Delete the packet and get the next
+ * packet in the flow.
+ */
+ j = delete_tcp_item(tbl->items, j,
+ &tbl->item_num, INVALID_ARRAY_INDEX);
+ tbl->flows[i].start_index = j;
+ if (j == INVALID_ARRAY_INDEX)
+ tbl->flow_num--;
+
+ if (unlikely(k == nb_out))
+ return k;
+ } else
+ /*
+ * The left packets in this flow won't be
+ * timeout. Go to check other flows.
+ */
+ break;
+ }
+ }
+ return k;
+}
+
+uint32_t
+gro_tcp6_tbl_pkt_count(void *tbl)
+{
+ struct gro_tcp6_tbl *gro_tbl = tbl;
+
+ if (gro_tbl)
+ return gro_tbl->item_num;
+
+ return 0;
+}
diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h
new file mode 100644
index 0000000000..073122f0ec
--- /dev/null
+++ b/lib/gro/gro_tcp6.h
@@ -0,0 +1,160 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+
+#ifndef _GRO_TCP6_H_
+#define _GRO_TCP6_H_
+
+#include "gro_tcp.h"
+
+#define GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
+
+/* Header fields representing a TCP/IPv6 flow */
+struct tcp6_flow_key {
+ struct cmn_tcp_key cmn_key;
+ uint8_t src_addr[16];
+ uint8_t dst_addr[16];
+ rte_be32_t vtc_flow;
+};
+
+struct gro_tcp6_flow {
+ struct tcp6_flow_key key;
+ /*
+ * The index of the first packet in the flow.
+ * INVALID_ARRAY_INDEX indicates an empty flow.
+ */
+ uint32_t start_index;
+};
+
+/*
+ * TCP/IPv6 reassembly table structure.
+ */
+struct gro_tcp6_tbl {
+ /* item array */
+ struct gro_tcp_item *items;
+ /* flow array */
+ struct gro_tcp6_flow *flows;
+ /* current item number */
+ uint32_t item_num;
+ /* current flow num */
+ uint32_t flow_num;
+ /* item array size */
+ uint32_t max_item_num;
+ /* flow array size */
+ uint32_t max_flow_num;
+};
+
+/**
+ * This function creates a TCP/IPv6 reassembly table.
+ *
+ * @param socket_id
+ * Socket index for allocating the TCP/IPv6 reassemble table
+ * @param max_flow_num
+ * The maximum number of flows in the TCP/IPv6 GRO table
+ * @param max_item_per_flow
+ * The maximum number of packets per flow
+ *
+ * @return
+ * - Return the table pointer on success.
+ * - Return NULL on failure.
+ */
+void *gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow);
+
+/**
+ * This function destroys a TCP/IPv6 reassembly table.
+ *
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table.
+ */
+void gro_tcp6_tbl_destroy(void *tbl);
+
+/**
+ * This function merges a TCP/IPv6 packet. It doesn't process the packet,
+ * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't have
+ * payload.
+ *
+ * This function doesn't check if the packet has correct checksums and
+ * doesn't re-calculate checksums for the merged packet. Additionally,
+ * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
+ * when IP fragmentation is possible (i.e., DF==0). It returns the
+ * packet, if the packet has invalid parameters (e.g. SYN bit is set)
+ * or there is no available space in the table.
+ *
+ * @param pkt
+ * Packet to reassemble
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table
+ * @start_time
+ * The time when the packet is inserted into the table
+ *
+ * @return
+ * - Return a positive value if the packet is merged.
+ * - Return zero if the packet isn't merged but stored in the table.
+ * - Return a negative value for invalid parameters or no available
+ * space in the table.
+ */
+int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time);
+
+/**
+ * This function flushes timeout packets in a TCP/IPv6 reassembly table,
+ * and without updating checksums.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ * @param flush_timestamp
+ * Flush packets which are inserted into the table before or at the
+ * flush_timestamp.
+ * @param out
+ * Pointer array used to keep flushed packets
+ * @param nb_out
+ * The element number in 'out'. It also determines the maximum number of
+ * packets that can be flushed finally.
+ *
+ * @return
+ * The number of flushed packets
+ */
+uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out);
+
+/**
+ * This function returns the number of the packets in a TCP/IPv6
+ * reassembly table.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ *
+ * @return
+ * The number of packets in the table
+ */
+uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
+
+/*
+ * Check if two TCP/IPv6 packets belong to the same flow.
+ */
+static inline int
+is_same_tcp6_flow(struct tcp6_flow_key *k1, struct tcp6_flow_key *k2)
+{
+ rte_be32_t vtc_flow_diff;
+
+ if (memcmp(&k1->src_addr, &k2->src_addr, 16))
+ return 0;
+ if (memcmp(&k1->dst_addr, &k2->dst_addr, 16))
+ return 0;
+ /*
+ * IP version (4) Traffic Class (8) Flow Label (20)
+ * All fields except Traffic class should be same
+ */
+ vtc_flow_diff = (k1->vtc_flow ^ k2->vtc_flow);
+ if (vtc_flow_diff & htonl(0xF00FFFFF))
+ return 0;
+
+ return is_same_common_tcp_key(&k1->cmn_key, &k2->cmn_key);
+}
+
+#endif
diff --git a/lib/gro/gro_tcp_internal.h b/lib/gro/gro_tcp_internal.h
new file mode 100644
index 0000000000..e38ffeda57
--- /dev/null
+++ b/lib/gro/gro_tcp_internal.h
@@ -0,0 +1,129 @@
+
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+
+#ifndef _GRO_TCP_INTERNAL_H_
+#define _GRO_TCP_INTERNAL_H_
+
+static inline uint32_t
+find_an_empty_item(struct gro_tcp_item *items,
+ uint32_t max_item_num)
+{
+ uint32_t i;
+
+ for (i = 0; i < max_item_num; i++)
+ if (items[i].firstseg == NULL)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic)
+{
+ uint32_t item_idx;
+
+ item_idx = find_an_empty_item(items, max_item_num);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return INVALID_ARRAY_INDEX;
+
+ items[item_idx].firstseg = pkt;
+ items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
+ items[item_idx].start_time = start_time;
+ items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
+ items[item_idx].sent_seq = sent_seq;
+ items[item_idx].l3.ip_id = ip_id;
+ items[item_idx].nb_merged = 1;
+ items[item_idx].is_atomic = is_atomic;
+ (*item_num) += 1;
+
+ /* if the previous packet exists, chain them together. */
+ if (prev_idx != INVALID_ARRAY_INDEX) {
+ items[item_idx].next_pkt_idx =
+ items[prev_idx].next_pkt_idx;
+ items[prev_idx].next_pkt_idx = item_idx;
+ }
+
+ return item_idx;
+}
+
+static inline uint32_t
+delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx)
+{
+ uint32_t next_idx = items[item_idx].next_pkt_idx;
+
+ /* NULL indicates an empty item */
+ items[item_idx].firstseg = NULL;
+ (*item_num) -= 1;
+ if (prev_item_idx != INVALID_ARRAY_INDEX)
+ items[prev_item_idx].next_pkt_idx = next_idx;
+
+ return next_idx;
+}
+
+static inline int32_t
+process_tcp_item(struct rte_mbuf *pkt,
+ struct rte_tcp_hdr *tcp_hdr,
+ int32_t tcp_dl,
+ struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time)
+{
+ uint32_t cur_idx;
+ uint32_t prev_idx;
+ int cmp;
+ uint32_t sent_seq;
+
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ /*
+ * Check all packets in the flow and try to find a neighbor for
+ * the input packet.
+ */
+ cur_idx = item_idx;
+ prev_idx = cur_idx;
+ do {
+ cmp = check_seq_option(&items[cur_idx], tcp_hdr,
+ sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
+ is_atomic);
+ if (cmp) {
+ if (merge_two_tcp_packets(&items[cur_idx],
+ pkt, cmp, sent_seq, ip_id, 0))
+ return 1;
+ /*
+ * Fail to merge the two packets, as the packet
+ * length is greater than the max value. Store
+ * the packet into the flow.
+ */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num,
+ start_time, cur_idx, sent_seq, ip_id, is_atomic) ==
+ INVALID_ARRAY_INDEX)
+ return -1;
+ return 0;
+ }
+ prev_idx = cur_idx;
+ cur_idx = items[cur_idx].next_pkt_idx;
+ } while (cur_idx != INVALID_ARRAY_INDEX);
+
+ /* Fail to find a neighbor, so store the packet into the flow. */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num, start_time, prev_idx, sent_seq,
+ ip_id, is_atomic) == INVALID_ARRAY_INDEX)
+ return -1;
+
+ return 0;
+}
+
+#endif
\ No newline at end of file
diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
index 3be4deb7c7..6ab7001922 100644
--- a/lib/gro/gro_vxlan_tcp4.c
+++ b/lib/gro/gro_vxlan_tcp4.c
@@ -116,7 +116,7 @@ insert_new_item(struct gro_vxlan_tcp4_tbl *tbl,
tbl->items[item_idx].inner_item.start_time = start_time;
tbl->items[item_idx].inner_item.next_pkt_idx = INVALID_ARRAY_INDEX;
tbl->items[item_idx].inner_item.sent_seq = sent_seq;
- tbl->items[item_idx].inner_item.ip_id = ip_id;
+ tbl->items[item_idx].inner_item.l3.ip_id = ip_id;
tbl->items[item_idx].inner_item.nb_merged = 1;
tbl->items[item_idx].inner_item.is_atomic = is_atomic;
tbl->items[item_idx].outer_ip_id = outer_ip_id;
@@ -163,15 +163,9 @@ insert_new_flow(struct gro_vxlan_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->inner_key.eth_saddr),
- &(dst->inner_key.eth_saddr));
- rte_ether_addr_copy(&(src->inner_key.eth_daddr),
- &(dst->inner_key.eth_daddr));
+ ASSIGN_COMMON_TCP_KEY((&(src->inner_key.cmn_key)), (&(dst->inner_key.cmn_key)));
dst->inner_key.ip_src_addr = src->inner_key.ip_src_addr;
dst->inner_key.ip_dst_addr = src->inner_key.ip_dst_addr;
- dst->inner_key.recv_ack = src->inner_key.recv_ack;
- dst->inner_key.src_port = src->inner_key.src_port;
- dst->inner_key.dst_port = src->inner_key.dst_port;
dst->vxlan_hdr.vx_flags = src->vxlan_hdr.vx_flags;
dst->vxlan_hdr.vx_vni = src->vxlan_hdr.vx_vni;
@@ -248,7 +242,7 @@ merge_two_vxlan_tcp4_packets(struct gro_vxlan_tcp4_item *item,
uint16_t outer_ip_id,
uint16_t ip_id)
{
- if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp, sent_seq,
+ if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
ip_id, pkt->outer_l2_len +
pkt->outer_l3_len)) {
/* Update the outer IPv4 ID to the large value. */
@@ -357,13 +351,13 @@ gro_vxlan_tcp4_reassemble(struct rte_mbuf *pkt,
sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.eth_daddr));
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.cmn_key.eth_daddr));
key.inner_key.ip_src_addr = ipv4_hdr->src_addr;
key.inner_key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.inner_key.recv_ack = tcp_hdr->recv_ack;
- key.inner_key.src_port = tcp_hdr->src_port;
- key.inner_key.dst_port = tcp_hdr->dst_port;
+ key.inner_key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+ key.inner_key.cmn_key.src_port = tcp_hdr->src_port;
+ key.inner_key.cmn_key.dst_port = tcp_hdr->dst_port;
key.vxlan_hdr.vx_flags = vxlan_hdr->vx_flags;
key.vxlan_hdr.vx_vni = vxlan_hdr->vx_vni;
diff --git a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h
index 7832942a68..662db01a88 100644
--- a/lib/gro/gro_vxlan_tcp4.h
+++ b/lib/gro/gro_vxlan_tcp4.h
@@ -36,7 +36,7 @@ struct gro_vxlan_tcp4_flow {
};
struct gro_vxlan_tcp4_item {
- struct gro_tcp4_item inner_item;
+ struct gro_tcp_item inner_item;
/* IPv4 ID in the outer IPv4 header */
uint16_t outer_ip_id;
/* Indicate if outer IPv4 ID can be ignored */
diff --git a/lib/gro/meson.build b/lib/gro/meson.build
index e4fa2958bd..dbce05220d 100644
--- a/lib/gro/meson.build
+++ b/lib/gro/meson.build
@@ -4,6 +4,7 @@
sources = files(
'rte_gro.c',
'gro_tcp4.c',
+ 'gro_tcp6.c',
'gro_udp4.c',
'gro_vxlan_tcp4.c',
'gro_vxlan_udp4.c',
diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c
index e35399fd42..d824eebd93 100644
--- a/lib/gro/rte_gro.c
+++ b/lib/gro/rte_gro.c
@@ -8,6 +8,7 @@
#include "rte_gro.h"
#include "gro_tcp4.h"
+#include "gro_tcp6.h"
#include "gro_udp4.h"
#include "gro_vxlan_tcp4.h"
#include "gro_vxlan_udp4.h"
@@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
- gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
+ gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, gro_tcp6_tbl_create, NULL};
static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
+ gro_tcp6_tbl_destroy,
NULL};
static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count,
+ gro_tcp6_tbl_pkt_count,
NULL};
#define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
@@ -35,6 +38,12 @@ static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+/* GRO with extension headers is not supported */
+#define IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
+ ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
+ ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
+ (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+
#define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
@@ -147,7 +156,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
/* allocate a reassembly table for TCP/IPv4 GRO */
struct gro_tcp4_tbl tcp_tbl;
struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
- struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+ struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+
+ struct gro_tcp6_tbl tcp6_tbl;
+ struct gro_tcp6_flow tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
+ struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
/* allocate a reassembly table for UDP/IPv4 GRO */
struct gro_udp4_tbl udp_tbl;
@@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
int32_t ret;
uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
- do_vxlan_udp_gro = 0;
+ do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
do_udp4_gro = 1;
}
+ if (param->gro_types & RTE_GRO_TCP_IPV6) {
+ for (i = 0; i < item_num; i++)
+ tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+ tcp6_tbl.flows = tcp6_flows;
+ tcp6_tbl.items = tcp6_items;
+ tcp6_tbl.flow_num = 0;
+ tcp6_tbl.item_num = 0;
+ tcp6_tbl.max_flow_num = item_num;
+ tcp6_tbl.max_item_num = item_num;
+ do_tcp6_gro = 1;
+ }
for (i = 0; i < nb_pkts; i++) {
/*
@@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
nb_after_gro--;
else if (ret < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
+ if (ret > 0)
+ /* merge successfully */
+ nb_after_gro--;
+ else if (ret < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
if ((nb_after_gro < nb_pkts)
|| (unprocess_num < nb_pkts)) {
i = 0;
+ /* Copy unprocessed packets */
+ if (unprocess_num > 0) {
+ memcpy(&pkts[i], unprocess_pkts,
+ sizeof(struct rte_mbuf *) *
+ unprocess_num);
+ i = unprocess_num;
+ }
+
/* Flush all packets from the tables */
if (do_vxlan_tcp_gro) {
- i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
+ i += gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
0, pkts, nb_pkts);
}
@@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
&pkts[i], nb_pkts - i);
}
- /* Copy unprocessed packets */
- if (unprocess_num > 0) {
- memcpy(&pkts[i], unprocess_pkts,
- sizeof(struct rte_mbuf *) *
- unprocess_num);
+
+ if (do_tcp6_gro) {
+ i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
+ &pkts[i], nb_pkts - i);
}
- nb_after_gro = i + unprocess_num;
}
return nb_after_gro;
@@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
{
struct rte_mbuf *unprocess_pkts[nb_pkts];
struct gro_ctx *gro_ctx = ctx;
- void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
+ void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
uint64_t current_time;
uint16_t i, unprocess_num = 0;
- uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro;
+ uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro, do_tcp6_gro;
if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
+ tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
RTE_GRO_TCP_IPV4;
@@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
RTE_GRO_UDP_IPV4;
do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
RTE_GRO_IPV4_VXLAN_UDP_IPV4;
+ do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) == RTE_GRO_TCP_IPV6;
current_time = rte_rdtsc();
@@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
if (gro_udp4_reassemble(pkts[i], udp_tbl,
current_time) < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
+ current_time) < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
flush_timestamp,
&out[num], left_nb_out);
+ left_nb_out = max_nb_out - num;
+ }
+
+ if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
+ num += gro_tcp6_tbl_timeout_flush(
+ gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
+ flush_timestamp,
+ &out[num], left_nb_out);
+
}
return num;
diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h
index 9f9ed4935a..c83dfd9ad1 100644
--- a/lib/gro/rte_gro.h
+++ b/lib/gro/rte_gro.h
@@ -38,6 +38,9 @@ extern "C" {
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL << RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
/**< VxLAN UDP/IPv4 GRO flag. */
+#define RTE_GRO_TCP_IPV6_INDEX 4
+#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX)
+/**< TCP/IPv6 GRO flag. */
/**
* Structure used to create GRO context objects or used to pass
--
2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* [PATCH v9] gro : ipv6 changes to support GRO for TCP/ipv6
2022-10-20 18:14 ` [PATCH v2] " Kumara Parameshwaran
` (7 preceding siblings ...)
2023-06-15 5:40 ` [PATCH v8] " Kumara Parameshwaran
@ 2023-06-15 6:20 ` Kumara Parameshwaran
2023-06-15 6:30 ` kumaraparameshwaran rathinavel
` (2 more replies)
2023-06-21 8:25 ` [PATCH v10 1/2] gro : refactor IPv4 to add GRO support for IPv6 Kumara Parameshwaran
2023-06-21 8:38 ` [PATCH v11 1/2] gro : refactor IPv4 to add GRO support for IPv6 Kumara Parameshwaran
10 siblings, 3 replies; 32+ messages in thread
From: Kumara Parameshwaran @ 2023-06-15 6:20 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev, Kumara Parameshwaran
The patch adds GRO support for TCP/ipv6 packets. This does not
include the support for vxlan, udp ipv6 packets.
Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
---
v1:
* Changes to support GRO for TCP/ipv6 packets. This does not include
vxlan changes.
* The GRO is performed only for ipv6 packets that does not contain
extension headers.
* The logic for the TCP coalescing remains the same, in ipv6 header
the source address, destination address, flow label, version fields
are expected to be the same.
* Re-organised the code to reuse certain tcp functions for both ipv4 and
ipv6 flows.
v2:
* Fix comments in gro_tcp6.h header file.
v3:
* Adderess review comments to fix code duplication for v4 and v6
v4:
* Addresses review comments for v3, do not use callbacks
v5:
* Address review comments
v6:
* Fix warning and coding style issues
v7:
* Fix build compilation issue
v8:
* Use gro_tcp_internal.h for functions used for gro_tcp4 and gro_tcp6 and use
gro_tcp.h for data structures and functions used across gro_vxlan_tcp4
v9:
* Resolve merge conflict and add gro_tcp.h in proper path
.../generic_receive_offload_lib.rst | 21 +-
doc/guides/rel_notes/release_23_07.rst | 2 +
lib/gro/gro_tcp.h | 187 ++++++++++++
lib/gro/gro_tcp4.c | 177 +++---------
lib/gro/gro_tcp4.h | 170 +----------
lib/gro/gro_tcp6.c | 268 ++++++++++++++++++
lib/gro/gro_tcp6.h | 160 +++++++++++
lib/gro/gro_tcp_internal.h | 128 +++++++++
lib/gro/gro_vxlan_tcp4.c | 22 +-
lib/gro/gro_vxlan_tcp4.h | 2 +-
lib/gro/meson.build | 1 +
lib/gro/rte_gro.c | 83 +++++-
lib/gro/rte_gro.h | 3 +
13 files changed, 883 insertions(+), 341 deletions(-)
create mode 100644 lib/gro/gro_tcp.h
create mode 100644 lib/gro/gro_tcp6.c
create mode 100644 lib/gro/gro_tcp6.h
create mode 100644 lib/gro/gro_tcp_internal.h
diff --git a/doc/guides/prog_guide/generic_receive_offload_lib.rst b/doc/guides/prog_guide/generic_receive_offload_lib.rst
index 98a5d29bbc..533cda7f5c 100644
--- a/doc/guides/prog_guide/generic_receive_offload_lib.rst
+++ b/doc/guides/prog_guide/generic_receive_offload_lib.rst
@@ -138,20 +138,21 @@ The key-based algorithm has two characters:
Key-based Reassembly Algorithm
-TCP/IPv4 GRO
-------------
+TCP-IPv4/IPv6 GRO
+-----------------
-The table structure used by TCP/IPv4 GRO contains two arrays: flow array
+The table structure used by TCP-IPv4/IPv6 GRO contains two arrays: flow array
and item array. The flow array keeps flow information, and the item array
-keeps packet information.
+keeps packet information. The flow array is different for IPv4 and IPv6 while
+the item array is the same.
-Header fields used to define a TCP/IPv4 flow include:
+Header fields used to define a TCP-IPv4/IPv6 flow include:
-- source and destination: Ethernet and IP address, TCP port
+- common tcp key fields : Ethernet address, TCP port, TCP acknowledge number
+- version specific IP address
+- IPv6 flow label for IPv6 flow
-- TCP acknowledge number
-
-TCP/IPv4 packets whose FIN, SYN, RST, URG, PSH, ECE or CWR bit is set
+TCP packets whose FIN, SYN, RST, URG, PSH, ECE or CWR bit is set
won't be processed.
Header fields deciding if two packets are neighbors include:
@@ -159,7 +160,7 @@ Header fields deciding if two packets are neighbors include:
- TCP sequence number
- IPv4 ID. The IPv4 ID fields of the packets, whose DF bit is 0, should
- be increased by 1.
+ be increased by 1. This is applicable only for IPv4
VxLAN GRO
---------
diff --git a/doc/guides/rel_notes/release_23_07.rst b/doc/guides/rel_notes/release_23_07.rst
index 027ae7bd2d..7124cf45c7 100644
--- a/doc/guides/rel_notes/release_23_07.rst
+++ b/doc/guides/rel_notes/release_23_07.rst
@@ -170,6 +170,8 @@ New Features
See :doc:`../prog_guide/pdcp_lib` for more information.
+* **Added support for TCP/IPv6 GRO.**
+ * Enhanced the existing library to support GRO for TCP packets over IPv6 network.
Removed Items
-------------
diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h
new file mode 100644
index 0000000000..7c7a6a3008
--- /dev/null
+++ b/lib/gro/gro_tcp.h
@@ -0,0 +1,187 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+#ifndef _GRO_TCP_H_
+#define _GRO_TCP_H_
+
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+
+#include <rte_tcp.h>
+
+/*
+ * The max length of a IPv4 packet, which includes the length of the L3
+ * header, the L4 header and the data payload.
+ */
+#define MAX_IP_PKT_LENGTH UINT16_MAX
+
+/* The maximum TCP header length */
+#define MAX_TCP_HLEN 60
+#define INVALID_TCP_HDRLEN(len) \
+ (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
+
+struct cmn_tcp_key {
+ struct rte_ether_addr eth_saddr;
+ struct rte_ether_addr eth_daddr;
+ uint32_t recv_ack;
+ uint16_t src_port;
+ uint16_t dst_port;
+};
+
+#define ASSIGN_COMMON_TCP_KEY(k1, k2) \
+ do {\
+ rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
+ rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr)); \
+ k2->recv_ack = k1->recv_ack; \
+ k2->src_port = k1->src_port; \
+ k2->dst_port = k1->dst_port; \
+ } while (0)
+
+struct gro_tcp_item {
+ /*
+ * The first MBUF segment of the packet. If the value
+ * is NULL, it means the item is empty.
+ */
+ struct rte_mbuf *firstseg;
+ /* The last MBUF segment of the packet */
+ struct rte_mbuf *lastseg;
+ /*
+ * The time when the first packet is inserted into the table.
+ * This value won't be updated, even if the packet is merged
+ * with other packets.
+ */
+ uint64_t start_time;
+ /*
+ * next_pkt_idx is used to chain the packets that
+ * are in the same flow but can't be merged together
+ * (e.g. caused by packet reordering).
+ */
+ uint32_t next_pkt_idx;
+ /* TCP sequence number of the packet */
+ uint32_t sent_seq;
+ union {
+ /* IPv4 ID of the packet */
+ uint16_t ip_id;
+ /* Unused field for IPv6 */
+ uint16_t unused;
+ } l3;
+ /* the number of merged packets */
+ uint16_t nb_merged;
+ /* Indicate if IPv4 ID can be ignored */
+ uint8_t is_atomic;
+};
+
+/*
+ * Merge two TCP packets without updating checksums.
+ * If cmp is larger than 0, append the new packet to the
+ * original packet. Otherwise, pre-pend the new packet to
+ * the original packet.
+ */
+static inline int
+merge_two_tcp_packets(struct gro_tcp_item *item,
+ struct rte_mbuf *pkt,
+ int cmp,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t l2_offset)
+{
+ struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
+ uint16_t hdr_len, l2_len;
+
+ if (cmp > 0) {
+ pkt_head = item->firstseg;
+ pkt_tail = pkt;
+ } else {
+ pkt_head = pkt;
+ pkt_tail = item->firstseg;
+ }
+
+ /* check if the IPv4 packet length is greater than the max value */
+ hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
+ pkt_head->l4_len;
+ l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
+ if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
+ hdr_len > MAX_IP_PKT_LENGTH))
+ return 0;
+
+ /* remove the packet header for the tail packet */
+ rte_pktmbuf_adj(pkt_tail, hdr_len);
+
+ /* chain two packets together */
+ if (cmp > 0) {
+ item->lastseg->next = pkt;
+ item->lastseg = rte_pktmbuf_lastseg(pkt);
+ /* update IP ID to the larger value */
+ item->l3.ip_id = ip_id;
+ } else {
+ lastseg = rte_pktmbuf_lastseg(pkt);
+ lastseg->next = item->firstseg;
+ item->firstseg = pkt;
+ /* update sent_seq to the smaller value */
+ item->sent_seq = sent_seq;
+ item->l3.ip_id = ip_id;
+ }
+ item->nb_merged++;
+
+ /* update MBUF metadata for the merged packet */
+ pkt_head->nb_segs += pkt_tail->nb_segs;
+ pkt_head->pkt_len += pkt_tail->pkt_len;
+
+ return 1;
+}
+
+/*
+ * Check if two TCP packets are neighbors.
+ */
+static inline int
+check_seq_option(struct gro_tcp_item *item,
+ struct rte_tcp_hdr *tcph,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t tcp_hl,
+ uint16_t tcp_dl,
+ uint16_t l2_offset,
+ uint8_t is_atomic)
+{
+ struct rte_mbuf *pkt_orig = item->firstseg;
+ char *iph_orig;
+ struct rte_tcp_hdr *tcph_orig;
+ uint16_t len, tcp_hl_orig;
+
+ iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
+ l2_offset + pkt_orig->l2_len);
+ tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
+ tcp_hl_orig = pkt_orig->l4_len;
+
+ /* Check if TCP option fields equal */
+ len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
+ if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
+ (memcmp(tcph + 1, tcph_orig + 1,
+ len) != 0)))
+ return 0;
+
+ /* Don't merge packets whose DF bits are different */
+ if (unlikely(item->is_atomic ^ is_atomic))
+ return 0;
+
+ /* check if the two packets are neighbors */
+ len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
+ pkt_orig->l3_len - tcp_hl_orig;
+ if ((sent_seq == item->sent_seq + len) && (is_atomic ||
+ (ip_id == item->l3.ip_id + 1)))
+ /* append the new packet */
+ return 1;
+ else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
+ (ip_id + item->nb_merged == item->l3.ip_id)))
+ /* pre-pend the new packet */
+ return -1;
+
+ return 0;
+}
+
+static inline int
+is_same_common_tcp_key(struct cmn_tcp_key *k1, struct cmn_tcp_key *k2)
+{
+ return (!memcmp(k1, k2, sizeof(struct cmn_tcp_key)));
+}
+
+#endif
diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c
index 0014096e63..6645de592b 100644
--- a/lib/gro/gro_tcp4.c
+++ b/lib/gro/gro_tcp4.c
@@ -7,6 +7,7 @@
#include <rte_ethdev.h>
#include "gro_tcp4.h"
+#include "gro_tcp_internal.h"
void *
gro_tcp4_tbl_create(uint16_t socket_id,
@@ -30,7 +31,7 @@ gro_tcp4_tbl_create(uint16_t socket_id,
if (tbl == NULL)
return NULL;
- size = sizeof(struct gro_tcp4_item) * entries_num;
+ size = sizeof(struct gro_tcp_item) * entries_num;
tbl->items = rte_zmalloc_socket(__func__,
size,
RTE_CACHE_LINE_SIZE,
@@ -71,18 +72,6 @@ gro_tcp4_tbl_destroy(void *tbl)
rte_free(tcp_tbl);
}
-static inline uint32_t
-find_an_empty_item(struct gro_tcp4_tbl *tbl)
-{
- uint32_t i;
- uint32_t max_item_num = tbl->max_item_num;
-
- for (i = 0; i < max_item_num; i++)
- if (tbl->items[i].firstseg == NULL)
- return i;
- return INVALID_ARRAY_INDEX;
-}
-
static inline uint32_t
find_an_empty_flow(struct gro_tcp4_tbl *tbl)
{
@@ -95,56 +84,6 @@ find_an_empty_flow(struct gro_tcp4_tbl *tbl)
return INVALID_ARRAY_INDEX;
}
-static inline uint32_t
-insert_new_item(struct gro_tcp4_tbl *tbl,
- struct rte_mbuf *pkt,
- uint64_t start_time,
- uint32_t prev_idx,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint8_t is_atomic)
-{
- uint32_t item_idx;
-
- item_idx = find_an_empty_item(tbl);
- if (item_idx == INVALID_ARRAY_INDEX)
- return INVALID_ARRAY_INDEX;
-
- tbl->items[item_idx].firstseg = pkt;
- tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
- tbl->items[item_idx].start_time = start_time;
- tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
- tbl->items[item_idx].sent_seq = sent_seq;
- tbl->items[item_idx].ip_id = ip_id;
- tbl->items[item_idx].nb_merged = 1;
- tbl->items[item_idx].is_atomic = is_atomic;
- tbl->item_num++;
-
- /* if the previous packet exists, chain them together. */
- if (prev_idx != INVALID_ARRAY_INDEX) {
- tbl->items[item_idx].next_pkt_idx =
- tbl->items[prev_idx].next_pkt_idx;
- tbl->items[prev_idx].next_pkt_idx = item_idx;
- }
-
- return item_idx;
-}
-
-static inline uint32_t
-delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
- uint32_t prev_item_idx)
-{
- uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
-
- /* NULL indicates an empty item */
- tbl->items[item_idx].firstseg = NULL;
- tbl->item_num--;
- if (prev_item_idx != INVALID_ARRAY_INDEX)
- tbl->items[prev_item_idx].next_pkt_idx = next_idx;
-
- return next_idx;
-}
-
static inline uint32_t
insert_new_flow(struct gro_tcp4_tbl *tbl,
struct tcp4_flow_key *src,
@@ -159,13 +98,10 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
- rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
+ ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
+
dst->ip_src_addr = src->ip_src_addr;
dst->ip_dst_addr = src->ip_dst_addr;
- dst->recv_ack = src->recv_ack;
- dst->src_port = src->src_port;
- dst->dst_port = src->dst_port;
tbl->flows[flow_idx].start_index = item_idx;
tbl->flow_num++;
@@ -173,21 +109,6 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
return flow_idx;
}
-/*
- * update the packet length for the flushed packet.
- */
-static inline void
-update_header(struct gro_tcp4_item *item)
-{
- struct rte_ipv4_hdr *ipv4_hdr;
- struct rte_mbuf *pkt = item->firstseg;
-
- ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
- pkt->l2_len);
- ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
- pkt->l2_len);
-}
-
int32_t
gro_tcp4_reassemble(struct rte_mbuf *pkt,
struct gro_tcp4_tbl *tbl,
@@ -202,9 +123,8 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
uint8_t is_atomic;
struct tcp4_flow_key key;
- uint32_t cur_idx, prev_idx, item_idx;
+ uint32_t item_idx;
uint32_t i, max_flow_num, remaining_flow_num;
- int cmp;
uint8_t find;
/*
@@ -239,6 +159,14 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
if (tcp_dl <= 0)
return -1;
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.cmn_key.eth_daddr));
+ key.ip_src_addr = ipv4_hdr->src_addr;
+ key.ip_dst_addr = ipv4_hdr->dst_addr;
+ key.cmn_key.src_port = tcp_hdr->src_port;
+ key.cmn_key.dst_port = tcp_hdr->dst_port;
+ key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+
/*
* Save IPv4 ID for the packet whose DF bit is 0. For the packet
* whose DF bit is 1, IPv4 ID is ignored.
@@ -246,15 +174,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) == RTE_IPV4_HDR_DF_FLAG;
ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
- sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
-
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
- key.ip_src_addr = ipv4_hdr->src_addr;
- key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.src_port = tcp_hdr->src_port;
- key.dst_port = tcp_hdr->dst_port;
- key.recv_ack = tcp_hdr->recv_ack;
/* Search for a matched flow. */
max_flow_num = tbl->max_flow_num;
@@ -270,63 +189,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
}
}
- /*
- * Fail to find a matched flow. Insert a new flow and store the
- * packet into the flow.
- */
if (find == 0) {
- item_idx = insert_new_item(tbl, pkt, start_time,
- INVALID_ARRAY_INDEX, sent_seq, ip_id,
- is_atomic);
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num,
+ tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, ip_id,
+ is_atomic);
if (item_idx == INVALID_ARRAY_INDEX)
return -1;
if (insert_new_flow(tbl, &key, item_idx) ==
- INVALID_ARRAY_INDEX) {
+ INVALID_ARRAY_INDEX) {
/*
* Fail to insert a new flow, so delete the
* stored packet.
- */
- delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
return -1;
}
return 0;
}
- /*
- * Check all packets in the flow and try to find a neighbor for
- * the input packet.
- */
- cur_idx = tbl->flows[i].start_index;
- prev_idx = cur_idx;
- do {
- cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
- sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
- is_atomic);
- if (cmp) {
- if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
- pkt, cmp, sent_seq, ip_id, 0))
- return 1;
- /*
- * Fail to merge the two packets, as the packet
- * length is greater than the max value. Store
- * the packet into the flow.
- */
- if (insert_new_item(tbl, pkt, start_time, cur_idx,
- sent_seq, ip_id, is_atomic) ==
- INVALID_ARRAY_INDEX)
- return -1;
- return 0;
- }
- prev_idx = cur_idx;
- cur_idx = tbl->items[cur_idx].next_pkt_idx;
- } while (cur_idx != INVALID_ARRAY_INDEX);
+ return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl->flows[i].start_index,
+ &tbl->item_num, tbl->max_item_num,
+ ip_id, is_atomic, start_time);
+}
- /* Fail to find a neighbor, so store the packet into the flow. */
- if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
- ip_id, is_atomic) == INVALID_ARRAY_INDEX)
- return -1;
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
- return 0;
+ ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len);
}
uint16_t
@@ -353,7 +253,8 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
* Delete the packet and get the next
* packet in the flow.
*/
- j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
+ j = delete_tcp_item(tbl->items, j,
+ &tbl->item_num, INVALID_ARRAY_INDEX);
tbl->flows[i].start_index = j;
if (j == INVALID_ARRAY_INDEX)
tbl->flow_num--;
diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h
index 212f97a042..245e5da486 100644
--- a/lib/gro/gro_tcp4.h
+++ b/lib/gro/gro_tcp4.h
@@ -5,32 +5,15 @@
#ifndef _GRO_TCP4_H_
#define _GRO_TCP4_H_
-#include <rte_tcp.h>
+#include "gro_tcp.h"
-#define INVALID_ARRAY_INDEX 0xffffffffUL
#define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
-/*
- * The max length of a IPv4 packet, which includes the length of the L3
- * header, the L4 header and the data payload.
- */
-#define MAX_IPV4_PKT_LENGTH UINT16_MAX
-
-/* The maximum TCP header length */
-#define MAX_TCP_HLEN 60
-#define INVALID_TCP_HDRLEN(len) \
- (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
-
-/* Header fields representing a TCP/IPv4 flow */
+/* Header fields representing common fields in TCP flow */
struct tcp4_flow_key {
- struct rte_ether_addr eth_saddr;
- struct rte_ether_addr eth_daddr;
+ struct cmn_tcp_key cmn_key;
uint32_t ip_src_addr;
uint32_t ip_dst_addr;
-
- uint32_t recv_ack;
- uint16_t src_port;
- uint16_t dst_port;
};
struct gro_tcp4_flow {
@@ -42,42 +25,12 @@ struct gro_tcp4_flow {
uint32_t start_index;
};
-struct gro_tcp4_item {
- /*
- * The first MBUF segment of the packet. If the value
- * is NULL, it means the item is empty.
- */
- struct rte_mbuf *firstseg;
- /* The last MBUF segment of the packet */
- struct rte_mbuf *lastseg;
- /*
- * The time when the first packet is inserted into the table.
- * This value won't be updated, even if the packet is merged
- * with other packets.
- */
- uint64_t start_time;
- /*
- * next_pkt_idx is used to chain the packets that
- * are in the same flow but can't be merged together
- * (e.g. caused by packet reordering).
- */
- uint32_t next_pkt_idx;
- /* TCP sequence number of the packet */
- uint32_t sent_seq;
- /* IPv4 ID of the packet */
- uint16_t ip_id;
- /* the number of merged packets */
- uint16_t nb_merged;
- /* Indicate if IPv4 ID can be ignored */
- uint8_t is_atomic;
-};
-
/*
* TCP/IPv4 reassembly table structure.
*/
struct gro_tcp4_tbl {
/* item array */
- struct gro_tcp4_item *items;
+ struct gro_tcp_item *items;
/* flow array */
struct gro_tcp4_flow *flows;
/* current item number */
@@ -186,120 +139,9 @@ uint32_t gro_tcp4_tbl_pkt_count(void *tbl);
static inline int
is_same_tcp4_flow(struct tcp4_flow_key k1, struct tcp4_flow_key k2)
{
- return (rte_is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) &&
- rte_is_same_ether_addr(&k1.eth_daddr, &k2.eth_daddr) &&
- (k1.ip_src_addr == k2.ip_src_addr) &&
+ return ((k1.ip_src_addr == k2.ip_src_addr) &&
(k1.ip_dst_addr == k2.ip_dst_addr) &&
- (k1.recv_ack == k2.recv_ack) &&
- (k1.src_port == k2.src_port) &&
- (k1.dst_port == k2.dst_port));
+ is_same_common_tcp_key(&k1.cmn_key, &k2.cmn_key));
}
-/*
- * Merge two TCP/IPv4 packets without updating checksums.
- * If cmp is larger than 0, append the new packet to the
- * original packet. Otherwise, pre-pend the new packet to
- * the original packet.
- */
-static inline int
-merge_two_tcp4_packets(struct gro_tcp4_item *item,
- struct rte_mbuf *pkt,
- int cmp,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t l2_offset)
-{
- struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
- uint16_t hdr_len, l2_len;
-
- if (cmp > 0) {
- pkt_head = item->firstseg;
- pkt_tail = pkt;
- } else {
- pkt_head = pkt;
- pkt_tail = item->firstseg;
- }
-
- /* check if the IPv4 packet length is greater than the max value */
- hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
- pkt_head->l4_len;
- l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
- if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
- hdr_len > MAX_IPV4_PKT_LENGTH))
- return 0;
-
- /* remove the packet header for the tail packet */
- rte_pktmbuf_adj(pkt_tail, hdr_len);
-
- /* chain two packets together */
- if (cmp > 0) {
- item->lastseg->next = pkt;
- item->lastseg = rte_pktmbuf_lastseg(pkt);
- /* update IP ID to the larger value */
- item->ip_id = ip_id;
- } else {
- lastseg = rte_pktmbuf_lastseg(pkt);
- lastseg->next = item->firstseg;
- item->firstseg = pkt;
- /* update sent_seq to the smaller value */
- item->sent_seq = sent_seq;
- item->ip_id = ip_id;
- }
- item->nb_merged++;
-
- /* update MBUF metadata for the merged packet */
- pkt_head->nb_segs += pkt_tail->nb_segs;
- pkt_head->pkt_len += pkt_tail->pkt_len;
-
- return 1;
-}
-
-/*
- * Check if two TCP/IPv4 packets are neighbors.
- */
-static inline int
-check_seq_option(struct gro_tcp4_item *item,
- struct rte_tcp_hdr *tcph,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t tcp_hl,
- uint16_t tcp_dl,
- uint16_t l2_offset,
- uint8_t is_atomic)
-{
- struct rte_mbuf *pkt_orig = item->firstseg;
- struct rte_ipv4_hdr *iph_orig;
- struct rte_tcp_hdr *tcph_orig;
- uint16_t len, tcp_hl_orig;
-
- iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char *) +
- l2_offset + pkt_orig->l2_len);
- tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig + pkt_orig->l3_len);
- tcp_hl_orig = pkt_orig->l4_len;
-
- /* Check if TCP option fields equal */
- len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
- if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
- (memcmp(tcph + 1, tcph_orig + 1,
- len) != 0)))
- return 0;
-
- /* Don't merge packets whose DF bits are different */
- if (unlikely(item->is_atomic ^ is_atomic))
- return 0;
-
- /* check if the two packets are neighbors */
- len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
- pkt_orig->l3_len - tcp_hl_orig;
- if ((sent_seq == item->sent_seq + len) && (is_atomic ||
- (ip_id == item->ip_id + 1)))
- /* append the new packet */
- return 1;
- else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
- (ip_id + item->nb_merged == item->ip_id)))
- /* pre-pend the new packet */
- return -1;
-
- return 0;
-}
#endif
diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c
new file mode 100644
index 0000000000..5aa39801e1
--- /dev/null
+++ b/lib/gro/gro_tcp6.c
@@ -0,0 +1,268 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp6.h"
+#include "gro_tcp_internal.h"
+
+void *
+gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow)
+{
+ struct gro_tcp6_tbl *tbl;
+ size_t size;
+ uint32_t entries_num, i;
+
+ entries_num = max_flow_num * max_item_per_flow;
+ entries_num = RTE_MIN(entries_num, GRO_TCP6_TBL_MAX_ITEM_NUM);
+
+ if (entries_num == 0)
+ return NULL;
+
+ tbl = rte_zmalloc_socket(__func__,
+ sizeof(struct gro_tcp6_tbl),
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl == NULL)
+ return NULL;
+
+ size = sizeof(struct gro_tcp_item) * entries_num;
+ tbl->items = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->items == NULL) {
+ rte_free(tbl);
+ return NULL;
+ }
+ tbl->max_item_num = entries_num;
+
+ size = sizeof(struct gro_tcp6_flow) * entries_num;
+ tbl->flows = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->flows == NULL) {
+ rte_free(tbl->items);
+ rte_free(tbl);
+ return NULL;
+ }
+ /* INVALID_ARRAY_INDEX indicates an empty flow */
+ for (i = 0; i < entries_num; i++)
+ tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
+ tbl->max_flow_num = entries_num;
+
+ return tbl;
+}
+
+void
+gro_tcp6_tbl_destroy(void *tbl)
+{
+ struct gro_tcp6_tbl *tcp_tbl = tbl;
+
+ if (tcp_tbl) {
+ rte_free(tcp_tbl->items);
+ rte_free(tcp_tbl->flows);
+ }
+ rte_free(tcp_tbl);
+}
+
+static inline uint32_t
+find_an_empty_flow(struct gro_tcp6_tbl *tbl)
+{
+ uint32_t i;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++)
+ if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+insert_new_flow(struct gro_tcp6_tbl *tbl,
+ struct tcp6_flow_key *src,
+ uint32_t item_idx)
+{
+ struct tcp6_flow_key *dst;
+ uint32_t flow_idx;
+
+ flow_idx = find_an_empty_flow(tbl);
+ if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
+ return INVALID_ARRAY_INDEX;
+
+ dst = &(tbl->flows[flow_idx].key);
+
+ ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
+ memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst->src_addr));
+ memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst->dst_addr));
+ dst->vtc_flow = src->vtc_flow;
+
+ tbl->flows[flow_idx].start_index = item_idx;
+ tbl->flow_num++;
+
+ return flow_idx;
+}
+
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
+
+ ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len - pkt->l3_len);
+}
+
+int32_t
+gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time)
+{
+ struct rte_ether_hdr *eth_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ int32_t tcp_dl;
+ uint16_t ip_tlen;
+ struct tcp6_flow_key key;
+ uint32_t i, max_flow_num, remaining_flow_num;
+ uint32_t sent_seq;
+ struct rte_tcp_hdr *tcp_hdr;
+ uint8_t find;
+ uint32_t item_idx;
+ /*
+ * Don't process the packet whose TCP header length is greater
+ * than 60 bytes or less than 20 bytes.
+ */
+ if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
+ return -1;
+
+ eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
+ ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
+ tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt->l2_len + pkt->l3_len);
+
+ /*
+ * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
+ * or CWR set.
+ */
+ if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
+ return -1;
+
+ ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
+ /*
+ * Don't process the packet whose payload length is less than or
+ * equal to 0.
+ */
+ tcp_dl = ip_tlen - pkt->l4_len;
+ if (tcp_dl <= 0)
+ return -1;
+
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.cmn_key.eth_daddr));
+ memcpy(&key.src_addr[0], &ipv6_hdr->src_addr, sizeof(key.src_addr));
+ memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr, sizeof(key.dst_addr));
+ key.cmn_key.src_port = tcp_hdr->src_port;
+ key.cmn_key.dst_port = tcp_hdr->dst_port;
+ key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+ key.vtc_flow = ipv6_hdr->vtc_flow;
+
+ /* Search for a matched flow. */
+ max_flow_num = tbl->max_flow_num;
+ remaining_flow_num = tbl->flow_num;
+ find = 0;
+ for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
+ if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
+ if (is_same_tcp6_flow(&tbl->flows[i].key, &key)) {
+ find = 1;
+ break;
+ }
+ remaining_flow_num--;
+ }
+ }
+
+ if (find == 0) {
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num,
+ tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, 0, true);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return -1;
+ if (insert_new_flow(tbl, &key, item_idx) ==
+ INVALID_ARRAY_INDEX) {
+ /*
+ * Fail to insert a new flow, so delete the
+ * stored packet.
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
+ return -1;
+ }
+ return 0;
+ }
+
+ return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl->flows[i].start_index,
+ &tbl->item_num, tbl->max_item_num,
+ 0, true, start_time);
+}
+
+uint16_t
+gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out)
+{
+ uint16_t k = 0;
+ uint32_t i, j;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++) {
+ if (unlikely(tbl->flow_num == 0))
+ return k;
+
+ j = tbl->flows[i].start_index;
+ while (j != INVALID_ARRAY_INDEX) {
+ if (tbl->items[j].start_time <= flush_timestamp) {
+ out[k++] = tbl->items[j].firstseg;
+ if (tbl->items[j].nb_merged > 1)
+ update_header(&(tbl->items[j]));
+ /*
+ * Delete the packet and get the next
+ * packet in the flow.
+ */
+ j = delete_tcp_item(tbl->items, j,
+ &tbl->item_num, INVALID_ARRAY_INDEX);
+ tbl->flows[i].start_index = j;
+ if (j == INVALID_ARRAY_INDEX)
+ tbl->flow_num--;
+
+ if (unlikely(k == nb_out))
+ return k;
+ } else
+ /*
+ * The left packets in this flow won't be
+ * timeout. Go to check other flows.
+ */
+ break;
+ }
+ }
+ return k;
+}
+
+uint32_t
+gro_tcp6_tbl_pkt_count(void *tbl)
+{
+ struct gro_tcp6_tbl *gro_tbl = tbl;
+
+ if (gro_tbl)
+ return gro_tbl->item_num;
+
+ return 0;
+}
diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h
new file mode 100644
index 0000000000..073122f0ec
--- /dev/null
+++ b/lib/gro/gro_tcp6.h
@@ -0,0 +1,160 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+
+#ifndef _GRO_TCP6_H_
+#define _GRO_TCP6_H_
+
+#include "gro_tcp.h"
+
+#define GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
+
+/* Header fields representing a TCP/IPv6 flow */
+struct tcp6_flow_key {
+ struct cmn_tcp_key cmn_key;
+ uint8_t src_addr[16];
+ uint8_t dst_addr[16];
+ rte_be32_t vtc_flow;
+};
+
+struct gro_tcp6_flow {
+ struct tcp6_flow_key key;
+ /*
+ * The index of the first packet in the flow.
+ * INVALID_ARRAY_INDEX indicates an empty flow.
+ */
+ uint32_t start_index;
+};
+
+/*
+ * TCP/IPv6 reassembly table structure.
+ */
+struct gro_tcp6_tbl {
+ /* item array */
+ struct gro_tcp_item *items;
+ /* flow array */
+ struct gro_tcp6_flow *flows;
+ /* current item number */
+ uint32_t item_num;
+ /* current flow num */
+ uint32_t flow_num;
+ /* item array size */
+ uint32_t max_item_num;
+ /* flow array size */
+ uint32_t max_flow_num;
+};
+
+/**
+ * This function creates a TCP/IPv6 reassembly table.
+ *
+ * @param socket_id
+ * Socket index for allocating the TCP/IPv6 reassemble table
+ * @param max_flow_num
+ * The maximum number of flows in the TCP/IPv6 GRO table
+ * @param max_item_per_flow
+ * The maximum number of packets per flow
+ *
+ * @return
+ * - Return the table pointer on success.
+ * - Return NULL on failure.
+ */
+void *gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow);
+
+/**
+ * This function destroys a TCP/IPv6 reassembly table.
+ *
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table.
+ */
+void gro_tcp6_tbl_destroy(void *tbl);
+
+/**
+ * This function merges a TCP/IPv6 packet. It doesn't process the packet,
+ * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't have
+ * payload.
+ *
+ * This function doesn't check if the packet has correct checksums and
+ * doesn't re-calculate checksums for the merged packet. Additionally,
+ * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
+ * when IP fragmentation is possible (i.e., DF==0). It returns the
+ * packet, if the packet has invalid parameters (e.g. SYN bit is set)
+ * or there is no available space in the table.
+ *
+ * @param pkt
+ * Packet to reassemble
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table
+ * @start_time
+ * The time when the packet is inserted into the table
+ *
+ * @return
+ * - Return a positive value if the packet is merged.
+ * - Return zero if the packet isn't merged but stored in the table.
+ * - Return a negative value for invalid parameters or no available
+ * space in the table.
+ */
+int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time);
+
+/**
+ * This function flushes timeout packets in a TCP/IPv6 reassembly table,
+ * and without updating checksums.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ * @param flush_timestamp
+ * Flush packets which are inserted into the table before or at the
+ * flush_timestamp.
+ * @param out
+ * Pointer array used to keep flushed packets
+ * @param nb_out
+ * The element number in 'out'. It also determines the maximum number of
+ * packets that can be flushed finally.
+ *
+ * @return
+ * The number of flushed packets
+ */
+uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out);
+
+/**
+ * This function returns the number of the packets in a TCP/IPv6
+ * reassembly table.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ *
+ * @return
+ * The number of packets in the table
+ */
+uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
+
+/*
+ * Check if two TCP/IPv6 packets belong to the same flow.
+ */
+static inline int
+is_same_tcp6_flow(struct tcp6_flow_key *k1, struct tcp6_flow_key *k2)
+{
+ rte_be32_t vtc_flow_diff;
+
+ if (memcmp(&k1->src_addr, &k2->src_addr, 16))
+ return 0;
+ if (memcmp(&k1->dst_addr, &k2->dst_addr, 16))
+ return 0;
+ /*
+ * IP version (4) Traffic Class (8) Flow Label (20)
+ * All fields except Traffic class should be same
+ */
+ vtc_flow_diff = (k1->vtc_flow ^ k2->vtc_flow);
+ if (vtc_flow_diff & htonl(0xF00FFFFF))
+ return 0;
+
+ return is_same_common_tcp_key(&k1->cmn_key, &k2->cmn_key);
+}
+
+#endif
diff --git a/lib/gro/gro_tcp_internal.h b/lib/gro/gro_tcp_internal.h
new file mode 100644
index 0000000000..cc84abeaeb
--- /dev/null
+++ b/lib/gro/gro_tcp_internal.h
@@ -0,0 +1,128 @@
+
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+
+#ifndef _GRO_TCP_INTERNAL_H_
+#define _GRO_TCP_INTERNAL_H_
+
+static inline uint32_t
+find_an_empty_item(struct gro_tcp_item *items,
+ uint32_t max_item_num)
+{
+ uint32_t i;
+
+ for (i = 0; i < max_item_num; i++)
+ if (items[i].firstseg == NULL)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic)
+{
+ uint32_t item_idx;
+
+ item_idx = find_an_empty_item(items, max_item_num);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return INVALID_ARRAY_INDEX;
+
+ items[item_idx].firstseg = pkt;
+ items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
+ items[item_idx].start_time = start_time;
+ items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
+ items[item_idx].sent_seq = sent_seq;
+ items[item_idx].l3.ip_id = ip_id;
+ items[item_idx].nb_merged = 1;
+ items[item_idx].is_atomic = is_atomic;
+ (*item_num) += 1;
+
+ /* if the previous packet exists, chain them together. */
+ if (prev_idx != INVALID_ARRAY_INDEX) {
+ items[item_idx].next_pkt_idx =
+ items[prev_idx].next_pkt_idx;
+ items[prev_idx].next_pkt_idx = item_idx;
+ }
+
+ return item_idx;
+}
+
+static inline uint32_t
+delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx)
+{
+ uint32_t next_idx = items[item_idx].next_pkt_idx;
+
+ /* NULL indicates an empty item */
+ items[item_idx].firstseg = NULL;
+ (*item_num) -= 1;
+ if (prev_item_idx != INVALID_ARRAY_INDEX)
+ items[prev_item_idx].next_pkt_idx = next_idx;
+
+ return next_idx;
+}
+
+static inline int32_t
+process_tcp_item(struct rte_mbuf *pkt,
+ struct rte_tcp_hdr *tcp_hdr,
+ int32_t tcp_dl,
+ struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time)
+{
+ uint32_t cur_idx;
+ uint32_t prev_idx;
+ int cmp;
+ uint32_t sent_seq;
+
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ /*
+ * Check all packets in the flow and try to find a neighbor for
+ * the input packet.
+ */
+ cur_idx = item_idx;
+ prev_idx = cur_idx;
+ do {
+ cmp = check_seq_option(&items[cur_idx], tcp_hdr,
+ sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
+ is_atomic);
+ if (cmp) {
+ if (merge_two_tcp_packets(&items[cur_idx],
+ pkt, cmp, sent_seq, ip_id, 0))
+ return 1;
+ /*
+ * Fail to merge the two packets, as the packet
+ * length is greater than the max value. Store
+ * the packet into the flow.
+ */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num,
+ start_time, cur_idx, sent_seq, ip_id, is_atomic) ==
+ INVALID_ARRAY_INDEX)
+ return -1;
+ return 0;
+ }
+ prev_idx = cur_idx;
+ cur_idx = items[cur_idx].next_pkt_idx;
+ } while (cur_idx != INVALID_ARRAY_INDEX);
+
+ /* Fail to find a neighbor, so store the packet into the flow. */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num, start_time, prev_idx, sent_seq,
+ ip_id, is_atomic) == INVALID_ARRAY_INDEX)
+ return -1;
+
+ return 0;
+}
+#endif
diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
index 3be4deb7c7..6ab7001922 100644
--- a/lib/gro/gro_vxlan_tcp4.c
+++ b/lib/gro/gro_vxlan_tcp4.c
@@ -116,7 +116,7 @@ insert_new_item(struct gro_vxlan_tcp4_tbl *tbl,
tbl->items[item_idx].inner_item.start_time = start_time;
tbl->items[item_idx].inner_item.next_pkt_idx = INVALID_ARRAY_INDEX;
tbl->items[item_idx].inner_item.sent_seq = sent_seq;
- tbl->items[item_idx].inner_item.ip_id = ip_id;
+ tbl->items[item_idx].inner_item.l3.ip_id = ip_id;
tbl->items[item_idx].inner_item.nb_merged = 1;
tbl->items[item_idx].inner_item.is_atomic = is_atomic;
tbl->items[item_idx].outer_ip_id = outer_ip_id;
@@ -163,15 +163,9 @@ insert_new_flow(struct gro_vxlan_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->inner_key.eth_saddr),
- &(dst->inner_key.eth_saddr));
- rte_ether_addr_copy(&(src->inner_key.eth_daddr),
- &(dst->inner_key.eth_daddr));
+ ASSIGN_COMMON_TCP_KEY((&(src->inner_key.cmn_key)), (&(dst->inner_key.cmn_key)));
dst->inner_key.ip_src_addr = src->inner_key.ip_src_addr;
dst->inner_key.ip_dst_addr = src->inner_key.ip_dst_addr;
- dst->inner_key.recv_ack = src->inner_key.recv_ack;
- dst->inner_key.src_port = src->inner_key.src_port;
- dst->inner_key.dst_port = src->inner_key.dst_port;
dst->vxlan_hdr.vx_flags = src->vxlan_hdr.vx_flags;
dst->vxlan_hdr.vx_vni = src->vxlan_hdr.vx_vni;
@@ -248,7 +242,7 @@ merge_two_vxlan_tcp4_packets(struct gro_vxlan_tcp4_item *item,
uint16_t outer_ip_id,
uint16_t ip_id)
{
- if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp, sent_seq,
+ if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
ip_id, pkt->outer_l2_len +
pkt->outer_l3_len)) {
/* Update the outer IPv4 ID to the large value. */
@@ -357,13 +351,13 @@ gro_vxlan_tcp4_reassemble(struct rte_mbuf *pkt,
sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.eth_daddr));
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.cmn_key.eth_daddr));
key.inner_key.ip_src_addr = ipv4_hdr->src_addr;
key.inner_key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.inner_key.recv_ack = tcp_hdr->recv_ack;
- key.inner_key.src_port = tcp_hdr->src_port;
- key.inner_key.dst_port = tcp_hdr->dst_port;
+ key.inner_key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+ key.inner_key.cmn_key.src_port = tcp_hdr->src_port;
+ key.inner_key.cmn_key.dst_port = tcp_hdr->dst_port;
key.vxlan_hdr.vx_flags = vxlan_hdr->vx_flags;
key.vxlan_hdr.vx_vni = vxlan_hdr->vx_vni;
diff --git a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h
index 7832942a68..662db01a88 100644
--- a/lib/gro/gro_vxlan_tcp4.h
+++ b/lib/gro/gro_vxlan_tcp4.h
@@ -36,7 +36,7 @@ struct gro_vxlan_tcp4_flow {
};
struct gro_vxlan_tcp4_item {
- struct gro_tcp4_item inner_item;
+ struct gro_tcp_item inner_item;
/* IPv4 ID in the outer IPv4 header */
uint16_t outer_ip_id;
/* Indicate if outer IPv4 ID can be ignored */
diff --git a/lib/gro/meson.build b/lib/gro/meson.build
index e4fa2958bd..dbce05220d 100644
--- a/lib/gro/meson.build
+++ b/lib/gro/meson.build
@@ -4,6 +4,7 @@
sources = files(
'rte_gro.c',
'gro_tcp4.c',
+ 'gro_tcp6.c',
'gro_udp4.c',
'gro_vxlan_tcp4.c',
'gro_vxlan_udp4.c',
diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c
index e35399fd42..d824eebd93 100644
--- a/lib/gro/rte_gro.c
+++ b/lib/gro/rte_gro.c
@@ -8,6 +8,7 @@
#include "rte_gro.h"
#include "gro_tcp4.h"
+#include "gro_tcp6.h"
#include "gro_udp4.h"
#include "gro_vxlan_tcp4.h"
#include "gro_vxlan_udp4.h"
@@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
- gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
+ gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, gro_tcp6_tbl_create, NULL};
static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
+ gro_tcp6_tbl_destroy,
NULL};
static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count,
+ gro_tcp6_tbl_pkt_count,
NULL};
#define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
@@ -35,6 +38,12 @@ static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+/* GRO with extension headers is not supported */
+#define IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
+ ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
+ ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
+ (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+
#define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
@@ -147,7 +156,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
/* allocate a reassembly table for TCP/IPv4 GRO */
struct gro_tcp4_tbl tcp_tbl;
struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
- struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+ struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+
+ struct gro_tcp6_tbl tcp6_tbl;
+ struct gro_tcp6_flow tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
+ struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
/* allocate a reassembly table for UDP/IPv4 GRO */
struct gro_udp4_tbl udp_tbl;
@@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
int32_t ret;
uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
- do_vxlan_udp_gro = 0;
+ do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
do_udp4_gro = 1;
}
+ if (param->gro_types & RTE_GRO_TCP_IPV6) {
+ for (i = 0; i < item_num; i++)
+ tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+ tcp6_tbl.flows = tcp6_flows;
+ tcp6_tbl.items = tcp6_items;
+ tcp6_tbl.flow_num = 0;
+ tcp6_tbl.item_num = 0;
+ tcp6_tbl.max_flow_num = item_num;
+ tcp6_tbl.max_item_num = item_num;
+ do_tcp6_gro = 1;
+ }
for (i = 0; i < nb_pkts; i++) {
/*
@@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
nb_after_gro--;
else if (ret < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
+ if (ret > 0)
+ /* merge successfully */
+ nb_after_gro--;
+ else if (ret < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
if ((nb_after_gro < nb_pkts)
|| (unprocess_num < nb_pkts)) {
i = 0;
+ /* Copy unprocessed packets */
+ if (unprocess_num > 0) {
+ memcpy(&pkts[i], unprocess_pkts,
+ sizeof(struct rte_mbuf *) *
+ unprocess_num);
+ i = unprocess_num;
+ }
+
/* Flush all packets from the tables */
if (do_vxlan_tcp_gro) {
- i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
+ i += gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
0, pkts, nb_pkts);
}
@@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
&pkts[i], nb_pkts - i);
}
- /* Copy unprocessed packets */
- if (unprocess_num > 0) {
- memcpy(&pkts[i], unprocess_pkts,
- sizeof(struct rte_mbuf *) *
- unprocess_num);
+
+ if (do_tcp6_gro) {
+ i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
+ &pkts[i], nb_pkts - i);
}
- nb_after_gro = i + unprocess_num;
}
return nb_after_gro;
@@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
{
struct rte_mbuf *unprocess_pkts[nb_pkts];
struct gro_ctx *gro_ctx = ctx;
- void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
+ void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
uint64_t current_time;
uint16_t i, unprocess_num = 0;
- uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro;
+ uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro, do_tcp6_gro;
if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
+ tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
RTE_GRO_TCP_IPV4;
@@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
RTE_GRO_UDP_IPV4;
do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
RTE_GRO_IPV4_VXLAN_UDP_IPV4;
+ do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) == RTE_GRO_TCP_IPV6;
current_time = rte_rdtsc();
@@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
if (gro_udp4_reassemble(pkts[i], udp_tbl,
current_time) < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
+ current_time) < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
flush_timestamp,
&out[num], left_nb_out);
+ left_nb_out = max_nb_out - num;
+ }
+
+ if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
+ num += gro_tcp6_tbl_timeout_flush(
+ gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
+ flush_timestamp,
+ &out[num], left_nb_out);
+
}
return num;
diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h
index 9f9ed4935a..c83dfd9ad1 100644
--- a/lib/gro/rte_gro.h
+++ b/lib/gro/rte_gro.h
@@ -38,6 +38,9 @@ extern "C" {
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL << RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
/**< VxLAN UDP/IPv4 GRO flag. */
+#define RTE_GRO_TCP_IPV6_INDEX 4
+#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX)
+/**< TCP/IPv6 GRO flag. */
/**
* Structure used to create GRO context objects or used to pass
--
2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [PATCH v9] gro : ipv6 changes to support GRO for TCP/ipv6
2023-06-15 6:20 ` [PATCH v9] " Kumara Parameshwaran
@ 2023-06-15 6:30 ` kumaraparameshwaran rathinavel
2023-06-15 8:01 ` Hu, Jiayu
2023-06-19 13:30 ` Thomas Monjalon
2 siblings, 0 replies; 32+ messages in thread
From: kumaraparameshwaran rathinavel @ 2023-06-15 6:30 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev
[-- Attachment #1: Type: text/plain, Size: 66384 bytes --]
On Thu, Jun 15, 2023 at 11:50 AM Kumara Parameshwaran <
kumaraparamesh92@gmail.com> wrote:
> The patch adds GRO support for TCP/ipv6 packets. This does not
> include the support for vxlan, udp ipv6 packets.
>
> Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> ---
> v1:
> * Changes to support GRO for TCP/ipv6 packets. This does not
> include
> vxlan changes.
> * The GRO is performed only for ipv6 packets that does not contain
> extension headers.
> * The logic for the TCP coalescing remains the same, in ipv6
> header
> the source address, destination address, flow label, version
> fields
> are expected to be the same.
> * Re-organised the code to reuse certain tcp functions for both
> ipv4 and
> ipv6 flows.
> v2:
> * Fix comments in gro_tcp6.h header file.
>
> v3:
> * Adderess review comments to fix code duplication for v4 and v6
>
> v4:
> * Addresses review comments for v3, do not use callbacks
>
> v5:
> * Address review comments
>
> v6:
> * Fix warning and coding style issues
>
> v7:
> * Fix build compilation issue
>
> v8:
> * Use gro_tcp_internal.h for functions used for gro_tcp4 and
> gro_tcp6 and use
> gro_tcp.h for data structures and functions used across
> gro_vxlan_tcp4
>
> v9:
> * Resolve merge conflict and add gro_tcp.h in proper path
>
> .../generic_receive_offload_lib.rst | 21 +-
> doc/guides/rel_notes/release_23_07.rst | 2 +
> lib/gro/gro_tcp.h | 187 ++++++++++++
> lib/gro/gro_tcp4.c | 177 +++---------
> lib/gro/gro_tcp4.h | 170 +----------
> lib/gro/gro_tcp6.c | 268 ++++++++++++++++++
> lib/gro/gro_tcp6.h | 160 +++++++++++
> lib/gro/gro_tcp_internal.h | 128 +++++++++
> lib/gro/gro_vxlan_tcp4.c | 22 +-
> lib/gro/gro_vxlan_tcp4.h | 2 +-
> lib/gro/meson.build | 1 +
> lib/gro/rte_gro.c | 83 +++++-
> lib/gro/rte_gro.h | 3 +
> 13 files changed, 883 insertions(+), 341 deletions(-)
> create mode 100644 lib/gro/gro_tcp.h
> create mode 100644 lib/gro/gro_tcp6.c
> create mode 100644 lib/gro/gro_tcp6.h
> create mode 100644 lib/gro/gro_tcp_internal.h
>
> diff --git a/doc/guides/prog_guide/generic_receive_offload_lib.rst
> b/doc/guides/prog_guide/generic_receive_offload_lib.rst
> index 98a5d29bbc..533cda7f5c 100644
> --- a/doc/guides/prog_guide/generic_receive_offload_lib.rst
> +++ b/doc/guides/prog_guide/generic_receive_offload_lib.rst
> @@ -138,20 +138,21 @@ The key-based algorithm has two characters:
>
> Key-based Reassembly Algorithm
>
> -TCP/IPv4 GRO
> -------------
> +TCP-IPv4/IPv6 GRO
> +-----------------
>
> -The table structure used by TCP/IPv4 GRO contains two arrays: flow array
> +The table structure used by TCP-IPv4/IPv6 GRO contains two arrays: flow
> array
> and item array. The flow array keeps flow information, and the item array
> -keeps packet information.
> +keeps packet information. The flow array is different for IPv4 and IPv6
> while
> +the item array is the same.
>
> -Header fields used to define a TCP/IPv4 flow include:
> +Header fields used to define a TCP-IPv4/IPv6 flow include:
>
> -- source and destination: Ethernet and IP address, TCP port
> +- common tcp key fields : Ethernet address, TCP port, TCP acknowledge
> number
> +- version specific IP address
> +- IPv6 flow label for IPv6 flow
>
> -- TCP acknowledge number
> -
> -TCP/IPv4 packets whose FIN, SYN, RST, URG, PSH, ECE or CWR bit is set
> +TCP packets whose FIN, SYN, RST, URG, PSH, ECE or CWR bit is set
> won't be processed.
>
> Header fields deciding if two packets are neighbors include:
> @@ -159,7 +160,7 @@ Header fields deciding if two packets are neighbors
> include:
> - TCP sequence number
>
> - IPv4 ID. The IPv4 ID fields of the packets, whose DF bit is 0, should
> - be increased by 1.
> + be increased by 1. This is applicable only for IPv4
>
> VxLAN GRO
> ---------
> diff --git a/doc/guides/rel_notes/release_23_07.rst
> b/doc/guides/rel_notes/release_23_07.rst
> index 027ae7bd2d..7124cf45c7 100644
> --- a/doc/guides/rel_notes/release_23_07.rst
> +++ b/doc/guides/rel_notes/release_23_07.rst
> @@ -170,6 +170,8 @@ New Features
>
> See :doc:`../prog_guide/pdcp_lib` for more information.
>
> +* **Added support for TCP/IPv6 GRO.**
> + * Enhanced the existing library to support GRO for TCP packets over
> IPv6 network.
>
> Removed Items
> -------------
> diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h
> new file mode 100644
> index 0000000000..7c7a6a3008
> --- /dev/null
> +++ b/lib/gro/gro_tcp.h
> @@ -0,0 +1,187 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2023 Intel Corporation
> + */
> +#ifndef _GRO_TCP_H_
> +#define _GRO_TCP_H_
> +
> +#define INVALID_ARRAY_INDEX 0xffffffffUL
> +
> +#include <rte_tcp.h>
> +
> +/*
> + * The max length of a IPv4 packet, which includes the length of the L3
> + * header, the L4 header and the data payload.
> + */
> +#define MAX_IP_PKT_LENGTH UINT16_MAX
> +
> +/* The maximum TCP header length */
> +#define MAX_TCP_HLEN 60
> +#define INVALID_TCP_HDRLEN(len) \
> + (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> +
> +struct cmn_tcp_key {
> + struct rte_ether_addr eth_saddr;
> + struct rte_ether_addr eth_daddr;
> + uint32_t recv_ack;
> + uint16_t src_port;
> + uint16_t dst_port;
> +};
> +
> +#define ASSIGN_COMMON_TCP_KEY(k1, k2) \
> + do {\
> + rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
> + rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr)); \
> + k2->recv_ack = k1->recv_ack; \
> + k2->src_port = k1->src_port; \
> + k2->dst_port = k1->dst_port; \
> + } while (0)
> +
> +struct gro_tcp_item {
> + /*
> + * The first MBUF segment of the packet. If the value
> + * is NULL, it means the item is empty.
> + */
> + struct rte_mbuf *firstseg;
> + /* The last MBUF segment of the packet */
> + struct rte_mbuf *lastseg;
> + /*
> + * The time when the first packet is inserted into the table.
> + * This value won't be updated, even if the packet is merged
> + * with other packets.
> + */
> + uint64_t start_time;
> + /*
> + * next_pkt_idx is used to chain the packets that
> + * are in the same flow but can't be merged together
> + * (e.g. caused by packet reordering).
> + */
> + uint32_t next_pkt_idx;
> + /* TCP sequence number of the packet */
> + uint32_t sent_seq;
> + union {
> + /* IPv4 ID of the packet */
> + uint16_t ip_id;
> + /* Unused field for IPv6 */
> + uint16_t unused;
> + } l3;
> + /* the number of merged packets */
> + uint16_t nb_merged;
> + /* Indicate if IPv4 ID can be ignored */
> + uint8_t is_atomic;
> +};
> +
> +/*
> + * Merge two TCP packets without updating checksums.
> + * If cmp is larger than 0, append the new packet to the
> + * original packet. Otherwise, pre-pend the new packet to
> + * the original packet.
> + */
> +static inline int
> +merge_two_tcp_packets(struct gro_tcp_item *item,
> + struct rte_mbuf *pkt,
> + int cmp,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint16_t l2_offset)
> +{
> + struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
> + uint16_t hdr_len, l2_len;
> +
> + if (cmp > 0) {
> + pkt_head = item->firstseg;
> + pkt_tail = pkt;
> + } else {
> + pkt_head = pkt;
> + pkt_tail = item->firstseg;
> + }
> +
> + /* check if the IPv4 packet length is greater than the max value */
> + hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
> + pkt_head->l4_len;
> + l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
> + if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
> + hdr_len > MAX_IP_PKT_LENGTH))
> + return 0;
> +
> + /* remove the packet header for the tail packet */
> + rte_pktmbuf_adj(pkt_tail, hdr_len);
> +
> + /* chain two packets together */
> + if (cmp > 0) {
> + item->lastseg->next = pkt;
> + item->lastseg = rte_pktmbuf_lastseg(pkt);
> + /* update IP ID to the larger value */
> + item->l3.ip_id = ip_id;
> + } else {
> + lastseg = rte_pktmbuf_lastseg(pkt);
> + lastseg->next = item->firstseg;
> + item->firstseg = pkt;
> + /* update sent_seq to the smaller value */
> + item->sent_seq = sent_seq;
> + item->l3.ip_id = ip_id;
> + }
> + item->nb_merged++;
> +
> + /* update MBUF metadata for the merged packet */
> + pkt_head->nb_segs += pkt_tail->nb_segs;
> + pkt_head->pkt_len += pkt_tail->pkt_len;
> +
> + return 1;
> +}
> +
> +/*
> + * Check if two TCP packets are neighbors.
> + */
> +static inline int
> +check_seq_option(struct gro_tcp_item *item,
> + struct rte_tcp_hdr *tcph,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint16_t tcp_hl,
> + uint16_t tcp_dl,
> + uint16_t l2_offset,
> + uint8_t is_atomic)
> +{
> + struct rte_mbuf *pkt_orig = item->firstseg;
> + char *iph_orig;
> + struct rte_tcp_hdr *tcph_orig;
> + uint16_t len, tcp_hl_orig;
> +
> + iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
> + l2_offset + pkt_orig->l2_len);
> + tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
> + tcp_hl_orig = pkt_orig->l4_len;
> +
> + /* Check if TCP option fields equal */
> + len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> + if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> + (memcmp(tcph + 1, tcph_orig + 1,
> + len) != 0)))
> + return 0;
> +
> + /* Don't merge packets whose DF bits are different */
> + if (unlikely(item->is_atomic ^ is_atomic))
> + return 0;
> +
> + /* check if the two packets are neighbors */
> + len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> + pkt_orig->l3_len - tcp_hl_orig;
> + if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> + (ip_id == item->l3.ip_id + 1)))
> + /* append the new packet */
> + return 1;
> + else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> + (ip_id + item->nb_merged ==
> item->l3.ip_id)))
> + /* pre-pend the new packet */
> + return -1;
> +
> + return 0;
> +}
> +
> +static inline int
> +is_same_common_tcp_key(struct cmn_tcp_key *k1, struct cmn_tcp_key *k2)
> +{
> + return (!memcmp(k1, k2, sizeof(struct cmn_tcp_key)));
> +}
> +
> +#endif
> diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c
> index 0014096e63..6645de592b 100644
> --- a/lib/gro/gro_tcp4.c
> +++ b/lib/gro/gro_tcp4.c
> @@ -7,6 +7,7 @@
> #include <rte_ethdev.h>
>
> #include "gro_tcp4.h"
> +#include "gro_tcp_internal.h"
>
> void *
> gro_tcp4_tbl_create(uint16_t socket_id,
> @@ -30,7 +31,7 @@ gro_tcp4_tbl_create(uint16_t socket_id,
> if (tbl == NULL)
> return NULL;
>
> - size = sizeof(struct gro_tcp4_item) * entries_num;
> + size = sizeof(struct gro_tcp_item) * entries_num;
> tbl->items = rte_zmalloc_socket(__func__,
> size,
> RTE_CACHE_LINE_SIZE,
> @@ -71,18 +72,6 @@ gro_tcp4_tbl_destroy(void *tbl)
> rte_free(tcp_tbl);
> }
>
> -static inline uint32_t
> -find_an_empty_item(struct gro_tcp4_tbl *tbl)
> -{
> - uint32_t i;
> - uint32_t max_item_num = tbl->max_item_num;
> -
> - for (i = 0; i < max_item_num; i++)
> - if (tbl->items[i].firstseg == NULL)
> - return i;
> - return INVALID_ARRAY_INDEX;
> -}
> -
> static inline uint32_t
> find_an_empty_flow(struct gro_tcp4_tbl *tbl)
> {
> @@ -95,56 +84,6 @@ find_an_empty_flow(struct gro_tcp4_tbl *tbl)
> return INVALID_ARRAY_INDEX;
> }
>
> -static inline uint32_t
> -insert_new_item(struct gro_tcp4_tbl *tbl,
> - struct rte_mbuf *pkt,
> - uint64_t start_time,
> - uint32_t prev_idx,
> - uint32_t sent_seq,
> - uint16_t ip_id,
> - uint8_t is_atomic)
> -{
> - uint32_t item_idx;
> -
> - item_idx = find_an_empty_item(tbl);
> - if (item_idx == INVALID_ARRAY_INDEX)
> - return INVALID_ARRAY_INDEX;
> -
> - tbl->items[item_idx].firstseg = pkt;
> - tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> - tbl->items[item_idx].start_time = start_time;
> - tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> - tbl->items[item_idx].sent_seq = sent_seq;
> - tbl->items[item_idx].ip_id = ip_id;
> - tbl->items[item_idx].nb_merged = 1;
> - tbl->items[item_idx].is_atomic = is_atomic;
> - tbl->item_num++;
> -
> - /* if the previous packet exists, chain them together. */
> - if (prev_idx != INVALID_ARRAY_INDEX) {
> - tbl->items[item_idx].next_pkt_idx =
> - tbl->items[prev_idx].next_pkt_idx;
> - tbl->items[prev_idx].next_pkt_idx = item_idx;
> - }
> -
> - return item_idx;
> -}
> -
> -static inline uint32_t
> -delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
> - uint32_t prev_item_idx)
> -{
> - uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
> -
> - /* NULL indicates an empty item */
> - tbl->items[item_idx].firstseg = NULL;
> - tbl->item_num--;
> - if (prev_item_idx != INVALID_ARRAY_INDEX)
> - tbl->items[prev_item_idx].next_pkt_idx = next_idx;
> -
> - return next_idx;
> -}
> -
> static inline uint32_t
> insert_new_flow(struct gro_tcp4_tbl *tbl,
> struct tcp4_flow_key *src,
> @@ -159,13 +98,10 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
>
> dst = &(tbl->flows[flow_idx].key);
>
> - rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
> - rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
> + ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
> +
> dst->ip_src_addr = src->ip_src_addr;
> dst->ip_dst_addr = src->ip_dst_addr;
> - dst->recv_ack = src->recv_ack;
> - dst->src_port = src->src_port;
> - dst->dst_port = src->dst_port;
>
> tbl->flows[flow_idx].start_index = item_idx;
> tbl->flow_num++;
> @@ -173,21 +109,6 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
> return flow_idx;
> }
>
> -/*
> - * update the packet length for the flushed packet.
> - */
> -static inline void
> -update_header(struct gro_tcp4_item *item)
> -{
> - struct rte_ipv4_hdr *ipv4_hdr;
> - struct rte_mbuf *pkt = item->firstseg;
> -
> - ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> - pkt->l2_len);
> - ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
> - pkt->l2_len);
> -}
> -
> int32_t
> gro_tcp4_reassemble(struct rte_mbuf *pkt,
> struct gro_tcp4_tbl *tbl,
> @@ -202,9 +123,8 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> uint8_t is_atomic;
>
> struct tcp4_flow_key key;
> - uint32_t cur_idx, prev_idx, item_idx;
> + uint32_t item_idx;
> uint32_t i, max_flow_num, remaining_flow_num;
> - int cmp;
> uint8_t find;
>
> /*
> @@ -239,6 +159,14 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> if (tcp_dl <= 0)
> return -1;
>
> + rte_ether_addr_copy(&(eth_hdr->src_addr),
> &(key.cmn_key.eth_saddr));
> + rte_ether_addr_copy(&(eth_hdr->dst_addr),
> &(key.cmn_key.eth_daddr));
> + key.ip_src_addr = ipv4_hdr->src_addr;
> + key.ip_dst_addr = ipv4_hdr->dst_addr;
> + key.cmn_key.src_port = tcp_hdr->src_port;
> + key.cmn_key.dst_port = tcp_hdr->dst_port;
> + key.cmn_key.recv_ack = tcp_hdr->recv_ack;
> +
> /*
> * Save IPv4 ID for the packet whose DF bit is 0. For the packet
> * whose DF bit is 1, IPv4 ID is ignored.
> @@ -246,15 +174,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
> is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) ==
> RTE_IPV4_HDR_DF_FLAG;
> ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
> - sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> -
> - rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
> - rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
> - key.ip_src_addr = ipv4_hdr->src_addr;
> - key.ip_dst_addr = ipv4_hdr->dst_addr;
> - key.src_port = tcp_hdr->src_port;
> - key.dst_port = tcp_hdr->dst_port;
> - key.recv_ack = tcp_hdr->recv_ack;
>
> /* Search for a matched flow. */
> max_flow_num = tbl->max_flow_num;
> @@ -270,63 +189,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
> }
> }
>
> - /*
> - * Fail to find a matched flow. Insert a new flow and store the
> - * packet into the flow.
> - */
> if (find == 0) {
> - item_idx = insert_new_item(tbl, pkt, start_time,
> - INVALID_ARRAY_INDEX, sent_seq, ip_id,
> - is_atomic);
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> + item_idx = insert_new_tcp_item(pkt, tbl->items,
> &tbl->item_num,
> + tbl->max_item_num,
> start_time,
> + INVALID_ARRAY_INDEX,
> sent_seq, ip_id,
> + is_atomic);
> if (item_idx == INVALID_ARRAY_INDEX)
> return -1;
> if (insert_new_flow(tbl, &key, item_idx) ==
> - INVALID_ARRAY_INDEX) {
> + INVALID_ARRAY_INDEX) {
> /*
> * Fail to insert a new flow, so delete the
> * stored packet.
> - */
> - delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
> + */
> + delete_tcp_item(tbl->items, item_idx,
> &tbl->item_num, INVALID_ARRAY_INDEX);
> return -1;
> }
> return 0;
> }
>
> - /*
> - * Check all packets in the flow and try to find a neighbor for
> - * the input packet.
> - */
> - cur_idx = tbl->flows[i].start_index;
> - prev_idx = cur_idx;
> - do {
> - cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
> - sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> - is_atomic);
> - if (cmp) {
> - if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
> - pkt, cmp, sent_seq, ip_id,
> 0))
> - return 1;
> - /*
> - * Fail to merge the two packets, as the packet
> - * length is greater than the max value. Store
> - * the packet into the flow.
> - */
> - if (insert_new_item(tbl, pkt, start_time, cur_idx,
> - sent_seq, ip_id,
> is_atomic) ==
> - INVALID_ARRAY_INDEX)
> - return -1;
> - return 0;
> - }
> - prev_idx = cur_idx;
> - cur_idx = tbl->items[cur_idx].next_pkt_idx;
> - } while (cur_idx != INVALID_ARRAY_INDEX);
> + return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items,
> tbl->flows[i].start_index,
> + &tbl->item_num,
> tbl->max_item_num,
> + ip_id, is_atomic,
> start_time);
> +}
>
> - /* Fail to find a neighbor, so store the packet into the flow. */
> - if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
> - ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> - return -1;
> +/*
> + * update the packet length for the flushed packet.
> + */
> +static inline void
> +update_header(struct gro_tcp_item *item)
> +{
> + struct rte_ipv4_hdr *ipv4_hdr;
> + struct rte_mbuf *pkt = item->firstseg;
>
> - return 0;
> + ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> + pkt->l2_len);
> + ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
> + pkt->l2_len);
> }
>
> uint16_t
> @@ -353,7 +253,8 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
> * Delete the packet and get the next
> * packet in the flow.
> */
> - j = delete_item(tbl, j,
> INVALID_ARRAY_INDEX);
> + j = delete_tcp_item(tbl->items, j,
> + &tbl->item_num,
> INVALID_ARRAY_INDEX);
> tbl->flows[i].start_index = j;
> if (j == INVALID_ARRAY_INDEX)
> tbl->flow_num--;
> diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h
> index 212f97a042..245e5da486 100644
> --- a/lib/gro/gro_tcp4.h
> +++ b/lib/gro/gro_tcp4.h
> @@ -5,32 +5,15 @@
> #ifndef _GRO_TCP4_H_
> #define _GRO_TCP4_H_
>
> -#include <rte_tcp.h>
> +#include "gro_tcp.h"
>
> -#define INVALID_ARRAY_INDEX 0xffffffffUL
> #define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
>
> -/*
> - * The max length of a IPv4 packet, which includes the length of the L3
> - * header, the L4 header and the data payload.
> - */
> -#define MAX_IPV4_PKT_LENGTH UINT16_MAX
> -
> -/* The maximum TCP header length */
> -#define MAX_TCP_HLEN 60
> -#define INVALID_TCP_HDRLEN(len) \
> - (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
> -
> -/* Header fields representing a TCP/IPv4 flow */
> +/* Header fields representing common fields in TCP flow */
> struct tcp4_flow_key {
> - struct rte_ether_addr eth_saddr;
> - struct rte_ether_addr eth_daddr;
> + struct cmn_tcp_key cmn_key;
> uint32_t ip_src_addr;
> uint32_t ip_dst_addr;
> -
> - uint32_t recv_ack;
> - uint16_t src_port;
> - uint16_t dst_port;
> };
>
> struct gro_tcp4_flow {
> @@ -42,42 +25,12 @@ struct gro_tcp4_flow {
> uint32_t start_index;
> };
>
> -struct gro_tcp4_item {
> - /*
> - * The first MBUF segment of the packet. If the value
> - * is NULL, it means the item is empty.
> - */
> - struct rte_mbuf *firstseg;
> - /* The last MBUF segment of the packet */
> - struct rte_mbuf *lastseg;
> - /*
> - * The time when the first packet is inserted into the table.
> - * This value won't be updated, even if the packet is merged
> - * with other packets.
> - */
> - uint64_t start_time;
> - /*
> - * next_pkt_idx is used to chain the packets that
> - * are in the same flow but can't be merged together
> - * (e.g. caused by packet reordering).
> - */
> - uint32_t next_pkt_idx;
> - /* TCP sequence number of the packet */
> - uint32_t sent_seq;
> - /* IPv4 ID of the packet */
> - uint16_t ip_id;
> - /* the number of merged packets */
> - uint16_t nb_merged;
> - /* Indicate if IPv4 ID can be ignored */
> - uint8_t is_atomic;
> -};
> -
> /*
> * TCP/IPv4 reassembly table structure.
> */
> struct gro_tcp4_tbl {
> /* item array */
> - struct gro_tcp4_item *items;
> + struct gro_tcp_item *items;
> /* flow array */
> struct gro_tcp4_flow *flows;
> /* current item number */
> @@ -186,120 +139,9 @@ uint32_t gro_tcp4_tbl_pkt_count(void *tbl);
> static inline int
> is_same_tcp4_flow(struct tcp4_flow_key k1, struct tcp4_flow_key k2)
> {
> - return (rte_is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) &&
> - rte_is_same_ether_addr(&k1.eth_daddr,
> &k2.eth_daddr) &&
> - (k1.ip_src_addr == k2.ip_src_addr) &&
> + return ((k1.ip_src_addr == k2.ip_src_addr) &&
> (k1.ip_dst_addr == k2.ip_dst_addr) &&
> - (k1.recv_ack == k2.recv_ack) &&
> - (k1.src_port == k2.src_port) &&
> - (k1.dst_port == k2.dst_port));
> + is_same_common_tcp_key(&k1.cmn_key, &k2.cmn_key));
> }
>
> -/*
> - * Merge two TCP/IPv4 packets without updating checksums.
> - * If cmp is larger than 0, append the new packet to the
> - * original packet. Otherwise, pre-pend the new packet to
> - * the original packet.
> - */
> -static inline int
> -merge_two_tcp4_packets(struct gro_tcp4_item *item,
> - struct rte_mbuf *pkt,
> - int cmp,
> - uint32_t sent_seq,
> - uint16_t ip_id,
> - uint16_t l2_offset)
> -{
> - struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
> - uint16_t hdr_len, l2_len;
> -
> - if (cmp > 0) {
> - pkt_head = item->firstseg;
> - pkt_tail = pkt;
> - } else {
> - pkt_head = pkt;
> - pkt_tail = item->firstseg;
> - }
> -
> - /* check if the IPv4 packet length is greater than the max value */
> - hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
> - pkt_head->l4_len;
> - l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
> - if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
> - hdr_len > MAX_IPV4_PKT_LENGTH))
> - return 0;
> -
> - /* remove the packet header for the tail packet */
> - rte_pktmbuf_adj(pkt_tail, hdr_len);
> -
> - /* chain two packets together */
> - if (cmp > 0) {
> - item->lastseg->next = pkt;
> - item->lastseg = rte_pktmbuf_lastseg(pkt);
> - /* update IP ID to the larger value */
> - item->ip_id = ip_id;
> - } else {
> - lastseg = rte_pktmbuf_lastseg(pkt);
> - lastseg->next = item->firstseg;
> - item->firstseg = pkt;
> - /* update sent_seq to the smaller value */
> - item->sent_seq = sent_seq;
> - item->ip_id = ip_id;
> - }
> - item->nb_merged++;
> -
> - /* update MBUF metadata for the merged packet */
> - pkt_head->nb_segs += pkt_tail->nb_segs;
> - pkt_head->pkt_len += pkt_tail->pkt_len;
> -
> - return 1;
> -}
> -
> -/*
> - * Check if two TCP/IPv4 packets are neighbors.
> - */
> -static inline int
> -check_seq_option(struct gro_tcp4_item *item,
> - struct rte_tcp_hdr *tcph,
> - uint32_t sent_seq,
> - uint16_t ip_id,
> - uint16_t tcp_hl,
> - uint16_t tcp_dl,
> - uint16_t l2_offset,
> - uint8_t is_atomic)
> -{
> - struct rte_mbuf *pkt_orig = item->firstseg;
> - struct rte_ipv4_hdr *iph_orig;
> - struct rte_tcp_hdr *tcph_orig;
> - uint16_t len, tcp_hl_orig;
> -
> - iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char
> *) +
> - l2_offset + pkt_orig->l2_len);
> - tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig +
> pkt_orig->l3_len);
> - tcp_hl_orig = pkt_orig->l4_len;
> -
> - /* Check if TCP option fields equal */
> - len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
> - if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
> - (memcmp(tcph + 1, tcph_orig + 1,
> - len) != 0)))
> - return 0;
> -
> - /* Don't merge packets whose DF bits are different */
> - if (unlikely(item->is_atomic ^ is_atomic))
> - return 0;
> -
> - /* check if the two packets are neighbors */
> - len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
> - pkt_orig->l3_len - tcp_hl_orig;
> - if ((sent_seq == item->sent_seq + len) && (is_atomic ||
> - (ip_id == item->ip_id + 1)))
> - /* append the new packet */
> - return 1;
> - else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
> - (ip_id + item->nb_merged == item->ip_id)))
> - /* pre-pend the new packet */
> - return -1;
> -
> - return 0;
> -}
> #endif
> diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c
> new file mode 100644
> index 0000000000..5aa39801e1
> --- /dev/null
> +++ b/lib/gro/gro_tcp6.c
> @@ -0,0 +1,268 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2023 Intel Corporation
> + */
> +
> +#include <rte_malloc.h>
> +#include <rte_mbuf.h>
> +#include <rte_ethdev.h>
> +
> +#include "gro_tcp6.h"
> +#include "gro_tcp_internal.h"
> +
> +void *
> +gro_tcp6_tbl_create(uint16_t socket_id,
> + uint16_t max_flow_num,
> + uint16_t max_item_per_flow)
> +{
> + struct gro_tcp6_tbl *tbl;
> + size_t size;
> + uint32_t entries_num, i;
> +
> + entries_num = max_flow_num * max_item_per_flow;
> + entries_num = RTE_MIN(entries_num, GRO_TCP6_TBL_MAX_ITEM_NUM);
> +
> + if (entries_num == 0)
> + return NULL;
> +
> + tbl = rte_zmalloc_socket(__func__,
> + sizeof(struct gro_tcp6_tbl),
> + RTE_CACHE_LINE_SIZE,
> + socket_id);
> + if (tbl == NULL)
> + return NULL;
> +
> + size = sizeof(struct gro_tcp_item) * entries_num;
> + tbl->items = rte_zmalloc_socket(__func__,
> + size,
> + RTE_CACHE_LINE_SIZE,
> + socket_id);
> + if (tbl->items == NULL) {
> + rte_free(tbl);
> + return NULL;
> + }
> + tbl->max_item_num = entries_num;
> +
> + size = sizeof(struct gro_tcp6_flow) * entries_num;
> + tbl->flows = rte_zmalloc_socket(__func__,
> + size,
> + RTE_CACHE_LINE_SIZE,
> + socket_id);
> + if (tbl->flows == NULL) {
> + rte_free(tbl->items);
> + rte_free(tbl);
> + return NULL;
> + }
> + /* INVALID_ARRAY_INDEX indicates an empty flow */
> + for (i = 0; i < entries_num; i++)
> + tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
> + tbl->max_flow_num = entries_num;
> +
> + return tbl;
> +}
> +
> +void
> +gro_tcp6_tbl_destroy(void *tbl)
> +{
> + struct gro_tcp6_tbl *tcp_tbl = tbl;
> +
> + if (tcp_tbl) {
> + rte_free(tcp_tbl->items);
> + rte_free(tcp_tbl->flows);
> + }
> + rte_free(tcp_tbl);
> +}
> +
> +static inline uint32_t
> +find_an_empty_flow(struct gro_tcp6_tbl *tbl)
> +{
> + uint32_t i;
> + uint32_t max_flow_num = tbl->max_flow_num;
> +
> + for (i = 0; i < max_flow_num; i++)
> + if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
> + return i;
> + return INVALID_ARRAY_INDEX;
> +}
> +
> +static inline uint32_t
> +insert_new_flow(struct gro_tcp6_tbl *tbl,
> + struct tcp6_flow_key *src,
> + uint32_t item_idx)
> +{
> + struct tcp6_flow_key *dst;
> + uint32_t flow_idx;
> +
> + flow_idx = find_an_empty_flow(tbl);
> + if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
> + return INVALID_ARRAY_INDEX;
> +
> + dst = &(tbl->flows[flow_idx].key);
> +
> + ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
> + memcpy(&dst->src_addr[0], &src->src_addr[0],
> sizeof(dst->src_addr));
> + memcpy(&dst->dst_addr[0], &src->dst_addr[0],
> sizeof(dst->dst_addr));
> + dst->vtc_flow = src->vtc_flow;
> +
> + tbl->flows[flow_idx].start_index = item_idx;
> + tbl->flow_num++;
> +
> + return flow_idx;
> +}
> +
> +/*
> + * update the packet length for the flushed packet.
> + */
> +static inline void
> +update_header(struct gro_tcp_item *item)
> +{
> + struct rte_ipv6_hdr *ipv6_hdr;
> + struct rte_mbuf *pkt = item->firstseg;
> +
> + ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
> + pkt->l2_len);
> + ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
> + pkt->l2_len - pkt->l3_len);
> +}
> +
> +int32_t
> +gro_tcp6_reassemble(struct rte_mbuf *pkt,
> + struct gro_tcp6_tbl *tbl,
> + uint64_t start_time)
> +{
> + struct rte_ether_hdr *eth_hdr;
> + struct rte_ipv6_hdr *ipv6_hdr;
> + int32_t tcp_dl;
> + uint16_t ip_tlen;
> + struct tcp6_flow_key key;
> + uint32_t i, max_flow_num, remaining_flow_num;
> + uint32_t sent_seq;
> + struct rte_tcp_hdr *tcp_hdr;
> + uint8_t find;
> + uint32_t item_idx;
> + /*
> + * Don't process the packet whose TCP header length is greater
> + * than 60 bytes or less than 20 bytes.
> + */
> + if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
> + return -1;
> +
> + eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
> + ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
> + tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *,
> pkt->l2_len + pkt->l3_len);
> +
> + /*
> + * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
> + * or CWR set.
> + */
> + if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
> + return -1;
> +
> + ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
> + /*
> + * Don't process the packet whose payload length is less than or
> + * equal to 0.
> + */
> + tcp_dl = ip_tlen - pkt->l4_len;
> + if (tcp_dl <= 0)
> + return -1;
> +
> + rte_ether_addr_copy(&(eth_hdr->src_addr),
> &(key.cmn_key.eth_saddr));
> + rte_ether_addr_copy(&(eth_hdr->dst_addr),
> &(key.cmn_key.eth_daddr));
> + memcpy(&key.src_addr[0], &ipv6_hdr->src_addr,
> sizeof(key.src_addr));
> + memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr,
> sizeof(key.dst_addr));
> + key.cmn_key.src_port = tcp_hdr->src_port;
> + key.cmn_key.dst_port = tcp_hdr->dst_port;
> + key.cmn_key.recv_ack = tcp_hdr->recv_ack;
> + key.vtc_flow = ipv6_hdr->vtc_flow;
> +
> + /* Search for a matched flow. */
> + max_flow_num = tbl->max_flow_num;
> + remaining_flow_num = tbl->flow_num;
> + find = 0;
> + for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
> + if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
> + if (is_same_tcp6_flow(&tbl->flows[i].key, &key)) {
> + find = 1;
> + break;
> + }
> + remaining_flow_num--;
> + }
> + }
> +
> + if (find == 0) {
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> + item_idx = insert_new_tcp_item(pkt, tbl->items,
> &tbl->item_num,
> + tbl->max_item_num,
> start_time,
> + INVALID_ARRAY_INDEX,
> sent_seq, 0, true);
> + if (item_idx == INVALID_ARRAY_INDEX)
> + return -1;
> + if (insert_new_flow(tbl, &key, item_idx) ==
> + INVALID_ARRAY_INDEX) {
> + /*
> + * Fail to insert a new flow, so delete the
> + * stored packet.
> + */
> + delete_tcp_item(tbl->items, item_idx,
> &tbl->item_num, INVALID_ARRAY_INDEX);
> + return -1;
> + }
> + return 0;
> + }
> +
> + return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items,
> tbl->flows[i].start_index,
> + &tbl->item_num,
> tbl->max_item_num,
> + 0, true, start_time);
> +}
> +
> +uint16_t
> +gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
> + uint64_t flush_timestamp,
> + struct rte_mbuf **out,
> + uint16_t nb_out)
> +{
> + uint16_t k = 0;
> + uint32_t i, j;
> + uint32_t max_flow_num = tbl->max_flow_num;
> +
> + for (i = 0; i < max_flow_num; i++) {
> + if (unlikely(tbl->flow_num == 0))
> + return k;
> +
> + j = tbl->flows[i].start_index;
> + while (j != INVALID_ARRAY_INDEX) {
> + if (tbl->items[j].start_time <= flush_timestamp) {
> + out[k++] = tbl->items[j].firstseg;
> + if (tbl->items[j].nb_merged > 1)
> + update_header(&(tbl->items[j]));
> + /*
> + * Delete the packet and get the next
> + * packet in the flow.
> + */
> + j = delete_tcp_item(tbl->items, j,
> + &tbl->item_num,
> INVALID_ARRAY_INDEX);
> + tbl->flows[i].start_index = j;
> + if (j == INVALID_ARRAY_INDEX)
> + tbl->flow_num--;
> +
> + if (unlikely(k == nb_out))
> + return k;
> + } else
> + /*
> + * The left packets in this flow won't be
> + * timeout. Go to check other flows.
> + */
> + break;
> + }
> + }
> + return k;
> +}
> +
> +uint32_t
> +gro_tcp6_tbl_pkt_count(void *tbl)
> +{
> + struct gro_tcp6_tbl *gro_tbl = tbl;
> +
> + if (gro_tbl)
> + return gro_tbl->item_num;
> +
> + return 0;
> +}
> diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h
> new file mode 100644
> index 0000000000..073122f0ec
> --- /dev/null
> +++ b/lib/gro/gro_tcp6.h
> @@ -0,0 +1,160 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2023 Intel Corporation
> + */
> +
> +#ifndef _GRO_TCP6_H_
> +#define _GRO_TCP6_H_
> +
> +#include "gro_tcp.h"
> +
> +#define GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
> +
> +/* Header fields representing a TCP/IPv6 flow */
> +struct tcp6_flow_key {
> + struct cmn_tcp_key cmn_key;
> + uint8_t src_addr[16];
> + uint8_t dst_addr[16];
> + rte_be32_t vtc_flow;
> +};
> +
> +struct gro_tcp6_flow {
> + struct tcp6_flow_key key;
> + /*
> + * The index of the first packet in the flow.
> + * INVALID_ARRAY_INDEX indicates an empty flow.
> + */
> + uint32_t start_index;
> +};
> +
> +/*
> + * TCP/IPv6 reassembly table structure.
> + */
> +struct gro_tcp6_tbl {
> + /* item array */
> + struct gro_tcp_item *items;
> + /* flow array */
> + struct gro_tcp6_flow *flows;
> + /* current item number */
> + uint32_t item_num;
> + /* current flow num */
> + uint32_t flow_num;
> + /* item array size */
> + uint32_t max_item_num;
> + /* flow array size */
> + uint32_t max_flow_num;
> +};
> +
> +/**
> + * This function creates a TCP/IPv6 reassembly table.
> + *
> + * @param socket_id
> + * Socket index for allocating the TCP/IPv6 reassemble table
> + * @param max_flow_num
> + * The maximum number of flows in the TCP/IPv6 GRO table
> + * @param max_item_per_flow
> + * The maximum number of packets per flow
> + *
> + * @return
> + * - Return the table pointer on success.
> + * - Return NULL on failure.
> + */
> +void *gro_tcp6_tbl_create(uint16_t socket_id,
> + uint16_t max_flow_num,
> + uint16_t max_item_per_flow);
> +
> +/**
> + * This function destroys a TCP/IPv6 reassembly table.
> + *
> + * @param tbl
> + * Pointer pointing to the TCP/IPv6 reassembly table.
> + */
> +void gro_tcp6_tbl_destroy(void *tbl);
> +
> +/**
> + * This function merges a TCP/IPv6 packet. It doesn't process the packet,
> + * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't have
> + * payload.
> + *
> + * This function doesn't check if the packet has correct checksums and
> + * doesn't re-calculate checksums for the merged packet. Additionally,
> + * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
> + * when IP fragmentation is possible (i.e., DF==0). It returns the
> + * packet, if the packet has invalid parameters (e.g. SYN bit is set)
> + * or there is no available space in the table.
> + *
> + * @param pkt
> + * Packet to reassemble
> + * @param tbl
> + * Pointer pointing to the TCP/IPv6 reassembly table
> + * @start_time
> + * The time when the packet is inserted into the table
> + *
> + * @return
> + * - Return a positive value if the packet is merged.
> + * - Return zero if the packet isn't merged but stored in the table.
> + * - Return a negative value for invalid parameters or no available
> + * space in the table.
> + */
> +int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
> + struct gro_tcp6_tbl *tbl,
> + uint64_t start_time);
> +
> +/**
> + * This function flushes timeout packets in a TCP/IPv6 reassembly table,
> + * and without updating checksums.
> + *
> + * @param tbl
> + * TCP/IPv6 reassembly table pointer
> + * @param flush_timestamp
> + * Flush packets which are inserted into the table before or at the
> + * flush_timestamp.
> + * @param out
> + * Pointer array used to keep flushed packets
> + * @param nb_out
> + * The element number in 'out'. It also determines the maximum number of
> + * packets that can be flushed finally.
> + *
> + * @return
> + * The number of flushed packets
> + */
> +uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
> + uint64_t flush_timestamp,
> + struct rte_mbuf **out,
> + uint16_t nb_out);
> +
> +/**
> + * This function returns the number of the packets in a TCP/IPv6
> + * reassembly table.
> + *
> + * @param tbl
> + * TCP/IPv6 reassembly table pointer
> + *
> + * @return
> + * The number of packets in the table
> + */
> +uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
> +
> +/*
> + * Check if two TCP/IPv6 packets belong to the same flow.
> + */
> +static inline int
> +is_same_tcp6_flow(struct tcp6_flow_key *k1, struct tcp6_flow_key *k2)
> +{
> + rte_be32_t vtc_flow_diff;
> +
> + if (memcmp(&k1->src_addr, &k2->src_addr, 16))
> + return 0;
> + if (memcmp(&k1->dst_addr, &k2->dst_addr, 16))
> + return 0;
> + /*
> + * IP version (4) Traffic Class (8) Flow Label (20)
> + * All fields except Traffic class should be same
> + */
> + vtc_flow_diff = (k1->vtc_flow ^ k2->vtc_flow);
> + if (vtc_flow_diff & htonl(0xF00FFFFF))
> + return 0;
> +
> + return is_same_common_tcp_key(&k1->cmn_key, &k2->cmn_key);
> +}
> +
> +#endif
> diff --git a/lib/gro/gro_tcp_internal.h b/lib/gro/gro_tcp_internal.h
> new file mode 100644
> index 0000000000..cc84abeaeb
> --- /dev/null
> +++ b/lib/gro/gro_tcp_internal.h
> @@ -0,0 +1,128 @@
> +
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2023 Intel Corporation
> + */
> +
> +#ifndef _GRO_TCP_INTERNAL_H_
> +#define _GRO_TCP_INTERNAL_H_
> +
> +static inline uint32_t
> +find_an_empty_item(struct gro_tcp_item *items,
> + uint32_t max_item_num)
> +{
> + uint32_t i;
> +
> + for (i = 0; i < max_item_num; i++)
> + if (items[i].firstseg == NULL)
> + return i;
> + return INVALID_ARRAY_INDEX;
> +}
> +
> +static inline uint32_t
> +insert_new_tcp_item(struct rte_mbuf *pkt,
> + struct gro_tcp_item *items,
> + uint32_t *item_num,
> + uint32_t max_item_num,
> + uint64_t start_time,
> + uint32_t prev_idx,
> + uint32_t sent_seq,
> + uint16_t ip_id,
> + uint8_t is_atomic)
> +{
> + uint32_t item_idx;
> +
> + item_idx = find_an_empty_item(items, max_item_num);
> + if (item_idx == INVALID_ARRAY_INDEX)
> + return INVALID_ARRAY_INDEX;
> +
> + items[item_idx].firstseg = pkt;
> + items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
> + items[item_idx].start_time = start_time;
> + items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
> + items[item_idx].sent_seq = sent_seq;
> + items[item_idx].l3.ip_id = ip_id;
> + items[item_idx].nb_merged = 1;
> + items[item_idx].is_atomic = is_atomic;
> + (*item_num) += 1;
> +
> + /* if the previous packet exists, chain them together. */
> + if (prev_idx != INVALID_ARRAY_INDEX) {
> + items[item_idx].next_pkt_idx =
> + items[prev_idx].next_pkt_idx;
> + items[prev_idx].next_pkt_idx = item_idx;
> + }
> +
> + return item_idx;
> +}
> +
> +static inline uint32_t
> +delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t prev_item_idx)
> +{
> + uint32_t next_idx = items[item_idx].next_pkt_idx;
> +
> + /* NULL indicates an empty item */
> + items[item_idx].firstseg = NULL;
> + (*item_num) -= 1;
> + if (prev_item_idx != INVALID_ARRAY_INDEX)
> + items[prev_item_idx].next_pkt_idx = next_idx;
> +
> + return next_idx;
> +}
> +
> +static inline int32_t
> +process_tcp_item(struct rte_mbuf *pkt,
> + struct rte_tcp_hdr *tcp_hdr,
> + int32_t tcp_dl,
> + struct gro_tcp_item *items,
> + uint32_t item_idx,
> + uint32_t *item_num,
> + uint32_t max_item_num,
> + uint16_t ip_id,
> + uint8_t is_atomic,
> + uint64_t start_time)
> +{
> + uint32_t cur_idx;
> + uint32_t prev_idx;
> + int cmp;
> + uint32_t sent_seq;
> +
> + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
> + /*
> + * Check all packets in the flow and try to find a neighbor for
> + * the input packet.
> + */
> + cur_idx = item_idx;
> + prev_idx = cur_idx;
> + do {
> + cmp = check_seq_option(&items[cur_idx], tcp_hdr,
> + sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
> + is_atomic);
> + if (cmp) {
> + if (merge_two_tcp_packets(&items[cur_idx],
> + pkt, cmp, sent_seq, ip_id,
> 0))
> + return 1;
> + /*
> + * Fail to merge the two packets, as the packet
> + * length is greater than the max value. Store
> + * the packet into the flow.
> + */
> + if (insert_new_tcp_item(pkt, items, item_num,
> max_item_num,
> + start_time, cur_idx,
> sent_seq, ip_id, is_atomic) ==
> + INVALID_ARRAY_INDEX)
> + return -1;
> + return 0;
> + }
> + prev_idx = cur_idx;
> + cur_idx = items[cur_idx].next_pkt_idx;
> + } while (cur_idx != INVALID_ARRAY_INDEX);
> +
> + /* Fail to find a neighbor, so store the packet into the flow. */
> + if (insert_new_tcp_item(pkt, items, item_num, max_item_num,
> start_time, prev_idx, sent_seq,
> + ip_id, is_atomic) == INVALID_ARRAY_INDEX)
> + return -1;
> +
> + return 0;
> +}
> +#endif
> diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
> index 3be4deb7c7..6ab7001922 100644
> --- a/lib/gro/gro_vxlan_tcp4.c
> +++ b/lib/gro/gro_vxlan_tcp4.c
> @@ -116,7 +116,7 @@ insert_new_item(struct gro_vxlan_tcp4_tbl *tbl,
> tbl->items[item_idx].inner_item.start_time = start_time;
> tbl->items[item_idx].inner_item.next_pkt_idx = INVALID_ARRAY_INDEX;
> tbl->items[item_idx].inner_item.sent_seq = sent_seq;
> - tbl->items[item_idx].inner_item.ip_id = ip_id;
> + tbl->items[item_idx].inner_item.l3.ip_id = ip_id;
> tbl->items[item_idx].inner_item.nb_merged = 1;
> tbl->items[item_idx].inner_item.is_atomic = is_atomic;
> tbl->items[item_idx].outer_ip_id = outer_ip_id;
> @@ -163,15 +163,9 @@ insert_new_flow(struct gro_vxlan_tcp4_tbl *tbl,
>
> dst = &(tbl->flows[flow_idx].key);
>
> - rte_ether_addr_copy(&(src->inner_key.eth_saddr),
> - &(dst->inner_key.eth_saddr));
> - rte_ether_addr_copy(&(src->inner_key.eth_daddr),
> - &(dst->inner_key.eth_daddr));
> + ASSIGN_COMMON_TCP_KEY((&(src->inner_key.cmn_key)),
> (&(dst->inner_key.cmn_key)));
> dst->inner_key.ip_src_addr = src->inner_key.ip_src_addr;
> dst->inner_key.ip_dst_addr = src->inner_key.ip_dst_addr;
> - dst->inner_key.recv_ack = src->inner_key.recv_ack;
> - dst->inner_key.src_port = src->inner_key.src_port;
> - dst->inner_key.dst_port = src->inner_key.dst_port;
>
> dst->vxlan_hdr.vx_flags = src->vxlan_hdr.vx_flags;
> dst->vxlan_hdr.vx_vni = src->vxlan_hdr.vx_vni;
> @@ -248,7 +242,7 @@ merge_two_vxlan_tcp4_packets(struct
> gro_vxlan_tcp4_item *item,
> uint16_t outer_ip_id,
> uint16_t ip_id)
> {
> - if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp, sent_seq,
> + if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
> ip_id, pkt->outer_l2_len +
> pkt->outer_l3_len)) {
> /* Update the outer IPv4 ID to the large value. */
> @@ -357,13 +351,13 @@ gro_vxlan_tcp4_reassemble(struct rte_mbuf *pkt,
>
> sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
>
> - rte_ether_addr_copy(&(eth_hdr->src_addr),
> &(key.inner_key.eth_saddr));
> - rte_ether_addr_copy(&(eth_hdr->dst_addr),
> &(key.inner_key.eth_daddr));
> + rte_ether_addr_copy(&(eth_hdr->src_addr),
> &(key.inner_key.cmn_key.eth_saddr));
> + rte_ether_addr_copy(&(eth_hdr->dst_addr),
> &(key.inner_key.cmn_key.eth_daddr));
> key.inner_key.ip_src_addr = ipv4_hdr->src_addr;
> key.inner_key.ip_dst_addr = ipv4_hdr->dst_addr;
> - key.inner_key.recv_ack = tcp_hdr->recv_ack;
> - key.inner_key.src_port = tcp_hdr->src_port;
> - key.inner_key.dst_port = tcp_hdr->dst_port;
> + key.inner_key.cmn_key.recv_ack = tcp_hdr->recv_ack;
> + key.inner_key.cmn_key.src_port = tcp_hdr->src_port;
> + key.inner_key.cmn_key.dst_port = tcp_hdr->dst_port;
>
> key.vxlan_hdr.vx_flags = vxlan_hdr->vx_flags;
> key.vxlan_hdr.vx_vni = vxlan_hdr->vx_vni;
> diff --git a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h
> index 7832942a68..662db01a88 100644
> --- a/lib/gro/gro_vxlan_tcp4.h
> +++ b/lib/gro/gro_vxlan_tcp4.h
> @@ -36,7 +36,7 @@ struct gro_vxlan_tcp4_flow {
> };
>
> struct gro_vxlan_tcp4_item {
> - struct gro_tcp4_item inner_item;
> + struct gro_tcp_item inner_item;
> /* IPv4 ID in the outer IPv4 header */
> uint16_t outer_ip_id;
> /* Indicate if outer IPv4 ID can be ignored */
> diff --git a/lib/gro/meson.build b/lib/gro/meson.build
> index e4fa2958bd..dbce05220d 100644
> --- a/lib/gro/meson.build
> +++ b/lib/gro/meson.build
> @@ -4,6 +4,7 @@
> sources = files(
> 'rte_gro.c',
> 'gro_tcp4.c',
> + 'gro_tcp6.c',
> 'gro_udp4.c',
> 'gro_vxlan_tcp4.c',
> 'gro_vxlan_udp4.c',
> diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c
> index e35399fd42..d824eebd93 100644
> --- a/lib/gro/rte_gro.c
> +++ b/lib/gro/rte_gro.c
> @@ -8,6 +8,7 @@
>
> #include "rte_gro.h"
> #include "gro_tcp4.h"
> +#include "gro_tcp6.h"
> #include "gro_udp4.h"
> #include "gro_vxlan_tcp4.h"
> #include "gro_vxlan_udp4.h"
> @@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
>
> static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
> gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
> - gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
> + gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create,
> gro_tcp6_tbl_create, NULL};
> static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
> gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
> gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
> + gro_tcp6_tbl_destroy,
> NULL};
> static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
> gro_tcp4_tbl_pkt_count,
> gro_vxlan_tcp4_tbl_pkt_count,
> gro_udp4_tbl_pkt_count,
> gro_vxlan_udp4_tbl_pkt_count,
> + gro_tcp6_tbl_pkt_count,
> NULL};
>
> #define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
> @@ -35,6 +38,12 @@ static gro_tbl_pkt_count_fn
> tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
> ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
> (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
>
> +/* GRO with extension headers is not supported */
> +#define IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
> + ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
> + ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
> + (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
> +
> #define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
> ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
> (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
> @@ -147,7 +156,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> /* allocate a reassembly table for TCP/IPv4 GRO */
> struct gro_tcp4_tbl tcp_tbl;
> struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
> - struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0}
> };
> + struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
> +
> + struct gro_tcp6_tbl tcp6_tbl;
> + struct gro_tcp6_flow tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
> + struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0}
> };
>
> /* allocate a reassembly table for UDP/IPv4 GRO */
> struct gro_udp4_tbl udp_tbl;
> @@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> int32_t ret;
> uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
> uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
> - do_vxlan_udp_gro = 0;
> + do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
>
> if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
> - RTE_GRO_TCP_IPV4 |
> + RTE_GRO_TCP_IPV4 |
> RTE_GRO_TCP_IPV6 |
> RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
> RTE_GRO_UDP_IPV4)) == 0))
> return nb_pkts;
> @@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> do_udp4_gro = 1;
> }
>
> + if (param->gro_types & RTE_GRO_TCP_IPV6) {
> + for (i = 0; i < item_num; i++)
> + tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
> +
> + tcp6_tbl.flows = tcp6_flows;
> + tcp6_tbl.items = tcp6_items;
> + tcp6_tbl.flow_num = 0;
> + tcp6_tbl.item_num = 0;
> + tcp6_tbl.max_flow_num = item_num;
> + tcp6_tbl.max_item_num = item_num;
> + do_tcp6_gro = 1;
> + }
>
> for (i = 0; i < nb_pkts; i++) {
> /*
> @@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> nb_after_gro--;
> else if (ret < 0)
> unprocess_pkts[unprocess_num++] = pkts[i];
> + } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
> + do_tcp6_gro) {
> + ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
> + if (ret > 0)
> + /* merge successfully */
> + nb_after_gro--;
> + else if (ret < 0)
> + unprocess_pkts[unprocess_num++] = pkts[i];
> } else
> unprocess_pkts[unprocess_num++] = pkts[i];
> }
> @@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> if ((nb_after_gro < nb_pkts)
> || (unprocess_num < nb_pkts)) {
> i = 0;
> + /* Copy unprocessed packets */
> + if (unprocess_num > 0) {
> + memcpy(&pkts[i], unprocess_pkts,
> + sizeof(struct rte_mbuf *) *
> + unprocess_num);
> + i = unprocess_num;
> + }
> +
> /* Flush all packets from the tables */
> if (do_vxlan_tcp_gro) {
> - i =
> gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
> + i +=
> gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
> 0, pkts, nb_pkts);
> }
>
> @@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
> i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
> &pkts[i], nb_pkts - i);
> }
> - /* Copy unprocessed packets */
> - if (unprocess_num > 0) {
> - memcpy(&pkts[i], unprocess_pkts,
> - sizeof(struct rte_mbuf *) *
> - unprocess_num);
> +
> + if (do_tcp6_gro) {
> + i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
> + &pkts[i], nb_pkts - i);
> }
> - nb_after_gro = i + unprocess_num;
> }
>
> return nb_after_gro;
> @@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> {
> struct rte_mbuf *unprocess_pkts[nb_pkts];
> struct gro_ctx *gro_ctx = ctx;
> - void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
> + void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
> uint64_t current_time;
> uint16_t i, unprocess_num = 0;
> - uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro,
> do_vxlan_udp_gro;
> + uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro,
> do_vxlan_udp_gro, do_tcp6_gro;
>
> if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
> - RTE_GRO_TCP_IPV4 |
> + RTE_GRO_TCP_IPV4 |
> RTE_GRO_TCP_IPV6 |
> RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
> RTE_GRO_UDP_IPV4)) == 0))
> return nb_pkts;
> @@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
> udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
> vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
> + tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
>
> do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
> RTE_GRO_TCP_IPV4;
> @@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> RTE_GRO_UDP_IPV4;
> do_vxlan_udp_gro = (gro_ctx->gro_types &
> RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
> RTE_GRO_IPV4_VXLAN_UDP_IPV4;
> + do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) ==
> RTE_GRO_TCP_IPV6;
>
> current_time = rte_rdtsc();
>
> @@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
> if (gro_udp4_reassemble(pkts[i], udp_tbl,
> current_time) < 0)
> unprocess_pkts[unprocess_num++] = pkts[i];
> + } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
> + do_tcp6_gro) {
> + if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
> + current_time) < 0)
> + unprocess_pkts[unprocess_num++] = pkts[i];
> } else
> unprocess_pkts[unprocess_num++] = pkts[i];
> }
> @@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
> gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
> flush_timestamp,
> &out[num], left_nb_out);
> + left_nb_out = max_nb_out - num;
> + }
> +
> + if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
> + num += gro_tcp6_tbl_timeout_flush(
> + gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
> + flush_timestamp,
> + &out[num], left_nb_out);
> +
> }
>
> return num;
> diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h
> index 9f9ed4935a..c83dfd9ad1 100644
> --- a/lib/gro/rte_gro.h
> +++ b/lib/gro/rte_gro.h
> @@ -38,6 +38,9 @@ extern "C" {
> #define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3
> #define RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL <<
> RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
> /**< VxLAN UDP/IPv4 GRO flag. */
> +#define RTE_GRO_TCP_IPV6_INDEX 4
> +#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX)
> +/**< TCP/IPv6 GRO flag. */
>
> /**
> * Structure used to create GRO context objects or used to pass
> --
> 2.25.1
>
>> Hi Jiyau, In the recent patch I have moved the functions that had the
>> error in Windows to the header file and since that is used by gro_tcp6.c
>> and gro_tcp4.c, I have moved it to gro_tcp_internal.h. There are other
>> functions and ds reused by gro_vxlan_tcp4.c, I have moved them to gro_tcp.h
>> so that the functions that are not used by gro_vxlan_tcp4.c will not be
>> included.
>>
>
[-- Attachment #2: Type: text/html, Size: 79372 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread
* RE: [PATCH v9] gro : ipv6 changes to support GRO for TCP/ipv6
2023-06-15 6:20 ` [PATCH v9] " Kumara Parameshwaran
2023-06-15 6:30 ` kumaraparameshwaran rathinavel
@ 2023-06-15 8:01 ` Hu, Jiayu
2023-06-15 9:16 ` kumaraparameshwaran rathinavel
2023-06-19 13:30 ` Thomas Monjalon
2 siblings, 1 reply; 32+ messages in thread
From: Hu, Jiayu @ 2023-06-15 8:01 UTC (permalink / raw)
To: Kumara Parameshwaran; +Cc: dev
Hi Kumara,
> -----Original Message-----
> From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> Sent: Thursday, June 15, 2023 2:20 PM
> To: Hu, Jiayu <jiayu.hu@intel.com>
> Cc: dev@dpdk.org; Kumara Parameshwaran
> <kumaraparamesh92@gmail.com>
> Subject: [PATCH v9] gro : ipv6 changes to support GRO for TCP/ipv6
>
> The patch adds GRO support for TCP/ipv6 packets. This does not include the
> support for vxlan, udp ipv6 packets.
>
> Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> ---
> v1:
> * Changes to support GRO for TCP/ipv6 packets. This does not
> include
> vxlan changes.
> * The GRO is performed only for ipv6 packets that does not contain
> extension headers.
> * The logic for the TCP coalescing remains the same, in ipv6 header
> the source address, destination address, flow label, version fields
> are expected to be the same.
> * Re-organised the code to reuse certain tcp functions for both ipv4
> and
> ipv6 flows.
> v2:
> * Fix comments in gro_tcp6.h header file.
>
> v3:
> * Adderess review comments to fix code duplication for v4 and v6
>
> v4:
> * Addresses review comments for v3, do not use callbacks
>
> v5:
> * Address review comments
>
> v6:
> * Fix warning and coding style issues
>
> v7:
> * Fix build compilation issue
>
> v8:
> * Use gro_tcp_internal.h for functions used for gro_tcp4 and
> gro_tcp6 and use
> gro_tcp.h for data structures and functions used across
> gro_vxlan_tcp4
>
> v9:
> * Resolve merge conflict and add gro_tcp.h in proper path
>
> .../generic_receive_offload_lib.rst | 21 +-
> doc/guides/rel_notes/release_23_07.rst | 2 +
> lib/gro/gro_tcp.h | 187 ++++++++++++
> lib/gro/gro_tcp4.c | 177 +++---------
> lib/gro/gro_tcp4.h | 170 +----------
> lib/gro/gro_tcp6.c | 268 ++++++++++++++++++
> lib/gro/gro_tcp6.h | 160 +++++++++++
> lib/gro/gro_tcp_internal.h | 128 +++++++++
> lib/gro/gro_vxlan_tcp4.c | 22 +-
> lib/gro/gro_vxlan_tcp4.h | 2 +-
> lib/gro/meson.build | 1 +
> lib/gro/rte_gro.c | 83 +++++-
> lib/gro/rte_gro.h | 3 +
> 13 files changed, 883 insertions(+), 341 deletions(-) create mode 100644
> lib/gro/gro_tcp.h create mode 100644 lib/gro/gro_tcp6.c create mode
> 100644 lib/gro/gro_tcp6.h create mode 100644 lib/gro/gro_tcp_internal.h
>
Reviewed-by: Jiayu Hu <Jiayu.hu@intel.com>
Hi Kumara,
I notice you write Intel in the license header. Is it intended or mistaken?
Thanks,
Jiayu
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [PATCH v9] gro : ipv6 changes to support GRO for TCP/ipv6
2023-06-15 8:01 ` Hu, Jiayu
@ 2023-06-15 9:16 ` kumaraparameshwaran rathinavel
0 siblings, 0 replies; 32+ messages in thread
From: kumaraparameshwaran rathinavel @ 2023-06-15 9:16 UTC (permalink / raw)
To: Hu, Jiayu; +Cc: dev
[-- Attachment #1: Type: text/plain, Size: 3217 bytes --]
On Thu, Jun 15, 2023 at 1:31 PM Hu, Jiayu <jiayu.hu@intel.com> wrote:
> Hi Kumara,
>
> > -----Original Message-----
> > From: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> > Sent: Thursday, June 15, 2023 2:20 PM
> > To: Hu, Jiayu <jiayu.hu@intel.com>
> > Cc: dev@dpdk.org; Kumara Parameshwaran
> > <kumaraparamesh92@gmail.com>
> > Subject: [PATCH v9] gro : ipv6 changes to support GRO for TCP/ipv6
> >
> > The patch adds GRO support for TCP/ipv6 packets. This does not include
> the
> > support for vxlan, udp ipv6 packets.
> >
> > Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> > ---
> > v1:
> > * Changes to support GRO for TCP/ipv6 packets. This does not
> > include
> > vxlan changes.
> > * The GRO is performed only for ipv6 packets that does not contain
> > extension headers.
> > * The logic for the TCP coalescing remains the same, in ipv6 header
> > the source address, destination address, flow label, version
> fields
> > are expected to be the same.
> > * Re-organised the code to reuse certain tcp functions for both
> ipv4
> > and
> > ipv6 flows.
> > v2:
> > * Fix comments in gro_tcp6.h header file.
> >
> > v3:
> > * Adderess review comments to fix code duplication for v4 and v6
> >
> > v4:
> > * Addresses review comments for v3, do not use callbacks
> >
> > v5:
> > * Address review comments
> >
> > v6:
> > * Fix warning and coding style issues
> >
> > v7:
> > * Fix build compilation issue
> >
> > v8:
> > * Use gro_tcp_internal.h for functions used for gro_tcp4 and
> > gro_tcp6 and use
> > gro_tcp.h for data structures and functions used across
> > gro_vxlan_tcp4
> >
> > v9:
> > * Resolve merge conflict and add gro_tcp.h in proper path
> >
> > .../generic_receive_offload_lib.rst | 21 +-
> > doc/guides/rel_notes/release_23_07.rst | 2 +
> > lib/gro/gro_tcp.h | 187 ++++++++++++
> > lib/gro/gro_tcp4.c | 177 +++---------
> > lib/gro/gro_tcp4.h | 170 +----------
> > lib/gro/gro_tcp6.c | 268 ++++++++++++++++++
> > lib/gro/gro_tcp6.h | 160 +++++++++++
> > lib/gro/gro_tcp_internal.h | 128 +++++++++
> > lib/gro/gro_vxlan_tcp4.c | 22 +-
> > lib/gro/gro_vxlan_tcp4.h | 2 +-
> > lib/gro/meson.build | 1 +
> > lib/gro/rte_gro.c | 83 +++++-
> > lib/gro/rte_gro.h | 3 +
> > 13 files changed, 883 insertions(+), 341 deletions(-) create mode
> 100644
> > lib/gro/gro_tcp.h create mode 100644 lib/gro/gro_tcp6.c create mode
> > 100644 lib/gro/gro_tcp6.h create mode 100644 lib/gro/gro_tcp_internal.h
> >
>
> Reviewed-by: Jiayu Hu <Jiayu.hu@intel.com>
>
> Hi Kumara,
>
> I notice you write Intel in the license header. Is it intended or mistaken?
>
>>
>> Hi Jiyau, This is intentional.
>>
>> Thanks,
>> Kumara.
>>
>
> Thanks,
> Jiayu
>
[-- Attachment #2: Type: text/html, Size: 4965 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [PATCH v9] gro : ipv6 changes to support GRO for TCP/ipv6
2023-06-15 6:20 ` [PATCH v9] " Kumara Parameshwaran
2023-06-15 6:30 ` kumaraparameshwaran rathinavel
2023-06-15 8:01 ` Hu, Jiayu
@ 2023-06-19 13:30 ` Thomas Monjalon
2 siblings, 0 replies; 32+ messages in thread
From: Thomas Monjalon @ 2023-06-19 13:30 UTC (permalink / raw)
To: Kumara Parameshwaran; +Cc: jiayu.hu, dev
15/06/2023 08:20, Kumara Parameshwaran:
> The patch adds GRO support for TCP/ipv6 packets. This does not
> include the support for vxlan, udp ipv6 packets.
>
> Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
> ---
> lib/gro/gro_tcp.h | 187 ++++++++++++
> lib/gro/gro_tcp4.c | 177 +++---------
> lib/gro/gro_tcp4.h | 170 +----------
> lib/gro/gro_tcp6.c | 268 ++++++++++++++++++
> lib/gro/gro_tcp6.h | 160 +++++++++++
> lib/gro/gro_tcp_internal.h | 128 +++++++++
> lib/gro/gro_vxlan_tcp4.c | 22 +-
> lib/gro/gro_vxlan_tcp4.h | 2 +-
> lib/gro/meson.build | 1 +
> lib/gro/rte_gro.c | 83 +++++-
> lib/gro/rte_gro.h | 3 +
Please would it be possible to have 2 patches:
- first would move IPv4 code in a new file (without any other change)
- second would add IPv6 support
It would allow to see more clearly what is changed for IPv6.
Thanks
^ permalink raw reply [flat|nested] 32+ messages in thread
* [PATCH v10 1/2] gro : refactor IPv4 to add GRO support for IPv6
2022-10-20 18:14 ` [PATCH v2] " Kumara Parameshwaran
` (8 preceding siblings ...)
2023-06-15 6:20 ` [PATCH v9] " Kumara Parameshwaran
@ 2023-06-21 8:25 ` Kumara Parameshwaran
2023-06-21 8:25 ` [PATCH v10 2/2] gro : add support for IPv6 GRO Kumara Parameshwaran
2023-06-21 8:38 ` [PATCH v11 1/2] gro : refactor IPv4 to add GRO support for IPv6 Kumara Parameshwaran
10 siblings, 1 reply; 32+ messages in thread
From: Kumara Parameshwaran @ 2023-06-21 8:25 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev, Kumara Parameshwaran
Refactor the existing code to reuse data strutures and functions
to add support for IPv6 GRO
Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
---
v1:
* Changes to support GRO for TCP/ipv6 packets. This does not include
vxlan changes.
* The GRO is performed only for ipv6 packets that does not contain
extension headers.
* The logic for the TCP coalescing remains the same, in ipv6 header
the source address, destination address, flow label, version fields
are expected to be the same.
* Re-organised the code to reuse certain tcp functions for both ipv4 and
ipv6 flows.
v2:
* Fix comments in gro_tcp6.h header file.
v3:
* Adderess review comments to fix code duplication for v4 and v6
v4:
* Addresses review comments for v3, do not use callbacks
v5:
* Address review comments
v6:
* Fix warning and coding style issues
v7:
* Fix build compilation issue
v8:
* Use gro_tcp_internal.h for functions used for gro_tcp4 and gro_tcp6 and use
gro_tcp.h for data structures and functions used across gro_vxlan_tcp4
v9:
* Resolve merge conflict and add gro_tcp.h in proper path
v10:
* Refactor the code to contain 2 patches. This patch contains code/data strcuture changes
in ipv4 code that can be reused.
lib/gro/gro_tcp.h | 190 +++++++++++++++++++++++++++++++++++++
lib/gro/gro_tcp4.c | 177 ++++++++--------------------------
lib/gro/gro_tcp4.h | 170 ++-------------------------------
lib/gro/gro_tcp_internal.h | 128 +++++++++++++++++++++++++
lib/gro/gro_vxlan_tcp4.c | 22 ++---
lib/gro/gro_vxlan_tcp4.h | 2 +-
lib/gro/rte_gro.c | 2 +-
7 files changed, 373 insertions(+), 318 deletions(-)
create mode 100644 lib/gro/gro_tcp.h
create mode 100644 lib/gro/gro_tcp_internal.h
diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h
new file mode 100644
index 0000000000..d926c4b8cc
--- /dev/null
+++ b/lib/gro/gro_tcp.h
@@ -0,0 +1,190 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+#ifndef _GRO_TCP_H_
+#define _GRO_TCP_H_
+
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+
+#include <rte_tcp.h>
+
+/*
+ * The max length of a IPv4 packet, which includes the length of the L3
+ * header, the L4 header and the data payload.
+ */
+#define MAX_IP_PKT_LENGTH UINT16_MAX
+
+/* The maximum TCP header length */
+#define MAX_TCP_HLEN 60
+#define INVALID_TCP_HDRLEN(len) \
+ (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
+
+struct cmn_tcp_key {
+ struct rte_ether_addr eth_saddr;
+ struct rte_ether_addr eth_daddr;
+ uint32_t recv_ack;
+ uint16_t src_port;
+ uint16_t dst_port;
+};
+
+#define ASSIGN_COMMON_TCP_KEY(k1, k2) \
+ do {\
+ rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
+ rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr)); \
+ k2->recv_ack = k1->recv_ack; \
+ k2->src_port = k1->src_port; \
+ k2->dst_port = k1->dst_port; \
+ } while (0)
+
+struct gro_tcp_item {
+ /*
+ * The first MBUF segment of the packet. If the value
+ * is NULL, it means the item is empty.
+ */
+ struct rte_mbuf *firstseg;
+ /* The last MBUF segment of the packet */
+ struct rte_mbuf *lastseg;
+ /*
+ * The time when the first packet is inserted into the table.
+ * This value won't be updated, even if the packet is merged
+ * with other packets.
+ */
+ uint64_t start_time;
+ /*
+ * next_pkt_idx is used to chain the packets that
+ * are in the same flow but can't be merged together
+ * (e.g. caused by packet reordering).
+ */
+ uint32_t next_pkt_idx;
+ /* TCP sequence number of the packet */
+ uint32_t sent_seq;
+ union {
+ /* IPv4 ID of the packet */
+ uint16_t ip_id;
+ /* Unused field for IPv6 */
+ uint16_t unused;
+ } l3;
+ /* the number of merged packets */
+ uint16_t nb_merged;
+ /* Indicate if IPv4 ID can be ignored */
+ uint8_t is_atomic;
+};
+
+/*
+ * Merge two TCP packets without updating checksums.
+ * If cmp is larger than 0, append the new packet to the
+ * original packet. Otherwise, pre-pend the new packet to
+ * the original packet.
+ */
+static inline int
+merge_two_tcp_packets(struct gro_tcp_item *item,
+ struct rte_mbuf *pkt,
+ int cmp,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t l2_offset)
+{
+ struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
+ uint16_t hdr_len, l2_len;
+
+ if (cmp > 0) {
+ pkt_head = item->firstseg;
+ pkt_tail = pkt;
+ } else {
+ pkt_head = pkt;
+ pkt_tail = item->firstseg;
+ }
+
+ /* check if the IPv4 packet length is greater than the max value */
+ hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
+ pkt_head->l4_len;
+ l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
+ if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
+ hdr_len > MAX_IP_PKT_LENGTH))
+ return 0;
+
+ if (unlikely(pkt_head->nb_segs >= 20))
+ return 0;
+
+ /* remove the packet header for the tail packet */
+ rte_pktmbuf_adj(pkt_tail, hdr_len);
+
+ /* chain two packets together */
+ if (cmp > 0) {
+ item->lastseg->next = pkt;
+ item->lastseg = rte_pktmbuf_lastseg(pkt);
+ /* update IP ID to the larger value */
+ item->l3.ip_id = ip_id;
+ } else {
+ lastseg = rte_pktmbuf_lastseg(pkt);
+ lastseg->next = item->firstseg;
+ item->firstseg = pkt;
+ /* update sent_seq to the smaller value */
+ item->sent_seq = sent_seq;
+ item->l3.ip_id = ip_id;
+ }
+ item->nb_merged++;
+
+ /* update MBUF metadata for the merged packet */
+ pkt_head->nb_segs += pkt_tail->nb_segs;
+ pkt_head->pkt_len += pkt_tail->pkt_len;
+
+ return 1;
+}
+
+/*
+ * Check if two TCP packets are neighbors.
+ */
+static inline int
+check_seq_option(struct gro_tcp_item *item,
+ struct rte_tcp_hdr *tcph,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t tcp_hl,
+ uint16_t tcp_dl,
+ uint16_t l2_offset,
+ uint8_t is_atomic)
+{
+ struct rte_mbuf *pkt_orig = item->firstseg;
+ char *iph_orig;
+ struct rte_tcp_hdr *tcph_orig;
+ uint16_t len, tcp_hl_orig;
+
+ iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
+ l2_offset + pkt_orig->l2_len);
+ tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
+ tcp_hl_orig = pkt_orig->l4_len;
+
+ /* Check if TCP option fields equal */
+ len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
+ if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
+ (memcmp(tcph + 1, tcph_orig + 1,
+ len) != 0)))
+ return 0;
+
+ /* Don't merge packets whose DF bits are different */
+ if (unlikely(item->is_atomic ^ is_atomic))
+ return 0;
+
+ /* check if the two packets are neighbors */
+ len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
+ pkt_orig->l3_len - tcp_hl_orig;
+ if ((sent_seq == item->sent_seq + len) && (is_atomic ||
+ (ip_id == item->l3.ip_id + 1)))
+ /* append the new packet */
+ return 1;
+ else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
+ (ip_id + item->nb_merged == item->l3.ip_id)))
+ /* pre-pend the new packet */
+ return -1;
+
+ return 0;
+}
+
+static inline int
+is_same_common_tcp_key(struct cmn_tcp_key *k1, struct cmn_tcp_key *k2)
+{
+ return (!memcmp(k1, k2, sizeof(struct cmn_tcp_key)));
+}
+
+#endif
diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c
index 0014096e63..6645de592b 100644
--- a/lib/gro/gro_tcp4.c
+++ b/lib/gro/gro_tcp4.c
@@ -7,6 +7,7 @@
#include <rte_ethdev.h>
#include "gro_tcp4.h"
+#include "gro_tcp_internal.h"
void *
gro_tcp4_tbl_create(uint16_t socket_id,
@@ -30,7 +31,7 @@ gro_tcp4_tbl_create(uint16_t socket_id,
if (tbl == NULL)
return NULL;
- size = sizeof(struct gro_tcp4_item) * entries_num;
+ size = sizeof(struct gro_tcp_item) * entries_num;
tbl->items = rte_zmalloc_socket(__func__,
size,
RTE_CACHE_LINE_SIZE,
@@ -71,18 +72,6 @@ gro_tcp4_tbl_destroy(void *tbl)
rte_free(tcp_tbl);
}
-static inline uint32_t
-find_an_empty_item(struct gro_tcp4_tbl *tbl)
-{
- uint32_t i;
- uint32_t max_item_num = tbl->max_item_num;
-
- for (i = 0; i < max_item_num; i++)
- if (tbl->items[i].firstseg == NULL)
- return i;
- return INVALID_ARRAY_INDEX;
-}
-
static inline uint32_t
find_an_empty_flow(struct gro_tcp4_tbl *tbl)
{
@@ -95,56 +84,6 @@ find_an_empty_flow(struct gro_tcp4_tbl *tbl)
return INVALID_ARRAY_INDEX;
}
-static inline uint32_t
-insert_new_item(struct gro_tcp4_tbl *tbl,
- struct rte_mbuf *pkt,
- uint64_t start_time,
- uint32_t prev_idx,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint8_t is_atomic)
-{
- uint32_t item_idx;
-
- item_idx = find_an_empty_item(tbl);
- if (item_idx == INVALID_ARRAY_INDEX)
- return INVALID_ARRAY_INDEX;
-
- tbl->items[item_idx].firstseg = pkt;
- tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
- tbl->items[item_idx].start_time = start_time;
- tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
- tbl->items[item_idx].sent_seq = sent_seq;
- tbl->items[item_idx].ip_id = ip_id;
- tbl->items[item_idx].nb_merged = 1;
- tbl->items[item_idx].is_atomic = is_atomic;
- tbl->item_num++;
-
- /* if the previous packet exists, chain them together. */
- if (prev_idx != INVALID_ARRAY_INDEX) {
- tbl->items[item_idx].next_pkt_idx =
- tbl->items[prev_idx].next_pkt_idx;
- tbl->items[prev_idx].next_pkt_idx = item_idx;
- }
-
- return item_idx;
-}
-
-static inline uint32_t
-delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
- uint32_t prev_item_idx)
-{
- uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
-
- /* NULL indicates an empty item */
- tbl->items[item_idx].firstseg = NULL;
- tbl->item_num--;
- if (prev_item_idx != INVALID_ARRAY_INDEX)
- tbl->items[prev_item_idx].next_pkt_idx = next_idx;
-
- return next_idx;
-}
-
static inline uint32_t
insert_new_flow(struct gro_tcp4_tbl *tbl,
struct tcp4_flow_key *src,
@@ -159,13 +98,10 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
- rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
+ ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
+
dst->ip_src_addr = src->ip_src_addr;
dst->ip_dst_addr = src->ip_dst_addr;
- dst->recv_ack = src->recv_ack;
- dst->src_port = src->src_port;
- dst->dst_port = src->dst_port;
tbl->flows[flow_idx].start_index = item_idx;
tbl->flow_num++;
@@ -173,21 +109,6 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
return flow_idx;
}
-/*
- * update the packet length for the flushed packet.
- */
-static inline void
-update_header(struct gro_tcp4_item *item)
-{
- struct rte_ipv4_hdr *ipv4_hdr;
- struct rte_mbuf *pkt = item->firstseg;
-
- ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
- pkt->l2_len);
- ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
- pkt->l2_len);
-}
-
int32_t
gro_tcp4_reassemble(struct rte_mbuf *pkt,
struct gro_tcp4_tbl *tbl,
@@ -202,9 +123,8 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
uint8_t is_atomic;
struct tcp4_flow_key key;
- uint32_t cur_idx, prev_idx, item_idx;
+ uint32_t item_idx;
uint32_t i, max_flow_num, remaining_flow_num;
- int cmp;
uint8_t find;
/*
@@ -239,6 +159,14 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
if (tcp_dl <= 0)
return -1;
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.cmn_key.eth_daddr));
+ key.ip_src_addr = ipv4_hdr->src_addr;
+ key.ip_dst_addr = ipv4_hdr->dst_addr;
+ key.cmn_key.src_port = tcp_hdr->src_port;
+ key.cmn_key.dst_port = tcp_hdr->dst_port;
+ key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+
/*
* Save IPv4 ID for the packet whose DF bit is 0. For the packet
* whose DF bit is 1, IPv4 ID is ignored.
@@ -246,15 +174,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) == RTE_IPV4_HDR_DF_FLAG;
ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
- sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
-
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
- key.ip_src_addr = ipv4_hdr->src_addr;
- key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.src_port = tcp_hdr->src_port;
- key.dst_port = tcp_hdr->dst_port;
- key.recv_ack = tcp_hdr->recv_ack;
/* Search for a matched flow. */
max_flow_num = tbl->max_flow_num;
@@ -270,63 +189,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
}
}
- /*
- * Fail to find a matched flow. Insert a new flow and store the
- * packet into the flow.
- */
if (find == 0) {
- item_idx = insert_new_item(tbl, pkt, start_time,
- INVALID_ARRAY_INDEX, sent_seq, ip_id,
- is_atomic);
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num,
+ tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, ip_id,
+ is_atomic);
if (item_idx == INVALID_ARRAY_INDEX)
return -1;
if (insert_new_flow(tbl, &key, item_idx) ==
- INVALID_ARRAY_INDEX) {
+ INVALID_ARRAY_INDEX) {
/*
* Fail to insert a new flow, so delete the
* stored packet.
- */
- delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
return -1;
}
return 0;
}
- /*
- * Check all packets in the flow and try to find a neighbor for
- * the input packet.
- */
- cur_idx = tbl->flows[i].start_index;
- prev_idx = cur_idx;
- do {
- cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
- sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
- is_atomic);
- if (cmp) {
- if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
- pkt, cmp, sent_seq, ip_id, 0))
- return 1;
- /*
- * Fail to merge the two packets, as the packet
- * length is greater than the max value. Store
- * the packet into the flow.
- */
- if (insert_new_item(tbl, pkt, start_time, cur_idx,
- sent_seq, ip_id, is_atomic) ==
- INVALID_ARRAY_INDEX)
- return -1;
- return 0;
- }
- prev_idx = cur_idx;
- cur_idx = tbl->items[cur_idx].next_pkt_idx;
- } while (cur_idx != INVALID_ARRAY_INDEX);
+ return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl->flows[i].start_index,
+ &tbl->item_num, tbl->max_item_num,
+ ip_id, is_atomic, start_time);
+}
- /* Fail to find a neighbor, so store the packet into the flow. */
- if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
- ip_id, is_atomic) == INVALID_ARRAY_INDEX)
- return -1;
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
- return 0;
+ ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len);
}
uint16_t
@@ -353,7 +253,8 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
* Delete the packet and get the next
* packet in the flow.
*/
- j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
+ j = delete_tcp_item(tbl->items, j,
+ &tbl->item_num, INVALID_ARRAY_INDEX);
tbl->flows[i].start_index = j;
if (j == INVALID_ARRAY_INDEX)
tbl->flow_num--;
diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h
index 212f97a042..245e5da486 100644
--- a/lib/gro/gro_tcp4.h
+++ b/lib/gro/gro_tcp4.h
@@ -5,32 +5,15 @@
#ifndef _GRO_TCP4_H_
#define _GRO_TCP4_H_
-#include <rte_tcp.h>
+#include "gro_tcp.h"
-#define INVALID_ARRAY_INDEX 0xffffffffUL
#define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
-/*
- * The max length of a IPv4 packet, which includes the length of the L3
- * header, the L4 header and the data payload.
- */
-#define MAX_IPV4_PKT_LENGTH UINT16_MAX
-
-/* The maximum TCP header length */
-#define MAX_TCP_HLEN 60
-#define INVALID_TCP_HDRLEN(len) \
- (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
-
-/* Header fields representing a TCP/IPv4 flow */
+/* Header fields representing common fields in TCP flow */
struct tcp4_flow_key {
- struct rte_ether_addr eth_saddr;
- struct rte_ether_addr eth_daddr;
+ struct cmn_tcp_key cmn_key;
uint32_t ip_src_addr;
uint32_t ip_dst_addr;
-
- uint32_t recv_ack;
- uint16_t src_port;
- uint16_t dst_port;
};
struct gro_tcp4_flow {
@@ -42,42 +25,12 @@ struct gro_tcp4_flow {
uint32_t start_index;
};
-struct gro_tcp4_item {
- /*
- * The first MBUF segment of the packet. If the value
- * is NULL, it means the item is empty.
- */
- struct rte_mbuf *firstseg;
- /* The last MBUF segment of the packet */
- struct rte_mbuf *lastseg;
- /*
- * The time when the first packet is inserted into the table.
- * This value won't be updated, even if the packet is merged
- * with other packets.
- */
- uint64_t start_time;
- /*
- * next_pkt_idx is used to chain the packets that
- * are in the same flow but can't be merged together
- * (e.g. caused by packet reordering).
- */
- uint32_t next_pkt_idx;
- /* TCP sequence number of the packet */
- uint32_t sent_seq;
- /* IPv4 ID of the packet */
- uint16_t ip_id;
- /* the number of merged packets */
- uint16_t nb_merged;
- /* Indicate if IPv4 ID can be ignored */
- uint8_t is_atomic;
-};
-
/*
* TCP/IPv4 reassembly table structure.
*/
struct gro_tcp4_tbl {
/* item array */
- struct gro_tcp4_item *items;
+ struct gro_tcp_item *items;
/* flow array */
struct gro_tcp4_flow *flows;
/* current item number */
@@ -186,120 +139,9 @@ uint32_t gro_tcp4_tbl_pkt_count(void *tbl);
static inline int
is_same_tcp4_flow(struct tcp4_flow_key k1, struct tcp4_flow_key k2)
{
- return (rte_is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) &&
- rte_is_same_ether_addr(&k1.eth_daddr, &k2.eth_daddr) &&
- (k1.ip_src_addr == k2.ip_src_addr) &&
+ return ((k1.ip_src_addr == k2.ip_src_addr) &&
(k1.ip_dst_addr == k2.ip_dst_addr) &&
- (k1.recv_ack == k2.recv_ack) &&
- (k1.src_port == k2.src_port) &&
- (k1.dst_port == k2.dst_port));
+ is_same_common_tcp_key(&k1.cmn_key, &k2.cmn_key));
}
-/*
- * Merge two TCP/IPv4 packets without updating checksums.
- * If cmp is larger than 0, append the new packet to the
- * original packet. Otherwise, pre-pend the new packet to
- * the original packet.
- */
-static inline int
-merge_two_tcp4_packets(struct gro_tcp4_item *item,
- struct rte_mbuf *pkt,
- int cmp,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t l2_offset)
-{
- struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
- uint16_t hdr_len, l2_len;
-
- if (cmp > 0) {
- pkt_head = item->firstseg;
- pkt_tail = pkt;
- } else {
- pkt_head = pkt;
- pkt_tail = item->firstseg;
- }
-
- /* check if the IPv4 packet length is greater than the max value */
- hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
- pkt_head->l4_len;
- l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
- if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
- hdr_len > MAX_IPV4_PKT_LENGTH))
- return 0;
-
- /* remove the packet header for the tail packet */
- rte_pktmbuf_adj(pkt_tail, hdr_len);
-
- /* chain two packets together */
- if (cmp > 0) {
- item->lastseg->next = pkt;
- item->lastseg = rte_pktmbuf_lastseg(pkt);
- /* update IP ID to the larger value */
- item->ip_id = ip_id;
- } else {
- lastseg = rte_pktmbuf_lastseg(pkt);
- lastseg->next = item->firstseg;
- item->firstseg = pkt;
- /* update sent_seq to the smaller value */
- item->sent_seq = sent_seq;
- item->ip_id = ip_id;
- }
- item->nb_merged++;
-
- /* update MBUF metadata for the merged packet */
- pkt_head->nb_segs += pkt_tail->nb_segs;
- pkt_head->pkt_len += pkt_tail->pkt_len;
-
- return 1;
-}
-
-/*
- * Check if two TCP/IPv4 packets are neighbors.
- */
-static inline int
-check_seq_option(struct gro_tcp4_item *item,
- struct rte_tcp_hdr *tcph,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t tcp_hl,
- uint16_t tcp_dl,
- uint16_t l2_offset,
- uint8_t is_atomic)
-{
- struct rte_mbuf *pkt_orig = item->firstseg;
- struct rte_ipv4_hdr *iph_orig;
- struct rte_tcp_hdr *tcph_orig;
- uint16_t len, tcp_hl_orig;
-
- iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char *) +
- l2_offset + pkt_orig->l2_len);
- tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig + pkt_orig->l3_len);
- tcp_hl_orig = pkt_orig->l4_len;
-
- /* Check if TCP option fields equal */
- len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
- if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
- (memcmp(tcph + 1, tcph_orig + 1,
- len) != 0)))
- return 0;
-
- /* Don't merge packets whose DF bits are different */
- if (unlikely(item->is_atomic ^ is_atomic))
- return 0;
-
- /* check if the two packets are neighbors */
- len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
- pkt_orig->l3_len - tcp_hl_orig;
- if ((sent_seq == item->sent_seq + len) && (is_atomic ||
- (ip_id == item->ip_id + 1)))
- /* append the new packet */
- return 1;
- else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
- (ip_id + item->nb_merged == item->ip_id)))
- /* pre-pend the new packet */
- return -1;
-
- return 0;
-}
#endif
diff --git a/lib/gro/gro_tcp_internal.h b/lib/gro/gro_tcp_internal.h
new file mode 100644
index 0000000000..cc84abeaeb
--- /dev/null
+++ b/lib/gro/gro_tcp_internal.h
@@ -0,0 +1,128 @@
+
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+
+#ifndef _GRO_TCP_INTERNAL_H_
+#define _GRO_TCP_INTERNAL_H_
+
+static inline uint32_t
+find_an_empty_item(struct gro_tcp_item *items,
+ uint32_t max_item_num)
+{
+ uint32_t i;
+
+ for (i = 0; i < max_item_num; i++)
+ if (items[i].firstseg == NULL)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic)
+{
+ uint32_t item_idx;
+
+ item_idx = find_an_empty_item(items, max_item_num);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return INVALID_ARRAY_INDEX;
+
+ items[item_idx].firstseg = pkt;
+ items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
+ items[item_idx].start_time = start_time;
+ items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
+ items[item_idx].sent_seq = sent_seq;
+ items[item_idx].l3.ip_id = ip_id;
+ items[item_idx].nb_merged = 1;
+ items[item_idx].is_atomic = is_atomic;
+ (*item_num) += 1;
+
+ /* if the previous packet exists, chain them together. */
+ if (prev_idx != INVALID_ARRAY_INDEX) {
+ items[item_idx].next_pkt_idx =
+ items[prev_idx].next_pkt_idx;
+ items[prev_idx].next_pkt_idx = item_idx;
+ }
+
+ return item_idx;
+}
+
+static inline uint32_t
+delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx)
+{
+ uint32_t next_idx = items[item_idx].next_pkt_idx;
+
+ /* NULL indicates an empty item */
+ items[item_idx].firstseg = NULL;
+ (*item_num) -= 1;
+ if (prev_item_idx != INVALID_ARRAY_INDEX)
+ items[prev_item_idx].next_pkt_idx = next_idx;
+
+ return next_idx;
+}
+
+static inline int32_t
+process_tcp_item(struct rte_mbuf *pkt,
+ struct rte_tcp_hdr *tcp_hdr,
+ int32_t tcp_dl,
+ struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time)
+{
+ uint32_t cur_idx;
+ uint32_t prev_idx;
+ int cmp;
+ uint32_t sent_seq;
+
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ /*
+ * Check all packets in the flow and try to find a neighbor for
+ * the input packet.
+ */
+ cur_idx = item_idx;
+ prev_idx = cur_idx;
+ do {
+ cmp = check_seq_option(&items[cur_idx], tcp_hdr,
+ sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
+ is_atomic);
+ if (cmp) {
+ if (merge_two_tcp_packets(&items[cur_idx],
+ pkt, cmp, sent_seq, ip_id, 0))
+ return 1;
+ /*
+ * Fail to merge the two packets, as the packet
+ * length is greater than the max value. Store
+ * the packet into the flow.
+ */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num,
+ start_time, cur_idx, sent_seq, ip_id, is_atomic) ==
+ INVALID_ARRAY_INDEX)
+ return -1;
+ return 0;
+ }
+ prev_idx = cur_idx;
+ cur_idx = items[cur_idx].next_pkt_idx;
+ } while (cur_idx != INVALID_ARRAY_INDEX);
+
+ /* Fail to find a neighbor, so store the packet into the flow. */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num, start_time, prev_idx, sent_seq,
+ ip_id, is_atomic) == INVALID_ARRAY_INDEX)
+ return -1;
+
+ return 0;
+}
+#endif
diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
index 3be4deb7c7..6ab7001922 100644
--- a/lib/gro/gro_vxlan_tcp4.c
+++ b/lib/gro/gro_vxlan_tcp4.c
@@ -116,7 +116,7 @@ insert_new_item(struct gro_vxlan_tcp4_tbl *tbl,
tbl->items[item_idx].inner_item.start_time = start_time;
tbl->items[item_idx].inner_item.next_pkt_idx = INVALID_ARRAY_INDEX;
tbl->items[item_idx].inner_item.sent_seq = sent_seq;
- tbl->items[item_idx].inner_item.ip_id = ip_id;
+ tbl->items[item_idx].inner_item.l3.ip_id = ip_id;
tbl->items[item_idx].inner_item.nb_merged = 1;
tbl->items[item_idx].inner_item.is_atomic = is_atomic;
tbl->items[item_idx].outer_ip_id = outer_ip_id;
@@ -163,15 +163,9 @@ insert_new_flow(struct gro_vxlan_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->inner_key.eth_saddr),
- &(dst->inner_key.eth_saddr));
- rte_ether_addr_copy(&(src->inner_key.eth_daddr),
- &(dst->inner_key.eth_daddr));
+ ASSIGN_COMMON_TCP_KEY((&(src->inner_key.cmn_key)), (&(dst->inner_key.cmn_key)));
dst->inner_key.ip_src_addr = src->inner_key.ip_src_addr;
dst->inner_key.ip_dst_addr = src->inner_key.ip_dst_addr;
- dst->inner_key.recv_ack = src->inner_key.recv_ack;
- dst->inner_key.src_port = src->inner_key.src_port;
- dst->inner_key.dst_port = src->inner_key.dst_port;
dst->vxlan_hdr.vx_flags = src->vxlan_hdr.vx_flags;
dst->vxlan_hdr.vx_vni = src->vxlan_hdr.vx_vni;
@@ -248,7 +242,7 @@ merge_two_vxlan_tcp4_packets(struct gro_vxlan_tcp4_item *item,
uint16_t outer_ip_id,
uint16_t ip_id)
{
- if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp, sent_seq,
+ if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
ip_id, pkt->outer_l2_len +
pkt->outer_l3_len)) {
/* Update the outer IPv4 ID to the large value. */
@@ -357,13 +351,13 @@ gro_vxlan_tcp4_reassemble(struct rte_mbuf *pkt,
sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.eth_daddr));
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.cmn_key.eth_daddr));
key.inner_key.ip_src_addr = ipv4_hdr->src_addr;
key.inner_key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.inner_key.recv_ack = tcp_hdr->recv_ack;
- key.inner_key.src_port = tcp_hdr->src_port;
- key.inner_key.dst_port = tcp_hdr->dst_port;
+ key.inner_key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+ key.inner_key.cmn_key.src_port = tcp_hdr->src_port;
+ key.inner_key.cmn_key.dst_port = tcp_hdr->dst_port;
key.vxlan_hdr.vx_flags = vxlan_hdr->vx_flags;
key.vxlan_hdr.vx_vni = vxlan_hdr->vx_vni;
diff --git a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h
index 7832942a68..662db01a88 100644
--- a/lib/gro/gro_vxlan_tcp4.h
+++ b/lib/gro/gro_vxlan_tcp4.h
@@ -36,7 +36,7 @@ struct gro_vxlan_tcp4_flow {
};
struct gro_vxlan_tcp4_item {
- struct gro_tcp4_item inner_item;
+ struct gro_tcp_item inner_item;
/* IPv4 ID in the outer IPv4 header */
uint16_t outer_ip_id;
/* Indicate if outer IPv4 ID can be ignored */
diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c
index e35399fd42..ac3d1cdc94 100644
--- a/lib/gro/rte_gro.c
+++ b/lib/gro/rte_gro.c
@@ -147,7 +147,7 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
/* allocate a reassembly table for TCP/IPv4 GRO */
struct gro_tcp4_tbl tcp_tbl;
struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
- struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+ struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
/* allocate a reassembly table for UDP/IPv4 GRO */
struct gro_udp4_tbl udp_tbl;
--
2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* [PATCH v10 2/2] gro : add support for IPv6 GRO
2023-06-21 8:25 ` [PATCH v10 1/2] gro : refactor IPv4 to add GRO support for IPv6 Kumara Parameshwaran
@ 2023-06-21 8:25 ` Kumara Parameshwaran
0 siblings, 0 replies; 32+ messages in thread
From: Kumara Parameshwaran @ 2023-06-21 8:25 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev, Kumara Parameshwaran
Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
---
v1:
* Changes to support GRO for TCP/ipv6 packets. This does not include
vxlan changes.
* The GRO is performed only for ipv6 packets that does not contain
extension headers.
* The logic for the TCP coalescing remains the same, in ipv6 header
the source address, destination address, flow label, version fields
are expected to be the same.
* Re-organised the code to reuse certain tcp functions for both ipv4 and
ipv6 flows.
v2:
* Fix comments in gro_tcp6.h header file.
v3:
* Adderess review comments to fix code duplication for v4 and v6
v4:
* Addresses review comments for v3, do not use callbacks
v5:
* Address review comments
v6:
* Fix warning and coding style issues
v7:
* Fix build compilation issue
v8:
* Use gro_tcp_internal.h for functions used for gro_tcp4 and gro_tcp6 and use
gro_tcp.h for data structures and functions used across gro_vxlan_tcp4
v9:
* Resolve merge conflict and add gro_tcp.h in proper path
v10:
* Refactor the code to contain 2 patches. This patch contains support for IPv6 GRO
.../generic_receive_offload_lib.rst | 21 ++---
doc/guides/rel_notes/release_23_07.rst | 2 +
lib/gro/meson.build | 1 +
lib/gro/rte_gro.c | 81 ++++++++++++++++---
lib/gro/rte_gro.h | 3 +
5 files changed, 85 insertions(+), 23 deletions(-)
diff --git a/doc/guides/prog_guide/generic_receive_offload_lib.rst b/doc/guides/prog_guide/generic_receive_offload_lib.rst
index 98a5d29bbc..533cda7f5c 100644
--- a/doc/guides/prog_guide/generic_receive_offload_lib.rst
+++ b/doc/guides/prog_guide/generic_receive_offload_lib.rst
@@ -138,20 +138,21 @@ The key-based algorithm has two characters:
Key-based Reassembly Algorithm
-TCP/IPv4 GRO
-------------
+TCP-IPv4/IPv6 GRO
+-----------------
-The table structure used by TCP/IPv4 GRO contains two arrays: flow array
+The table structure used by TCP-IPv4/IPv6 GRO contains two arrays: flow array
and item array. The flow array keeps flow information, and the item array
-keeps packet information.
+keeps packet information. The flow array is different for IPv4 and IPv6 while
+the item array is the same.
-Header fields used to define a TCP/IPv4 flow include:
+Header fields used to define a TCP-IPv4/IPv6 flow include:
-- source and destination: Ethernet and IP address, TCP port
+- common tcp key fields : Ethernet address, TCP port, TCP acknowledge number
+- version specific IP address
+- IPv6 flow label for IPv6 flow
-- TCP acknowledge number
-
-TCP/IPv4 packets whose FIN, SYN, RST, URG, PSH, ECE or CWR bit is set
+TCP packets whose FIN, SYN, RST, URG, PSH, ECE or CWR bit is set
won't be processed.
Header fields deciding if two packets are neighbors include:
@@ -159,7 +160,7 @@ Header fields deciding if two packets are neighbors include:
- TCP sequence number
- IPv4 ID. The IPv4 ID fields of the packets, whose DF bit is 0, should
- be increased by 1.
+ be increased by 1. This is applicable only for IPv4
VxLAN GRO
---------
diff --git a/doc/guides/rel_notes/release_23_07.rst b/doc/guides/rel_notes/release_23_07.rst
index 027ae7bd2d..7124cf45c7 100644
--- a/doc/guides/rel_notes/release_23_07.rst
+++ b/doc/guides/rel_notes/release_23_07.rst
@@ -170,6 +170,8 @@ New Features
See :doc:`../prog_guide/pdcp_lib` for more information.
+* **Added support for TCP/IPv6 GRO.**
+ * Enhanced the existing library to support GRO for TCP packets over IPv6 network.
Removed Items
-------------
diff --git a/lib/gro/meson.build b/lib/gro/meson.build
index e4fa2958bd..dbce05220d 100644
--- a/lib/gro/meson.build
+++ b/lib/gro/meson.build
@@ -4,6 +4,7 @@
sources = files(
'rte_gro.c',
'gro_tcp4.c',
+ 'gro_tcp6.c',
'gro_udp4.c',
'gro_vxlan_tcp4.c',
'gro_vxlan_udp4.c',
diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c
index ac3d1cdc94..d824eebd93 100644
--- a/lib/gro/rte_gro.c
+++ b/lib/gro/rte_gro.c
@@ -8,6 +8,7 @@
#include "rte_gro.h"
#include "gro_tcp4.h"
+#include "gro_tcp6.h"
#include "gro_udp4.h"
#include "gro_vxlan_tcp4.h"
#include "gro_vxlan_udp4.h"
@@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
- gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
+ gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, gro_tcp6_tbl_create, NULL};
static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
+ gro_tcp6_tbl_destroy,
NULL};
static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count,
+ gro_tcp6_tbl_pkt_count,
NULL};
#define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
@@ -35,6 +38,12 @@ static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+/* GRO with extension headers is not supported */
+#define IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
+ ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
+ ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
+ (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+
#define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
@@ -149,6 +158,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+ struct gro_tcp6_tbl tcp6_tbl;
+ struct gro_tcp6_flow tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
+ struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+
/* allocate a reassembly table for UDP/IPv4 GRO */
struct gro_udp4_tbl udp_tbl;
struct gro_udp4_flow udp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
@@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
int32_t ret;
uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
- do_vxlan_udp_gro = 0;
+ do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
do_udp4_gro = 1;
}
+ if (param->gro_types & RTE_GRO_TCP_IPV6) {
+ for (i = 0; i < item_num; i++)
+ tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+ tcp6_tbl.flows = tcp6_flows;
+ tcp6_tbl.items = tcp6_items;
+ tcp6_tbl.flow_num = 0;
+ tcp6_tbl.item_num = 0;
+ tcp6_tbl.max_flow_num = item_num;
+ tcp6_tbl.max_item_num = item_num;
+ do_tcp6_gro = 1;
+ }
for (i = 0; i < nb_pkts; i++) {
/*
@@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
nb_after_gro--;
else if (ret < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
+ if (ret > 0)
+ /* merge successfully */
+ nb_after_gro--;
+ else if (ret < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
if ((nb_after_gro < nb_pkts)
|| (unprocess_num < nb_pkts)) {
i = 0;
+ /* Copy unprocessed packets */
+ if (unprocess_num > 0) {
+ memcpy(&pkts[i], unprocess_pkts,
+ sizeof(struct rte_mbuf *) *
+ unprocess_num);
+ i = unprocess_num;
+ }
+
/* Flush all packets from the tables */
if (do_vxlan_tcp_gro) {
- i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
+ i += gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
0, pkts, nb_pkts);
}
@@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
&pkts[i], nb_pkts - i);
}
- /* Copy unprocessed packets */
- if (unprocess_num > 0) {
- memcpy(&pkts[i], unprocess_pkts,
- sizeof(struct rte_mbuf *) *
- unprocess_num);
+
+ if (do_tcp6_gro) {
+ i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
+ &pkts[i], nb_pkts - i);
}
- nb_after_gro = i + unprocess_num;
}
return nb_after_gro;
@@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
{
struct rte_mbuf *unprocess_pkts[nb_pkts];
struct gro_ctx *gro_ctx = ctx;
- void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
+ void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
uint64_t current_time;
uint16_t i, unprocess_num = 0;
- uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro;
+ uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro, do_tcp6_gro;
if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
+ tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
RTE_GRO_TCP_IPV4;
@@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
RTE_GRO_UDP_IPV4;
do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
RTE_GRO_IPV4_VXLAN_UDP_IPV4;
+ do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) == RTE_GRO_TCP_IPV6;
current_time = rte_rdtsc();
@@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
if (gro_udp4_reassemble(pkts[i], udp_tbl,
current_time) < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
+ current_time) < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
flush_timestamp,
&out[num], left_nb_out);
+ left_nb_out = max_nb_out - num;
+ }
+
+ if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
+ num += gro_tcp6_tbl_timeout_flush(
+ gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
+ flush_timestamp,
+ &out[num], left_nb_out);
+
}
return num;
diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h
index 9f9ed4935a..c83dfd9ad1 100644
--- a/lib/gro/rte_gro.h
+++ b/lib/gro/rte_gro.h
@@ -38,6 +38,9 @@ extern "C" {
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL << RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
/**< VxLAN UDP/IPv4 GRO flag. */
+#define RTE_GRO_TCP_IPV6_INDEX 4
+#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX)
+/**< TCP/IPv6 GRO flag. */
/**
* Structure used to create GRO context objects or used to pass
--
2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* [PATCH v11 1/2] gro : refactor IPv4 to add GRO support for IPv6
2022-10-20 18:14 ` [PATCH v2] " Kumara Parameshwaran
` (9 preceding siblings ...)
2023-06-21 8:25 ` [PATCH v10 1/2] gro : refactor IPv4 to add GRO support for IPv6 Kumara Parameshwaran
@ 2023-06-21 8:38 ` Kumara Parameshwaran
2023-06-21 8:38 ` [PATCH v11 2/2] gro : add support for IPv6 GRO Kumara Parameshwaran
10 siblings, 1 reply; 32+ messages in thread
From: Kumara Parameshwaran @ 2023-06-21 8:38 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev, Kumara Parameshwaran
Refactor the existing code to reuse data strutures and functions
to add support for IPv6 GRO
Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
---
v1:
* Changes to support GRO for TCP/ipv6 packets. This does not include
vxlan changes.
* The GRO is performed only for ipv6 packets that does not contain
extension headers.
* The logic for the TCP coalescing remains the same, in ipv6 header
the source address, destination address, flow label, version fields
are expected to be the same.
* Re-organised the code to reuse certain tcp functions for both ipv4 and
ipv6 flows.
v2:
* Fix comments in gro_tcp6.h header file.
v3:
* Adderess review comments to fix code duplication for v4 and v6
v4:
* Addresses review comments for v3, do not use callbacks
v5:
* Address review comments
v6:
* Fix warning and coding style issues
v7:
* Fix build compilation issue
v8:
* Use gro_tcp_internal.h for functions used for gro_tcp4 and gro_tcp6 and use
gro_tcp.h for data structures and functions used across gro_vxlan_tcp4
v9:
* Resolve merge conflict and add gro_tcp.h in proper path
v10:
* Refactor the code to contain 2 patches. This patch contains code/data strcuture changes
in ipv4 code that can be reused.
lib/gro/gro_tcp.h | 190 +++++++++++++++++++++++++++++++++++++
lib/gro/gro_tcp4.c | 177 ++++++++--------------------------
lib/gro/gro_tcp4.h | 170 ++-------------------------------
lib/gro/gro_tcp_internal.h | 128 +++++++++++++++++++++++++
lib/gro/gro_vxlan_tcp4.c | 22 ++---
lib/gro/gro_vxlan_tcp4.h | 2 +-
lib/gro/rte_gro.c | 2 +-
7 files changed, 373 insertions(+), 318 deletions(-)
create mode 100644 lib/gro/gro_tcp.h
create mode 100644 lib/gro/gro_tcp_internal.h
diff --git a/lib/gro/gro_tcp.h b/lib/gro/gro_tcp.h
new file mode 100644
index 0000000000..d926c4b8cc
--- /dev/null
+++ b/lib/gro/gro_tcp.h
@@ -0,0 +1,190 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+#ifndef _GRO_TCP_H_
+#define _GRO_TCP_H_
+
+#define INVALID_ARRAY_INDEX 0xffffffffUL
+
+#include <rte_tcp.h>
+
+/*
+ * The max length of a IPv4 packet, which includes the length of the L3
+ * header, the L4 header and the data payload.
+ */
+#define MAX_IP_PKT_LENGTH UINT16_MAX
+
+/* The maximum TCP header length */
+#define MAX_TCP_HLEN 60
+#define INVALID_TCP_HDRLEN(len) \
+ (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
+
+struct cmn_tcp_key {
+ struct rte_ether_addr eth_saddr;
+ struct rte_ether_addr eth_daddr;
+ uint32_t recv_ack;
+ uint16_t src_port;
+ uint16_t dst_port;
+};
+
+#define ASSIGN_COMMON_TCP_KEY(k1, k2) \
+ do {\
+ rte_ether_addr_copy(&(k1->eth_saddr), &(k2->eth_saddr)); \
+ rte_ether_addr_copy(&(k1->eth_daddr), &(k2->eth_daddr)); \
+ k2->recv_ack = k1->recv_ack; \
+ k2->src_port = k1->src_port; \
+ k2->dst_port = k1->dst_port; \
+ } while (0)
+
+struct gro_tcp_item {
+ /*
+ * The first MBUF segment of the packet. If the value
+ * is NULL, it means the item is empty.
+ */
+ struct rte_mbuf *firstseg;
+ /* The last MBUF segment of the packet */
+ struct rte_mbuf *lastseg;
+ /*
+ * The time when the first packet is inserted into the table.
+ * This value won't be updated, even if the packet is merged
+ * with other packets.
+ */
+ uint64_t start_time;
+ /*
+ * next_pkt_idx is used to chain the packets that
+ * are in the same flow but can't be merged together
+ * (e.g. caused by packet reordering).
+ */
+ uint32_t next_pkt_idx;
+ /* TCP sequence number of the packet */
+ uint32_t sent_seq;
+ union {
+ /* IPv4 ID of the packet */
+ uint16_t ip_id;
+ /* Unused field for IPv6 */
+ uint16_t unused;
+ } l3;
+ /* the number of merged packets */
+ uint16_t nb_merged;
+ /* Indicate if IPv4 ID can be ignored */
+ uint8_t is_atomic;
+};
+
+/*
+ * Merge two TCP packets without updating checksums.
+ * If cmp is larger than 0, append the new packet to the
+ * original packet. Otherwise, pre-pend the new packet to
+ * the original packet.
+ */
+static inline int
+merge_two_tcp_packets(struct gro_tcp_item *item,
+ struct rte_mbuf *pkt,
+ int cmp,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t l2_offset)
+{
+ struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
+ uint16_t hdr_len, l2_len;
+
+ if (cmp > 0) {
+ pkt_head = item->firstseg;
+ pkt_tail = pkt;
+ } else {
+ pkt_head = pkt;
+ pkt_tail = item->firstseg;
+ }
+
+ /* check if the IPv4 packet length is greater than the max value */
+ hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
+ pkt_head->l4_len;
+ l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
+ if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
+ hdr_len > MAX_IP_PKT_LENGTH))
+ return 0;
+
+ if (unlikely(pkt_head->nb_segs >= 20))
+ return 0;
+
+ /* remove the packet header for the tail packet */
+ rte_pktmbuf_adj(pkt_tail, hdr_len);
+
+ /* chain two packets together */
+ if (cmp > 0) {
+ item->lastseg->next = pkt;
+ item->lastseg = rte_pktmbuf_lastseg(pkt);
+ /* update IP ID to the larger value */
+ item->l3.ip_id = ip_id;
+ } else {
+ lastseg = rte_pktmbuf_lastseg(pkt);
+ lastseg->next = item->firstseg;
+ item->firstseg = pkt;
+ /* update sent_seq to the smaller value */
+ item->sent_seq = sent_seq;
+ item->l3.ip_id = ip_id;
+ }
+ item->nb_merged++;
+
+ /* update MBUF metadata for the merged packet */
+ pkt_head->nb_segs += pkt_tail->nb_segs;
+ pkt_head->pkt_len += pkt_tail->pkt_len;
+
+ return 1;
+}
+
+/*
+ * Check if two TCP packets are neighbors.
+ */
+static inline int
+check_seq_option(struct gro_tcp_item *item,
+ struct rte_tcp_hdr *tcph,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint16_t tcp_hl,
+ uint16_t tcp_dl,
+ uint16_t l2_offset,
+ uint8_t is_atomic)
+{
+ struct rte_mbuf *pkt_orig = item->firstseg;
+ char *iph_orig;
+ struct rte_tcp_hdr *tcph_orig;
+ uint16_t len, tcp_hl_orig;
+
+ iph_orig = (char *)(rte_pktmbuf_mtod(pkt_orig, char *) +
+ l2_offset + pkt_orig->l2_len);
+ tcph_orig = (struct rte_tcp_hdr *)(iph_orig + pkt_orig->l3_len);
+ tcp_hl_orig = pkt_orig->l4_len;
+
+ /* Check if TCP option fields equal */
+ len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
+ if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
+ (memcmp(tcph + 1, tcph_orig + 1,
+ len) != 0)))
+ return 0;
+
+ /* Don't merge packets whose DF bits are different */
+ if (unlikely(item->is_atomic ^ is_atomic))
+ return 0;
+
+ /* check if the two packets are neighbors */
+ len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
+ pkt_orig->l3_len - tcp_hl_orig;
+ if ((sent_seq == item->sent_seq + len) && (is_atomic ||
+ (ip_id == item->l3.ip_id + 1)))
+ /* append the new packet */
+ return 1;
+ else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
+ (ip_id + item->nb_merged == item->l3.ip_id)))
+ /* pre-pend the new packet */
+ return -1;
+
+ return 0;
+}
+
+static inline int
+is_same_common_tcp_key(struct cmn_tcp_key *k1, struct cmn_tcp_key *k2)
+{
+ return (!memcmp(k1, k2, sizeof(struct cmn_tcp_key)));
+}
+
+#endif
diff --git a/lib/gro/gro_tcp4.c b/lib/gro/gro_tcp4.c
index 0014096e63..6645de592b 100644
--- a/lib/gro/gro_tcp4.c
+++ b/lib/gro/gro_tcp4.c
@@ -7,6 +7,7 @@
#include <rte_ethdev.h>
#include "gro_tcp4.h"
+#include "gro_tcp_internal.h"
void *
gro_tcp4_tbl_create(uint16_t socket_id,
@@ -30,7 +31,7 @@ gro_tcp4_tbl_create(uint16_t socket_id,
if (tbl == NULL)
return NULL;
- size = sizeof(struct gro_tcp4_item) * entries_num;
+ size = sizeof(struct gro_tcp_item) * entries_num;
tbl->items = rte_zmalloc_socket(__func__,
size,
RTE_CACHE_LINE_SIZE,
@@ -71,18 +72,6 @@ gro_tcp4_tbl_destroy(void *tbl)
rte_free(tcp_tbl);
}
-static inline uint32_t
-find_an_empty_item(struct gro_tcp4_tbl *tbl)
-{
- uint32_t i;
- uint32_t max_item_num = tbl->max_item_num;
-
- for (i = 0; i < max_item_num; i++)
- if (tbl->items[i].firstseg == NULL)
- return i;
- return INVALID_ARRAY_INDEX;
-}
-
static inline uint32_t
find_an_empty_flow(struct gro_tcp4_tbl *tbl)
{
@@ -95,56 +84,6 @@ find_an_empty_flow(struct gro_tcp4_tbl *tbl)
return INVALID_ARRAY_INDEX;
}
-static inline uint32_t
-insert_new_item(struct gro_tcp4_tbl *tbl,
- struct rte_mbuf *pkt,
- uint64_t start_time,
- uint32_t prev_idx,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint8_t is_atomic)
-{
- uint32_t item_idx;
-
- item_idx = find_an_empty_item(tbl);
- if (item_idx == INVALID_ARRAY_INDEX)
- return INVALID_ARRAY_INDEX;
-
- tbl->items[item_idx].firstseg = pkt;
- tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
- tbl->items[item_idx].start_time = start_time;
- tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
- tbl->items[item_idx].sent_seq = sent_seq;
- tbl->items[item_idx].ip_id = ip_id;
- tbl->items[item_idx].nb_merged = 1;
- tbl->items[item_idx].is_atomic = is_atomic;
- tbl->item_num++;
-
- /* if the previous packet exists, chain them together. */
- if (prev_idx != INVALID_ARRAY_INDEX) {
- tbl->items[item_idx].next_pkt_idx =
- tbl->items[prev_idx].next_pkt_idx;
- tbl->items[prev_idx].next_pkt_idx = item_idx;
- }
-
- return item_idx;
-}
-
-static inline uint32_t
-delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx,
- uint32_t prev_item_idx)
-{
- uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
-
- /* NULL indicates an empty item */
- tbl->items[item_idx].firstseg = NULL;
- tbl->item_num--;
- if (prev_item_idx != INVALID_ARRAY_INDEX)
- tbl->items[prev_item_idx].next_pkt_idx = next_idx;
-
- return next_idx;
-}
-
static inline uint32_t
insert_new_flow(struct gro_tcp4_tbl *tbl,
struct tcp4_flow_key *src,
@@ -159,13 +98,10 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
- rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
+ ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
+
dst->ip_src_addr = src->ip_src_addr;
dst->ip_dst_addr = src->ip_dst_addr;
- dst->recv_ack = src->recv_ack;
- dst->src_port = src->src_port;
- dst->dst_port = src->dst_port;
tbl->flows[flow_idx].start_index = item_idx;
tbl->flow_num++;
@@ -173,21 +109,6 @@ insert_new_flow(struct gro_tcp4_tbl *tbl,
return flow_idx;
}
-/*
- * update the packet length for the flushed packet.
- */
-static inline void
-update_header(struct gro_tcp4_item *item)
-{
- struct rte_ipv4_hdr *ipv4_hdr;
- struct rte_mbuf *pkt = item->firstseg;
-
- ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
- pkt->l2_len);
- ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
- pkt->l2_len);
-}
-
int32_t
gro_tcp4_reassemble(struct rte_mbuf *pkt,
struct gro_tcp4_tbl *tbl,
@@ -202,9 +123,8 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
uint8_t is_atomic;
struct tcp4_flow_key key;
- uint32_t cur_idx, prev_idx, item_idx;
+ uint32_t item_idx;
uint32_t i, max_flow_num, remaining_flow_num;
- int cmp;
uint8_t find;
/*
@@ -239,6 +159,14 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
if (tcp_dl <= 0)
return -1;
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.cmn_key.eth_daddr));
+ key.ip_src_addr = ipv4_hdr->src_addr;
+ key.ip_dst_addr = ipv4_hdr->dst_addr;
+ key.cmn_key.src_port = tcp_hdr->src_port;
+ key.cmn_key.dst_port = tcp_hdr->dst_port;
+ key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+
/*
* Save IPv4 ID for the packet whose DF bit is 0. For the packet
* whose DF bit is 1, IPv4 ID is ignored.
@@ -246,15 +174,6 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
is_atomic = (frag_off & RTE_IPV4_HDR_DF_FLAG) == RTE_IPV4_HDR_DF_FLAG;
ip_id = is_atomic ? 0 : rte_be_to_cpu_16(ipv4_hdr->packet_id);
- sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
-
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.eth_daddr));
- key.ip_src_addr = ipv4_hdr->src_addr;
- key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.src_port = tcp_hdr->src_port;
- key.dst_port = tcp_hdr->dst_port;
- key.recv_ack = tcp_hdr->recv_ack;
/* Search for a matched flow. */
max_flow_num = tbl->max_flow_num;
@@ -270,63 +189,44 @@ gro_tcp4_reassemble(struct rte_mbuf *pkt,
}
}
- /*
- * Fail to find a matched flow. Insert a new flow and store the
- * packet into the flow.
- */
if (find == 0) {
- item_idx = insert_new_item(tbl, pkt, start_time,
- INVALID_ARRAY_INDEX, sent_seq, ip_id,
- is_atomic);
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num,
+ tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, ip_id,
+ is_atomic);
if (item_idx == INVALID_ARRAY_INDEX)
return -1;
if (insert_new_flow(tbl, &key, item_idx) ==
- INVALID_ARRAY_INDEX) {
+ INVALID_ARRAY_INDEX) {
/*
* Fail to insert a new flow, so delete the
* stored packet.
- */
- delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
return -1;
}
return 0;
}
- /*
- * Check all packets in the flow and try to find a neighbor for
- * the input packet.
- */
- cur_idx = tbl->flows[i].start_index;
- prev_idx = cur_idx;
- do {
- cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr,
- sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
- is_atomic);
- if (cmp) {
- if (merge_two_tcp4_packets(&(tbl->items[cur_idx]),
- pkt, cmp, sent_seq, ip_id, 0))
- return 1;
- /*
- * Fail to merge the two packets, as the packet
- * length is greater than the max value. Store
- * the packet into the flow.
- */
- if (insert_new_item(tbl, pkt, start_time, cur_idx,
- sent_seq, ip_id, is_atomic) ==
- INVALID_ARRAY_INDEX)
- return -1;
- return 0;
- }
- prev_idx = cur_idx;
- cur_idx = tbl->items[cur_idx].next_pkt_idx;
- } while (cur_idx != INVALID_ARRAY_INDEX);
+ return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl->flows[i].start_index,
+ &tbl->item_num, tbl->max_item_num,
+ ip_id, is_atomic, start_time);
+}
- /* Fail to find a neighbor, so store the packet into the flow. */
- if (insert_new_item(tbl, pkt, start_time, prev_idx, sent_seq,
- ip_id, is_atomic) == INVALID_ARRAY_INDEX)
- return -1;
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
- return 0;
+ ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len);
}
uint16_t
@@ -353,7 +253,8 @@ gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl,
* Delete the packet and get the next
* packet in the flow.
*/
- j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
+ j = delete_tcp_item(tbl->items, j,
+ &tbl->item_num, INVALID_ARRAY_INDEX);
tbl->flows[i].start_index = j;
if (j == INVALID_ARRAY_INDEX)
tbl->flow_num--;
diff --git a/lib/gro/gro_tcp4.h b/lib/gro/gro_tcp4.h
index 212f97a042..245e5da486 100644
--- a/lib/gro/gro_tcp4.h
+++ b/lib/gro/gro_tcp4.h
@@ -5,32 +5,15 @@
#ifndef _GRO_TCP4_H_
#define _GRO_TCP4_H_
-#include <rte_tcp.h>
+#include "gro_tcp.h"
-#define INVALID_ARRAY_INDEX 0xffffffffUL
#define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
-/*
- * The max length of a IPv4 packet, which includes the length of the L3
- * header, the L4 header and the data payload.
- */
-#define MAX_IPV4_PKT_LENGTH UINT16_MAX
-
-/* The maximum TCP header length */
-#define MAX_TCP_HLEN 60
-#define INVALID_TCP_HDRLEN(len) \
- (((len) < sizeof(struct rte_tcp_hdr)) || ((len) > MAX_TCP_HLEN))
-
-/* Header fields representing a TCP/IPv4 flow */
+/* Header fields representing common fields in TCP flow */
struct tcp4_flow_key {
- struct rte_ether_addr eth_saddr;
- struct rte_ether_addr eth_daddr;
+ struct cmn_tcp_key cmn_key;
uint32_t ip_src_addr;
uint32_t ip_dst_addr;
-
- uint32_t recv_ack;
- uint16_t src_port;
- uint16_t dst_port;
};
struct gro_tcp4_flow {
@@ -42,42 +25,12 @@ struct gro_tcp4_flow {
uint32_t start_index;
};
-struct gro_tcp4_item {
- /*
- * The first MBUF segment of the packet. If the value
- * is NULL, it means the item is empty.
- */
- struct rte_mbuf *firstseg;
- /* The last MBUF segment of the packet */
- struct rte_mbuf *lastseg;
- /*
- * The time when the first packet is inserted into the table.
- * This value won't be updated, even if the packet is merged
- * with other packets.
- */
- uint64_t start_time;
- /*
- * next_pkt_idx is used to chain the packets that
- * are in the same flow but can't be merged together
- * (e.g. caused by packet reordering).
- */
- uint32_t next_pkt_idx;
- /* TCP sequence number of the packet */
- uint32_t sent_seq;
- /* IPv4 ID of the packet */
- uint16_t ip_id;
- /* the number of merged packets */
- uint16_t nb_merged;
- /* Indicate if IPv4 ID can be ignored */
- uint8_t is_atomic;
-};
-
/*
* TCP/IPv4 reassembly table structure.
*/
struct gro_tcp4_tbl {
/* item array */
- struct gro_tcp4_item *items;
+ struct gro_tcp_item *items;
/* flow array */
struct gro_tcp4_flow *flows;
/* current item number */
@@ -186,120 +139,9 @@ uint32_t gro_tcp4_tbl_pkt_count(void *tbl);
static inline int
is_same_tcp4_flow(struct tcp4_flow_key k1, struct tcp4_flow_key k2)
{
- return (rte_is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) &&
- rte_is_same_ether_addr(&k1.eth_daddr, &k2.eth_daddr) &&
- (k1.ip_src_addr == k2.ip_src_addr) &&
+ return ((k1.ip_src_addr == k2.ip_src_addr) &&
(k1.ip_dst_addr == k2.ip_dst_addr) &&
- (k1.recv_ack == k2.recv_ack) &&
- (k1.src_port == k2.src_port) &&
- (k1.dst_port == k2.dst_port));
+ is_same_common_tcp_key(&k1.cmn_key, &k2.cmn_key));
}
-/*
- * Merge two TCP/IPv4 packets without updating checksums.
- * If cmp is larger than 0, append the new packet to the
- * original packet. Otherwise, pre-pend the new packet to
- * the original packet.
- */
-static inline int
-merge_two_tcp4_packets(struct gro_tcp4_item *item,
- struct rte_mbuf *pkt,
- int cmp,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t l2_offset)
-{
- struct rte_mbuf *pkt_head, *pkt_tail, *lastseg;
- uint16_t hdr_len, l2_len;
-
- if (cmp > 0) {
- pkt_head = item->firstseg;
- pkt_tail = pkt;
- } else {
- pkt_head = pkt;
- pkt_tail = item->firstseg;
- }
-
- /* check if the IPv4 packet length is greater than the max value */
- hdr_len = l2_offset + pkt_head->l2_len + pkt_head->l3_len +
- pkt_head->l4_len;
- l2_len = l2_offset > 0 ? pkt_head->outer_l2_len : pkt_head->l2_len;
- if (unlikely(pkt_head->pkt_len - l2_len + pkt_tail->pkt_len -
- hdr_len > MAX_IPV4_PKT_LENGTH))
- return 0;
-
- /* remove the packet header for the tail packet */
- rte_pktmbuf_adj(pkt_tail, hdr_len);
-
- /* chain two packets together */
- if (cmp > 0) {
- item->lastseg->next = pkt;
- item->lastseg = rte_pktmbuf_lastseg(pkt);
- /* update IP ID to the larger value */
- item->ip_id = ip_id;
- } else {
- lastseg = rte_pktmbuf_lastseg(pkt);
- lastseg->next = item->firstseg;
- item->firstseg = pkt;
- /* update sent_seq to the smaller value */
- item->sent_seq = sent_seq;
- item->ip_id = ip_id;
- }
- item->nb_merged++;
-
- /* update MBUF metadata for the merged packet */
- pkt_head->nb_segs += pkt_tail->nb_segs;
- pkt_head->pkt_len += pkt_tail->pkt_len;
-
- return 1;
-}
-
-/*
- * Check if two TCP/IPv4 packets are neighbors.
- */
-static inline int
-check_seq_option(struct gro_tcp4_item *item,
- struct rte_tcp_hdr *tcph,
- uint32_t sent_seq,
- uint16_t ip_id,
- uint16_t tcp_hl,
- uint16_t tcp_dl,
- uint16_t l2_offset,
- uint8_t is_atomic)
-{
- struct rte_mbuf *pkt_orig = item->firstseg;
- struct rte_ipv4_hdr *iph_orig;
- struct rte_tcp_hdr *tcph_orig;
- uint16_t len, tcp_hl_orig;
-
- iph_orig = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt_orig, char *) +
- l2_offset + pkt_orig->l2_len);
- tcph_orig = (struct rte_tcp_hdr *)((char *)iph_orig + pkt_orig->l3_len);
- tcp_hl_orig = pkt_orig->l4_len;
-
- /* Check if TCP option fields equal */
- len = RTE_MAX(tcp_hl, tcp_hl_orig) - sizeof(struct rte_tcp_hdr);
- if ((tcp_hl != tcp_hl_orig) || ((len > 0) &&
- (memcmp(tcph + 1, tcph_orig + 1,
- len) != 0)))
- return 0;
-
- /* Don't merge packets whose DF bits are different */
- if (unlikely(item->is_atomic ^ is_atomic))
- return 0;
-
- /* check if the two packets are neighbors */
- len = pkt_orig->pkt_len - l2_offset - pkt_orig->l2_len -
- pkt_orig->l3_len - tcp_hl_orig;
- if ((sent_seq == item->sent_seq + len) && (is_atomic ||
- (ip_id == item->ip_id + 1)))
- /* append the new packet */
- return 1;
- else if ((sent_seq + tcp_dl == item->sent_seq) && (is_atomic ||
- (ip_id + item->nb_merged == item->ip_id)))
- /* pre-pend the new packet */
- return -1;
-
- return 0;
-}
#endif
diff --git a/lib/gro/gro_tcp_internal.h b/lib/gro/gro_tcp_internal.h
new file mode 100644
index 0000000000..cc84abeaeb
--- /dev/null
+++ b/lib/gro/gro_tcp_internal.h
@@ -0,0 +1,128 @@
+
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+
+#ifndef _GRO_TCP_INTERNAL_H_
+#define _GRO_TCP_INTERNAL_H_
+
+static inline uint32_t
+find_an_empty_item(struct gro_tcp_item *items,
+ uint32_t max_item_num)
+{
+ uint32_t i;
+
+ for (i = 0; i < max_item_num; i++)
+ if (items[i].firstseg == NULL)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+insert_new_tcp_item(struct rte_mbuf *pkt,
+ struct gro_tcp_item *items,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint64_t start_time,
+ uint32_t prev_idx,
+ uint32_t sent_seq,
+ uint16_t ip_id,
+ uint8_t is_atomic)
+{
+ uint32_t item_idx;
+
+ item_idx = find_an_empty_item(items, max_item_num);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return INVALID_ARRAY_INDEX;
+
+ items[item_idx].firstseg = pkt;
+ items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
+ items[item_idx].start_time = start_time;
+ items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
+ items[item_idx].sent_seq = sent_seq;
+ items[item_idx].l3.ip_id = ip_id;
+ items[item_idx].nb_merged = 1;
+ items[item_idx].is_atomic = is_atomic;
+ (*item_num) += 1;
+
+ /* if the previous packet exists, chain them together. */
+ if (prev_idx != INVALID_ARRAY_INDEX) {
+ items[item_idx].next_pkt_idx =
+ items[prev_idx].next_pkt_idx;
+ items[prev_idx].next_pkt_idx = item_idx;
+ }
+
+ return item_idx;
+}
+
+static inline uint32_t
+delete_tcp_item(struct gro_tcp_item *items, uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t prev_item_idx)
+{
+ uint32_t next_idx = items[item_idx].next_pkt_idx;
+
+ /* NULL indicates an empty item */
+ items[item_idx].firstseg = NULL;
+ (*item_num) -= 1;
+ if (prev_item_idx != INVALID_ARRAY_INDEX)
+ items[prev_item_idx].next_pkt_idx = next_idx;
+
+ return next_idx;
+}
+
+static inline int32_t
+process_tcp_item(struct rte_mbuf *pkt,
+ struct rte_tcp_hdr *tcp_hdr,
+ int32_t tcp_dl,
+ struct gro_tcp_item *items,
+ uint32_t item_idx,
+ uint32_t *item_num,
+ uint32_t max_item_num,
+ uint16_t ip_id,
+ uint8_t is_atomic,
+ uint64_t start_time)
+{
+ uint32_t cur_idx;
+ uint32_t prev_idx;
+ int cmp;
+ uint32_t sent_seq;
+
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ /*
+ * Check all packets in the flow and try to find a neighbor for
+ * the input packet.
+ */
+ cur_idx = item_idx;
+ prev_idx = cur_idx;
+ do {
+ cmp = check_seq_option(&items[cur_idx], tcp_hdr,
+ sent_seq, ip_id, pkt->l4_len, tcp_dl, 0,
+ is_atomic);
+ if (cmp) {
+ if (merge_two_tcp_packets(&items[cur_idx],
+ pkt, cmp, sent_seq, ip_id, 0))
+ return 1;
+ /*
+ * Fail to merge the two packets, as the packet
+ * length is greater than the max value. Store
+ * the packet into the flow.
+ */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num,
+ start_time, cur_idx, sent_seq, ip_id, is_atomic) ==
+ INVALID_ARRAY_INDEX)
+ return -1;
+ return 0;
+ }
+ prev_idx = cur_idx;
+ cur_idx = items[cur_idx].next_pkt_idx;
+ } while (cur_idx != INVALID_ARRAY_INDEX);
+
+ /* Fail to find a neighbor, so store the packet into the flow. */
+ if (insert_new_tcp_item(pkt, items, item_num, max_item_num, start_time, prev_idx, sent_seq,
+ ip_id, is_atomic) == INVALID_ARRAY_INDEX)
+ return -1;
+
+ return 0;
+}
+#endif
diff --git a/lib/gro/gro_vxlan_tcp4.c b/lib/gro/gro_vxlan_tcp4.c
index 3be4deb7c7..6ab7001922 100644
--- a/lib/gro/gro_vxlan_tcp4.c
+++ b/lib/gro/gro_vxlan_tcp4.c
@@ -116,7 +116,7 @@ insert_new_item(struct gro_vxlan_tcp4_tbl *tbl,
tbl->items[item_idx].inner_item.start_time = start_time;
tbl->items[item_idx].inner_item.next_pkt_idx = INVALID_ARRAY_INDEX;
tbl->items[item_idx].inner_item.sent_seq = sent_seq;
- tbl->items[item_idx].inner_item.ip_id = ip_id;
+ tbl->items[item_idx].inner_item.l3.ip_id = ip_id;
tbl->items[item_idx].inner_item.nb_merged = 1;
tbl->items[item_idx].inner_item.is_atomic = is_atomic;
tbl->items[item_idx].outer_ip_id = outer_ip_id;
@@ -163,15 +163,9 @@ insert_new_flow(struct gro_vxlan_tcp4_tbl *tbl,
dst = &(tbl->flows[flow_idx].key);
- rte_ether_addr_copy(&(src->inner_key.eth_saddr),
- &(dst->inner_key.eth_saddr));
- rte_ether_addr_copy(&(src->inner_key.eth_daddr),
- &(dst->inner_key.eth_daddr));
+ ASSIGN_COMMON_TCP_KEY((&(src->inner_key.cmn_key)), (&(dst->inner_key.cmn_key)));
dst->inner_key.ip_src_addr = src->inner_key.ip_src_addr;
dst->inner_key.ip_dst_addr = src->inner_key.ip_dst_addr;
- dst->inner_key.recv_ack = src->inner_key.recv_ack;
- dst->inner_key.src_port = src->inner_key.src_port;
- dst->inner_key.dst_port = src->inner_key.dst_port;
dst->vxlan_hdr.vx_flags = src->vxlan_hdr.vx_flags;
dst->vxlan_hdr.vx_vni = src->vxlan_hdr.vx_vni;
@@ -248,7 +242,7 @@ merge_two_vxlan_tcp4_packets(struct gro_vxlan_tcp4_item *item,
uint16_t outer_ip_id,
uint16_t ip_id)
{
- if (merge_two_tcp4_packets(&item->inner_item, pkt, cmp, sent_seq,
+ if (merge_two_tcp_packets(&item->inner_item, pkt, cmp, sent_seq,
ip_id, pkt->outer_l2_len +
pkt->outer_l3_len)) {
/* Update the outer IPv4 ID to the large value. */
@@ -357,13 +351,13 @@ gro_vxlan_tcp4_reassemble(struct rte_mbuf *pkt,
sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
- rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.eth_saddr));
- rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.eth_daddr));
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.inner_key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.inner_key.cmn_key.eth_daddr));
key.inner_key.ip_src_addr = ipv4_hdr->src_addr;
key.inner_key.ip_dst_addr = ipv4_hdr->dst_addr;
- key.inner_key.recv_ack = tcp_hdr->recv_ack;
- key.inner_key.src_port = tcp_hdr->src_port;
- key.inner_key.dst_port = tcp_hdr->dst_port;
+ key.inner_key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+ key.inner_key.cmn_key.src_port = tcp_hdr->src_port;
+ key.inner_key.cmn_key.dst_port = tcp_hdr->dst_port;
key.vxlan_hdr.vx_flags = vxlan_hdr->vx_flags;
key.vxlan_hdr.vx_vni = vxlan_hdr->vx_vni;
diff --git a/lib/gro/gro_vxlan_tcp4.h b/lib/gro/gro_vxlan_tcp4.h
index 7832942a68..662db01a88 100644
--- a/lib/gro/gro_vxlan_tcp4.h
+++ b/lib/gro/gro_vxlan_tcp4.h
@@ -36,7 +36,7 @@ struct gro_vxlan_tcp4_flow {
};
struct gro_vxlan_tcp4_item {
- struct gro_tcp4_item inner_item;
+ struct gro_tcp_item inner_item;
/* IPv4 ID in the outer IPv4 header */
uint16_t outer_ip_id;
/* Indicate if outer IPv4 ID can be ignored */
diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c
index e35399fd42..ac3d1cdc94 100644
--- a/lib/gro/rte_gro.c
+++ b/lib/gro/rte_gro.c
@@ -147,7 +147,7 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
/* allocate a reassembly table for TCP/IPv4 GRO */
struct gro_tcp4_tbl tcp_tbl;
struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
- struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+ struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
/* allocate a reassembly table for UDP/IPv4 GRO */
struct gro_udp4_tbl udp_tbl;
--
2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* [PATCH v11 2/2] gro : add support for IPv6 GRO
2023-06-21 8:38 ` [PATCH v11 1/2] gro : refactor IPv4 to add GRO support for IPv6 Kumara Parameshwaran
@ 2023-06-21 8:38 ` Kumara Parameshwaran
2023-06-27 15:47 ` Thomas Monjalon
0 siblings, 1 reply; 32+ messages in thread
From: Kumara Parameshwaran @ 2023-06-21 8:38 UTC (permalink / raw)
To: jiayu.hu; +Cc: dev, Kumara Parameshwaran
Add support for IPv6 GRO for TCP packets
Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
---
v1:
* Changes to support GRO for TCP/ipv6 packets. This does not include
vxlan changes.
* The GRO is performed only for ipv6 packets that does not contain
extension headers.
* The logic for the TCP coalescing remains the same, in ipv6 header
the source address, destination address, flow label, version fields
are expected to be the same.
* Re-organised the code to reuse certain tcp functions for both ipv4 and
ipv6 flows.
v2:
* Fix comments in gro_tcp6.h header file.
v3:
* Adderess review comments to fix code duplication for v4 and v6
v4:
* Addresses review comments for v3, do not use callbacks
v5:
* Address review comments
v6:
* Fix warning and coding style issues
v7:
* Fix build compilation issue
v8:
* Use gro_tcp_internal.h for functions used for gro_tcp4 and gro_tcp6 and use
gro_tcp.h for data structures and functions used across gro_vxlan_tcp4
v9:
* Resolve merge conflict and add gro_tcp.h in proper path
v10:
* Refactor the code to contain 2 patches. This patch contains support for IPv6 GRO
v11:
* Fix commit to add missed files
.../generic_receive_offload_lib.rst | 21 +-
doc/guides/rel_notes/release_23_07.rst | 2 +
lib/gro/gro_tcp6.c | 268 ++++++++++++++++++
lib/gro/gro_tcp6.h | 160 +++++++++++
lib/gro/meson.build | 1 +
lib/gro/rte_gro.c | 81 +++++-
lib/gro/rte_gro.h | 3 +
7 files changed, 513 insertions(+), 23 deletions(-)
create mode 100644 lib/gro/gro_tcp6.c
create mode 100644 lib/gro/gro_tcp6.h
diff --git a/doc/guides/prog_guide/generic_receive_offload_lib.rst b/doc/guides/prog_guide/generic_receive_offload_lib.rst
index 98a5d29bbc..533cda7f5c 100644
--- a/doc/guides/prog_guide/generic_receive_offload_lib.rst
+++ b/doc/guides/prog_guide/generic_receive_offload_lib.rst
@@ -138,20 +138,21 @@ The key-based algorithm has two characters:
Key-based Reassembly Algorithm
-TCP/IPv4 GRO
-------------
+TCP-IPv4/IPv6 GRO
+-----------------
-The table structure used by TCP/IPv4 GRO contains two arrays: flow array
+The table structure used by TCP-IPv4/IPv6 GRO contains two arrays: flow array
and item array. The flow array keeps flow information, and the item array
-keeps packet information.
+keeps packet information. The flow array is different for IPv4 and IPv6 while
+the item array is the same.
-Header fields used to define a TCP/IPv4 flow include:
+Header fields used to define a TCP-IPv4/IPv6 flow include:
-- source and destination: Ethernet and IP address, TCP port
+- common tcp key fields : Ethernet address, TCP port, TCP acknowledge number
+- version specific IP address
+- IPv6 flow label for IPv6 flow
-- TCP acknowledge number
-
-TCP/IPv4 packets whose FIN, SYN, RST, URG, PSH, ECE or CWR bit is set
+TCP packets whose FIN, SYN, RST, URG, PSH, ECE or CWR bit is set
won't be processed.
Header fields deciding if two packets are neighbors include:
@@ -159,7 +160,7 @@ Header fields deciding if two packets are neighbors include:
- TCP sequence number
- IPv4 ID. The IPv4 ID fields of the packets, whose DF bit is 0, should
- be increased by 1.
+ be increased by 1. This is applicable only for IPv4
VxLAN GRO
---------
diff --git a/doc/guides/rel_notes/release_23_07.rst b/doc/guides/rel_notes/release_23_07.rst
index 027ae7bd2d..7124cf45c7 100644
--- a/doc/guides/rel_notes/release_23_07.rst
+++ b/doc/guides/rel_notes/release_23_07.rst
@@ -170,6 +170,8 @@ New Features
See :doc:`../prog_guide/pdcp_lib` for more information.
+* **Added support for TCP/IPv6 GRO.**
+ * Enhanced the existing library to support GRO for TCP packets over IPv6 network.
Removed Items
-------------
diff --git a/lib/gro/gro_tcp6.c b/lib/gro/gro_tcp6.c
new file mode 100644
index 0000000000..5aa39801e1
--- /dev/null
+++ b/lib/gro/gro_tcp6.c
@@ -0,0 +1,268 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+
+#include "gro_tcp6.h"
+#include "gro_tcp_internal.h"
+
+void *
+gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow)
+{
+ struct gro_tcp6_tbl *tbl;
+ size_t size;
+ uint32_t entries_num, i;
+
+ entries_num = max_flow_num * max_item_per_flow;
+ entries_num = RTE_MIN(entries_num, GRO_TCP6_TBL_MAX_ITEM_NUM);
+
+ if (entries_num == 0)
+ return NULL;
+
+ tbl = rte_zmalloc_socket(__func__,
+ sizeof(struct gro_tcp6_tbl),
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl == NULL)
+ return NULL;
+
+ size = sizeof(struct gro_tcp_item) * entries_num;
+ tbl->items = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->items == NULL) {
+ rte_free(tbl);
+ return NULL;
+ }
+ tbl->max_item_num = entries_num;
+
+ size = sizeof(struct gro_tcp6_flow) * entries_num;
+ tbl->flows = rte_zmalloc_socket(__func__,
+ size,
+ RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (tbl->flows == NULL) {
+ rte_free(tbl->items);
+ rte_free(tbl);
+ return NULL;
+ }
+ /* INVALID_ARRAY_INDEX indicates an empty flow */
+ for (i = 0; i < entries_num; i++)
+ tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
+ tbl->max_flow_num = entries_num;
+
+ return tbl;
+}
+
+void
+gro_tcp6_tbl_destroy(void *tbl)
+{
+ struct gro_tcp6_tbl *tcp_tbl = tbl;
+
+ if (tcp_tbl) {
+ rte_free(tcp_tbl->items);
+ rte_free(tcp_tbl->flows);
+ }
+ rte_free(tcp_tbl);
+}
+
+static inline uint32_t
+find_an_empty_flow(struct gro_tcp6_tbl *tbl)
+{
+ uint32_t i;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++)
+ if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
+ return i;
+ return INVALID_ARRAY_INDEX;
+}
+
+static inline uint32_t
+insert_new_flow(struct gro_tcp6_tbl *tbl,
+ struct tcp6_flow_key *src,
+ uint32_t item_idx)
+{
+ struct tcp6_flow_key *dst;
+ uint32_t flow_idx;
+
+ flow_idx = find_an_empty_flow(tbl);
+ if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
+ return INVALID_ARRAY_INDEX;
+
+ dst = &(tbl->flows[flow_idx].key);
+
+ ASSIGN_COMMON_TCP_KEY((&src->cmn_key), (&dst->cmn_key));
+ memcpy(&dst->src_addr[0], &src->src_addr[0], sizeof(dst->src_addr));
+ memcpy(&dst->dst_addr[0], &src->dst_addr[0], sizeof(dst->dst_addr));
+ dst->vtc_flow = src->vtc_flow;
+
+ tbl->flows[flow_idx].start_index = item_idx;
+ tbl->flow_num++;
+
+ return flow_idx;
+}
+
+/*
+ * update the packet length for the flushed packet.
+ */
+static inline void
+update_header(struct gro_tcp_item *item)
+{
+ struct rte_ipv6_hdr *ipv6_hdr;
+ struct rte_mbuf *pkt = item->firstseg;
+
+ ipv6_hdr = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
+ pkt->l2_len);
+ ipv6_hdr->payload_len = rte_cpu_to_be_16(pkt->pkt_len -
+ pkt->l2_len - pkt->l3_len);
+}
+
+int32_t
+gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time)
+{
+ struct rte_ether_hdr *eth_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ int32_t tcp_dl;
+ uint16_t ip_tlen;
+ struct tcp6_flow_key key;
+ uint32_t i, max_flow_num, remaining_flow_num;
+ uint32_t sent_seq;
+ struct rte_tcp_hdr *tcp_hdr;
+ uint8_t find;
+ uint32_t item_idx;
+ /*
+ * Don't process the packet whose TCP header length is greater
+ * than 60 bytes or less than 20 bytes.
+ */
+ if (unlikely(INVALID_TCP_HDRLEN(pkt->l4_len)))
+ return -1;
+
+ eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
+ ipv6_hdr = (struct rte_ipv6_hdr *)((char *)eth_hdr + pkt->l2_len);
+ tcp_hdr = rte_pktmbuf_mtod_offset(pkt, struct rte_tcp_hdr *, pkt->l2_len + pkt->l3_len);
+
+ /*
+ * Don't process the packet which has FIN, SYN, RST, PSH, URG, ECE
+ * or CWR set.
+ */
+ if (tcp_hdr->tcp_flags != RTE_TCP_ACK_FLAG)
+ return -1;
+
+ ip_tlen = rte_be_to_cpu_16(ipv6_hdr->payload_len);
+ /*
+ * Don't process the packet whose payload length is less than or
+ * equal to 0.
+ */
+ tcp_dl = ip_tlen - pkt->l4_len;
+ if (tcp_dl <= 0)
+ return -1;
+
+ rte_ether_addr_copy(&(eth_hdr->src_addr), &(key.cmn_key.eth_saddr));
+ rte_ether_addr_copy(&(eth_hdr->dst_addr), &(key.cmn_key.eth_daddr));
+ memcpy(&key.src_addr[0], &ipv6_hdr->src_addr, sizeof(key.src_addr));
+ memcpy(&key.dst_addr[0], &ipv6_hdr->dst_addr, sizeof(key.dst_addr));
+ key.cmn_key.src_port = tcp_hdr->src_port;
+ key.cmn_key.dst_port = tcp_hdr->dst_port;
+ key.cmn_key.recv_ack = tcp_hdr->recv_ack;
+ key.vtc_flow = ipv6_hdr->vtc_flow;
+
+ /* Search for a matched flow. */
+ max_flow_num = tbl->max_flow_num;
+ remaining_flow_num = tbl->flow_num;
+ find = 0;
+ for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
+ if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
+ if (is_same_tcp6_flow(&tbl->flows[i].key, &key)) {
+ find = 1;
+ break;
+ }
+ remaining_flow_num--;
+ }
+ }
+
+ if (find == 0) {
+ sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq);
+ item_idx = insert_new_tcp_item(pkt, tbl->items, &tbl->item_num,
+ tbl->max_item_num, start_time,
+ INVALID_ARRAY_INDEX, sent_seq, 0, true);
+ if (item_idx == INVALID_ARRAY_INDEX)
+ return -1;
+ if (insert_new_flow(tbl, &key, item_idx) ==
+ INVALID_ARRAY_INDEX) {
+ /*
+ * Fail to insert a new flow, so delete the
+ * stored packet.
+ */
+ delete_tcp_item(tbl->items, item_idx, &tbl->item_num, INVALID_ARRAY_INDEX);
+ return -1;
+ }
+ return 0;
+ }
+
+ return process_tcp_item(pkt, tcp_hdr, tcp_dl, tbl->items, tbl->flows[i].start_index,
+ &tbl->item_num, tbl->max_item_num,
+ 0, true, start_time);
+}
+
+uint16_t
+gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out)
+{
+ uint16_t k = 0;
+ uint32_t i, j;
+ uint32_t max_flow_num = tbl->max_flow_num;
+
+ for (i = 0; i < max_flow_num; i++) {
+ if (unlikely(tbl->flow_num == 0))
+ return k;
+
+ j = tbl->flows[i].start_index;
+ while (j != INVALID_ARRAY_INDEX) {
+ if (tbl->items[j].start_time <= flush_timestamp) {
+ out[k++] = tbl->items[j].firstseg;
+ if (tbl->items[j].nb_merged > 1)
+ update_header(&(tbl->items[j]));
+ /*
+ * Delete the packet and get the next
+ * packet in the flow.
+ */
+ j = delete_tcp_item(tbl->items, j,
+ &tbl->item_num, INVALID_ARRAY_INDEX);
+ tbl->flows[i].start_index = j;
+ if (j == INVALID_ARRAY_INDEX)
+ tbl->flow_num--;
+
+ if (unlikely(k == nb_out))
+ return k;
+ } else
+ /*
+ * The left packets in this flow won't be
+ * timeout. Go to check other flows.
+ */
+ break;
+ }
+ }
+ return k;
+}
+
+uint32_t
+gro_tcp6_tbl_pkt_count(void *tbl)
+{
+ struct gro_tcp6_tbl *gro_tbl = tbl;
+
+ if (gro_tbl)
+ return gro_tbl->item_num;
+
+ return 0;
+}
diff --git a/lib/gro/gro_tcp6.h b/lib/gro/gro_tcp6.h
new file mode 100644
index 0000000000..073122f0ec
--- /dev/null
+++ b/lib/gro/gro_tcp6.h
@@ -0,0 +1,160 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Intel Corporation
+ */
+
+#ifndef _GRO_TCP6_H_
+#define _GRO_TCP6_H_
+
+#include "gro_tcp.h"
+
+#define GRO_TCP6_TBL_MAX_ITEM_NUM (1024UL * 1024UL)
+
+/* Header fields representing a TCP/IPv6 flow */
+struct tcp6_flow_key {
+ struct cmn_tcp_key cmn_key;
+ uint8_t src_addr[16];
+ uint8_t dst_addr[16];
+ rte_be32_t vtc_flow;
+};
+
+struct gro_tcp6_flow {
+ struct tcp6_flow_key key;
+ /*
+ * The index of the first packet in the flow.
+ * INVALID_ARRAY_INDEX indicates an empty flow.
+ */
+ uint32_t start_index;
+};
+
+/*
+ * TCP/IPv6 reassembly table structure.
+ */
+struct gro_tcp6_tbl {
+ /* item array */
+ struct gro_tcp_item *items;
+ /* flow array */
+ struct gro_tcp6_flow *flows;
+ /* current item number */
+ uint32_t item_num;
+ /* current flow num */
+ uint32_t flow_num;
+ /* item array size */
+ uint32_t max_item_num;
+ /* flow array size */
+ uint32_t max_flow_num;
+};
+
+/**
+ * This function creates a TCP/IPv6 reassembly table.
+ *
+ * @param socket_id
+ * Socket index for allocating the TCP/IPv6 reassemble table
+ * @param max_flow_num
+ * The maximum number of flows in the TCP/IPv6 GRO table
+ * @param max_item_per_flow
+ * The maximum number of packets per flow
+ *
+ * @return
+ * - Return the table pointer on success.
+ * - Return NULL on failure.
+ */
+void *gro_tcp6_tbl_create(uint16_t socket_id,
+ uint16_t max_flow_num,
+ uint16_t max_item_per_flow);
+
+/**
+ * This function destroys a TCP/IPv6 reassembly table.
+ *
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table.
+ */
+void gro_tcp6_tbl_destroy(void *tbl);
+
+/**
+ * This function merges a TCP/IPv6 packet. It doesn't process the packet,
+ * which has SYN, FIN, RST, PSH, CWR, ECE or URG set, or doesn't have
+ * payload.
+ *
+ * This function doesn't check if the packet has correct checksums and
+ * doesn't re-calculate checksums for the merged packet. Additionally,
+ * it assumes the packets are complete (i.e., MF==0 && frag_off==0),
+ * when IP fragmentation is possible (i.e., DF==0). It returns the
+ * packet, if the packet has invalid parameters (e.g. SYN bit is set)
+ * or there is no available space in the table.
+ *
+ * @param pkt
+ * Packet to reassemble
+ * @param tbl
+ * Pointer pointing to the TCP/IPv6 reassembly table
+ * @start_time
+ * The time when the packet is inserted into the table
+ *
+ * @return
+ * - Return a positive value if the packet is merged.
+ * - Return zero if the packet isn't merged but stored in the table.
+ * - Return a negative value for invalid parameters or no available
+ * space in the table.
+ */
+int32_t gro_tcp6_reassemble(struct rte_mbuf *pkt,
+ struct gro_tcp6_tbl *tbl,
+ uint64_t start_time);
+
+/**
+ * This function flushes timeout packets in a TCP/IPv6 reassembly table,
+ * and without updating checksums.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ * @param flush_timestamp
+ * Flush packets which are inserted into the table before or at the
+ * flush_timestamp.
+ * @param out
+ * Pointer array used to keep flushed packets
+ * @param nb_out
+ * The element number in 'out'. It also determines the maximum number of
+ * packets that can be flushed finally.
+ *
+ * @return
+ * The number of flushed packets
+ */
+uint16_t gro_tcp6_tbl_timeout_flush(struct gro_tcp6_tbl *tbl,
+ uint64_t flush_timestamp,
+ struct rte_mbuf **out,
+ uint16_t nb_out);
+
+/**
+ * This function returns the number of the packets in a TCP/IPv6
+ * reassembly table.
+ *
+ * @param tbl
+ * TCP/IPv6 reassembly table pointer
+ *
+ * @return
+ * The number of packets in the table
+ */
+uint32_t gro_tcp6_tbl_pkt_count(void *tbl);
+
+/*
+ * Check if two TCP/IPv6 packets belong to the same flow.
+ */
+static inline int
+is_same_tcp6_flow(struct tcp6_flow_key *k1, struct tcp6_flow_key *k2)
+{
+ rte_be32_t vtc_flow_diff;
+
+ if (memcmp(&k1->src_addr, &k2->src_addr, 16))
+ return 0;
+ if (memcmp(&k1->dst_addr, &k2->dst_addr, 16))
+ return 0;
+ /*
+ * IP version (4) Traffic Class (8) Flow Label (20)
+ * All fields except Traffic class should be same
+ */
+ vtc_flow_diff = (k1->vtc_flow ^ k2->vtc_flow);
+ if (vtc_flow_diff & htonl(0xF00FFFFF))
+ return 0;
+
+ return is_same_common_tcp_key(&k1->cmn_key, &k2->cmn_key);
+}
+
+#endif
diff --git a/lib/gro/meson.build b/lib/gro/meson.build
index e4fa2958bd..dbce05220d 100644
--- a/lib/gro/meson.build
+++ b/lib/gro/meson.build
@@ -4,6 +4,7 @@
sources = files(
'rte_gro.c',
'gro_tcp4.c',
+ 'gro_tcp6.c',
'gro_udp4.c',
'gro_vxlan_tcp4.c',
'gro_vxlan_udp4.c',
diff --git a/lib/gro/rte_gro.c b/lib/gro/rte_gro.c
index ac3d1cdc94..d824eebd93 100644
--- a/lib/gro/rte_gro.c
+++ b/lib/gro/rte_gro.c
@@ -8,6 +8,7 @@
#include "rte_gro.h"
#include "gro_tcp4.h"
+#include "gro_tcp6.h"
#include "gro_udp4.h"
#include "gro_vxlan_tcp4.h"
#include "gro_vxlan_udp4.h"
@@ -20,14 +21,16 @@ typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
- gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
+ gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, gro_tcp6_tbl_create, NULL};
static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
+ gro_tcp6_tbl_destroy,
NULL};
static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count,
+ gro_tcp6_tbl_pkt_count,
NULL};
#define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
@@ -35,6 +38,12 @@ static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+/* GRO with extension headers is not supported */
+#define IS_IPV6_TCP_PKT(ptype) (RTE_ETH_IS_IPV6_HDR(ptype) && \
+ ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
+ ((ptype & RTE_PTYPE_L4_FRAG) != RTE_PTYPE_L4_FRAG) && \
+ (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
+
#define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
(RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
@@ -149,6 +158,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
struct gro_tcp_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+ struct gro_tcp6_tbl tcp6_tbl;
+ struct gro_tcp6_flow tcp6_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
+ struct gro_tcp_item tcp6_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
+
/* allocate a reassembly table for UDP/IPv4 GRO */
struct gro_udp4_tbl udp_tbl;
struct gro_udp4_flow udp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
@@ -171,10 +184,10 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
int32_t ret;
uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
- do_vxlan_udp_gro = 0;
+ do_vxlan_udp_gro = 0, do_tcp6_gro = 0;
if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -236,6 +249,18 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
do_udp4_gro = 1;
}
+ if (param->gro_types & RTE_GRO_TCP_IPV6) {
+ for (i = 0; i < item_num; i++)
+ tcp6_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+ tcp6_tbl.flows = tcp6_flows;
+ tcp6_tbl.items = tcp6_items;
+ tcp6_tbl.flow_num = 0;
+ tcp6_tbl.item_num = 0;
+ tcp6_tbl.max_flow_num = item_num;
+ tcp6_tbl.max_item_num = item_num;
+ do_tcp6_gro = 1;
+ }
for (i = 0; i < nb_pkts; i++) {
/*
@@ -276,6 +301,14 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
nb_after_gro--;
else if (ret < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ ret = gro_tcp6_reassemble(pkts[i], &tcp6_tbl, 0);
+ if (ret > 0)
+ /* merge successfully */
+ nb_after_gro--;
+ else if (ret < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -283,9 +316,17 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
if ((nb_after_gro < nb_pkts)
|| (unprocess_num < nb_pkts)) {
i = 0;
+ /* Copy unprocessed packets */
+ if (unprocess_num > 0) {
+ memcpy(&pkts[i], unprocess_pkts,
+ sizeof(struct rte_mbuf *) *
+ unprocess_num);
+ i = unprocess_num;
+ }
+
/* Flush all packets from the tables */
if (do_vxlan_tcp_gro) {
- i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
+ i += gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
0, pkts, nb_pkts);
}
@@ -304,13 +345,11 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
&pkts[i], nb_pkts - i);
}
- /* Copy unprocessed packets */
- if (unprocess_num > 0) {
- memcpy(&pkts[i], unprocess_pkts,
- sizeof(struct rte_mbuf *) *
- unprocess_num);
+
+ if (do_tcp6_gro) {
+ i += gro_tcp6_tbl_timeout_flush(&tcp6_tbl, 0,
+ &pkts[i], nb_pkts - i);
}
- nb_after_gro = i + unprocess_num;
}
return nb_after_gro;
@@ -323,13 +362,13 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
{
struct rte_mbuf *unprocess_pkts[nb_pkts];
struct gro_ctx *gro_ctx = ctx;
- void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
+ void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl, *tcp6_tbl;
uint64_t current_time;
uint16_t i, unprocess_num = 0;
- uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro;
+ uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro, do_tcp6_gro;
if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
- RTE_GRO_TCP_IPV4 |
+ RTE_GRO_TCP_IPV4 | RTE_GRO_TCP_IPV6 |
RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
RTE_GRO_UDP_IPV4)) == 0))
return nb_pkts;
@@ -338,6 +377,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
+ tcp6_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX];
do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
RTE_GRO_TCP_IPV4;
@@ -347,6 +387,7 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
RTE_GRO_UDP_IPV4;
do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
RTE_GRO_IPV4_VXLAN_UDP_IPV4;
+ do_tcp6_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV6) == RTE_GRO_TCP_IPV6;
current_time = rte_rdtsc();
@@ -371,6 +412,11 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
if (gro_udp4_reassemble(pkts[i], udp_tbl,
current_time) < 0)
unprocess_pkts[unprocess_num++] = pkts[i];
+ } else if (IS_IPV6_TCP_PKT(pkts[i]->packet_type) &&
+ do_tcp6_gro) {
+ if (gro_tcp6_reassemble(pkts[i], tcp6_tbl,
+ current_time) < 0)
+ unprocess_pkts[unprocess_num++] = pkts[i];
} else
unprocess_pkts[unprocess_num++] = pkts[i];
}
@@ -426,6 +472,15 @@ rte_gro_timeout_flush(void *ctx,
gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
flush_timestamp,
&out[num], left_nb_out);
+ left_nb_out = max_nb_out - num;
+ }
+
+ if ((gro_types & RTE_GRO_TCP_IPV6) && left_nb_out > 0) {
+ num += gro_tcp6_tbl_timeout_flush(
+ gro_ctx->tbls[RTE_GRO_TCP_IPV6_INDEX],
+ flush_timestamp,
+ &out[num], left_nb_out);
+
}
return num;
diff --git a/lib/gro/rte_gro.h b/lib/gro/rte_gro.h
index 9f9ed4935a..c83dfd9ad1 100644
--- a/lib/gro/rte_gro.h
+++ b/lib/gro/rte_gro.h
@@ -38,6 +38,9 @@ extern "C" {
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX 3
#define RTE_GRO_IPV4_VXLAN_UDP_IPV4 (1ULL << RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX)
/**< VxLAN UDP/IPv4 GRO flag. */
+#define RTE_GRO_TCP_IPV6_INDEX 4
+#define RTE_GRO_TCP_IPV6 (1ULL << RTE_GRO_TCP_IPV6_INDEX)
+/**< TCP/IPv6 GRO flag. */
/**
* Structure used to create GRO context objects or used to pass
--
2.25.1
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [PATCH v11 2/2] gro : add support for IPv6 GRO
2023-06-21 8:38 ` [PATCH v11 2/2] gro : add support for IPv6 GRO Kumara Parameshwaran
@ 2023-06-27 15:47 ` Thomas Monjalon
0 siblings, 0 replies; 32+ messages in thread
From: Thomas Monjalon @ 2023-06-27 15:47 UTC (permalink / raw)
To: Kumara Parameshwaran; +Cc: jiayu.hu, dev
21/06/2023 10:38, Kumara Parameshwaran:
> Add support for IPv6 GRO for TCP packets
>
> Signed-off-by: Kumara Parameshwaran <kumaraparamesh92@gmail.com>
Did minor fixes in documentation,
added review tag from Jiayu (please keep it in your patch when received)
and applied, thanks.
^ permalink raw reply [flat|nested] 32+ messages in thread
end of thread, other threads:[~2023-06-27 15:47 UTC | newest]
Thread overview: 32+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-10-20 18:07 [PATCH] gro : ipv6 changes to support GRO for TCP/ipv6 Kumara Parameshwaran
2022-10-20 18:14 ` [PATCH v2] " Kumara Parameshwaran
2023-05-12 2:47 ` Hu, Jiayu
2023-05-16 9:28 ` kumaraparameshwaran rathinavel
2023-05-25 11:22 ` kumaraparameshwaran rathinavel
2023-05-31 8:20 ` Hu, Jiayu
2023-06-02 6:02 ` [PATCH v3] gro : ipv6-gro review comments to reduce code duplication across v4 and v6 Kumara Parameshwaran
2023-06-02 6:34 ` [PATCH v3] gro : ipv6 changes to support GRO for TCP/ipv6 Kumara Parameshwaran
2023-06-06 4:35 ` Hu, Jiayu
2023-06-06 9:31 ` kumaraparameshwaran rathinavel
2023-06-06 14:58 ` [PATCH v4] " Kumara Parameshwaran
2023-06-08 4:05 ` Hu, Jiayu
2023-06-08 16:52 ` kumaraparameshwaran rathinavel
2023-06-09 1:04 ` Hu, Jiayu
2023-06-12 11:05 ` [PATCH v5] " Kumara Parameshwaran
2023-06-12 11:23 ` [PATCH v6] " Kumara Parameshwaran
2023-06-12 11:31 ` [PATCH v7] " Kumara Parameshwaran
2023-06-12 12:04 ` kumaraparameshwaran rathinavel
2023-06-13 2:25 ` Hu, Jiayu
2023-06-14 3:43 ` kumaraparameshwaran rathinavel
2023-06-14 4:56 ` Hu, Jiayu
2023-06-15 5:40 ` [PATCH v8] " Kumara Parameshwaran
2023-06-15 6:20 ` [PATCH v9] " Kumara Parameshwaran
2023-06-15 6:30 ` kumaraparameshwaran rathinavel
2023-06-15 8:01 ` Hu, Jiayu
2023-06-15 9:16 ` kumaraparameshwaran rathinavel
2023-06-19 13:30 ` Thomas Monjalon
2023-06-21 8:25 ` [PATCH v10 1/2] gro : refactor IPv4 to add GRO support for IPv6 Kumara Parameshwaran
2023-06-21 8:25 ` [PATCH v10 2/2] gro : add support for IPv6 GRO Kumara Parameshwaran
2023-06-21 8:38 ` [PATCH v11 1/2] gro : refactor IPv4 to add GRO support for IPv6 Kumara Parameshwaran
2023-06-21 8:38 ` [PATCH v11 2/2] gro : add support for IPv6 GRO Kumara Parameshwaran
2023-06-27 15:47 ` Thomas Monjalon
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).