[dpdk-dev] [PATCH 0/2] Support the Heavyweight Mode GRO in Testpmd

[dpdk-dev] [PATCH 0/2] Support the Heavyweight Mode GRO in Testpmd

[Jiayu Hu]

The GRO library provides two reassembly modes: lightweight mode and
heavyweight mode. The lightweight mode has been supported by testpmd.
This patchset is to support the heavyweight mode in testpmd.

Jiayu Hu (2):
  app/testpmd: support the heavywight mode GRO
  doc: update testpmd user guide for the heavyweight mode GRO

 app/test-pmd/cmdline.c                      | 79 +++++++++++++++++++++++++++--
 app/test-pmd/config.c                       | 33 +++++++-----
 app/test-pmd/csumonly.c                     | 29 +++++++++--
 app/test-pmd/testpmd.c                      | 17 +++++++
 app/test-pmd/testpmd.h                      | 12 ++++-
 doc/guides/testpmd_app_ug/testpmd_funcs.rst | 14 ++++-
 6 files changed, 160 insertions(+), 24 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH 1/2] app/testpmd: support the heavywight mode GRO

[Jiayu Hu]

The GRO library provides two reassembly modes: lightweight mode and
heavyweight mode. This patch is to support the heavyweight mode in
csum forwarding engine.

With the command "gro (heavymode|lightmode) (on|off) <port id>", users
can select the lightweight mode or the heavyweight mode to use. With
the command "gro flush interval <num>", users can set the interval of
flushing GROed packets from the reassembly tables for the heavyweight
mode.

Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
---
 app/test-pmd/cmdline.c  | 79 ++++++++++++++++++++++++++++++++++++++++++++++---
 app/test-pmd/config.c   | 33 +++++++++++++--------
 app/test-pmd/csumonly.c | 29 ++++++++++++++----
 app/test-pmd/testpmd.c  | 17 +++++++++++
 app/test-pmd/testpmd.h  | 12 +++++++-
 5 files changed, 147 insertions(+), 23 deletions(-)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index cd8c358..3224ce1 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -423,7 +423,7 @@ static void cmd_help_long_parsed(void *parsed_result,
 			"tso show (portid)"
 			"    Display the status of TCP Segmentation Offload.\n\n"
 
-			"gro (on|off) (port_id)"
+			"gro (heavymode|lightmode) (on|off) (port_id)\n"
 			"    Enable or disable Generic Receive Offload in"
 			" csum forwarding engine.\n\n"
 
@@ -431,6 +431,10 @@ static void cmd_help_long_parsed(void *parsed_result,
 			"    Set max flow number and max packet number per-flow"
 			" for GRO.\n\n"
 
+			"gro flush interval (num)\n"
+			"    Set the interval of flushing GROed packets from"
+			" reassembly tables.\n\n"
+
 			"set fwd (%s)\n"
 			"    Set packet forwarding mode.\n\n"
 
@@ -3853,6 +3857,7 @@ cmdline_parse_inst_t cmd_tunnel_tso_show = {
 struct cmd_gro_result {
 	cmdline_fixed_string_t cmd_keyword;
 	cmdline_fixed_string_t mode;
+	cmdline_fixed_string_t onoff;
 	uint8_t port_id;
 };
 
@@ -3864,7 +3869,7 @@ cmd_enable_gro_parsed(void *parsed_result,
 	struct cmd_gro_result *res;
 
 	res = parsed_result;
-	setup_gro(res->mode, res->port_id);
+	setup_gro(res->mode, res->onoff, res->port_id);
 }
 
 cmdline_parse_token_string_t cmd_gro_keyword =
@@ -3872,7 +3877,10 @@ cmdline_parse_token_string_t cmd_gro_keyword =
 			cmd_keyword, "gro");
 cmdline_parse_token_string_t cmd_gro_mode =
 	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
-			mode, "on#off");
+			mode, "heavymode#lightmode");
+cmdline_parse_token_string_t cmd_gro_onoff =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
+			onoff, "on#off");
 cmdline_parse_token_num_t cmd_gro_pid =
 	TOKEN_NUM_INITIALIZER(struct cmd_gro_result,
 			port_id, UINT8);
@@ -3880,15 +3888,77 @@ cmdline_parse_token_num_t cmd_gro_pid =
 cmdline_parse_inst_t cmd_enable_gro = {
 	.f = cmd_enable_gro_parsed,
 	.data = NULL,
-	.help_str = "gro (on|off) (port_id)",
+	.help_str = "gro (heavymode|lightmode) (on|off) <port_id>",
 	.tokens = {
 		(void *)&cmd_gro_keyword,
 		(void *)&cmd_gro_mode,
+		(void *)&cmd_gro_onoff,
 		(void *)&cmd_gro_pid,
 		NULL,
 	},
 };
 
+/* *** SET FLUSH INTERVAL FOR THE HEAVYWEIGHT MODE GRO *** */
+struct cmd_gro_flush_result {
+	cmdline_fixed_string_t cmd_keyword;
+	cmdline_fixed_string_t cmd_flush;
+	cmdline_fixed_string_t cmd_interval;
+	uint32_t cmd_num;
+};
+
+static void
+cmd_gro_flush_parsed(void *parsed_result,
+		__attribute__((unused)) struct cmdline *cl,
+		__attribute__((unused)) void *data)
+{
+	struct cmd_gro_flush_result *res;
+
+	res = parsed_result;
+	if (test_done == 0) {
+		printf("Before set flushing interval for the heavyweight"
+				" mode GRO, please stop forwarding first\n");
+		return;
+	}
+
+	if (!strcmp(res->cmd_interval, "interval")) {
+		if (res->cmd_num > GRO_DEFAULT_FLUSH_INTERVAL) {
+			printf("The interval value should be in the range"
+					" of 0 to %u. Revert to the default"
+					" value %u\n",
+					GRO_MAX_FLUSH_INTERVAL,
+					GRO_DEFAULT_FLUSH_INTERVAL);
+			gro_flush_interval = GRO_DEFAULT_FLUSH_INTERVAL;
+		} else
+			gro_flush_interval = res->cmd_num;
+	}
+}
+
+cmdline_parse_token_string_t cmd_gro_flush_keyword =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_keyword, "gro");
+cmdline_parse_token_string_t cmd_gro_flush_flush =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_flush, "flush");
+cmdline_parse_token_string_t cmd_gro_flush_interval =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_interval, "interval");
+cmdline_parse_token_num_t cmd_gro_flush_num =
+	TOKEN_NUM_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_num, UINT32);
+
+cmdline_parse_inst_t cmd_gro_flush = {
+	.f = cmd_gro_flush_parsed,
+	.data = NULL,
+	.help_str = "gro flush interval <num>",
+	.tokens = {
+		(void *)&cmd_gro_flush_keyword,
+		(void *)&cmd_gro_flush_flush,
+		(void *)&cmd_gro_flush_interval,
+		(void *)&cmd_gro_flush_num,
+		NULL,
+	},
+};
+
 /* *** SET MAX FLOW NUMBER AND ITEM NUM PER FLOW FOR GRO *** */
 struct cmd_gro_set_result {
 	cmdline_fixed_string_t gro;
@@ -14251,6 +14321,7 @@ cmdline_parse_ctx_t main_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_tunnel_tso_show,
 	(cmdline_parse_inst_t *)&cmd_enable_gro,
 	(cmdline_parse_inst_t *)&cmd_gro_set,
+	(cmdline_parse_inst_t *)&cmd_gro_flush,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_rx,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_tx,
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index 3ae3e1c..3a1321a 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -2420,7 +2420,7 @@ set_tx_pkt_segments(unsigned *seg_lengths, unsigned nb_segs)
 }
 
 void
-setup_gro(const char *mode, uint8_t port_id)
+setup_gro(const char *mode, const char *onoff, uint8_t port_id)
 {
 	if (!rte_eth_dev_is_valid_port(port_id)) {
 		printf("invalid port id %u\n", port_id);
@@ -2431,20 +2431,27 @@ setup_gro(const char *mode, uint8_t port_id)
 				" please stop forwarding first\n");
 		return;
 	}
-	if (strcmp(mode, "on") == 0) {
-		if (gro_ports[port_id].enable) {
-			printf("port %u has enabled GRO\n", port_id);
+	if (strcmp(onoff, "on") == 0) {
+		if (gro_ports[port_id].enable != 0) {
+			printf("port %u has enabled GRO. Please"
+					" disable GRO first\n", port_id);
 			return;
 		}
-		gro_ports[port_id].enable = 1;
-		gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4;
-
-		if (gro_ports[port_id].param.max_flow_num == 0)
-			gro_ports[port_id].param.max_flow_num =
-				GRO_DEFAULT_FLOW_NUM;
-		if (gro_ports[port_id].param.max_item_per_flow == 0)
-			gro_ports[port_id].param.max_item_per_flow =
-				GRO_DEFAULT_ITEM_NUM_PER_FLOW;
+		if (strcmp(mode, "heavymode") == 0)
+			gro_ports[port_id].enable = GRO_HEAVYMODE;
+		else {
+			gro_ports[port_id].enable = GRO_LIGHTMODE;
+			gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4;
+
+			if (gro_ports[port_id].param.max_flow_num == 0) {
+				gro_ports[port_id].param.max_flow_num =
+					GRO_DEFAULT_FLOW_NUM;
+			}
+			if (gro_ports[port_id].param.max_item_per_flow == 0) {
+				gro_ports[port_id].param.max_item_per_flow =
+					GRO_DEFAULT_ITEM_NUM_PER_FLOW;
+			}
+		}
 	} else {
 		if (gro_ports[port_id].enable == 0) {
 			printf("port %u has disabled GRO\n", port_id);
diff --git a/app/test-pmd/csumonly.c b/app/test-pmd/csumonly.c
index 90c8119..f9d818c 100644
--- a/app/test-pmd/csumonly.c
+++ b/app/test-pmd/csumonly.c
@@ -631,6 +631,9 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 	struct rte_mbuf *m, *p;
 	struct ether_hdr *eth_hdr;
 	void *l3_hdr = NULL, *outer_l3_hdr = NULL; /* can be IPv4 or IPv6 */
+	void *gro_ctx;
+	uint16_t gro_pkts_num;
+	uint8_t gro_enable;
 	uint16_t nb_rx;
 	uint16_t nb_tx;
 	uint16_t nb_prep;
@@ -657,17 +660,33 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 				 nb_pkt_per_burst);
 	if (unlikely(nb_rx == 0))
 		return;
-	if (unlikely(gro_ports[fs->rx_port].enable))
-		nb_rx = rte_gro_reassemble_burst(pkts_burst,
-				nb_rx,
-				&(gro_ports[fs->rx_port].param));
-
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 	fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
 #endif
 	fs->rx_packets += nb_rx;
 	rx_bad_ip_csum = 0;
 	rx_bad_l4_csum = 0;
+	gro_enable = gro_ports[fs->rx_port].enable;
+
+	if (unlikely(gro_enable == GRO_HEAVYMODE)) {
+		gro_ctx = current_fwd_lcore()->gro_ctx;
+		nb_rx = rte_gro_reassemble(pkts_burst, nb_rx, gro_ctx);
+
+		if (fs->gro_times++ >= gro_flush_interval) {
+			gro_pkts_num = rte_gro_get_pkt_count(gro_ctx);
+			if (gro_pkts_num > MAX_PKT_BURST - nb_rx)
+				gro_pkts_num = MAX_PKT_BURST - nb_rx;
+
+			nb_rx += rte_gro_timeout_flush(gro_ctx, 0,
+					gro_ports[fs->rx_port].param.gro_types,
+					&pkts_burst[nb_rx],
+					gro_pkts_num);
+			fs->gro_times = 0;
+		}
+	} else if (unlikely(gro_enable == GRO_LIGHTMODE)) {
+		nb_rx = rte_gro_reassemble_burst(pkts_burst, nb_rx,
+				&(gro_ports[fs->rx_port].param));
+	}
 
 	txp = &ports[fs->tx_port];
 	testpmd_ol_flags = txp->tx_ol_flags;
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index 7d40139..f06c158 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -386,6 +386,7 @@ uint8_t bitrate_enabled;
 #endif
 
 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
+uint32_t gro_flush_interval = GRO_DEFAULT_FLUSH_INTERVAL;
 
 /* Forward function declarations */
 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port);
@@ -570,6 +571,7 @@ init_config(void)
 	unsigned int nb_mbuf_per_pool;
 	lcoreid_t  lc_id;
 	uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
+	struct rte_gro_param gro_param;
 
 	memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
 
@@ -671,6 +673,20 @@ init_config(void)
 		rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
 
 	fwd_config_setup();
+
+	/* create a gro context for each lcore */
+	gro_param.gro_types = RTE_GRO_TCP_IPV4;
+	gro_param.max_flow_num = GRO_MAX_FLUSH_INTERVAL;
+	gro_param.max_item_per_flow = MAX_PKT_BURST;
+	for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
+		gro_param.socket_id = rte_lcore_to_socket_id(
+				fwd_lcores_cpuids[lc_id]);
+		fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
+		if (fwd_lcores[lc_id]->gro_ctx == NULL) {
+			rte_exit(EXIT_FAILURE,
+					"rte_gro_ctx_create() failed\n");
+		}
+	}
 }
 
 
@@ -1165,6 +1181,7 @@ start_packet_forwarding(int with_tx_first)
 		fwd_streams[sm_id]->fwd_dropped = 0;
 		fwd_streams[sm_id]->rx_bad_ip_csum = 0;
 		fwd_streams[sm_id]->rx_bad_l4_csum = 0;
+		fwd_streams[sm_id]->gro_times = 0;
 
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 		memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index c9d7739..7c6c2eb 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -120,6 +120,7 @@ struct fwd_stream {
 	unsigned int fwd_dropped; /**< received packets not forwarded */
 	unsigned int rx_bad_ip_csum ; /**< received packets has bad ip checksum */
 	unsigned int rx_bad_l4_csum ; /**< received packets has bad l4 checksum */
+	unsigned int gro_times;	/**< reassembly times in heavyweight mode */
 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 	uint64_t     core_cycles; /**< used for RX and TX processing */
 #endif
@@ -206,6 +207,7 @@ struct rte_port {
  */
 struct fwd_lcore {
 	struct rte_mempool *mbp; /**< The mbuf pool to use by this core */
+	void *gro_ctx;		/**< GRO context */
 	streamid_t stream_idx;   /**< index of 1st stream in "fwd_streams" */
 	streamid_t stream_nb;    /**< number of streams in "fwd_streams" */
 	lcoreid_t  cpuid_idx;    /**< index of logical core in CPU id table */
@@ -434,13 +436,21 @@ extern struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
 extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
 
+#define GRO_HEAVYMODE 0x1
+#define GRO_LIGHTMODE 0x2
+
 #define GRO_DEFAULT_FLOW_NUM 4
 #define GRO_DEFAULT_ITEM_NUM_PER_FLOW DEF_PKT_BURST
+
+#define GRO_DEFAULT_FLUSH_INTERVAL 4
+#define GRO_MAX_FLUSH_INTERVAL 8
+
 struct gro_status {
 	struct rte_gro_param param;
 	uint8_t enable;
 };
 extern struct gro_status gro_ports[RTE_MAX_ETHPORTS];
+extern uint32_t gro_flush_interval;
 
 static inline unsigned int
 lcore_num(void)
@@ -640,7 +650,7 @@ void get_2tuple_filter(uint8_t port_id, uint16_t index);
 void get_5tuple_filter(uint8_t port_id, uint16_t index);
 int rx_queue_id_is_invalid(queueid_t rxq_id);
 int tx_queue_id_is_invalid(queueid_t txq_id);
-void setup_gro(const char *mode, uint8_t port_id);
+void setup_gro(const char *mode, const char *onoff, uint8_t port_id);
 
 /* Functions to manage the set of filtered Multicast MAC addresses */
 void mcast_addr_add(uint8_t port_id, struct ether_addr *mc_addr);
-- 
2.7.4

[dpdk-dev] [PATCH 2/2] doc: update testpmd user guide for the heavyweight mode GRO

[Jiayu Hu]

Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
---
 doc/guides/testpmd_app_ug/testpmd_funcs.rst | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 2ed62f5..bc8f7f8 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -903,7 +903,7 @@ gro
 
 Enable or disable GRO in ``csum`` forwarding engine::
 
-   testpmd> gro (on|off) (port_id)
+   testpmd> gro (heavymode|lightmode) (on|off) (port_id)
 
 If enabled, the csum forwarding engine will perform GRO on the TCP/IPv4
 packets received from the given port.
@@ -932,6 +932,18 @@ number of packets a GRO table can store.
 If current packet number is greater than or equal to the max value, GRO
 will stop processing incoming packets.
 
+gro flush interval
+~~~~~~~~~~~~~~~~~~
+
+Set the interval of flushing GROed packets from reassembly tables::
+
+   testpmd> gro flush interval (num)
+
+In the heavyweight mode, GROed packets are flushed from the reassembly
+tables and sent out when perform GRO every ``num`` times.
+
+The value of ``num`` should be in the range of 0 to ``GRO_MAX_FLUSH_INTERVAL``.
+
 mac_addr add
 ~~~~~~~~~~~~
 
-- 
2.7.4

Re: [dpdk-dev] [PATCH 1/2] app/testpmd: support the heavywight mode GRO

[Ferruh Yigit]

On 8/10/2017 3:50 AM, Jiayu Hu wrote:
> The GRO library provides two reassembly modes: lightweight mode and
> heavyweight mode. This patch is to support the heavyweight mode in
> csum forwarding engine.
> 
> With the command "gro (heavymode|lightmode) (on|off) <port id>", users
> can select the lightweight mode or the heavyweight mode to use. With
> the command "gro flush interval <num>", users can set the interval of
> flushing GROed packets from the reassembly tables for the heavyweight
> mode.
> 
> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
> ---
>  app/test-pmd/cmdline.c  | 79 ++++++++++++++++++++++++++++++++++++++++++++++---
>  app/test-pmd/config.c   | 33 +++++++++++++--------
>  app/test-pmd/csumonly.c | 29 ++++++++++++++----
>  app/test-pmd/testpmd.c  | 17 +++++++++++
>  app/test-pmd/testpmd.h  | 12 +++++++-
>  5 files changed, 147 insertions(+), 23 deletions(-)
> 
> diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
> index cd8c358..3224ce1 100644
> --- a/app/test-pmd/cmdline.c
> +++ b/app/test-pmd/cmdline.c
> @@ -423,7 +423,7 @@ static void cmd_help_long_parsed(void *parsed_result,
>  			"tso show (portid)"
>  			"    Display the status of TCP Segmentation Offload.\n\n"
>  
> -			"gro (on|off) (port_id)"
> +			"gro (heavymode|lightmode) (on|off) (port_id)\n"

Not specific / limited to gro, but I have a few testpmd usability questions:

1) to update some settings, there are two root level commands already:
a) set ... (like: "set fwd mac", "set promisc #P on")
b) port config ... (like: "port config 	#P speed auto duplex auto")

I don't know what it difference between above two, but thinking as "port
config" is to configure ports and "set" is to set rest of testpmd config
makes sense to me. (but "set vf .." and "set port .."  doesn't fit to
this statement)

Instead of adding "gro" root level command, why not add this under "port
config", like: "port config #P gro on" ? (or "set port #P gro on")


2) Should each configuration set have a corresponding show command?

How a user can see the current gro setting?
Do we need a "show port gro" ?


3) Where to place #P in the command:
There are inconsistencies about where to put it, like:
"port config #P l2-tunnel enable"
"port config mtu #P value"
"port #P rxq #Q start"

or

"show port info #P"
"show port #P rss-hash ip4 key"

It can be good to define a place for it for consistency.


4) Using "port" keyword in some commands before port_id:
Like: "set link-down port #P", "reset port #P mirror-rule value"

Mostly we don't have it:
"show txq info #P #Q", "set bonding mode #M #P", "set stat_qmap rx #P
#Q", "vlan set strip on #P"

It can be good to define one or other.

And I guess having need to use "port" keyword can be an indication that
command should move under "port" command:
"reset port #P mirror-rule #id" can be "port reset #P mirror-rule #id"


Thanks,
ferruh

Re: [dpdk-dev] [PATCH 1/2] app/testpmd: support the heavywight mode GRO

[Jiayu Hu]

Hi Ferruh,

On Thu, Aug 10, 2017 at 10:50:02AM +0100, Ferruh Yigit wrote:
> On 8/10/2017 3:50 AM, Jiayu Hu wrote:
> > The GRO library provides two reassembly modes: lightweight mode and
> > heavyweight mode. This patch is to support the heavyweight mode in
> > csum forwarding engine.
> > 
> > With the command "gro (heavymode|lightmode) (on|off) <port id>", users
> > can select the lightweight mode or the heavyweight mode to use. With
> > the command "gro flush interval <num>", users can set the interval of
> > flushing GROed packets from the reassembly tables for the heavyweight
> > mode.
> > 
> > Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
> > ---
> >  app/test-pmd/cmdline.c  | 79 ++++++++++++++++++++++++++++++++++++++++++++++---
> >  app/test-pmd/config.c   | 33 +++++++++++++--------
> >  app/test-pmd/csumonly.c | 29 ++++++++++++++----
> >  app/test-pmd/testpmd.c  | 17 +++++++++++
> >  app/test-pmd/testpmd.h  | 12 +++++++-
> >  5 files changed, 147 insertions(+), 23 deletions(-)
> > 
> > diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
> > index cd8c358..3224ce1 100644
> > --- a/app/test-pmd/cmdline.c
> > +++ b/app/test-pmd/cmdline.c
> > @@ -423,7 +423,7 @@ static void cmd_help_long_parsed(void *parsed_result,
> >  			"tso show (portid)"
> >  			"    Display the status of TCP Segmentation Offload.\n\n"
> >  
> > -			"gro (on|off) (port_id)"
> > +			"gro (heavymode|lightmode) (on|off) (port_id)\n"
> 
> Not specific / limited to gro, but I have a few testpmd usability questions:
> 
> 1) to update some settings, there are two root level commands already:
> a) set ... (like: "set fwd mac", "set promisc #P on")
> b) port config ... (like: "port config 	#P speed auto duplex auto")
> 
> I don't know what it difference between above two, but thinking as "port
> config" is to configure ports and "set" is to set rest of testpmd config
> makes sense to me. (but "set vf .." and "set port .."  doesn't fit to
> this statement)
> 
> Instead of adding "gro" root level command, why not add this under "port
> config", like: "port config #P gro on" ? (or "set port #P gro on")

Yes, using 'gro' as root command will make the usage of testpmd inconsistent.
I will change the GSO command in the next patch. Besides, there are other root
commands, like 'tso', and maybe we need to change them too.

> 
> 
> 2) Should each configuration set have a corresponding show command?
> 
> How a user can see the current gro setting?
> Do we need a "show port gro" ?

Yes, I will add one in the next patch.

> 
> 
> 3) Where to place #P in the command:
> There are inconsistencies about where to put it, like:
> "port config #P l2-tunnel enable"
> "port config mtu #P value"
> "port #P rxq #Q start"
> 
> or
> 
> "show port info #P"
> "show port #P rss-hash ip4 key"
> 
> It can be good to define a place for it for consistency.

Make sense.

> 
> 
> 4) Using "port" keyword in some commands before port_id:
> Like: "set link-down port #P", "reset port #P mirror-rule value"
> 
> Mostly we don't have it:
> "show txq info #P #Q", "set bonding mode #M #P", "set stat_qmap rx #P
> #Q", "vlan set strip on #P"
> 
> It can be good to define one or other.

Agree, and I will add "port" to GRO related commands.

Thanks,
Jiayu

> 
> And I guess having need to use "port" keyword can be an indication that
> command should move under "port" command:
> "reset port #P mirror-rule #id" can be "port reset #P mirror-rule #id"
> 
> 
> Thanks,
> ferruh

[dpdk-dev] [PATCH v2 0/2] Support the Heavyweight Mode GRO in Testpmd

[Jiayu Hu]

The GRO library provides two reassembly modes: lightweight mode and
heavyweight mode. The lightweight mode has been supported by testpmd.
This patchset is to support the heavyweight mode in testpmd.

Change log
==========
v2:
- use "set" and "show" as the root level command
- add a new command to show GRO configuration
- fix l2_len/l3_len/l4_len unset etc. bugs

Jiayu Hu (2):
  app/testpmd: support the heavywight mode GRO
  doc: update testpmd user guide for the heavyweight mode GRO

 app/test-pmd/cmdline.c                      | 181 ++++++++++++++++++++++++----
 app/test-pmd/config.c                       |  72 ++++++++---
 app/test-pmd/csumonly.c                     |  29 ++++-
 app/test-pmd/testpmd.c                      |  18 ++-
 app/test-pmd/testpmd.h                      |  13 +-
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  37 ++++--
 6 files changed, 300 insertions(+), 50 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v2 1/2] app/testpmd: support the heavywight mode GRO

[Jiayu Hu]

The GRO library provides two reassembly modes: lightweight mode and
heavyweight mode. This patch is to support the heavyweight mode in
csum forwarding engine.

With the command "set port <port_id> gro (heavymode|lightmode) (on|off)",
users can select the lightweight mode or the heavyweight mode to use. With
the command "set gro flush interval <num>", users can set the interval of
flushing GROed packets from reassembly tables for the heavyweight mode.
With the command "show port <port_id> gro", users can display GRO
configuration.

Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
---
 app/test-pmd/cmdline.c  | 181 ++++++++++++++++++++++++++++++++++++++++++------
 app/test-pmd/config.c   |  72 +++++++++++++++----
 app/test-pmd/csumonly.c |  29 ++++++--
 app/test-pmd/testpmd.c  |  18 ++++-
 app/test-pmd/testpmd.h  |  13 +++-
 5 files changed, 270 insertions(+), 43 deletions(-)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index cd8c358..9876604 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -423,14 +423,22 @@ static void cmd_help_long_parsed(void *parsed_result,
 			"tso show (portid)"
 			"    Display the status of TCP Segmentation Offload.\n\n"
 
-			"gro (on|off) (port_id)"
+			"set port (port_id) gro (heavymode|lightmode) "
+			"(on|off)\n"
 			"    Enable or disable Generic Receive Offload in"
 			" csum forwarding engine.\n\n"
 
+			"show port (port_id) gro\n"
+			"    Display GRO configuration.\n\n"
+
 			"gro set (max_flow_num) (max_item_num_per_flow) (port_id)\n"
 			"    Set max flow number and max packet number per-flow"
 			" for GRO.\n\n"
 
+			"set gro flush interval (num)\n"
+			"    Set the interval of flushing GROed packets from"
+			" reassembly tables.\n\n"
+
 			"set fwd (%s)\n"
 			"    Set packet forwarding mode.\n\n"
 
@@ -3850,41 +3858,168 @@ cmdline_parse_inst_t cmd_tunnel_tso_show = {
 };
 
 /* *** SET GRO FOR A PORT *** */
-struct cmd_gro_result {
+struct cmd_gro_enable_result {
+	cmdline_fixed_string_t cmd_set;
+	cmdline_fixed_string_t cmd_port;
 	cmdline_fixed_string_t cmd_keyword;
-	cmdline_fixed_string_t mode;
-	uint8_t port_id;
+	cmdline_fixed_string_t cmd_mode;
+	cmdline_fixed_string_t cmd_onoff;
+	uint8_t cmd_pid;
+};
+
+static void
+cmd_gro_enable_parsed(void *parsed_result,
+		__attribute__((unused)) struct cmdline *cl,
+		__attribute__((unused)) void *data)
+{
+	struct cmd_gro_enable_result *res;
+
+	res = parsed_result;
+	setup_gro(res->cmd_mode, res->cmd_onoff, res->cmd_pid);
+}
+
+cmdline_parse_token_string_t cmd_gro_enable_set =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_set, "set");
+cmdline_parse_token_string_t cmd_gro_enable_port =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_keyword, "port");
+cmdline_parse_token_num_t cmd_gro_enable_pid =
+	TOKEN_NUM_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_pid, UINT8);
+cmdline_parse_token_string_t cmd_gro_enable_keyword =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_keyword, "gro");
+cmdline_parse_token_string_t cmd_gro_enable_mode =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_mode, "heavymode#lightmode");
+cmdline_parse_token_string_t cmd_gro_enable_onoff =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_onoff, "on#off");
+
+cmdline_parse_inst_t cmd_gro_enable = {
+	.f = cmd_gro_enable_parsed,
+	.data = NULL,
+	.help_str = "set port <port_id> gro (heavymode|lightmode) (on|off)",
+	.tokens = {
+		(void *)&cmd_gro_enable_set,
+		(void *)&cmd_gro_enable_port,
+		(void *)&cmd_gro_enable_pid,
+		(void *)&cmd_gro_enable_keyword,
+		(void *)&cmd_gro_enable_mode,
+		(void *)&cmd_gro_enable_onoff,
+		NULL,
+	},
+};
+
+/* *** DISPLAY GRO CONFIGURATION *** */
+struct cmd_gro_show_result {
+	cmdline_fixed_string_t cmd_show;
+	cmdline_fixed_string_t cmd_port;
+	cmdline_fixed_string_t cmd_keyword;
+	uint8_t cmd_pid;
+};
+
+static void
+cmd_gro_show_parsed(void *parsed_result,
+		__attribute__((unused)) struct cmdline *cl,
+		__attribute__((unused)) void *data)
+{
+	struct cmd_gro_show_result *res;
+
+	res = parsed_result;
+	if (!strcmp(res->cmd_keyword, "gro"))
+		show_gro(res->cmd_pid);
+}
+
+cmdline_parse_token_string_t cmd_gro_show_show =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
+			cmd_show, "show");
+cmdline_parse_token_string_t cmd_gro_show_port =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
+			cmd_port, "port");
+cmdline_parse_token_num_t cmd_gro_show_pid =
+	TOKEN_NUM_INITIALIZER(struct cmd_gro_show_result,
+			cmd_pid, UINT8);
+cmdline_parse_token_string_t cmd_gro_show_keyword =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
+			cmd_keyword, "gro");
+
+cmdline_parse_inst_t cmd_gro_show = {
+	.f = cmd_gro_show_parsed,
+	.data = NULL,
+	.help_str = "show port <port_id> gro",
+	.tokens = {
+		(void *)&cmd_gro_show_show,
+		(void *)&cmd_gro_show_port,
+		(void *)&cmd_gro_show_pid,
+		(void *)&cmd_gro_show_keyword,
+		NULL,
+	},
+};
+
+/* *** SET FLUSH INTERVAL FOR THE HEAVYWEIGHT MODE GRO *** */
+struct cmd_gro_flush_result {
+	cmdline_fixed_string_t cmd_set;
+	cmdline_fixed_string_t cmd_keyword;
+	cmdline_fixed_string_t cmd_flush;
+	cmdline_fixed_string_t cmd_interval;
+	uint32_t cmd_num;
 };
 
 static void
-cmd_enable_gro_parsed(void *parsed_result,
+cmd_gro_flush_parsed(void *parsed_result,
 		__attribute__((unused)) struct cmdline *cl,
 		__attribute__((unused)) void *data)
 {
-	struct cmd_gro_result *res;
+	struct cmd_gro_flush_result *res;
 
 	res = parsed_result;
-	setup_gro(res->mode, res->port_id);
+	if (test_done == 0) {
+		printf("Before set flushing interval for heavyweight"
+				" mode GRO, please stop forwarding first\n");
+		return;
+	}
+
+	if (!strcmp(res->cmd_interval, "interval")) {
+		if (res->cmd_num > GRO_MAX_FLUSH_INTERVAL) {
+			printf("The interval value should be in the range"
+					" of 0 to %u. Revert to the default"
+					" value %u\n",
+					GRO_MAX_FLUSH_INTERVAL,
+					GRO_DEFAULT_FLUSH_INTERVAL);
+			gro_flush_interval = GRO_DEFAULT_FLUSH_INTERVAL;
+		} else
+			gro_flush_interval = res->cmd_num;
+	}
 }
 
-cmdline_parse_token_string_t cmd_gro_keyword =
-	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
+cmdline_parse_token_string_t cmd_gro_flush_set =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_set, "set");
+cmdline_parse_token_string_t cmd_gro_flush_keyword =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
 			cmd_keyword, "gro");
-cmdline_parse_token_string_t cmd_gro_mode =
-	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
-			mode, "on#off");
-cmdline_parse_token_num_t cmd_gro_pid =
-	TOKEN_NUM_INITIALIZER(struct cmd_gro_result,
-			port_id, UINT8);
+cmdline_parse_token_string_t cmd_gro_flush_flush =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_flush, "flush");
+cmdline_parse_token_string_t cmd_gro_flush_interval =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_interval, "interval");
+cmdline_parse_token_num_t cmd_gro_flush_num =
+	TOKEN_NUM_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_num, UINT32);
 
-cmdline_parse_inst_t cmd_enable_gro = {
-	.f = cmd_enable_gro_parsed,
+cmdline_parse_inst_t cmd_gro_flush = {
+	.f = cmd_gro_flush_parsed,
 	.data = NULL,
-	.help_str = "gro (on|off) (port_id)",
+	.help_str = "set gro flush interval <num>",
 	.tokens = {
-		(void *)&cmd_gro_keyword,
-		(void *)&cmd_gro_mode,
-		(void *)&cmd_gro_pid,
+		(void *)&cmd_gro_flush_set,
+		(void *)&cmd_gro_flush_keyword,
+		(void *)&cmd_gro_flush_flush,
+		(void *)&cmd_gro_flush_interval,
+		(void *)&cmd_gro_flush_num,
 		NULL,
 	},
 };
@@ -14249,8 +14384,10 @@ cmdline_parse_ctx_t main_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_tso_show,
 	(cmdline_parse_inst_t *)&cmd_tunnel_tso_set,
 	(cmdline_parse_inst_t *)&cmd_tunnel_tso_show,
-	(cmdline_parse_inst_t *)&cmd_enable_gro,
+	(cmdline_parse_inst_t *)&cmd_gro_enable,
+	(cmdline_parse_inst_t *)&cmd_gro_show,
 	(cmdline_parse_inst_t *)&cmd_gro_set,
+	(cmdline_parse_inst_t *)&cmd_gro_flush,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_rx,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_tx,
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index 3ae3e1c..9ee4e59 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -2420,7 +2420,7 @@ set_tx_pkt_segments(unsigned *seg_lengths, unsigned nb_segs)
 }
 
 void
-setup_gro(const char *mode, uint8_t port_id)
+setup_gro(const char *mode, const char *onoff, uint8_t port_id)
 {
 	if (!rte_eth_dev_is_valid_port(port_id)) {
 		printf("invalid port id %u\n", port_id);
@@ -2431,29 +2431,73 @@ setup_gro(const char *mode, uint8_t port_id)
 				" please stop forwarding first\n");
 		return;
 	}
-	if (strcmp(mode, "on") == 0) {
-		if (gro_ports[port_id].enable) {
-			printf("port %u has enabled GRO\n", port_id);
+	if (strcmp(onoff, "on") == 0) {
+		if (gro_ports[port_id].enable != 0) {
+			printf("Port %u has enabled GRO. Please"
+					" disable GRO first\n", port_id);
 			return;
 		}
-		gro_ports[port_id].enable = 1;
-		gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4;
-
-		if (gro_ports[port_id].param.max_flow_num == 0)
-			gro_ports[port_id].param.max_flow_num =
-				GRO_DEFAULT_FLOW_NUM;
-		if (gro_ports[port_id].param.max_item_per_flow == 0)
-			gro_ports[port_id].param.max_item_per_flow =
-				GRO_DEFAULT_ITEM_NUM_PER_FLOW;
+		if (strcmp(mode, "heavymode") == 0)
+			gro_ports[port_id].enable = GRO_HEAVYMODE;
+		else {
+			gro_ports[port_id].enable = GRO_LIGHTMODE;
+			gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4;
+
+			if (gro_ports[port_id].param.max_flow_num == 0) {
+				gro_ports[port_id].param.max_flow_num =
+					GRO_DEFAULT_FLOW_NUM;
+			}
+			if (gro_ports[port_id].param.max_item_per_flow == 0) {
+				gro_ports[port_id].param.max_item_per_flow =
+					GRO_DEFAULT_ITEM_NUM_PER_FLOW;
+			}
+		}
 	} else {
 		if (gro_ports[port_id].enable == 0) {
-			printf("port %u has disabled GRO\n", port_id);
+			printf("Port %u has disabled GRO\n", port_id);
 			return;
 		}
 		gro_ports[port_id].enable = 0;
 	}
 }
 
+void
+show_gro(uint8_t port_id)
+{
+	struct rte_gro_param *param;
+
+	param = &gro_ports[port_id].param;
+
+	if (!rte_eth_dev_is_valid_port(port_id)) {
+		printf("Invalid port id %u\n", port_id);
+		return;
+	}
+	switch (gro_ports[port_id].enable) {
+	case GRO_HEAVYMODE:
+		printf("GRO mode: heavyweight\n"
+				"Flushing interval: %u\n"
+				"GRO type: TCP/IPv4\n"
+				"Max flow number (fixed): %u\n"
+				"Max item per flow (fixed): %u\n",
+				gro_flush_interval,
+				GRO_MAX_FLUSH_INTERVAL,
+				MAX_PKT_BURST);
+		break;
+	case GRO_LIGHTMODE:
+		printf("GRO mode: lightweight\n"
+				"Flushing interval: N/A\n"
+				"GRO type: TCP/IPv4\n"
+				"Max flow number: %u\n"
+				"Max item per flow: %u\n",
+				param->max_flow_num,
+				param->max_item_per_flow);
+		break;
+	case 0:
+		printf("Port %u doesn't enable GRO\n", port_id);
+		break;
+	}
+}
+
 char*
 list_pkt_forwarding_modes(void)
 {
diff --git a/app/test-pmd/csumonly.c b/app/test-pmd/csumonly.c
index 90c8119..6e98938 100644
--- a/app/test-pmd/csumonly.c
+++ b/app/test-pmd/csumonly.c
@@ -631,6 +631,9 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 	struct rte_mbuf *m, *p;
 	struct ether_hdr *eth_hdr;
 	void *l3_hdr = NULL, *outer_l3_hdr = NULL; /* can be IPv4 or IPv6 */
+	void *gro_ctx;
+	uint16_t gro_pkts_num;
+	uint8_t gro_enable;
 	uint16_t nb_rx;
 	uint16_t nb_tx;
 	uint16_t nb_prep;
@@ -657,17 +660,13 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 				 nb_pkt_per_burst);
 	if (unlikely(nb_rx == 0))
 		return;
-	if (unlikely(gro_ports[fs->rx_port].enable))
-		nb_rx = rte_gro_reassemble_burst(pkts_burst,
-				nb_rx,
-				&(gro_ports[fs->rx_port].param));
-
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 	fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
 #endif
 	fs->rx_packets += nb_rx;
 	rx_bad_ip_csum = 0;
 	rx_bad_l4_csum = 0;
+	gro_enable = gro_ports[fs->rx_port].enable;
 
 	txp = &ports[fs->tx_port];
 	testpmd_ol_flags = txp->tx_ol_flags;
@@ -851,6 +850,26 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 		}
 	}
 
+	if (unlikely(gro_enable == GRO_HEAVYMODE)) {
+		gro_ctx = current_fwd_lcore()->gro_ctx;
+		nb_rx = rte_gro_reassemble(pkts_burst, nb_rx, gro_ctx);
+
+		if (fs->gro_times++ >= gro_flush_interval) {
+			gro_pkts_num = rte_gro_get_pkt_count(gro_ctx);
+			if (gro_pkts_num > MAX_PKT_BURST - nb_rx)
+				gro_pkts_num = MAX_PKT_BURST - nb_rx;
+
+			nb_rx += rte_gro_timeout_flush(gro_ctx, 0,
+					RTE_GRO_TCP_IPV4,
+					&pkts_burst[nb_rx],
+					gro_pkts_num);
+			fs->gro_times = 0;
+		}
+	} else if (unlikely(gro_enable == GRO_LIGHTMODE)) {
+		nb_rx = rte_gro_reassemble_burst(pkts_burst, nb_rx,
+				&(gro_ports[fs->rx_port].param));
+	}
+
 	nb_prep = rte_eth_tx_prepare(fs->tx_port, fs->tx_queue,
 			pkts_burst, nb_rx);
 	if (nb_prep != nb_rx)
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index 7d40139..3946a45 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -90,7 +90,6 @@
 #ifdef RTE_LIBRTE_LATENCY_STATS
 #include <rte_latencystats.h>
 #endif
-#include <rte_gro.h>
 
 #include "testpmd.h"
 
@@ -386,6 +385,7 @@ uint8_t bitrate_enabled;
 #endif
 
 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
+uint32_t gro_flush_interval = GRO_DEFAULT_FLUSH_INTERVAL;
 
 /* Forward function declarations */
 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port);
@@ -570,6 +570,7 @@ init_config(void)
 	unsigned int nb_mbuf_per_pool;
 	lcoreid_t  lc_id;
 	uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
+	struct rte_gro_param gro_param;
 
 	memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
 
@@ -671,6 +672,20 @@ init_config(void)
 		rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
 
 	fwd_config_setup();
+
+	/* create a gro context for each lcore */
+	gro_param.gro_types = RTE_GRO_TCP_IPV4;
+	gro_param.max_flow_num = GRO_MAX_FLUSH_INTERVAL;
+	gro_param.max_item_per_flow = MAX_PKT_BURST;
+	for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
+		gro_param.socket_id = rte_lcore_to_socket_id(
+				fwd_lcores_cpuids[lc_id]);
+		fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
+		if (fwd_lcores[lc_id]->gro_ctx == NULL) {
+			rte_exit(EXIT_FAILURE,
+					"rte_gro_ctx_create() failed\n");
+		}
+	}
 }
 
 
@@ -1165,6 +1180,7 @@ start_packet_forwarding(int with_tx_first)
 		fwd_streams[sm_id]->fwd_dropped = 0;
 		fwd_streams[sm_id]->rx_bad_ip_csum = 0;
 		fwd_streams[sm_id]->rx_bad_l4_csum = 0;
+		fwd_streams[sm_id]->gro_times = 0;
 
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 		memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index c9d7739..8b71a1f 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -120,6 +120,7 @@ struct fwd_stream {
 	unsigned int fwd_dropped; /**< received packets not forwarded */
 	unsigned int rx_bad_ip_csum ; /**< received packets has bad ip checksum */
 	unsigned int rx_bad_l4_csum ; /**< received packets has bad l4 checksum */
+	unsigned int gro_times;	/**< reassembly times in heavyweight mode */
 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 	uint64_t     core_cycles; /**< used for RX and TX processing */
 #endif
@@ -206,6 +207,7 @@ struct rte_port {
  */
 struct fwd_lcore {
 	struct rte_mempool *mbp; /**< The mbuf pool to use by this core */
+	void *gro_ctx;		/**< GRO context */
 	streamid_t stream_idx;   /**< index of 1st stream in "fwd_streams" */
 	streamid_t stream_nb;    /**< number of streams in "fwd_streams" */
 	lcoreid_t  cpuid_idx;    /**< index of logical core in CPU id table */
@@ -434,13 +436,21 @@ extern struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
 extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
 
+#define GRO_HEAVYMODE 0x1
+#define GRO_LIGHTMODE 0x2
+
 #define GRO_DEFAULT_FLOW_NUM 4
 #define GRO_DEFAULT_ITEM_NUM_PER_FLOW DEF_PKT_BURST
+
+#define GRO_DEFAULT_FLUSH_INTERVAL 2
+#define GRO_MAX_FLUSH_INTERVAL 4
+
 struct gro_status {
 	struct rte_gro_param param;
 	uint8_t enable;
 };
 extern struct gro_status gro_ports[RTE_MAX_ETHPORTS];
+extern uint32_t gro_flush_interval;
 
 static inline unsigned int
 lcore_num(void)
@@ -640,7 +650,8 @@ void get_2tuple_filter(uint8_t port_id, uint16_t index);
 void get_5tuple_filter(uint8_t port_id, uint16_t index);
 int rx_queue_id_is_invalid(queueid_t rxq_id);
 int tx_queue_id_is_invalid(queueid_t txq_id);
-void setup_gro(const char *mode, uint8_t port_id);
+void setup_gro(const char *mode, const char *onoff, uint8_t port_id);
+void show_gro(uint8_t port_id);
 
 /* Functions to manage the set of filtered Multicast MAC addresses */
 void mcast_addr_add(uint8_t port_id, struct ether_addr *mc_addr);
-- 
2.7.4

[dpdk-dev] [PATCH v2 2/2] doc: update testpmd user guide for the heavyweight mode GRO

[Jiayu Hu]

This patch is to update testpmd user guide for the heavyweight mode GRO.

Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
---
 doc/guides/testpmd_app_ug/testpmd_funcs.rst | 37 +++++++++++++++++++++++------
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 2ed62f5..fa507ef 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -898,12 +898,12 @@ Display the status of TCP Segmentation Offload::
 
    testpmd> tso show (port_id)
 
-gro
-~~~
+set port - gro
+~~~~~~~~~~~~~~
 
 Enable or disable GRO in ``csum`` forwarding engine::
 
-   testpmd> gro (on|off) (port_id)
+   testpmd> set port (port_id) gro (heavymode|lightmode) (on|off)
 
 If enabled, the csum forwarding engine will perform GRO on the TCP/IPv4
 packets received from the given port.
@@ -914,10 +914,18 @@ GRO. By default, GRO is disabled for all ports.
 .. note::
 
    When enable GRO for a port, TCP/IPv4 packets received from the port
-   will be performed GRO. After GRO, the merged packets are multi-segments.
-   But csum forwarding engine doesn't support to calculate TCP checksum
-   for multi-segment packets in SW. So please select TCP HW checksum
-   calculation for the port which GROed packets are transmitted to.
+   will be performed GRO. After GRO, all merged packets have bad
+   checksums, since the GRO library doesn't re-calculate checksums for
+   the merged packets. Therefore, if users want the merged packets to
+   have correct checksums, please select IP and TCP HW checksum calculation
+   for the port which the merged packets are transmitted to.
+
+show port - gro
+~~~~~~~~~~~~~~~
+
+Display GRO configuration::
+
+   testpmd> show port (port_id) gro
 
 gro set
 ~~~~~~~
@@ -932,6 +940,21 @@ number of packets a GRO table can store.
 If current packet number is greater than or equal to the max value, GRO
 will stop processing incoming packets.
 
+set gro flush interval
+~~~~~~~~~~~~~~~~~~~~~~
+
+Set the interval of flushing GROed packets from reassembly tables::
+
+   testpmd> set gro flush interval (num)
+
+In the heavyweight mode, GROed packets are stored in reassembly tables
+and need flushing from the tables manually. This command is to set the
+number of performing GRO operations between two flushing operations.
+
+The value of ``num`` should be in the range of 0 to ``GRO_MAX_FLUSH_INTERVAL``.
+``0`` means flush GROed packets from the tables when one GRO operation
+finishes.
+
 mac_addr add
 ~~~~~~~~~~~~
 
-- 
2.7.4

Re: [dpdk-dev] [PATCH v2 2/2] doc: update testpmd user guide for the heavyweight mode GRO

[Ferruh Yigit]

On 8/17/2017 10:08 AM, Jiayu Hu wrote:
> This patch is to update testpmd user guide for the heavyweight mode GRO.

Documentation can be part of the implementation patch.
It is good to split patches logically, but I don't see the benefit of
splitting patch as documentation and implementation.

> 
> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
> ---

<...>

>  
> +set gro flush interval
> +~~~~~~~~~~~~~~~~~~~~~~
> +
> +Set the interval of flushing GROed packets from reassembly tables::
> +
> +   testpmd> set gro flush interval (num)
> +
> +In the heavyweight mode, GROed packets are stored in reassembly tables
> +and need flushing from the tables manually. This command is to set the
> +number of performing GRO operations between two flushing operations.
> +
> +The value of ``num`` should be in the range of 0 to ``GRO_MAX_FLUSH_INTERVAL``.
> +``0`` means flush GROed packets from the tables when one GRO operation
> +finishes.

It can be good to mention default interval value is
GRO_DEFAULT_FLUSH_INTERVAL if this command not issued.

> +
>  mac_addr add
>  ~~~~~~~~~~~~
>  
> 

Re: [dpdk-dev] [PATCH v2 1/2] app/testpmd: support the heavywight mode GRO

[Ferruh Yigit]

On 8/17/2017 10:08 AM, Jiayu Hu wrote:
> The GRO library provides two reassembly modes: lightweight mode and
> heavyweight mode. This patch is to support the heavyweight mode in
> csum forwarding engine.
> 
> With the command "set port <port_id> gro (heavymode|lightmode) (on|off)",
> users can select the lightweight mode or the heavyweight mode to use. With
> the command "set gro flush interval <num>", users can set the interval of
> flushing GROed packets from reassembly tables for the heavyweight mode.
> With the command "show port <port_id> gro", users can display GRO
> configuration.
> 
> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>

<...>

>  	lcoreid_t  cpuid_idx;    /**< index of logical core in CPU id table */
> @@ -434,13 +436,21 @@ extern struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
>  extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
>  extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
>  
> +#define GRO_HEAVYMODE 0x1
> +#define GRO_LIGHTMODE 0x2

Why these are not part of the gro library?
Is the concept "lightweight mode and heavyweight mode" part of gro
library or implemented only in testpmd?

> +
>  #define GRO_DEFAULT_FLOW_NUM 4
>  #define GRO_DEFAULT_ITEM_NUM_PER_FLOW DEF_PKT_BURST
> +
> +#define GRO_DEFAULT_FLUSH_INTERVAL 2
> +#define GRO_MAX_FLUSH_INTERVAL 4
> +
>  struct gro_status {
>  	struct rte_gro_param param;
>  	uint8_t enable;
>  };
>  extern struct gro_status gro_ports[RTE_MAX_ETHPORTS];
> +extern uint32_t gro_flush_interval;

<...>

Re: [dpdk-dev] [PATCH v2 1/2] app/testpmd: support the heavywight mode GRO

[Ferruh Yigit]

On 8/17/2017 10:08 AM, Jiayu Hu wrote:
> The GRO library provides two reassembly modes: lightweight mode and
> heavyweight mode. This patch is to support the heavyweight mode in
> csum forwarding engine.
> 
> With the command "set port <port_id> gro (heavymode|lightmode) (on|off)",
> users can select the lightweight mode or the heavyweight mode to use. With
> the command "set gro flush interval <num>", users can set the interval of
> flushing GROed packets from reassembly tables for the heavyweight mode.
> With the command "show port <port_id> gro", users can display GRO
> configuration.
> 
> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>

<...>

> +	/* create a gro context for each lcore */
> +	gro_param.gro_types = RTE_GRO_TCP_IPV4;
> +	gro_param.max_flow_num = GRO_MAX_FLUSH_INTERVAL;
> +	gro_param.max_item_per_flow = MAX_PKT_BURST;
> +	for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
> +		gro_param.socket_id = rte_lcore_to_socket_id(
> +				fwd_lcores_cpuids[lc_id]);
> +		fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);

The gro library .map file has wrong function name "rte_gro_ctrl_create"
instead of "rte_gro_ctx_create". So this is giving a build error with
shared library config.

Both "rte_gro_ctrl_create" and "rte_gro_ctrl_destroy" need to be fixed.

> +		if (fwd_lcores[lc_id]->gro_ctx == NULL) {
> +			rte_exit(EXIT_FAILURE,
> +					"rte_gro_ctx_create() failed\n");
> +		}
> +	}
>  }

<...>

Re: [dpdk-dev] [PATCH v2 2/2] doc: update testpmd user guide for the heavyweight mode GRO

[Hu, Jiayu]

Hi,

> -----Original Message-----
> From: Yigit, Ferruh
> Sent: Monday, August 21, 2017 7:04 PM
> To: Hu, Jiayu <jiayu.hu@intel.com>; dev@dpdk.org
> Cc: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Tan, Jianfeng
> <jianfeng.tan@intel.com>; thomas@monjalon.net; Wu, Jingjing
> <jingjing.wu@intel.com>; Yao, Lei A <lei.a.yao@intel.com>
> Subject: Re: [PATCH v2 2/2] doc: update testpmd user guide for the
> heavyweight mode GRO
> 
> On 8/17/2017 10:08 AM, Jiayu Hu wrote:
> > This patch is to update testpmd user guide for the heavyweight mode GRO.
> 
> Documentation can be part of the implementation patch.
> It is good to split patches logically, but I don't see the benefit of
> splitting patch as documentation and implementation.

OK, I will combine these two patches in the next version.

> 
> >
> > Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
> > ---
> 
> <...>
> 
> >
> > +set gro flush interval
> > +~~~~~~~~~~~~~~~~~~~~~~
> > +
> > +Set the interval of flushing GROed packets from reassembly tables::
> > +
> > +   testpmd> set gro flush interval (num)
> > +
> > +In the heavyweight mode, GROed packets are stored in reassembly tables
> > +and need flushing from the tables manually. This command is to set the
> > +number of performing GRO operations between two flushing operations.
> > +
> > +The value of ``num`` should be in the range of 0 to
> ``GRO_MAX_FLUSH_INTERVAL``.
> > +``0`` means flush GROed packets from the tables when one GRO operation
> > +finishes.
> 
> It can be good to mention default interval value is
> GRO_DEFAULT_FLUSH_INTERVAL if this command not issued.

Thanks, I will add it in the next patch.

BRs,
Jiayu

> 
> > +
> >  mac_addr add
> >  ~~~~~~~~~~~~
> >
> >

Re: [dpdk-dev] [PATCH v2 1/2] app/testpmd: support the heavywight mode GRO

[Hu, Jiayu]

Hi,

> -----Original Message-----
> From: Yigit, Ferruh
> Sent: Monday, August 21, 2017 7:16 PM
> To: Hu, Jiayu <jiayu.hu@intel.com>; dev@dpdk.org
> Cc: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Tan, Jianfeng
> <jianfeng.tan@intel.com>; thomas@monjalon.net; Wu, Jingjing
> <jingjing.wu@intel.com>; Yao, Lei A <lei.a.yao@intel.com>
> Subject: Re: [PATCH v2 1/2] app/testpmd: support the heavywight mode GRO
> 
> On 8/17/2017 10:08 AM, Jiayu Hu wrote:
> > The GRO library provides two reassembly modes: lightweight mode and
> > heavyweight mode. This patch is to support the heavyweight mode in
> > csum forwarding engine.
> >
> > With the command "set port <port_id> gro (heavymode|lightmode)
> (on|off)",
> > users can select the lightweight mode or the heavyweight mode to use.
> With
> > the command "set gro flush interval <num>", users can set the interval of
> > flushing GROed packets from reassembly tables for the heavyweight mode.
> > With the command "show port <port_id> gro", users can display GRO
> > configuration.
> >
> > Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
> 
> <...>
> 
> > +	/* create a gro context for each lcore */
> > +	gro_param.gro_types = RTE_GRO_TCP_IPV4;
> > +	gro_param.max_flow_num = GRO_MAX_FLUSH_INTERVAL;
> > +	gro_param.max_item_per_flow = MAX_PKT_BURST;
> > +	for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
> > +		gro_param.socket_id = rte_lcore_to_socket_id(
> > +				fwd_lcores_cpuids[lc_id]);
> > +		fwd_lcores[lc_id]->gro_ctx =
> rte_gro_ctx_create(&gro_param);
> 
> The gro library .map file has wrong function name "rte_gro_ctrl_create"
> instead of "rte_gro_ctx_create". So this is giving a build error with
> shared library config.
> 
> Both "rte_gro_ctrl_create" and "rte_gro_ctrl_destroy" need to be fixed.

Thanks a lot. I will send a patch to fix them.

BRs,
Jiayu

> 
> > +		if (fwd_lcores[lc_id]->gro_ctx == NULL) {
> > +			rte_exit(EXIT_FAILURE,
> > +					"rte_gro_ctx_create() failed\n");
> > +		}
> > +	}
> >  }
> 
> <...>

Re: [dpdk-dev] [PATCH v2 1/2] app/testpmd: support the heavywight mode GRO

[Hu, Jiayu]

Hi,
> -----Original Message-----
> From: Yigit, Ferruh
> Sent: Monday, August 21, 2017 7:04 PM
> To: Hu, Jiayu <jiayu.hu@intel.com>; dev@dpdk.org
> Cc: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Tan, Jianfeng
> <jianfeng.tan@intel.com>; thomas@monjalon.net; Wu, Jingjing
> <jingjing.wu@intel.com>; Yao, Lei A <lei.a.yao@intel.com>
> Subject: Re: [PATCH v2 1/2] app/testpmd: support the heavywight mode GRO
> 
> On 8/17/2017 10:08 AM, Jiayu Hu wrote:
> > The GRO library provides two reassembly modes: lightweight mode and
> > heavyweight mode. This patch is to support the heavyweight mode in
> > csum forwarding engine.
> >
> > With the command "set port <port_id> gro (heavymode|lightmode)
> (on|off)",
> > users can select the lightweight mode or the heavyweight mode to use.
> With
> > the command "set gro flush interval <num>", users can set the interval of
> > flushing GROed packets from reassembly tables for the heavyweight mode.
> > With the command "show port <port_id> gro", users can display GRO
> > configuration.
> >
> > Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
> 
> <...>
> 
> >  	lcoreid_t  cpuid_idx;    /**< index of logical core in CPU id table */
> > @@ -434,13 +436,21 @@ extern struct ether_addr
> peer_eth_addrs[RTE_MAX_ETHPORTS];
> >  extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-
> retry. */
> >  extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-
> retry. */
> >
> > +#define GRO_HEAVYMODE 0x1
> > +#define GRO_LIGHTMODE 0x2
> 
> Why these are not part of the gro library?
> Is the concept "lightweight mode and heavyweight mode" part of gro
> library or implemented only in testpmd?

Lightweight mode and heavyweight mode are two reassembly methods we
provided in the GRO library. For applications, they are just two kinds of APIs.
Applications can select any of them to merge packets.

In testpmd, we want to show how to use these two reassembly modes, so
I define two macros to present them. Users can select which one to use via
command line.

> 
> > +
> >  #define GRO_DEFAULT_FLOW_NUM 4
> >  #define GRO_DEFAULT_ITEM_NUM_PER_FLOW DEF_PKT_BURST
> > +
> > +#define GRO_DEFAULT_FLUSH_INTERVAL 2
> > +#define GRO_MAX_FLUSH_INTERVAL 4
> > +
> >  struct gro_status {
> >  	struct rte_gro_param param;
> >  	uint8_t enable;
> >  };
> >  extern struct gro_status gro_ports[RTE_MAX_ETHPORTS];
> > +extern uint32_t gro_flush_interval;
> 
> <...>

Re: [dpdk-dev] [PATCH v2 1/2] app/testpmd: support the heavywight mode GRO

[Jiayu Hu]

Hi Ferruh,

On Tue, Aug 22, 2017 at 09:30:32AM +0100, Ferruh Yigit wrote:
> On 8/22/2017 2:00 AM, Hu, Jiayu wrote:
> > Hi,
> >> -----Original Message-----
> >> From: Yigit, Ferruh
> >> Sent: Monday, August 21, 2017 7:04 PM
> >> To: Hu, Jiayu <jiayu.hu@intel.com>; dev@dpdk.org
> >> Cc: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Tan, Jianfeng
> >> <jianfeng.tan@intel.com>; thomas@monjalon.net; Wu, Jingjing
> >> <jingjing.wu@intel.com>; Yao, Lei A <lei.a.yao@intel.com>
> >> Subject: Re: [PATCH v2 1/2] app/testpmd: support the heavywight mode GRO
> >>
> >> On 8/17/2017 10:08 AM, Jiayu Hu wrote:
> >>> The GRO library provides two reassembly modes: lightweight mode and
> >>> heavyweight mode. This patch is to support the heavyweight mode in
> >>> csum forwarding engine.
> >>>
> >>> With the command "set port <port_id> gro (heavymode|lightmode)
> >> (on|off)",
> >>> users can select the lightweight mode or the heavyweight mode to use.
> >> With
> >>> the command "set gro flush interval <num>", users can set the interval of
> >>> flushing GROed packets from reassembly tables for the heavyweight mode.
> >>> With the command "show port <port_id> gro", users can display GRO
> >>> configuration.
> >>>
> >>> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
> >>
> >> <...>
> >>
> >>>  	lcoreid_t  cpuid_idx;    /**< index of logical core in CPU id table */
> >>> @@ -434,13 +436,21 @@ extern struct ether_addr
> >> peer_eth_addrs[RTE_MAX_ETHPORTS];
> >>>  extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-
> >> retry. */
> >>>  extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-
> >> retry. */
> >>>
> >>> +#define GRO_HEAVYMODE 0x1
> >>> +#define GRO_LIGHTMODE 0x2
> >>
> >> Why these are not part of the gro library?
> >> Is the concept "lightweight mode and heavyweight mode" part of gro
> >> library or implemented only in testpmd?
> > 
> > Lightweight mode and heavyweight mode are two reassembly methods we
> > provided in the GRO library. For applications, they are just two kinds of APIs.
> > Applications can select any of them to merge packets.
> 
> GRO modes are defined in testpmd, and kept in testpmd variables, library
> seems not aware of these modes.
> 
> What are these two APIs, rte_gro_reassemble() and
> rte_gro_reassemble_burst() ?
> Perhaps you can detail what Lightweight mode and heavyweight mode are,
> doc also don't have much about it.
> 
> This still looks like gro library provides common API and testpmd calls
> these API with different parameters and calls these lightweight and
> heavyweight, if these modes are common use case, I believe they should

Yes, the GRO API doesn't show the concept of 'heavyweight' and 'lightweight'.
This concept is only used to describe the supported reassembly modes in the
GRO library.

> be part of library. If not, instead of saying different gro modes, it
> can be presented as different gro usage samples in testpmd.

What does "different gro usage samples" mean? Different forwarding
engines?

> 
> testpmd good for testing dpdk, and good for providing usage sample for
> APIs, but I believe it shouldn't have the concepts coded in it,
> libraries should have it, that is what end user uses.
> 
> > 
> > In testpmd, we want to show how to use these two reassembly modes, so
> > I define two macros to present them. Users can select which one to use via
> > command line.
> > 
> >>
> >>> +
> >>>  #define GRO_DEFAULT_FLOW_NUM 4
> >>>  #define GRO_DEFAULT_ITEM_NUM_PER_FLOW DEF_PKT_BURST
> >>> +
> >>> +#define GRO_DEFAULT_FLUSH_INTERVAL 2
> >>> +#define GRO_MAX_FLUSH_INTERVAL 4
> >>> +
> >>>  struct gro_status {
> >>>  	struct rte_gro_param param;
> >>>  	uint8_t enable;
> >>>  };
> >>>  extern struct gro_status gro_ports[RTE_MAX_ETHPORTS];
> >>> +extern uint32_t gro_flush_interval;
> >>
> >> <...>

[dpdk-dev] [PATCH v3] app/testpmd: enable the heavyweight mode TCP/IPv4 GRO

[Jiayu Hu]

The GRO library provides two modes to reassemble packets. Currently, the
csum forwarding engine has supported to use the lightweight mode to
reassemble TCP/IPv4 packets. This patch introduces the heavyweight mode
for TCP/IPv4 GRO in the csum forwarding engine.

With the command "set port <port_id> gro on|off", users can enable
TCP/IPv4 GRO for a given port. With the command "set gro flush <cycles>",
users can determine when the GROed TCP/IPv4 packets are flushed from
reassembly tables. With the command "show port <port_id> gro", users can
display GRO configuration.

Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
---
changes in v3:
- remove "heavyweight mode" and "lightweight mode" from GRO commands
- combine two patches into one
- use consistent help string for GRO commands 
- remove the unnecessary command "gro set (max_flow_num)
	(max_item_num_per_flow) (port_id)"
changes in v2:
- use "set" and "show" as the root level command
- add a new command to show GRO configuration
- fix l2_len/l3_len/l4_len unset etc. bugs

 app/test-pmd/cmdline.c                      | 206 ++++++++++++++++------------
 app/test-pmd/config.c                       |  67 +++++++--
 app/test-pmd/csumonly.c                     |  31 ++++-
 app/test-pmd/testpmd.c                      |  18 ++-
 app/test-pmd/testpmd.h                      |  16 ++-
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  45 ++++--
 6 files changed, 263 insertions(+), 120 deletions(-)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index cd8c358..d628250 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -423,13 +423,16 @@ static void cmd_help_long_parsed(void *parsed_result,
 			"tso show (portid)"
 			"    Display the status of TCP Segmentation Offload.\n\n"
 
-			"gro (on|off) (port_id)"
+			"set port (port_id) gro on|off\n"
 			"    Enable or disable Generic Receive Offload in"
 			" csum forwarding engine.\n\n"
 
-			"gro set (max_flow_num) (max_item_num_per_flow) (port_id)\n"
-			"    Set max flow number and max packet number per-flow"
-			" for GRO.\n\n"
+			"show port (port_id) gro\n"
+			"    Display GRO configuration.\n\n"
+
+			"set gro flush (cycles)\n"
+			"    Set the cycle to flush GROed packets from"
+			" reassembly tables.\n\n"
 
 			"set fwd (%s)\n"
 			"    Set packet forwarding mode.\n\n"
@@ -3850,115 +3853,145 @@ cmdline_parse_inst_t cmd_tunnel_tso_show = {
 };
 
 /* *** SET GRO FOR A PORT *** */
-struct cmd_gro_result {
+struct cmd_gro_enable_result {
+	cmdline_fixed_string_t cmd_set;
+	cmdline_fixed_string_t cmd_port;
 	cmdline_fixed_string_t cmd_keyword;
-	cmdline_fixed_string_t mode;
-	uint8_t port_id;
+	cmdline_fixed_string_t cmd_onoff;
+	uint8_t cmd_pid;
 };
 
 static void
-cmd_enable_gro_parsed(void *parsed_result,
+cmd_gro_enable_parsed(void *parsed_result,
 		__attribute__((unused)) struct cmdline *cl,
 		__attribute__((unused)) void *data)
 {
-	struct cmd_gro_result *res;
+	struct cmd_gro_enable_result *res;
 
 	res = parsed_result;
-	setup_gro(res->mode, res->port_id);
-}
-
-cmdline_parse_token_string_t cmd_gro_keyword =
-	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
+	if (!strcmp(res->cmd_keyword, "gro"))
+		setup_gro(res->cmd_onoff, res->cmd_pid);
+}
+
+cmdline_parse_token_string_t cmd_gro_enable_set =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_set, "set");
+cmdline_parse_token_string_t cmd_gro_enable_port =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_keyword, "port");
+cmdline_parse_token_num_t cmd_gro_enable_pid =
+	TOKEN_NUM_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_pid, UINT8);
+cmdline_parse_token_string_t cmd_gro_enable_keyword =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
 			cmd_keyword, "gro");
-cmdline_parse_token_string_t cmd_gro_mode =
-	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
-			mode, "on#off");
-cmdline_parse_token_num_t cmd_gro_pid =
-	TOKEN_NUM_INITIALIZER(struct cmd_gro_result,
-			port_id, UINT8);
+cmdline_parse_token_string_t cmd_gro_enable_onoff =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_onoff, "on#off");
 
-cmdline_parse_inst_t cmd_enable_gro = {
-	.f = cmd_enable_gro_parsed,
+cmdline_parse_inst_t cmd_gro_enable = {
+	.f = cmd_gro_enable_parsed,
 	.data = NULL,
-	.help_str = "gro (on|off) (port_id)",
+	.help_str = "set port <port_id> gro on|off",
 	.tokens = {
-		(void *)&cmd_gro_keyword,
-		(void *)&cmd_gro_mode,
-		(void *)&cmd_gro_pid,
+		(void *)&cmd_gro_enable_set,
+		(void *)&cmd_gro_enable_port,
+		(void *)&cmd_gro_enable_pid,
+		(void *)&cmd_gro_enable_keyword,
+		(void *)&cmd_gro_enable_onoff,
 		NULL,
 	},
 };
 
-/* *** SET MAX FLOW NUMBER AND ITEM NUM PER FLOW FOR GRO *** */
-struct cmd_gro_set_result {
-	cmdline_fixed_string_t gro;
-	cmdline_fixed_string_t mode;
-	uint16_t flow_num;
-	uint16_t item_num_per_flow;
-	uint8_t port_id;
+/* *** DISPLAY GRO CONFIGURATION *** */
+struct cmd_gro_show_result {
+	cmdline_fixed_string_t cmd_show;
+	cmdline_fixed_string_t cmd_port;
+	cmdline_fixed_string_t cmd_keyword;
+	uint8_t cmd_pid;
 };
 
 static void
-cmd_gro_set_parsed(void *parsed_result,
-		       __attribute__((unused)) struct cmdline *cl,
-		       __attribute__((unused)) void *data)
+cmd_gro_show_parsed(void *parsed_result,
+		__attribute__((unused)) struct cmdline *cl,
+		__attribute__((unused)) void *data)
 {
-	struct cmd_gro_set_result *res = parsed_result;
+	struct cmd_gro_show_result *res;
 
-	if (port_id_is_invalid(res->port_id, ENABLED_WARN))
-		return;
-	if (test_done == 0) {
-		printf("Before set GRO flow_num and item_num_per_flow,"
-				" please stop forwarding first\n");
-		return;
-	}
+	res = parsed_result;
+	if (!strcmp(res->cmd_keyword, "gro"))
+		show_gro(res->cmd_pid);
+}
+
+cmdline_parse_token_string_t cmd_gro_show_show =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
+			cmd_show, "show");
+cmdline_parse_token_string_t cmd_gro_show_port =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
+			cmd_port, "port");
+cmdline_parse_token_num_t cmd_gro_show_pid =
+	TOKEN_NUM_INITIALIZER(struct cmd_gro_show_result,
+			cmd_pid, UINT8);
+cmdline_parse_token_string_t cmd_gro_show_keyword =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
+			cmd_keyword, "gro");
 
-	if (!strcmp(res->mode, "set")) {
-		if (res->flow_num == 0)
-			printf("Invalid flow number. Revert to default value:"
-					" %u.\n", GRO_DEFAULT_FLOW_NUM);
-		else
-			gro_ports[res->port_id].param.max_flow_num =
-				res->flow_num;
+cmdline_parse_inst_t cmd_gro_show = {
+	.f = cmd_gro_show_parsed,
+	.data = NULL,
+	.help_str = "show port <port_id> gro",
+	.tokens = {
+		(void *)&cmd_gro_show_show,
+		(void *)&cmd_gro_show_port,
+		(void *)&cmd_gro_show_pid,
+		(void *)&cmd_gro_show_keyword,
+		NULL,
+	},
+};
 
-		if (res->item_num_per_flow == 0)
-			printf("Invalid item number per-flow. Revert"
-					" to default value:%u.\n",
-					GRO_DEFAULT_ITEM_NUM_PER_FLOW);
-		else
-			gro_ports[res->port_id].param.max_item_per_flow =
-				res->item_num_per_flow;
-	}
+/* *** SET FLUSH CYCLES FOR GRO *** */
+struct cmd_gro_flush_result {
+	cmdline_fixed_string_t cmd_set;
+	cmdline_fixed_string_t cmd_keyword;
+	cmdline_fixed_string_t cmd_flush;
+	uint8_t cmd_cycles;
+};
+
+static void
+cmd_gro_flush_parsed(void *parsed_result,
+		__attribute__((unused)) struct cmdline *cl,
+		__attribute__((unused)) void *data)
+{
+	struct cmd_gro_flush_result *res;
+
+	res = parsed_result;
+	if ((!strcmp(res->cmd_keyword, "gro")) &&
+			(!strcmp(res->cmd_flush, "flush")))
+		setup_gro_flush_cycles(res->cmd_cycles);
 }
 
-cmdline_parse_token_string_t cmd_gro_set_gro =
-	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
-				gro, "gro");
-cmdline_parse_token_string_t cmd_gro_set_mode =
-	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
-				mode, "set");
-cmdline_parse_token_num_t cmd_gro_set_flow_num =
-	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
-				flow_num, UINT16);
-cmdline_parse_token_num_t cmd_gro_set_item_num_per_flow =
-	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
-				item_num_per_flow, UINT16);
-cmdline_parse_token_num_t cmd_gro_set_portid =
-	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
-				port_id, UINT8);
+cmdline_parse_token_string_t cmd_gro_flush_set =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_set, "set");
+cmdline_parse_token_string_t cmd_gro_flush_keyword =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_keyword, "gro");
+cmdline_parse_token_string_t cmd_gro_flush_flush =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_flush, "flush");
+cmdline_parse_token_num_t cmd_gro_flush_cycles =
+	TOKEN_NUM_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_cycles, UINT8);
 
-cmdline_parse_inst_t cmd_gro_set = {
-	.f = cmd_gro_set_parsed,
+cmdline_parse_inst_t cmd_gro_flush = {
+	.f = cmd_gro_flush_parsed,
 	.data = NULL,
-	.help_str = "gro set <max_flow_num> <max_item_num_per_flow> "
-		"<port_id>: set max flow number and max packet number per-flow "
-		"for GRO",
+	.help_str = "set gro flush <cycles>",
 	.tokens = {
-		(void *)&cmd_gro_set_gro,
-		(void *)&cmd_gro_set_mode,
-		(void *)&cmd_gro_set_flow_num,
-		(void *)&cmd_gro_set_item_num_per_flow,
-		(void *)&cmd_gro_set_portid,
+		(void *)&cmd_gro_flush_set,
+		(void *)&cmd_gro_flush_keyword,
+		(void *)&cmd_gro_flush_flush,
+		(void *)&cmd_gro_flush_cycles,
 		NULL,
 	},
 };
@@ -14249,8 +14282,9 @@ cmdline_parse_ctx_t main_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_tso_show,
 	(cmdline_parse_inst_t *)&cmd_tunnel_tso_set,
 	(cmdline_parse_inst_t *)&cmd_tunnel_tso_show,
-	(cmdline_parse_inst_t *)&cmd_enable_gro,
-	(cmdline_parse_inst_t *)&cmd_gro_set,
+	(cmdline_parse_inst_t *)&cmd_gro_enable,
+	(cmdline_parse_inst_t *)&cmd_gro_flush,
+	(cmdline_parse_inst_t *)&cmd_gro_show,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_rx,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_tx,
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index 3ae3e1c..d97d291 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -2420,7 +2420,7 @@ set_tx_pkt_segments(unsigned *seg_lengths, unsigned nb_segs)
 }
 
 void
-setup_gro(const char *mode, uint8_t port_id)
+setup_gro(const char *onoff, uint8_t port_id)
 {
 	if (!rte_eth_dev_is_valid_port(port_id)) {
 		printf("invalid port id %u\n", port_id);
@@ -2431,29 +2431,76 @@ setup_gro(const char *mode, uint8_t port_id)
 				" please stop forwarding first\n");
 		return;
 	}
-	if (strcmp(mode, "on") == 0) {
-		if (gro_ports[port_id].enable) {
-			printf("port %u has enabled GRO\n", port_id);
+	if (strcmp(onoff, "on") == 0) {
+		if (gro_ports[port_id].enable != 0) {
+			printf("Port %u has enabled GRO. Please"
+					" disable GRO first\n", port_id);
 			return;
 		}
-		gro_ports[port_id].enable = 1;
-		gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4;
-
-		if (gro_ports[port_id].param.max_flow_num == 0)
+		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
+			gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4;
 			gro_ports[port_id].param.max_flow_num =
 				GRO_DEFAULT_FLOW_NUM;
-		if (gro_ports[port_id].param.max_item_per_flow == 0)
 			gro_ports[port_id].param.max_item_per_flow =
 				GRO_DEFAULT_ITEM_NUM_PER_FLOW;
+		}
+		gro_ports[port_id].enable = 1;
 	} else {
 		if (gro_ports[port_id].enable == 0) {
-			printf("port %u has disabled GRO\n", port_id);
+			printf("Port %u has disabled GRO\n", port_id);
 			return;
 		}
 		gro_ports[port_id].enable = 0;
 	}
 }
 
+void
+setup_gro_flush_cycles(uint8_t cycles)
+{
+	if (test_done == 0) {
+		printf("Before change flush interval for GRO,"
+				" please stop forwarding first\n");
+		return;
+	}
+
+	if (cycles > GRO_MAX_FLUSH_CYCLES) {
+		printf("The flushing cycle be in the range"
+				" of 1 to %u. Revert to the default"
+				" value %u\n",
+				GRO_MAX_FLUSH_CYCLES,
+				GRO_DEFAULT_FLUSH_CYCLES);
+		cycles = GRO_DEFAULT_FLUSH_CYCLES;
+	}
+
+	gro_flush_cycles = cycles;
+}
+
+void
+show_gro(uint8_t port_id)
+{
+	struct rte_gro_param *param;
+	uint32_t max_pkts_num;
+
+	param = &gro_ports[port_id].param;
+
+	if (!rte_eth_dev_is_valid_port(port_id)) {
+		printf("Invalid port id %u\n", port_id);
+		return;
+	}
+	if (gro_ports[port_id].enable) {
+		printf("GRO type: TCP/IPv4\n");
+		if (gro_flush_cycles > 0)
+			max_pkts_num = MAX_PKT_BURST * GRO_MAX_FLUSH_CYCLES;
+		else {
+			max_pkts_num = param->max_flow_num *
+				param->max_item_per_flow;
+		}
+		printf("Max packet number to GRO: %u\n", max_pkts_num);
+		printf("Flushing cycles: %u\n", gro_flush_cycles);
+	} else
+		printf("Port %u doesn't enable GRO\n", port_id);
+}
+
 char*
 list_pkt_forwarding_modes(void)
 {
diff --git a/app/test-pmd/csumonly.c b/app/test-pmd/csumonly.c
index 90c8119..ca50ab7 100644
--- a/app/test-pmd/csumonly.c
+++ b/app/test-pmd/csumonly.c
@@ -631,6 +631,9 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 	struct rte_mbuf *m, *p;
 	struct ether_hdr *eth_hdr;
 	void *l3_hdr = NULL, *outer_l3_hdr = NULL; /* can be IPv4 or IPv6 */
+	void **gro_ctx;
+	uint16_t gro_pkts_num;
+	uint8_t gro_enable;
 	uint16_t nb_rx;
 	uint16_t nb_tx;
 	uint16_t nb_prep;
@@ -657,17 +660,13 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 				 nb_pkt_per_burst);
 	if (unlikely(nb_rx == 0))
 		return;
-	if (unlikely(gro_ports[fs->rx_port].enable))
-		nb_rx = rte_gro_reassemble_burst(pkts_burst,
-				nb_rx,
-				&(gro_ports[fs->rx_port].param));
-
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 	fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
 #endif
 	fs->rx_packets += nb_rx;
 	rx_bad_ip_csum = 0;
 	rx_bad_l4_csum = 0;
+	gro_enable = gro_ports[fs->rx_port].enable;
 
 	txp = &ports[fs->tx_port];
 	testpmd_ol_flags = txp->tx_ol_flags;
@@ -851,6 +850,28 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 		}
 	}
 
+	if (unlikely(gro_enable)) {
+		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
+			nb_rx = rte_gro_reassemble_burst(pkts_burst, nb_rx,
+					&(gro_ports[fs->rx_port].param));
+		} else {
+			gro_ctx = current_fwd_lcore()->gro_ctx;
+			nb_rx = rte_gro_reassemble(pkts_burst, nb_rx, gro_ctx);
+
+			if (++fs->gro_times >= gro_flush_cycles) {
+				gro_pkts_num = rte_gro_get_pkt_count(gro_ctx);
+				if (gro_pkts_num > MAX_PKT_BURST - nb_rx)
+					gro_pkts_num = MAX_PKT_BURST - nb_rx;
+
+				nb_rx += rte_gro_timeout_flush(gro_ctx, 0,
+						RTE_GRO_TCP_IPV4,
+						&pkts_burst[nb_rx],
+						gro_pkts_num);
+				fs->gro_times = 0;
+			}
+		}
+	}
+
 	nb_prep = rte_eth_tx_prepare(fs->tx_port, fs->tx_queue,
 			pkts_burst, nb_rx);
 	if (nb_prep != nb_rx)
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index 7d40139..c1423d5 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -90,7 +90,6 @@
 #ifdef RTE_LIBRTE_LATENCY_STATS
 #include <rte_latencystats.h>
 #endif
-#include <rte_gro.h>
 
 #include "testpmd.h"
 
@@ -386,6 +385,7 @@ uint8_t bitrate_enabled;
 #endif
 
 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
+uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 
 /* Forward function declarations */
 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port);
@@ -570,6 +570,7 @@ init_config(void)
 	unsigned int nb_mbuf_per_pool;
 	lcoreid_t  lc_id;
 	uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
+	struct rte_gro_param gro_param;
 
 	memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
 
@@ -671,6 +672,20 @@ init_config(void)
 		rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
 
 	fwd_config_setup();
+
+	/* create a gro context for each lcore */
+	gro_param.gro_types = RTE_GRO_TCP_IPV4;
+	gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
+	gro_param.max_item_per_flow = MAX_PKT_BURST;
+	for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
+		gro_param.socket_id = rte_lcore_to_socket_id(
+				fwd_lcores_cpuids[lc_id]);
+		fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
+		if (fwd_lcores[lc_id]->gro_ctx == NULL) {
+			rte_exit(EXIT_FAILURE,
+					"rte_gro_ctx_create() failed\n");
+		}
+	}
 }
 
 
@@ -1165,6 +1180,7 @@ start_packet_forwarding(int with_tx_first)
 		fwd_streams[sm_id]->fwd_dropped = 0;
 		fwd_streams[sm_id]->rx_bad_ip_csum = 0;
 		fwd_streams[sm_id]->rx_bad_l4_csum = 0;
+		fwd_streams[sm_id]->gro_times = 0;
 
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 		memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index c9d7739..e878bcb 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -120,6 +120,7 @@ struct fwd_stream {
 	unsigned int fwd_dropped; /**< received packets not forwarded */
 	unsigned int rx_bad_ip_csum ; /**< received packets has bad ip checksum */
 	unsigned int rx_bad_l4_csum ; /**< received packets has bad l4 checksum */
+	unsigned int gro_times;	/**< GRO operation times */
 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 	uint64_t     core_cycles; /**< used for RX and TX processing */
 #endif
@@ -206,6 +207,7 @@ struct rte_port {
  */
 struct fwd_lcore {
 	struct rte_mempool *mbp; /**< The mbuf pool to use by this core */
+	void *gro_ctx;		/**< GRO context */
 	streamid_t stream_idx;   /**< index of 1st stream in "fwd_streams" */
 	streamid_t stream_nb;    /**< number of streams in "fwd_streams" */
 	lcoreid_t  cpuid_idx;    /**< index of logical core in CPU id table */
@@ -434,13 +436,19 @@ extern struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
 extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
 
-#define GRO_DEFAULT_FLOW_NUM 4
-#define GRO_DEFAULT_ITEM_NUM_PER_FLOW DEF_PKT_BURST
+#define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
+#define GRO_DEFAULT_FLOW_NUM (RTE_GRO_MAX_BURST_ITEM_NUM / \
+		GRO_DEFAULT_ITEM_NUM_PER_FLOW)
+
+#define GRO_DEFAULT_FLUSH_CYCLES 1
+#define GRO_MAX_FLUSH_CYCLES 4
+
 struct gro_status {
 	struct rte_gro_param param;
 	uint8_t enable;
 };
 extern struct gro_status gro_ports[RTE_MAX_ETHPORTS];
+extern uint8_t gro_flush_cycles;
 
 static inline unsigned int
 lcore_num(void)
@@ -640,7 +648,9 @@ void get_2tuple_filter(uint8_t port_id, uint16_t index);
 void get_5tuple_filter(uint8_t port_id, uint16_t index);
 int rx_queue_id_is_invalid(queueid_t rxq_id);
 int tx_queue_id_is_invalid(queueid_t txq_id);
-void setup_gro(const char *mode, uint8_t port_id);
+void setup_gro(const char *onoff, uint8_t port_id);
+void setup_gro_flush_cycles(uint8_t cycles);
+void show_gro(uint8_t port_id);
 
 /* Functions to manage the set of filtered Multicast MAC addresses */
 void mcast_addr_add(uint8_t port_id, struct ether_addr *mc_addr);
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 2ed62f5..7f21b7d 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -898,12 +898,12 @@ Display the status of TCP Segmentation Offload::
 
    testpmd> tso show (port_id)
 
-gro
-~~~
+set port - gro
+~~~~~~~~~~~~~~
 
 Enable or disable GRO in ``csum`` forwarding engine::
 
-   testpmd> gro (on|off) (port_id)
+   testpmd> set port <port_id> gro on|off
 
 If enabled, the csum forwarding engine will perform GRO on the TCP/IPv4
 packets received from the given port.
@@ -914,23 +914,38 @@ GRO. By default, GRO is disabled for all ports.
 .. note::
 
    When enable GRO for a port, TCP/IPv4 packets received from the port
-   will be performed GRO. After GRO, the merged packets are multi-segments.
-   But csum forwarding engine doesn't support to calculate TCP checksum
-   for multi-segment packets in SW. So please select TCP HW checksum
-   calculation for the port which GROed packets are transmitted to.
+   will be performed GRO. After GRO, all merged packets have bad
+   checksums, since the GRO library doesn't re-calculate checksums for
+   the merged packets. Therefore, if users want the merged packets to
+   have correct checksums, please select IP and TCP HW checksum calculation
+   for the port which the merged packets are transmitted to.
 
-gro set
-~~~~~~~
+show port - gro
+~~~~~~~~~~~~~~~
+
+Display GRO configuration for a given port::
+
+   testpmd> show port <port_id> gro
+
+set gro flush
+~~~~~~~~~~~~~
+
+Set the cycle to flush the GROed packets from reassembly tables::
 
-Set max flow number and max packet number per-flow for GRO::
+   testpmd> set gro flush <cycles>
 
-   testpmd> gro set (max_flow_num) (max_item_num_per_flow) (port_id)
+When enable GRO, the csum forwarding engine performs GRO on received
+packets, and the GROed packets are stored in reassembly tables. Users
+can use this command to determine when the GROed packets are flushed
+from the reassembly tables.
 
-The product of ``max_flow_num`` and ``max_item_num_per_flow`` is the max
-number of packets a GRO table can store.
+The ``cycles`` is measured in GRO operation times. The csum forwarding
+engine flushes the GROed packets from the tables every ``cycles`` GRO
+operations.
 
-If current packet number is greater than or equal to the max value, GRO
-will stop processing incoming packets.
+By default, the value of ``cycles`` is 1, which means flush GROed packets
+from the reassembly tables as soon as one GRO operation finishes. The value
+of ``cycles`` should be in the range of 1 to ``GRO_MAX_FLUSH_CYCLES``.
 
 mac_addr add
 ~~~~~~~~~~~~
-- 
2.7.4

Re: [dpdk-dev] [PATCH v3] app/testpmd: enable the heavyweight mode TCP/IPv4 GRO

[Yao, Lei A]

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Jiayu Hu
> Sent: Sunday, September 3, 2017 2:30 PM
> To: dev@dpdk.org
> Cc: Yigit, Ferruh <ferruh.yigit@intel.com>; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>; Tan, Jianfeng <jianfeng.tan@intel.com>;
> Wu, Jingjing <jingjing.wu@intel.com>; Hu, Jiayu <jiayu.hu@intel.com>
> Subject: [dpdk-dev] [PATCH v3] app/testpmd: enable the heavyweight
> mode TCP/IPv4 GRO
> 
> The GRO library provides two modes to reassemble packets. Currently, the
> csum forwarding engine has supported to use the lightweight mode to
> reassemble TCP/IPv4 packets. This patch introduces the heavyweight mode
> for TCP/IPv4 GRO in the csum forwarding engine.
> 
> With the command "set port <port_id> gro on|off", users can enable
> TCP/IPv4 GRO for a given port. With the command "set gro flush <cycles>",
> users can determine when the GROed TCP/IPv4 packets are flushed from
> reassembly tables. With the command "show port <port_id> gro", users can
> display GRO configuration.
> 
> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
Tested-by : Lei Yao<lei.a.yao@intel.com>

This patch has been tested on my bench, iperf test result  is as following: 
No-GRO: 8 Gbps
Kernel GRO: 14.3 Gbps
GRO flush 0 : 12.7 Gbps
GRO flush 1: 16.8 Gbps
But when I use 40G NIC and set GRO flush cycle as 2, sometimes
the iperf traffic will stall for several seconds. Still need investigate.

> ---
> changes in v3:
> - remove "heavyweight mode" and "lightweight mode" from GRO
> commands
> - combine two patches into one
> - use consistent help string for GRO commands
> - remove the unnecessary command "gro set (max_flow_num)
> 	(max_item_num_per_flow) (port_id)"
> changes in v2:
> - use "set" and "show" as the root level command
> - add a new command to show GRO configuration
> - fix l2_len/l3_len/l4_len unset etc. bugs
> 
>  app/test-pmd/cmdline.c                      | 206 ++++++++++++++++------------
>  app/test-pmd/config.c                       |  67 +++++++--
>  app/test-pmd/csumonly.c                     |  31 ++++-
>  app/test-pmd/testpmd.c                      |  18 ++-
>  app/test-pmd/testpmd.h                      |  16 ++-
>  doc/guides/testpmd_app_ug/testpmd_funcs.rst |  45 ++++--
>  6 files changed, 263 insertions(+), 120 deletions(-)
> 
> diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
> index cd8c358..d628250 100644
> --- a/app/test-pmd/cmdline.c
> +++ b/app/test-pmd/cmdline.c
> @@ -423,13 +423,16 @@ static void cmd_help_long_parsed(void
> *parsed_result,
>  			"tso show (portid)"
>  			"    Display the status of TCP Segmentation
> Offload.\n\n"
> 
> -			"gro (on|off) (port_id)"
> +			"set port (port_id) gro on|off\n"
>  			"    Enable or disable Generic Receive Offload in"
>  			" csum forwarding engine.\n\n"
> 
> -			"gro set (max_flow_num)
> (max_item_num_per_flow) (port_id)\n"
> -			"    Set max flow number and max packet number
> per-flow"
> -			" for GRO.\n\n"
> +			"show port (port_id) gro\n"
> +			"    Display GRO configuration.\n\n"
> +
> +			"set gro flush (cycles)\n"
> +			"    Set the cycle to flush GROed packets from"
> +			" reassembly tables.\n\n"
> 
>  			"set fwd (%s)\n"
>  			"    Set packet forwarding mode.\n\n"
> @@ -3850,115 +3853,145 @@ cmdline_parse_inst_t cmd_tunnel_tso_show
> = {
>  };
> 
>  /* *** SET GRO FOR A PORT *** */
> -struct cmd_gro_result {
> +struct cmd_gro_enable_result {
> +	cmdline_fixed_string_t cmd_set;
> +	cmdline_fixed_string_t cmd_port;
>  	cmdline_fixed_string_t cmd_keyword;
> -	cmdline_fixed_string_t mode;
> -	uint8_t port_id;
> +	cmdline_fixed_string_t cmd_onoff;
> +	uint8_t cmd_pid;
>  };
> 
>  static void
> -cmd_enable_gro_parsed(void *parsed_result,
> +cmd_gro_enable_parsed(void *parsed_result,
>  		__attribute__((unused)) struct cmdline *cl,
>  		__attribute__((unused)) void *data)
>  {
> -	struct cmd_gro_result *res;
> +	struct cmd_gro_enable_result *res;
> 
>  	res = parsed_result;
> -	setup_gro(res->mode, res->port_id);
> -}
> -
> -cmdline_parse_token_string_t cmd_gro_keyword =
> -	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
> +	if (!strcmp(res->cmd_keyword, "gro"))
> +		setup_gro(res->cmd_onoff, res->cmd_pid);
> +}
> +
> +cmdline_parse_token_string_t cmd_gro_enable_set =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
> +			cmd_set, "set");
> +cmdline_parse_token_string_t cmd_gro_enable_port =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
> +			cmd_keyword, "port");
> +cmdline_parse_token_num_t cmd_gro_enable_pid =
> +	TOKEN_NUM_INITIALIZER(struct cmd_gro_enable_result,
> +			cmd_pid, UINT8);
> +cmdline_parse_token_string_t cmd_gro_enable_keyword =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
>  			cmd_keyword, "gro");
> -cmdline_parse_token_string_t cmd_gro_mode =
> -	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
> -			mode, "on#off");
> -cmdline_parse_token_num_t cmd_gro_pid =
> -	TOKEN_NUM_INITIALIZER(struct cmd_gro_result,
> -			port_id, UINT8);
> +cmdline_parse_token_string_t cmd_gro_enable_onoff =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
> +			cmd_onoff, "on#off");
> 
> -cmdline_parse_inst_t cmd_enable_gro = {
> -	.f = cmd_enable_gro_parsed,
> +cmdline_parse_inst_t cmd_gro_enable = {
> +	.f = cmd_gro_enable_parsed,
>  	.data = NULL,
> -	.help_str = "gro (on|off) (port_id)",
> +	.help_str = "set port <port_id> gro on|off",
>  	.tokens = {
> -		(void *)&cmd_gro_keyword,
> -		(void *)&cmd_gro_mode,
> -		(void *)&cmd_gro_pid,
> +		(void *)&cmd_gro_enable_set,
> +		(void *)&cmd_gro_enable_port,
> +		(void *)&cmd_gro_enable_pid,
> +		(void *)&cmd_gro_enable_keyword,
> +		(void *)&cmd_gro_enable_onoff,
>  		NULL,
>  	},
>  };
> 
> -/* *** SET MAX FLOW NUMBER AND ITEM NUM PER FLOW FOR GRO *** */
> -struct cmd_gro_set_result {
> -	cmdline_fixed_string_t gro;
> -	cmdline_fixed_string_t mode;
> -	uint16_t flow_num;
> -	uint16_t item_num_per_flow;
> -	uint8_t port_id;
> +/* *** DISPLAY GRO CONFIGURATION *** */
> +struct cmd_gro_show_result {
> +	cmdline_fixed_string_t cmd_show;
> +	cmdline_fixed_string_t cmd_port;
> +	cmdline_fixed_string_t cmd_keyword;
> +	uint8_t cmd_pid;
>  };
> 
>  static void
> -cmd_gro_set_parsed(void *parsed_result,
> -		       __attribute__((unused)) struct cmdline *cl,
> -		       __attribute__((unused)) void *data)
> +cmd_gro_show_parsed(void *parsed_result,
> +		__attribute__((unused)) struct cmdline *cl,
> +		__attribute__((unused)) void *data)
>  {
> -	struct cmd_gro_set_result *res = parsed_result;
> +	struct cmd_gro_show_result *res;
> 
> -	if (port_id_is_invalid(res->port_id, ENABLED_WARN))
> -		return;
> -	if (test_done == 0) {
> -		printf("Before set GRO flow_num and item_num_per_flow,"
> -				" please stop forwarding first\n");
> -		return;
> -	}
> +	res = parsed_result;
> +	if (!strcmp(res->cmd_keyword, "gro"))
> +		show_gro(res->cmd_pid);
> +}
> +
> +cmdline_parse_token_string_t cmd_gro_show_show =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
> +			cmd_show, "show");
> +cmdline_parse_token_string_t cmd_gro_show_port =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
> +			cmd_port, "port");
> +cmdline_parse_token_num_t cmd_gro_show_pid =
> +	TOKEN_NUM_INITIALIZER(struct cmd_gro_show_result,
> +			cmd_pid, UINT8);
> +cmdline_parse_token_string_t cmd_gro_show_keyword =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
> +			cmd_keyword, "gro");
> 
> -	if (!strcmp(res->mode, "set")) {
> -		if (res->flow_num == 0)
> -			printf("Invalid flow number. Revert to default value:"
> -					" %u.\n",
> GRO_DEFAULT_FLOW_NUM);
> -		else
> -			gro_ports[res->port_id].param.max_flow_num =
> -				res->flow_num;
> +cmdline_parse_inst_t cmd_gro_show = {
> +	.f = cmd_gro_show_parsed,
> +	.data = NULL,
> +	.help_str = "show port <port_id> gro",
> +	.tokens = {
> +		(void *)&cmd_gro_show_show,
> +		(void *)&cmd_gro_show_port,
> +		(void *)&cmd_gro_show_pid,
> +		(void *)&cmd_gro_show_keyword,
> +		NULL,
> +	},
> +};
> 
> -		if (res->item_num_per_flow == 0)
> -			printf("Invalid item number per-flow. Revert"
> -					" to default value:%u.\n",
> -
> 	GRO_DEFAULT_ITEM_NUM_PER_FLOW);
> -		else
> -			gro_ports[res->port_id].param.max_item_per_flow
> =
> -				res->item_num_per_flow;
> -	}
> +/* *** SET FLUSH CYCLES FOR GRO *** */
> +struct cmd_gro_flush_result {
> +	cmdline_fixed_string_t cmd_set;
> +	cmdline_fixed_string_t cmd_keyword;
> +	cmdline_fixed_string_t cmd_flush;
> +	uint8_t cmd_cycles;
> +};
> +
> +static void
> +cmd_gro_flush_parsed(void *parsed_result,
> +		__attribute__((unused)) struct cmdline *cl,
> +		__attribute__((unused)) void *data)
> +{
> +	struct cmd_gro_flush_result *res;
> +
> +	res = parsed_result;
> +	if ((!strcmp(res->cmd_keyword, "gro")) &&
> +			(!strcmp(res->cmd_flush, "flush")))
> +		setup_gro_flush_cycles(res->cmd_cycles);
>  }
> 
> -cmdline_parse_token_string_t cmd_gro_set_gro =
> -	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
> -				gro, "gro");
> -cmdline_parse_token_string_t cmd_gro_set_mode =
> -	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
> -				mode, "set");
> -cmdline_parse_token_num_t cmd_gro_set_flow_num =
> -	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> -				flow_num, UINT16);
> -cmdline_parse_token_num_t cmd_gro_set_item_num_per_flow =
> -	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> -				item_num_per_flow, UINT16);
> -cmdline_parse_token_num_t cmd_gro_set_portid =
> -	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> -				port_id, UINT8);
> +cmdline_parse_token_string_t cmd_gro_flush_set =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
> +			cmd_set, "set");
> +cmdline_parse_token_string_t cmd_gro_flush_keyword =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
> +			cmd_keyword, "gro");
> +cmdline_parse_token_string_t cmd_gro_flush_flush =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
> +			cmd_flush, "flush");
> +cmdline_parse_token_num_t cmd_gro_flush_cycles =
> +	TOKEN_NUM_INITIALIZER(struct cmd_gro_flush_result,
> +			cmd_cycles, UINT8);
> 
> -cmdline_parse_inst_t cmd_gro_set = {
> -	.f = cmd_gro_set_parsed,
> +cmdline_parse_inst_t cmd_gro_flush = {
> +	.f = cmd_gro_flush_parsed,
>  	.data = NULL,
> -	.help_str = "gro set <max_flow_num> <max_item_num_per_flow>
> "
> -		"<port_id>: set max flow number and max packet number
> per-flow "
> -		"for GRO",
> +	.help_str = "set gro flush <cycles>",
>  	.tokens = {
> -		(void *)&cmd_gro_set_gro,
> -		(void *)&cmd_gro_set_mode,
> -		(void *)&cmd_gro_set_flow_num,
> -		(void *)&cmd_gro_set_item_num_per_flow,
> -		(void *)&cmd_gro_set_portid,
> +		(void *)&cmd_gro_flush_set,
> +		(void *)&cmd_gro_flush_keyword,
> +		(void *)&cmd_gro_flush_flush,
> +		(void *)&cmd_gro_flush_cycles,
>  		NULL,
>  	},
>  };
> @@ -14249,8 +14282,9 @@ cmdline_parse_ctx_t main_ctx[] = {
>  	(cmdline_parse_inst_t *)&cmd_tso_show,
>  	(cmdline_parse_inst_t *)&cmd_tunnel_tso_set,
>  	(cmdline_parse_inst_t *)&cmd_tunnel_tso_show,
> -	(cmdline_parse_inst_t *)&cmd_enable_gro,
> -	(cmdline_parse_inst_t *)&cmd_gro_set,
> +	(cmdline_parse_inst_t *)&cmd_gro_enable,
> +	(cmdline_parse_inst_t *)&cmd_gro_flush,
> +	(cmdline_parse_inst_t *)&cmd_gro_show,
>  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set,
>  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_rx,
>  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_tx,
> diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
> index 3ae3e1c..d97d291 100644
> --- a/app/test-pmd/config.c
> +++ b/app/test-pmd/config.c
> @@ -2420,7 +2420,7 @@ set_tx_pkt_segments(unsigned *seg_lengths,
> unsigned nb_segs)
>  }
> 
>  void
> -setup_gro(const char *mode, uint8_t port_id)
> +setup_gro(const char *onoff, uint8_t port_id)
>  {
>  	if (!rte_eth_dev_is_valid_port(port_id)) {
>  		printf("invalid port id %u\n", port_id);
> @@ -2431,29 +2431,76 @@ setup_gro(const char *mode, uint8_t port_id)
>  				" please stop forwarding first\n");
>  		return;
>  	}
> -	if (strcmp(mode, "on") == 0) {
> -		if (gro_ports[port_id].enable) {
> -			printf("port %u has enabled GRO\n", port_id);
> +	if (strcmp(onoff, "on") == 0) {
> +		if (gro_ports[port_id].enable != 0) {
> +			printf("Port %u has enabled GRO. Please"
> +					" disable GRO first\n", port_id);
>  			return;
>  		}
> -		gro_ports[port_id].enable = 1;
> -		gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4;
> -
> -		if (gro_ports[port_id].param.max_flow_num == 0)
> +		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
> +			gro_ports[port_id].param.gro_types =
> RTE_GRO_TCP_IPV4;
>  			gro_ports[port_id].param.max_flow_num =
>  				GRO_DEFAULT_FLOW_NUM;
> -		if (gro_ports[port_id].param.max_item_per_flow == 0)
>  			gro_ports[port_id].param.max_item_per_flow =
>  				GRO_DEFAULT_ITEM_NUM_PER_FLOW;
> +		}
> +		gro_ports[port_id].enable = 1;
>  	} else {
>  		if (gro_ports[port_id].enable == 0) {
> -			printf("port %u has disabled GRO\n", port_id);
> +			printf("Port %u has disabled GRO\n", port_id);
>  			return;
>  		}
>  		gro_ports[port_id].enable = 0;
>  	}
>  }
> 
> +void
> +setup_gro_flush_cycles(uint8_t cycles)
> +{
> +	if (test_done == 0) {
> +		printf("Before change flush interval for GRO,"
> +				" please stop forwarding first\n");
> +		return;
> +	}
> +
> +	if (cycles > GRO_MAX_FLUSH_CYCLES) {
> +		printf("The flushing cycle be in the range"
> +				" of 1 to %u. Revert to the default"
> +				" value %u\n",
> +				GRO_MAX_FLUSH_CYCLES,
> +				GRO_DEFAULT_FLUSH_CYCLES);
> +		cycles = GRO_DEFAULT_FLUSH_CYCLES;
> +	}
> +
> +	gro_flush_cycles = cycles;
> +}
> +
> +void
> +show_gro(uint8_t port_id)
> +{
> +	struct rte_gro_param *param;
> +	uint32_t max_pkts_num;
> +
> +	param = &gro_ports[port_id].param;
> +
> +	if (!rte_eth_dev_is_valid_port(port_id)) {
> +		printf("Invalid port id %u\n", port_id);
> +		return;
> +	}
> +	if (gro_ports[port_id].enable) {
> +		printf("GRO type: TCP/IPv4\n");
> +		if (gro_flush_cycles > 0)
> +			max_pkts_num = MAX_PKT_BURST *
> GRO_MAX_FLUSH_CYCLES;
> +		else {
> +			max_pkts_num = param->max_flow_num *
> +				param->max_item_per_flow;
> +		}
> +		printf("Max packet number to GRO: %u\n", max_pkts_num);
> +		printf("Flushing cycles: %u\n", gro_flush_cycles);
> +	} else
> +		printf("Port %u doesn't enable GRO\n", port_id);
> +}
> +
>  char*
>  list_pkt_forwarding_modes(void)
>  {
> diff --git a/app/test-pmd/csumonly.c b/app/test-pmd/csumonly.c
> index 90c8119..ca50ab7 100644
> --- a/app/test-pmd/csumonly.c
> +++ b/app/test-pmd/csumonly.c
> @@ -631,6 +631,9 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
>  	struct rte_mbuf *m, *p;
>  	struct ether_hdr *eth_hdr;
>  	void *l3_hdr = NULL, *outer_l3_hdr = NULL; /* can be IPv4 or IPv6 */
> +	void **gro_ctx;
> +	uint16_t gro_pkts_num;
> +	uint8_t gro_enable;
>  	uint16_t nb_rx;
>  	uint16_t nb_tx;
>  	uint16_t nb_prep;
> @@ -657,17 +660,13 @@ pkt_burst_checksum_forward(struct fwd_stream
> *fs)
>  				 nb_pkt_per_burst);
>  	if (unlikely(nb_rx == 0))
>  		return;
> -	if (unlikely(gro_ports[fs->rx_port].enable))
> -		nb_rx = rte_gro_reassemble_burst(pkts_burst,
> -				nb_rx,
> -				&(gro_ports[fs->rx_port].param));
> -
>  #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
>  	fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
>  #endif
>  	fs->rx_packets += nb_rx;
>  	rx_bad_ip_csum = 0;
>  	rx_bad_l4_csum = 0;
> +	gro_enable = gro_ports[fs->rx_port].enable;
> 
>  	txp = &ports[fs->tx_port];
>  	testpmd_ol_flags = txp->tx_ol_flags;
> @@ -851,6 +850,28 @@ pkt_burst_checksum_forward(struct fwd_stream
> *fs)
>  		}
>  	}
> 
> +	if (unlikely(gro_enable)) {
> +		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
> +			nb_rx = rte_gro_reassemble_burst(pkts_burst,
> nb_rx,
> +					&(gro_ports[fs->rx_port].param));
> +		} else {
> +			gro_ctx = current_fwd_lcore()->gro_ctx;
> +			nb_rx = rte_gro_reassemble(pkts_burst, nb_rx,
> gro_ctx);
> +
> +			if (++fs->gro_times >= gro_flush_cycles) {
> +				gro_pkts_num =
> rte_gro_get_pkt_count(gro_ctx);
> +				if (gro_pkts_num > MAX_PKT_BURST - nb_rx)
> +					gro_pkts_num = MAX_PKT_BURST -
> nb_rx;
> +
> +				nb_rx += rte_gro_timeout_flush(gro_ctx, 0,
> +						RTE_GRO_TCP_IPV4,
> +						&pkts_burst[nb_rx],
> +						gro_pkts_num);
> +				fs->gro_times = 0;
> +			}
> +		}
> +	}
> +
>  	nb_prep = rte_eth_tx_prepare(fs->tx_port, fs->tx_queue,
>  			pkts_burst, nb_rx);
>  	if (nb_prep != nb_rx)
> diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
> index 7d40139..c1423d5 100644
> --- a/app/test-pmd/testpmd.c
> +++ b/app/test-pmd/testpmd.c
> @@ -90,7 +90,6 @@
>  #ifdef RTE_LIBRTE_LATENCY_STATS
>  #include <rte_latencystats.h>
>  #endif
> -#include <rte_gro.h>
> 
>  #include "testpmd.h"
> 
> @@ -386,6 +385,7 @@ uint8_t bitrate_enabled;
>  #endif
> 
>  struct gro_status gro_ports[RTE_MAX_ETHPORTS];
> +uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
> 
>  /* Forward function declarations */
>  static void map_port_queue_stats_mapping_registers(uint8_t pi, struct
> rte_port *port);
> @@ -570,6 +570,7 @@ init_config(void)
>  	unsigned int nb_mbuf_per_pool;
>  	lcoreid_t  lc_id;
>  	uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
> +	struct rte_gro_param gro_param;
> 
>  	memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
> 
> @@ -671,6 +672,20 @@ init_config(void)
>  		rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
> 
>  	fwd_config_setup();
> +
> +	/* create a gro context for each lcore */
> +	gro_param.gro_types = RTE_GRO_TCP_IPV4;
> +	gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
> +	gro_param.max_item_per_flow = MAX_PKT_BURST;
> +	for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
> +		gro_param.socket_id = rte_lcore_to_socket_id(
> +				fwd_lcores_cpuids[lc_id]);
> +		fwd_lcores[lc_id]->gro_ctx =
> rte_gro_ctx_create(&gro_param);
> +		if (fwd_lcores[lc_id]->gro_ctx == NULL) {
> +			rte_exit(EXIT_FAILURE,
> +					"rte_gro_ctx_create() failed\n");
> +		}
> +	}
>  }
> 
> 
> @@ -1165,6 +1180,7 @@ start_packet_forwarding(int with_tx_first)
>  		fwd_streams[sm_id]->fwd_dropped = 0;
>  		fwd_streams[sm_id]->rx_bad_ip_csum = 0;
>  		fwd_streams[sm_id]->rx_bad_l4_csum = 0;
> +		fwd_streams[sm_id]->gro_times = 0;
> 
>  #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
>  		memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
> diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
> index c9d7739..e878bcb 100644
> --- a/app/test-pmd/testpmd.h
> +++ b/app/test-pmd/testpmd.h
> @@ -120,6 +120,7 @@ struct fwd_stream {
>  	unsigned int fwd_dropped; /**< received packets not forwarded */
>  	unsigned int rx_bad_ip_csum ; /**< received packets has bad ip
> checksum */
>  	unsigned int rx_bad_l4_csum ; /**< received packets has bad l4
> checksum */
> +	unsigned int gro_times;	/**< GRO operation times */
>  #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
>  	uint64_t     core_cycles; /**< used for RX and TX processing */
>  #endif
> @@ -206,6 +207,7 @@ struct rte_port {
>   */
>  struct fwd_lcore {
>  	struct rte_mempool *mbp; /**< The mbuf pool to use by this core */
> +	void *gro_ctx;		/**< GRO context */
>  	streamid_t stream_idx;   /**< index of 1st stream in "fwd_streams"
> */
>  	streamid_t stream_nb;    /**< number of streams in "fwd_streams"
> */
>  	lcoreid_t  cpuid_idx;    /**< index of logical core in CPU id table */
> @@ -434,13 +436,19 @@ extern struct ether_addr
> peer_eth_addrs[RTE_MAX_ETHPORTS];
>  extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-
> retry. */
>  extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-
> retry. */
> 
> -#define GRO_DEFAULT_FLOW_NUM 4
> -#define GRO_DEFAULT_ITEM_NUM_PER_FLOW DEF_PKT_BURST
> +#define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
> +#define GRO_DEFAULT_FLOW_NUM (RTE_GRO_MAX_BURST_ITEM_NUM
> / \
> +		GRO_DEFAULT_ITEM_NUM_PER_FLOW)
> +
> +#define GRO_DEFAULT_FLUSH_CYCLES 1
> +#define GRO_MAX_FLUSH_CYCLES 4
> +
>  struct gro_status {
>  	struct rte_gro_param param;
>  	uint8_t enable;
>  };
>  extern struct gro_status gro_ports[RTE_MAX_ETHPORTS];
> +extern uint8_t gro_flush_cycles;
> 
>  static inline unsigned int
>  lcore_num(void)
> @@ -640,7 +648,9 @@ void get_2tuple_filter(uint8_t port_id, uint16_t
> index);
>  void get_5tuple_filter(uint8_t port_id, uint16_t index);
>  int rx_queue_id_is_invalid(queueid_t rxq_id);
>  int tx_queue_id_is_invalid(queueid_t txq_id);
> -void setup_gro(const char *mode, uint8_t port_id);
> +void setup_gro(const char *onoff, uint8_t port_id);
> +void setup_gro_flush_cycles(uint8_t cycles);
> +void show_gro(uint8_t port_id);
> 
>  /* Functions to manage the set of filtered Multicast MAC addresses */
>  void mcast_addr_add(uint8_t port_id, struct ether_addr *mc_addr);
> diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> index 2ed62f5..7f21b7d 100644
> --- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> +++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> @@ -898,12 +898,12 @@ Display the status of TCP Segmentation Offload::
> 
>     testpmd> tso show (port_id)
> 
> -gro
> -~~~
> +set port - gro
> +~~~~~~~~~~~~~~
> 
>  Enable or disable GRO in ``csum`` forwarding engine::
> 
> -   testpmd> gro (on|off) (port_id)
> +   testpmd> set port <port_id> gro on|off
> 
>  If enabled, the csum forwarding engine will perform GRO on the TCP/IPv4
>  packets received from the given port.
> @@ -914,23 +914,38 @@ GRO. By default, GRO is disabled for all ports.
>  .. note::
> 
>     When enable GRO for a port, TCP/IPv4 packets received from the port
> -   will be performed GRO. After GRO, the merged packets are multi-
> segments.
> -   But csum forwarding engine doesn't support to calculate TCP checksum
> -   for multi-segment packets in SW. So please select TCP HW checksum
> -   calculation for the port which GROed packets are transmitted to.
> +   will be performed GRO. After GRO, all merged packets have bad
> +   checksums, since the GRO library doesn't re-calculate checksums for
> +   the merged packets. Therefore, if users want the merged packets to
> +   have correct checksums, please select IP and TCP HW checksum
> calculation
> +   for the port which the merged packets are transmitted to.
> 
> -gro set
> -~~~~~~~
> +show port - gro
> +~~~~~~~~~~~~~~~
> +
> +Display GRO configuration for a given port::
> +
> +   testpmd> show port <port_id> gro
> +
> +set gro flush
> +~~~~~~~~~~~~~
> +
> +Set the cycle to flush the GROed packets from reassembly tables::
> 
> -Set max flow number and max packet number per-flow for GRO::
> +   testpmd> set gro flush <cycles>
> 
> -   testpmd> gro set (max_flow_num) (max_item_num_per_flow) (port_id)
> +When enable GRO, the csum forwarding engine performs GRO on received
> +packets, and the GROed packets are stored in reassembly tables. Users
> +can use this command to determine when the GROed packets are flushed
> +from the reassembly tables.
> 
> -The product of ``max_flow_num`` and ``max_item_num_per_flow`` is the
> max
> -number of packets a GRO table can store.
> +The ``cycles`` is measured in GRO operation times. The csum forwarding
> +engine flushes the GROed packets from the tables every ``cycles`` GRO
> +operations.
> 
> -If current packet number is greater than or equal to the max value, GRO
> -will stop processing incoming packets.
> +By default, the value of ``cycles`` is 1, which means flush GROed packets
> +from the reassembly tables as soon as one GRO operation finishes. The
> value
> +of ``cycles`` should be in the range of 1 to ``GRO_MAX_FLUSH_CYCLES``.
> 
>  mac_addr add
>  ~~~~~~~~~~~~
> --
> 2.7.4

Re: [dpdk-dev] [PATCH v3] app/testpmd: enable the heavyweight mode TCP/IPv4 GRO

[Hu, Jiayu]

Hi Lei,

> -----Original Message-----
> From: Yao, Lei A
> Sent: Wednesday, September 20, 2017 3:00 PM
> To: Hu, Jiayu <jiayu.hu@intel.com>; dev@dpdk.org
> Cc: Yigit, Ferruh <ferruh.yigit@intel.com>; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>; Tan, Jianfeng <jianfeng.tan@intel.com>;
> Wu, Jingjing <jingjing.wu@intel.com>; Hu, Jiayu <jiayu.hu@intel.com>
> Subject: RE: [dpdk-dev] [PATCH v3] app/testpmd: enable the heavyweight
> mode TCP/IPv4 GRO
> 
> 
> 
> > -----Original Message-----
> > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Jiayu Hu
> > Sent: Sunday, September 3, 2017 2:30 PM
> > To: dev@dpdk.org
> > Cc: Yigit, Ferruh <ferruh.yigit@intel.com>; Ananyev, Konstantin
> > <konstantin.ananyev@intel.com>; Tan, Jianfeng <jianfeng.tan@intel.com>;
> > Wu, Jingjing <jingjing.wu@intel.com>; Hu, Jiayu <jiayu.hu@intel.com>
> > Subject: [dpdk-dev] [PATCH v3] app/testpmd: enable the heavyweight
> > mode TCP/IPv4 GRO
> >
> > The GRO library provides two modes to reassemble packets. Currently, the
> > csum forwarding engine has supported to use the lightweight mode to
> > reassemble TCP/IPv4 packets. This patch introduces the heavyweight mode
> > for TCP/IPv4 GRO in the csum forwarding engine.
> >
> > With the command "set port <port_id> gro on|off", users can enable
> > TCP/IPv4 GRO for a given port. With the command "set gro flush <cycles>",
> > users can determine when the GROed TCP/IPv4 packets are flushed from
> > reassembly tables. With the command "show port <port_id> gro", users
> can
> > display GRO configuration.
> >
> > Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
> Tested-by : Lei Yao<lei.a.yao@intel.com>
> 
> This patch has been tested on my bench, iperf test result  is as following:
> No-GRO: 8 Gbps
> Kernel GRO: 14.3 Gbps
> GRO flush 0 : 12.7 Gbps
> GRO flush 1: 16.8 Gbps
> But when I use 40G NIC and set GRO flush cycle as 2, sometimes
> the iperf traffic will stall for several seconds. Still need investigate.

This issue happens in high link speed environment, like 40Gbps. I am
looking into this issue now.

Current experiment data shows that the large flush cycle value causes
lots of duplicated TCP ACKs with same ACK numbers, which may be one
of the reasons for the poor TCP/IP stack performance.

Thanks,
Jiayu
> 
> > ---
> > changes in v3:
> > - remove "heavyweight mode" and "lightweight mode" from GRO
> > commands
> > - combine two patches into one
> > - use consistent help string for GRO commands
> > - remove the unnecessary command "gro set (max_flow_num)
> > 	(max_item_num_per_flow) (port_id)"
> > changes in v2:
> > - use "set" and "show" as the root level command
> > - add a new command to show GRO configuration
> > - fix l2_len/l3_len/l4_len unset etc. bugs
> >
> >  app/test-pmd/cmdline.c                      | 206 ++++++++++++++++------------
> >  app/test-pmd/config.c                       |  67 +++++++--
> >  app/test-pmd/csumonly.c                     |  31 ++++-
> >  app/test-pmd/testpmd.c                      |  18 ++-
> >  app/test-pmd/testpmd.h                      |  16 ++-
> >  doc/guides/testpmd_app_ug/testpmd_funcs.rst |  45 ++++--
> >  6 files changed, 263 insertions(+), 120 deletions(-)
> >
> > diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
> > index cd8c358..d628250 100644
> > --- a/app/test-pmd/cmdline.c
> > +++ b/app/test-pmd/cmdline.c
> > @@ -423,13 +423,16 @@ static void cmd_help_long_parsed(void
> > *parsed_result,
> >  			"tso show (portid)"
> >  			"    Display the status of TCP Segmentation
> > Offload.\n\n"
> >
> > -			"gro (on|off) (port_id)"
> > +			"set port (port_id) gro on|off\n"
> >  			"    Enable or disable Generic Receive Offload in"
> >  			" csum forwarding engine.\n\n"
> >
> > -			"gro set (max_flow_num)
> > (max_item_num_per_flow) (port_id)\n"
> > -			"    Set max flow number and max packet number
> > per-flow"
> > -			" for GRO.\n\n"
> > +			"show port (port_id) gro\n"
> > +			"    Display GRO configuration.\n\n"
> > +
> > +			"set gro flush (cycles)\n"
> > +			"    Set the cycle to flush GROed packets from"
> > +			" reassembly tables.\n\n"
> >
> >  			"set fwd (%s)\n"
> >  			"    Set packet forwarding mode.\n\n"
> > @@ -3850,115 +3853,145 @@ cmdline_parse_inst_t
> cmd_tunnel_tso_show
> > = {
> >  };
> >
> >  /* *** SET GRO FOR A PORT *** */
> > -struct cmd_gro_result {
> > +struct cmd_gro_enable_result {
> > +	cmdline_fixed_string_t cmd_set;
> > +	cmdline_fixed_string_t cmd_port;
> >  	cmdline_fixed_string_t cmd_keyword;
> > -	cmdline_fixed_string_t mode;
> > -	uint8_t port_id;
> > +	cmdline_fixed_string_t cmd_onoff;
> > +	uint8_t cmd_pid;
> >  };
> >
> >  static void
> > -cmd_enable_gro_parsed(void *parsed_result,
> > +cmd_gro_enable_parsed(void *parsed_result,
> >  		__attribute__((unused)) struct cmdline *cl,
> >  		__attribute__((unused)) void *data)
> >  {
> > -	struct cmd_gro_result *res;
> > +	struct cmd_gro_enable_result *res;
> >
> >  	res = parsed_result;
> > -	setup_gro(res->mode, res->port_id);
> > -}
> > -
> > -cmdline_parse_token_string_t cmd_gro_keyword =
> > -	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
> > +	if (!strcmp(res->cmd_keyword, "gro"))
> > +		setup_gro(res->cmd_onoff, res->cmd_pid);
> > +}
> > +
> > +cmdline_parse_token_string_t cmd_gro_enable_set =
> > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
> > +			cmd_set, "set");
> > +cmdline_parse_token_string_t cmd_gro_enable_port =
> > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
> > +			cmd_keyword, "port");
> > +cmdline_parse_token_num_t cmd_gro_enable_pid =
> > +	TOKEN_NUM_INITIALIZER(struct cmd_gro_enable_result,
> > +			cmd_pid, UINT8);
> > +cmdline_parse_token_string_t cmd_gro_enable_keyword =
> > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
> >  			cmd_keyword, "gro");
> > -cmdline_parse_token_string_t cmd_gro_mode =
> > -	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
> > -			mode, "on#off");
> > -cmdline_parse_token_num_t cmd_gro_pid =
> > -	TOKEN_NUM_INITIALIZER(struct cmd_gro_result,
> > -			port_id, UINT8);
> > +cmdline_parse_token_string_t cmd_gro_enable_onoff =
> > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
> > +			cmd_onoff, "on#off");
> >
> > -cmdline_parse_inst_t cmd_enable_gro = {
> > -	.f = cmd_enable_gro_parsed,
> > +cmdline_parse_inst_t cmd_gro_enable = {
> > +	.f = cmd_gro_enable_parsed,
> >  	.data = NULL,
> > -	.help_str = "gro (on|off) (port_id)",
> > +	.help_str = "set port <port_id> gro on|off",
> >  	.tokens = {
> > -		(void *)&cmd_gro_keyword,
> > -		(void *)&cmd_gro_mode,
> > -		(void *)&cmd_gro_pid,
> > +		(void *)&cmd_gro_enable_set,
> > +		(void *)&cmd_gro_enable_port,
> > +		(void *)&cmd_gro_enable_pid,
> > +		(void *)&cmd_gro_enable_keyword,
> > +		(void *)&cmd_gro_enable_onoff,
> >  		NULL,
> >  	},
> >  };
> >
> > -/* *** SET MAX FLOW NUMBER AND ITEM NUM PER FLOW FOR GRO ***
> */
> > -struct cmd_gro_set_result {
> > -	cmdline_fixed_string_t gro;
> > -	cmdline_fixed_string_t mode;
> > -	uint16_t flow_num;
> > -	uint16_t item_num_per_flow;
> > -	uint8_t port_id;
> > +/* *** DISPLAY GRO CONFIGURATION *** */
> > +struct cmd_gro_show_result {
> > +	cmdline_fixed_string_t cmd_show;
> > +	cmdline_fixed_string_t cmd_port;
> > +	cmdline_fixed_string_t cmd_keyword;
> > +	uint8_t cmd_pid;
> >  };
> >
> >  static void
> > -cmd_gro_set_parsed(void *parsed_result,
> > -		       __attribute__((unused)) struct cmdline *cl,
> > -		       __attribute__((unused)) void *data)
> > +cmd_gro_show_parsed(void *parsed_result,
> > +		__attribute__((unused)) struct cmdline *cl,
> > +		__attribute__((unused)) void *data)
> >  {
> > -	struct cmd_gro_set_result *res = parsed_result;
> > +	struct cmd_gro_show_result *res;
> >
> > -	if (port_id_is_invalid(res->port_id, ENABLED_WARN))
> > -		return;
> > -	if (test_done == 0) {
> > -		printf("Before set GRO flow_num and item_num_per_flow,"
> > -				" please stop forwarding first\n");
> > -		return;
> > -	}
> > +	res = parsed_result;
> > +	if (!strcmp(res->cmd_keyword, "gro"))
> > +		show_gro(res->cmd_pid);
> > +}
> > +
> > +cmdline_parse_token_string_t cmd_gro_show_show =
> > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
> > +			cmd_show, "show");
> > +cmdline_parse_token_string_t cmd_gro_show_port =
> > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
> > +			cmd_port, "port");
> > +cmdline_parse_token_num_t cmd_gro_show_pid =
> > +	TOKEN_NUM_INITIALIZER(struct cmd_gro_show_result,
> > +			cmd_pid, UINT8);
> > +cmdline_parse_token_string_t cmd_gro_show_keyword =
> > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
> > +			cmd_keyword, "gro");
> >
> > -	if (!strcmp(res->mode, "set")) {
> > -		if (res->flow_num == 0)
> > -			printf("Invalid flow number. Revert to default
> value:"
> > -					" %u.\n",
> > GRO_DEFAULT_FLOW_NUM);
> > -		else
> > -			gro_ports[res->port_id].param.max_flow_num =
> > -				res->flow_num;
> > +cmdline_parse_inst_t cmd_gro_show = {
> > +	.f = cmd_gro_show_parsed,
> > +	.data = NULL,
> > +	.help_str = "show port <port_id> gro",
> > +	.tokens = {
> > +		(void *)&cmd_gro_show_show,
> > +		(void *)&cmd_gro_show_port,
> > +		(void *)&cmd_gro_show_pid,
> > +		(void *)&cmd_gro_show_keyword,
> > +		NULL,
> > +	},
> > +};
> >
> > -		if (res->item_num_per_flow == 0)
> > -			printf("Invalid item number per-flow. Revert"
> > -					" to default value:%u.\n",
> > -
> > 	GRO_DEFAULT_ITEM_NUM_PER_FLOW);
> > -		else
> > -			gro_ports[res->port_id].param.max_item_per_flow
> > =
> > -				res->item_num_per_flow;
> > -	}
> > +/* *** SET FLUSH CYCLES FOR GRO *** */
> > +struct cmd_gro_flush_result {
> > +	cmdline_fixed_string_t cmd_set;
> > +	cmdline_fixed_string_t cmd_keyword;
> > +	cmdline_fixed_string_t cmd_flush;
> > +	uint8_t cmd_cycles;
> > +};
> > +
> > +static void
> > +cmd_gro_flush_parsed(void *parsed_result,
> > +		__attribute__((unused)) struct cmdline *cl,
> > +		__attribute__((unused)) void *data)
> > +{
> > +	struct cmd_gro_flush_result *res;
> > +
> > +	res = parsed_result;
> > +	if ((!strcmp(res->cmd_keyword, "gro")) &&
> > +			(!strcmp(res->cmd_flush, "flush")))
> > +		setup_gro_flush_cycles(res->cmd_cycles);
> >  }
> >
> > -cmdline_parse_token_string_t cmd_gro_set_gro =
> > -	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
> > -				gro, "gro");
> > -cmdline_parse_token_string_t cmd_gro_set_mode =
> > -	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
> > -				mode, "set");
> > -cmdline_parse_token_num_t cmd_gro_set_flow_num =
> > -	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> > -				flow_num, UINT16);
> > -cmdline_parse_token_num_t cmd_gro_set_item_num_per_flow =
> > -	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> > -				item_num_per_flow, UINT16);
> > -cmdline_parse_token_num_t cmd_gro_set_portid =
> > -	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> > -				port_id, UINT8);
> > +cmdline_parse_token_string_t cmd_gro_flush_set =
> > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
> > +			cmd_set, "set");
> > +cmdline_parse_token_string_t cmd_gro_flush_keyword =
> > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
> > +			cmd_keyword, "gro");
> > +cmdline_parse_token_string_t cmd_gro_flush_flush =
> > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
> > +			cmd_flush, "flush");
> > +cmdline_parse_token_num_t cmd_gro_flush_cycles =
> > +	TOKEN_NUM_INITIALIZER(struct cmd_gro_flush_result,
> > +			cmd_cycles, UINT8);
> >
> > -cmdline_parse_inst_t cmd_gro_set = {
> > -	.f = cmd_gro_set_parsed,
> > +cmdline_parse_inst_t cmd_gro_flush = {
> > +	.f = cmd_gro_flush_parsed,
> >  	.data = NULL,
> > -	.help_str = "gro set <max_flow_num> <max_item_num_per_flow>
> > "
> > -		"<port_id>: set max flow number and max packet number
> > per-flow "
> > -		"for GRO",
> > +	.help_str = "set gro flush <cycles>",
> >  	.tokens = {
> > -		(void *)&cmd_gro_set_gro,
> > -		(void *)&cmd_gro_set_mode,
> > -		(void *)&cmd_gro_set_flow_num,
> > -		(void *)&cmd_gro_set_item_num_per_flow,
> > -		(void *)&cmd_gro_set_portid,
> > +		(void *)&cmd_gro_flush_set,
> > +		(void *)&cmd_gro_flush_keyword,
> > +		(void *)&cmd_gro_flush_flush,
> > +		(void *)&cmd_gro_flush_cycles,
> >  		NULL,
> >  	},
> >  };
> > @@ -14249,8 +14282,9 @@ cmdline_parse_ctx_t main_ctx[] = {
> >  	(cmdline_parse_inst_t *)&cmd_tso_show,
> >  	(cmdline_parse_inst_t *)&cmd_tunnel_tso_set,
> >  	(cmdline_parse_inst_t *)&cmd_tunnel_tso_show,
> > -	(cmdline_parse_inst_t *)&cmd_enable_gro,
> > -	(cmdline_parse_inst_t *)&cmd_gro_set,
> > +	(cmdline_parse_inst_t *)&cmd_gro_enable,
> > +	(cmdline_parse_inst_t *)&cmd_gro_flush,
> > +	(cmdline_parse_inst_t *)&cmd_gro_show,
> >  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set,
> >  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_rx,
> >  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_tx,
> > diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
> > index 3ae3e1c..d97d291 100644
> > --- a/app/test-pmd/config.c
> > +++ b/app/test-pmd/config.c
> > @@ -2420,7 +2420,7 @@ set_tx_pkt_segments(unsigned *seg_lengths,
> > unsigned nb_segs)
> >  }
> >
> >  void
> > -setup_gro(const char *mode, uint8_t port_id)
> > +setup_gro(const char *onoff, uint8_t port_id)
> >  {
> >  	if (!rte_eth_dev_is_valid_port(port_id)) {
> >  		printf("invalid port id %u\n", port_id);
> > @@ -2431,29 +2431,76 @@ setup_gro(const char *mode, uint8_t port_id)
> >  				" please stop forwarding first\n");
> >  		return;
> >  	}
> > -	if (strcmp(mode, "on") == 0) {
> > -		if (gro_ports[port_id].enable) {
> > -			printf("port %u has enabled GRO\n", port_id);
> > +	if (strcmp(onoff, "on") == 0) {
> > +		if (gro_ports[port_id].enable != 0) {
> > +			printf("Port %u has enabled GRO. Please"
> > +					" disable GRO first\n", port_id);
> >  			return;
> >  		}
> > -		gro_ports[port_id].enable = 1;
> > -		gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4;
> > -
> > -		if (gro_ports[port_id].param.max_flow_num == 0)
> > +		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
> > +			gro_ports[port_id].param.gro_types =
> > RTE_GRO_TCP_IPV4;
> >  			gro_ports[port_id].param.max_flow_num =
> >  				GRO_DEFAULT_FLOW_NUM;
> > -		if (gro_ports[port_id].param.max_item_per_flow == 0)
> >  			gro_ports[port_id].param.max_item_per_flow =
> >  				GRO_DEFAULT_ITEM_NUM_PER_FLOW;
> > +		}
> > +		gro_ports[port_id].enable = 1;
> >  	} else {
> >  		if (gro_ports[port_id].enable == 0) {
> > -			printf("port %u has disabled GRO\n", port_id);
> > +			printf("Port %u has disabled GRO\n", port_id);
> >  			return;
> >  		}
> >  		gro_ports[port_id].enable = 0;
> >  	}
> >  }
> >
> > +void
> > +setup_gro_flush_cycles(uint8_t cycles)
> > +{
> > +	if (test_done == 0) {
> > +		printf("Before change flush interval for GRO,"
> > +				" please stop forwarding first\n");
> > +		return;
> > +	}
> > +
> > +	if (cycles > GRO_MAX_FLUSH_CYCLES) {
> > +		printf("The flushing cycle be in the range"
> > +				" of 1 to %u. Revert to the default"
> > +				" value %u\n",
> > +				GRO_MAX_FLUSH_CYCLES,
> > +				GRO_DEFAULT_FLUSH_CYCLES);
> > +		cycles = GRO_DEFAULT_FLUSH_CYCLES;
> > +	}
> > +
> > +	gro_flush_cycles = cycles;
> > +}
> > +
> > +void
> > +show_gro(uint8_t port_id)
> > +{
> > +	struct rte_gro_param *param;
> > +	uint32_t max_pkts_num;
> > +
> > +	param = &gro_ports[port_id].param;
> > +
> > +	if (!rte_eth_dev_is_valid_port(port_id)) {
> > +		printf("Invalid port id %u\n", port_id);
> > +		return;
> > +	}
> > +	if (gro_ports[port_id].enable) {
> > +		printf("GRO type: TCP/IPv4\n");
> > +		if (gro_flush_cycles > 0)
> > +			max_pkts_num = MAX_PKT_BURST *
> > GRO_MAX_FLUSH_CYCLES;
> > +		else {
> > +			max_pkts_num = param->max_flow_num *
> > +				param->max_item_per_flow;
> > +		}
> > +		printf("Max packet number to GRO: %u\n", max_pkts_num);
> > +		printf("Flushing cycles: %u\n", gro_flush_cycles);
> > +	} else
> > +		printf("Port %u doesn't enable GRO\n", port_id);
> > +}
> > +
> >  char*
> >  list_pkt_forwarding_modes(void)
> >  {
> > diff --git a/app/test-pmd/csumonly.c b/app/test-pmd/csumonly.c
> > index 90c8119..ca50ab7 100644
> > --- a/app/test-pmd/csumonly.c
> > +++ b/app/test-pmd/csumonly.c
> > @@ -631,6 +631,9 @@ pkt_burst_checksum_forward(struct fwd_stream
> *fs)
> >  	struct rte_mbuf *m, *p;
> >  	struct ether_hdr *eth_hdr;
> >  	void *l3_hdr = NULL, *outer_l3_hdr = NULL; /* can be IPv4 or IPv6 */
> > +	void **gro_ctx;
> > +	uint16_t gro_pkts_num;
> > +	uint8_t gro_enable;
> >  	uint16_t nb_rx;
> >  	uint16_t nb_tx;
> >  	uint16_t nb_prep;
> > @@ -657,17 +660,13 @@ pkt_burst_checksum_forward(struct
> fwd_stream
> > *fs)
> >  				 nb_pkt_per_burst);
> >  	if (unlikely(nb_rx == 0))
> >  		return;
> > -	if (unlikely(gro_ports[fs->rx_port].enable))
> > -		nb_rx = rte_gro_reassemble_burst(pkts_burst,
> > -				nb_rx,
> > -				&(gro_ports[fs->rx_port].param));
> > -
> >  #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
> >  	fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
> >  #endif
> >  	fs->rx_packets += nb_rx;
> >  	rx_bad_ip_csum = 0;
> >  	rx_bad_l4_csum = 0;
> > +	gro_enable = gro_ports[fs->rx_port].enable;
> >
> >  	txp = &ports[fs->tx_port];
> >  	testpmd_ol_flags = txp->tx_ol_flags;
> > @@ -851,6 +850,28 @@ pkt_burst_checksum_forward(struct fwd_stream
> > *fs)
> >  		}
> >  	}
> >
> > +	if (unlikely(gro_enable)) {
> > +		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
> > +			nb_rx = rte_gro_reassemble_burst(pkts_burst,
> > nb_rx,
> > +					&(gro_ports[fs->rx_port].param));
> > +		} else {
> > +			gro_ctx = current_fwd_lcore()->gro_ctx;
> > +			nb_rx = rte_gro_reassemble(pkts_burst, nb_rx,
> > gro_ctx);
> > +
> > +			if (++fs->gro_times >= gro_flush_cycles) {
> > +				gro_pkts_num =
> > rte_gro_get_pkt_count(gro_ctx);
> > +				if (gro_pkts_num > MAX_PKT_BURST - nb_rx)
> > +					gro_pkts_num = MAX_PKT_BURST -
> > nb_rx;
> > +
> > +				nb_rx += rte_gro_timeout_flush(gro_ctx, 0,
> > +						RTE_GRO_TCP_IPV4,
> > +						&pkts_burst[nb_rx],
> > +						gro_pkts_num);
> > +				fs->gro_times = 0;
> > +			}
> > +		}
> > +	}
> > +
> >  	nb_prep = rte_eth_tx_prepare(fs->tx_port, fs->tx_queue,
> >  			pkts_burst, nb_rx);
> >  	if (nb_prep != nb_rx)
> > diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
> > index 7d40139..c1423d5 100644
> > --- a/app/test-pmd/testpmd.c
> > +++ b/app/test-pmd/testpmd.c
> > @@ -90,7 +90,6 @@
> >  #ifdef RTE_LIBRTE_LATENCY_STATS
> >  #include <rte_latencystats.h>
> >  #endif
> > -#include <rte_gro.h>
> >
> >  #include "testpmd.h"
> >
> > @@ -386,6 +385,7 @@ uint8_t bitrate_enabled;
> >  #endif
> >
> >  struct gro_status gro_ports[RTE_MAX_ETHPORTS];
> > +uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
> >
> >  /* Forward function declarations */
> >  static void map_port_queue_stats_mapping_registers(uint8_t pi, struct
> > rte_port *port);
> > @@ -570,6 +570,7 @@ init_config(void)
> >  	unsigned int nb_mbuf_per_pool;
> >  	lcoreid_t  lc_id;
> >  	uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
> > +	struct rte_gro_param gro_param;
> >
> >  	memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
> >
> > @@ -671,6 +672,20 @@ init_config(void)
> >  		rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
> >
> >  	fwd_config_setup();
> > +
> > +	/* create a gro context for each lcore */
> > +	gro_param.gro_types = RTE_GRO_TCP_IPV4;
> > +	gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
> > +	gro_param.max_item_per_flow = MAX_PKT_BURST;
> > +	for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
> > +		gro_param.socket_id = rte_lcore_to_socket_id(
> > +				fwd_lcores_cpuids[lc_id]);
> > +		fwd_lcores[lc_id]->gro_ctx =
> > rte_gro_ctx_create(&gro_param);
> > +		if (fwd_lcores[lc_id]->gro_ctx == NULL) {
> > +			rte_exit(EXIT_FAILURE,
> > +					"rte_gro_ctx_create() failed\n");
> > +		}
> > +	}
> >  }
> >
> >
> > @@ -1165,6 +1180,7 @@ start_packet_forwarding(int with_tx_first)
> >  		fwd_streams[sm_id]->fwd_dropped = 0;
> >  		fwd_streams[sm_id]->rx_bad_ip_csum = 0;
> >  		fwd_streams[sm_id]->rx_bad_l4_csum = 0;
> > +		fwd_streams[sm_id]->gro_times = 0;
> >
> >  #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
> >  		memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
> > diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
> > index c9d7739..e878bcb 100644
> > --- a/app/test-pmd/testpmd.h
> > +++ b/app/test-pmd/testpmd.h
> > @@ -120,6 +120,7 @@ struct fwd_stream {
> >  	unsigned int fwd_dropped; /**< received packets not forwarded */
> >  	unsigned int rx_bad_ip_csum ; /**< received packets has bad ip
> > checksum */
> >  	unsigned int rx_bad_l4_csum ; /**< received packets has bad l4
> > checksum */
> > +	unsigned int gro_times;	/**< GRO operation times */
> >  #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
> >  	uint64_t     core_cycles; /**< used for RX and TX processing */
> >  #endif
> > @@ -206,6 +207,7 @@ struct rte_port {
> >   */
> >  struct fwd_lcore {
> >  	struct rte_mempool *mbp; /**< The mbuf pool to use by this core */
> > +	void *gro_ctx;		/**< GRO context */
> >  	streamid_t stream_idx;   /**< index of 1st stream in "fwd_streams"
> > */
> >  	streamid_t stream_nb;    /**< number of streams in "fwd_streams"
> > */
> >  	lcoreid_t  cpuid_idx;    /**< index of logical core in CPU id table */
> > @@ -434,13 +436,19 @@ extern struct ether_addr
> > peer_eth_addrs[RTE_MAX_ETHPORTS];
> >  extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-
> > retry. */
> >  extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-
> > retry. */
> >
> > -#define GRO_DEFAULT_FLOW_NUM 4
> > -#define GRO_DEFAULT_ITEM_NUM_PER_FLOW DEF_PKT_BURST
> > +#define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
> > +#define GRO_DEFAULT_FLOW_NUM
> (RTE_GRO_MAX_BURST_ITEM_NUM
> > / \
> > +		GRO_DEFAULT_ITEM_NUM_PER_FLOW)
> > +
> > +#define GRO_DEFAULT_FLUSH_CYCLES 1
> > +#define GRO_MAX_FLUSH_CYCLES 4
> > +
> >  struct gro_status {
> >  	struct rte_gro_param param;
> >  	uint8_t enable;
> >  };
> >  extern struct gro_status gro_ports[RTE_MAX_ETHPORTS];
> > +extern uint8_t gro_flush_cycles;
> >
> >  static inline unsigned int
> >  lcore_num(void)
> > @@ -640,7 +648,9 @@ void get_2tuple_filter(uint8_t port_id, uint16_t
> > index);
> >  void get_5tuple_filter(uint8_t port_id, uint16_t index);
> >  int rx_queue_id_is_invalid(queueid_t rxq_id);
> >  int tx_queue_id_is_invalid(queueid_t txq_id);
> > -void setup_gro(const char *mode, uint8_t port_id);
> > +void setup_gro(const char *onoff, uint8_t port_id);
> > +void setup_gro_flush_cycles(uint8_t cycles);
> > +void show_gro(uint8_t port_id);
> >
> >  /* Functions to manage the set of filtered Multicast MAC addresses */
> >  void mcast_addr_add(uint8_t port_id, struct ether_addr *mc_addr);
> > diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> > b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> > index 2ed62f5..7f21b7d 100644
> > --- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> > +++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> > @@ -898,12 +898,12 @@ Display the status of TCP Segmentation Offload::
> >
> >     testpmd> tso show (port_id)
> >
> > -gro
> > -~~~
> > +set port - gro
> > +~~~~~~~~~~~~~~
> >
> >  Enable or disable GRO in ``csum`` forwarding engine::
> >
> > -   testpmd> gro (on|off) (port_id)
> > +   testpmd> set port <port_id> gro on|off
> >
> >  If enabled, the csum forwarding engine will perform GRO on the TCP/IPv4
> >  packets received from the given port.
> > @@ -914,23 +914,38 @@ GRO. By default, GRO is disabled for all ports.
> >  .. note::
> >
> >     When enable GRO for a port, TCP/IPv4 packets received from the port
> > -   will be performed GRO. After GRO, the merged packets are multi-
> > segments.
> > -   But csum forwarding engine doesn't support to calculate TCP checksum
> > -   for multi-segment packets in SW. So please select TCP HW checksum
> > -   calculation for the port which GROed packets are transmitted to.
> > +   will be performed GRO. After GRO, all merged packets have bad
> > +   checksums, since the GRO library doesn't re-calculate checksums for
> > +   the merged packets. Therefore, if users want the merged packets to
> > +   have correct checksums, please select IP and TCP HW checksum
> > calculation
> > +   for the port which the merged packets are transmitted to.
> >
> > -gro set
> > -~~~~~~~
> > +show port - gro
> > +~~~~~~~~~~~~~~~
> > +
> > +Display GRO configuration for a given port::
> > +
> > +   testpmd> show port <port_id> gro
> > +
> > +set gro flush
> > +~~~~~~~~~~~~~
> > +
> > +Set the cycle to flush the GROed packets from reassembly tables::
> >
> > -Set max flow number and max packet number per-flow for GRO::
> > +   testpmd> set gro flush <cycles>
> >
> > -   testpmd> gro set (max_flow_num) (max_item_num_per_flow) (port_id)
> > +When enable GRO, the csum forwarding engine performs GRO on
> received
> > +packets, and the GROed packets are stored in reassembly tables. Users
> > +can use this command to determine when the GROed packets are flushed
> > +from the reassembly tables.
> >
> > -The product of ``max_flow_num`` and ``max_item_num_per_flow`` is the
> > max
> > -number of packets a GRO table can store.
> > +The ``cycles`` is measured in GRO operation times. The csum forwarding
> > +engine flushes the GROed packets from the tables every ``cycles`` GRO
> > +operations.
> >
> > -If current packet number is greater than or equal to the max value, GRO
> > -will stop processing incoming packets.
> > +By default, the value of ``cycles`` is 1, which means flush GROed packets
> > +from the reassembly tables as soon as one GRO operation finishes. The
> > value
> > +of ``cycles`` should be in the range of 1 to ``GRO_MAX_FLUSH_CYCLES``.
> >
> >  mac_addr add
> >  ~~~~~~~~~~~~
> > --
> > 2.7.4

Re: [dpdk-dev] [PATCH v3] app/testpmd: enable the heavyweight mode TCP/IPv4 GRO

[Ferruh Yigit]

On 9/3/2017 7:30 AM, Jiayu Hu wrote:
> The GRO library provides two modes to reassemble packets. Currently, the
> csum forwarding engine has supported to use the lightweight mode to
> reassemble TCP/IPv4 packets. This patch introduces the heavyweight mode
> for TCP/IPv4 GRO in the csum forwarding engine.
> 
> With the command "set port <port_id> gro on|off", users can enable
> TCP/IPv4 GRO for a given port. With the command "set gro flush <cycles>",
> users can determine when the GROed TCP/IPv4 packets are flushed from
> reassembly tables. With the command "show port <port_id> gro", users can
> display GRO configuration.
> 
> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>

Reviewed-by: Ferruh Yigit <ferruh.yigit@intel.com>

Re: [dpdk-dev] [PATCH v3] app/testpmd: enable the heavyweight mode TCP/IPv4 GRO

[Hu, Jiayu]

Hi Lei,

> -----Original Message-----
> From: Hu, Jiayu
> Sent: Monday, September 25, 2017 6:20 PM
> To: Yao, Lei A <lei.a.yao@intel.com>; dev@dpdk.org
> Cc: Yigit, Ferruh <ferruh.yigit@intel.com>; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>; Tan, Jianfeng <jianfeng.tan@intel.com>;
> Wu, Jingjing <jingjing.wu@intel.com>
> Subject: RE: [dpdk-dev] [PATCH v3] app/testpmd: enable the heavyweight
> mode TCP/IPv4 GRO
> 
> Hi Lei,
> 
> > -----Original Message-----
> > From: Yao, Lei A
> > Sent: Wednesday, September 20, 2017 3:00 PM
> > To: Hu, Jiayu <jiayu.hu@intel.com>; dev@dpdk.org
> > Cc: Yigit, Ferruh <ferruh.yigit@intel.com>; Ananyev, Konstantin
> > <konstantin.ananyev@intel.com>; Tan, Jianfeng <jianfeng.tan@intel.com>;
> > Wu, Jingjing <jingjing.wu@intel.com>; Hu, Jiayu <jiayu.hu@intel.com>
> > Subject: RE: [dpdk-dev] [PATCH v3] app/testpmd: enable the heavyweight
> > mode TCP/IPv4 GRO
> >
> >
> >
> > > -----Original Message-----
> > > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Jiayu Hu
> > > Sent: Sunday, September 3, 2017 2:30 PM
> > > To: dev@dpdk.org
> > > Cc: Yigit, Ferruh <ferruh.yigit@intel.com>; Ananyev, Konstantin
> > > <konstantin.ananyev@intel.com>; Tan, Jianfeng
> <jianfeng.tan@intel.com>;
> > > Wu, Jingjing <jingjing.wu@intel.com>; Hu, Jiayu <jiayu.hu@intel.com>
> > > Subject: [dpdk-dev] [PATCH v3] app/testpmd: enable the heavyweight
> > > mode TCP/IPv4 GRO
> > >
> > > The GRO library provides two modes to reassemble packets. Currently,
> the
> > > csum forwarding engine has supported to use the lightweight mode to
> > > reassemble TCP/IPv4 packets. This patch introduces the heavyweight
> mode
> > > for TCP/IPv4 GRO in the csum forwarding engine.
> > >
> > > With the command "set port <port_id> gro on|off", users can enable
> > > TCP/IPv4 GRO for a given port. With the command "set gro flush
> <cycles>",
> > > users can determine when the GROed TCP/IPv4 packets are flushed from
> > > reassembly tables. With the command "show port <port_id> gro", users
> > can
> > > display GRO configuration.
> > >
> > > Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
> > Tested-by : Lei Yao<lei.a.yao@intel.com>
> >
> > This patch has been tested on my bench, iperf test result  is as following:
> > No-GRO: 8 Gbps
> > Kernel GRO: 14.3 Gbps
> > GRO flush 0 : 12.7 Gbps
> > GRO flush 1: 16.8 Gbps
> > But when I use 40G NIC and set GRO flush cycle as 2, sometimes
> > the iperf traffic will stall for several seconds. Still need investigate.

The default descriptor number per queue is 128. Given the flush
cycle is 3, the demanded descriptor number is up to 3x32, even if we
perform GRO. This is because each mbuf of a merged packet demands
one descriptor. Therefore, we'd better set large descriptor number
per-queue when launch testpmd, instead of using the default value 128.

After setting rxd and txd to 256, the iperf performance is stable at
16Gbps with different flush cycle settings.

Thanks,
Jiayu

> 
> This issue happens in high link speed environment, like 40Gbps. I am
> looking into this issue now.
> 
> Current experiment data shows that the large flush cycle value causes
> lots of duplicated TCP ACKs with same ACK numbers, which may be one
> of the reasons for the poor TCP/IP stack performance.
> 
> Thanks,
> Jiayu
> >
> > > ---
> > > changes in v3:
> > > - remove "heavyweight mode" and "lightweight mode" from GRO
> > > commands
> > > - combine two patches into one
> > > - use consistent help string for GRO commands
> > > - remove the unnecessary command "gro set (max_flow_num)
> > > 	(max_item_num_per_flow) (port_id)"
> > > changes in v2:
> > > - use "set" and "show" as the root level command
> > > - add a new command to show GRO configuration
> > > - fix l2_len/l3_len/l4_len unset etc. bugs
> > >
> > >  app/test-pmd/cmdline.c                      | 206 ++++++++++++++++------------
> > >  app/test-pmd/config.c                       |  67 +++++++--
> > >  app/test-pmd/csumonly.c                     |  31 ++++-
> > >  app/test-pmd/testpmd.c                      |  18 ++-
> > >  app/test-pmd/testpmd.h                      |  16 ++-
> > >  doc/guides/testpmd_app_ug/testpmd_funcs.rst |  45 ++++--
> > >  6 files changed, 263 insertions(+), 120 deletions(-)
> > >
> > > diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
> > > index cd8c358..d628250 100644
> > > --- a/app/test-pmd/cmdline.c
> > > +++ b/app/test-pmd/cmdline.c
> > > @@ -423,13 +423,16 @@ static void cmd_help_long_parsed(void
> > > *parsed_result,
> > >  			"tso show (portid)"
> > >  			"    Display the status of TCP Segmentation
> > > Offload.\n\n"
> > >
> > > -			"gro (on|off) (port_id)"
> > > +			"set port (port_id) gro on|off\n"
> > >  			"    Enable or disable Generic Receive Offload in"
> > >  			" csum forwarding engine.\n\n"
> > >
> > > -			"gro set (max_flow_num)
> > > (max_item_num_per_flow) (port_id)\n"
> > > -			"    Set max flow number and max packet number
> > > per-flow"
> > > -			" for GRO.\n\n"
> > > +			"show port (port_id) gro\n"
> > > +			"    Display GRO configuration.\n\n"
> > > +
> > > +			"set gro flush (cycles)\n"
> > > +			"    Set the cycle to flush GROed packets from"
> > > +			" reassembly tables.\n\n"
> > >
> > >  			"set fwd (%s)\n"
> > >  			"    Set packet forwarding mode.\n\n"
> > > @@ -3850,115 +3853,145 @@ cmdline_parse_inst_t
> > cmd_tunnel_tso_show
> > > = {
> > >  };
> > >
> > >  /* *** SET GRO FOR A PORT *** */
> > > -struct cmd_gro_result {
> > > +struct cmd_gro_enable_result {
> > > +	cmdline_fixed_string_t cmd_set;
> > > +	cmdline_fixed_string_t cmd_port;
> > >  	cmdline_fixed_string_t cmd_keyword;
> > > -	cmdline_fixed_string_t mode;
> > > -	uint8_t port_id;
> > > +	cmdline_fixed_string_t cmd_onoff;
> > > +	uint8_t cmd_pid;
> > >  };
> > >
> > >  static void
> > > -cmd_enable_gro_parsed(void *parsed_result,
> > > +cmd_gro_enable_parsed(void *parsed_result,
> > >  		__attribute__((unused)) struct cmdline *cl,
> > >  		__attribute__((unused)) void *data)
> > >  {
> > > -	struct cmd_gro_result *res;
> > > +	struct cmd_gro_enable_result *res;
> > >
> > >  	res = parsed_result;
> > > -	setup_gro(res->mode, res->port_id);
> > > -}
> > > -
> > > -cmdline_parse_token_string_t cmd_gro_keyword =
> > > -	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
> > > +	if (!strcmp(res->cmd_keyword, "gro"))
> > > +		setup_gro(res->cmd_onoff, res->cmd_pid);
> > > +}
> > > +
> > > +cmdline_parse_token_string_t cmd_gro_enable_set =
> > > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
> > > +			cmd_set, "set");
> > > +cmdline_parse_token_string_t cmd_gro_enable_port =
> > > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
> > > +			cmd_keyword, "port");
> > > +cmdline_parse_token_num_t cmd_gro_enable_pid =
> > > +	TOKEN_NUM_INITIALIZER(struct cmd_gro_enable_result,
> > > +			cmd_pid, UINT8);
> > > +cmdline_parse_token_string_t cmd_gro_enable_keyword =
> > > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
> > >  			cmd_keyword, "gro");
> > > -cmdline_parse_token_string_t cmd_gro_mode =
> > > -	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
> > > -			mode, "on#off");
> > > -cmdline_parse_token_num_t cmd_gro_pid =
> > > -	TOKEN_NUM_INITIALIZER(struct cmd_gro_result,
> > > -			port_id, UINT8);
> > > +cmdline_parse_token_string_t cmd_gro_enable_onoff =
> > > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
> > > +			cmd_onoff, "on#off");
> > >
> > > -cmdline_parse_inst_t cmd_enable_gro = {
> > > -	.f = cmd_enable_gro_parsed,
> > > +cmdline_parse_inst_t cmd_gro_enable = {
> > > +	.f = cmd_gro_enable_parsed,
> > >  	.data = NULL,
> > > -	.help_str = "gro (on|off) (port_id)",
> > > +	.help_str = "set port <port_id> gro on|off",
> > >  	.tokens = {
> > > -		(void *)&cmd_gro_keyword,
> > > -		(void *)&cmd_gro_mode,
> > > -		(void *)&cmd_gro_pid,
> > > +		(void *)&cmd_gro_enable_set,
> > > +		(void *)&cmd_gro_enable_port,
> > > +		(void *)&cmd_gro_enable_pid,
> > > +		(void *)&cmd_gro_enable_keyword,
> > > +		(void *)&cmd_gro_enable_onoff,
> > >  		NULL,
> > >  	},
> > >  };
> > >
> > > -/* *** SET MAX FLOW NUMBER AND ITEM NUM PER FLOW FOR GRO
> ***
> > */
> > > -struct cmd_gro_set_result {
> > > -	cmdline_fixed_string_t gro;
> > > -	cmdline_fixed_string_t mode;
> > > -	uint16_t flow_num;
> > > -	uint16_t item_num_per_flow;
> > > -	uint8_t port_id;
> > > +/* *** DISPLAY GRO CONFIGURATION *** */
> > > +struct cmd_gro_show_result {
> > > +	cmdline_fixed_string_t cmd_show;
> > > +	cmdline_fixed_string_t cmd_port;
> > > +	cmdline_fixed_string_t cmd_keyword;
> > > +	uint8_t cmd_pid;
> > >  };
> > >
> > >  static void
> > > -cmd_gro_set_parsed(void *parsed_result,
> > > -		       __attribute__((unused)) struct cmdline *cl,
> > > -		       __attribute__((unused)) void *data)
> > > +cmd_gro_show_parsed(void *parsed_result,
> > > +		__attribute__((unused)) struct cmdline *cl,
> > > +		__attribute__((unused)) void *data)
> > >  {
> > > -	struct cmd_gro_set_result *res = parsed_result;
> > > +	struct cmd_gro_show_result *res;
> > >
> > > -	if (port_id_is_invalid(res->port_id, ENABLED_WARN))
> > > -		return;
> > > -	if (test_done == 0) {
> > > -		printf("Before set GRO flow_num and item_num_per_flow,"
> > > -				" please stop forwarding first\n");
> > > -		return;
> > > -	}
> > > +	res = parsed_result;
> > > +	if (!strcmp(res->cmd_keyword, "gro"))
> > > +		show_gro(res->cmd_pid);
> > > +}
> > > +
> > > +cmdline_parse_token_string_t cmd_gro_show_show =
> > > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
> > > +			cmd_show, "show");
> > > +cmdline_parse_token_string_t cmd_gro_show_port =
> > > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
> > > +			cmd_port, "port");
> > > +cmdline_parse_token_num_t cmd_gro_show_pid =
> > > +	TOKEN_NUM_INITIALIZER(struct cmd_gro_show_result,
> > > +			cmd_pid, UINT8);
> > > +cmdline_parse_token_string_t cmd_gro_show_keyword =
> > > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
> > > +			cmd_keyword, "gro");
> > >
> > > -	if (!strcmp(res->mode, "set")) {
> > > -		if (res->flow_num == 0)
> > > -			printf("Invalid flow number. Revert to default
> > value:"
> > > -					" %u.\n",
> > > GRO_DEFAULT_FLOW_NUM);
> > > -		else
> > > -			gro_ports[res->port_id].param.max_flow_num =
> > > -				res->flow_num;
> > > +cmdline_parse_inst_t cmd_gro_show = {
> > > +	.f = cmd_gro_show_parsed,
> > > +	.data = NULL,
> > > +	.help_str = "show port <port_id> gro",
> > > +	.tokens = {
> > > +		(void *)&cmd_gro_show_show,
> > > +		(void *)&cmd_gro_show_port,
> > > +		(void *)&cmd_gro_show_pid,
> > > +		(void *)&cmd_gro_show_keyword,
> > > +		NULL,
> > > +	},
> > > +};
> > >
> > > -		if (res->item_num_per_flow == 0)
> > > -			printf("Invalid item number per-flow. Revert"
> > > -					" to default value:%u.\n",
> > > -
> > > 	GRO_DEFAULT_ITEM_NUM_PER_FLOW);
> > > -		else
> > > -			gro_ports[res->port_id].param.max_item_per_flow
> > > =
> > > -				res->item_num_per_flow;
> > > -	}
> > > +/* *** SET FLUSH CYCLES FOR GRO *** */
> > > +struct cmd_gro_flush_result {
> > > +	cmdline_fixed_string_t cmd_set;
> > > +	cmdline_fixed_string_t cmd_keyword;
> > > +	cmdline_fixed_string_t cmd_flush;
> > > +	uint8_t cmd_cycles;
> > > +};
> > > +
> > > +static void
> > > +cmd_gro_flush_parsed(void *parsed_result,
> > > +		__attribute__((unused)) struct cmdline *cl,
> > > +		__attribute__((unused)) void *data)
> > > +{
> > > +	struct cmd_gro_flush_result *res;
> > > +
> > > +	res = parsed_result;
> > > +	if ((!strcmp(res->cmd_keyword, "gro")) &&
> > > +			(!strcmp(res->cmd_flush, "flush")))
> > > +		setup_gro_flush_cycles(res->cmd_cycles);
> > >  }
> > >
> > > -cmdline_parse_token_string_t cmd_gro_set_gro =
> > > -	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
> > > -				gro, "gro");
> > > -cmdline_parse_token_string_t cmd_gro_set_mode =
> > > -	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
> > > -				mode, "set");
> > > -cmdline_parse_token_num_t cmd_gro_set_flow_num =
> > > -	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> > > -				flow_num, UINT16);
> > > -cmdline_parse_token_num_t cmd_gro_set_item_num_per_flow =
> > > -	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> > > -				item_num_per_flow, UINT16);
> > > -cmdline_parse_token_num_t cmd_gro_set_portid =
> > > -	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> > > -				port_id, UINT8);
> > > +cmdline_parse_token_string_t cmd_gro_flush_set =
> > > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
> > > +			cmd_set, "set");
> > > +cmdline_parse_token_string_t cmd_gro_flush_keyword =
> > > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
> > > +			cmd_keyword, "gro");
> > > +cmdline_parse_token_string_t cmd_gro_flush_flush =
> > > +	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
> > > +			cmd_flush, "flush");
> > > +cmdline_parse_token_num_t cmd_gro_flush_cycles =
> > > +	TOKEN_NUM_INITIALIZER(struct cmd_gro_flush_result,
> > > +			cmd_cycles, UINT8);
> > >
> > > -cmdline_parse_inst_t cmd_gro_set = {
> > > -	.f = cmd_gro_set_parsed,
> > > +cmdline_parse_inst_t cmd_gro_flush = {
> > > +	.f = cmd_gro_flush_parsed,
> > >  	.data = NULL,
> > > -	.help_str = "gro set <max_flow_num> <max_item_num_per_flow>
> > > "
> > > -		"<port_id>: set max flow number and max packet number
> > > per-flow "
> > > -		"for GRO",
> > > +	.help_str = "set gro flush <cycles>",
> > >  	.tokens = {
> > > -		(void *)&cmd_gro_set_gro,
> > > -		(void *)&cmd_gro_set_mode,
> > > -		(void *)&cmd_gro_set_flow_num,
> > > -		(void *)&cmd_gro_set_item_num_per_flow,
> > > -		(void *)&cmd_gro_set_portid,
> > > +		(void *)&cmd_gro_flush_set,
> > > +		(void *)&cmd_gro_flush_keyword,
> > > +		(void *)&cmd_gro_flush_flush,
> > > +		(void *)&cmd_gro_flush_cycles,
> > >  		NULL,
> > >  	},
> > >  };
> > > @@ -14249,8 +14282,9 @@ cmdline_parse_ctx_t main_ctx[] = {
> > >  	(cmdline_parse_inst_t *)&cmd_tso_show,
> > >  	(cmdline_parse_inst_t *)&cmd_tunnel_tso_set,
> > >  	(cmdline_parse_inst_t *)&cmd_tunnel_tso_show,
> > > -	(cmdline_parse_inst_t *)&cmd_enable_gro,
> > > -	(cmdline_parse_inst_t *)&cmd_gro_set,
> > > +	(cmdline_parse_inst_t *)&cmd_gro_enable,
> > > +	(cmdline_parse_inst_t *)&cmd_gro_flush,
> > > +	(cmdline_parse_inst_t *)&cmd_gro_show,
> > >  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set,
> > >  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_rx,
> > >  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_tx,
> > > diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
> > > index 3ae3e1c..d97d291 100644
> > > --- a/app/test-pmd/config.c
> > > +++ b/app/test-pmd/config.c
> > > @@ -2420,7 +2420,7 @@ set_tx_pkt_segments(unsigned *seg_lengths,
> > > unsigned nb_segs)
> > >  }
> > >
> > >  void
> > > -setup_gro(const char *mode, uint8_t port_id)
> > > +setup_gro(const char *onoff, uint8_t port_id)
> > >  {
> > >  	if (!rte_eth_dev_is_valid_port(port_id)) {
> > >  		printf("invalid port id %u\n", port_id);
> > > @@ -2431,29 +2431,76 @@ setup_gro(const char *mode, uint8_t
> port_id)
> > >  				" please stop forwarding first\n");
> > >  		return;
> > >  	}
> > > -	if (strcmp(mode, "on") == 0) {
> > > -		if (gro_ports[port_id].enable) {
> > > -			printf("port %u has enabled GRO\n", port_id);
> > > +	if (strcmp(onoff, "on") == 0) {
> > > +		if (gro_ports[port_id].enable != 0) {
> > > +			printf("Port %u has enabled GRO. Please"
> > > +					" disable GRO first\n", port_id);
> > >  			return;
> > >  		}
> > > -		gro_ports[port_id].enable = 1;
> > > -		gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4;
> > > -
> > > -		if (gro_ports[port_id].param.max_flow_num == 0)
> > > +		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
> > > +			gro_ports[port_id].param.gro_types =
> > > RTE_GRO_TCP_IPV4;
> > >  			gro_ports[port_id].param.max_flow_num =
> > >  				GRO_DEFAULT_FLOW_NUM;
> > > -		if (gro_ports[port_id].param.max_item_per_flow == 0)
> > >  			gro_ports[port_id].param.max_item_per_flow =
> > >  				GRO_DEFAULT_ITEM_NUM_PER_FLOW;
> > > +		}
> > > +		gro_ports[port_id].enable = 1;
> > >  	} else {
> > >  		if (gro_ports[port_id].enable == 0) {
> > > -			printf("port %u has disabled GRO\n", port_id);
> > > +			printf("Port %u has disabled GRO\n", port_id);
> > >  			return;
> > >  		}
> > >  		gro_ports[port_id].enable = 0;
> > >  	}
> > >  }
> > >
> > > +void
> > > +setup_gro_flush_cycles(uint8_t cycles)
> > > +{
> > > +	if (test_done == 0) {
> > > +		printf("Before change flush interval for GRO,"
> > > +				" please stop forwarding first\n");
> > > +		return;
> > > +	}
> > > +
> > > +	if (cycles > GRO_MAX_FLUSH_CYCLES) {
> > > +		printf("The flushing cycle be in the range"
> > > +				" of 1 to %u. Revert to the default"
> > > +				" value %u\n",
> > > +				GRO_MAX_FLUSH_CYCLES,
> > > +				GRO_DEFAULT_FLUSH_CYCLES);
> > > +		cycles = GRO_DEFAULT_FLUSH_CYCLES;
> > > +	}
> > > +
> > > +	gro_flush_cycles = cycles;
> > > +}
> > > +
> > > +void
> > > +show_gro(uint8_t port_id)
> > > +{
> > > +	struct rte_gro_param *param;
> > > +	uint32_t max_pkts_num;
> > > +
> > > +	param = &gro_ports[port_id].param;
> > > +
> > > +	if (!rte_eth_dev_is_valid_port(port_id)) {
> > > +		printf("Invalid port id %u\n", port_id);
> > > +		return;
> > > +	}
> > > +	if (gro_ports[port_id].enable) {
> > > +		printf("GRO type: TCP/IPv4\n");
> > > +		if (gro_flush_cycles > 0)
> > > +			max_pkts_num = MAX_PKT_BURST *
> > > GRO_MAX_FLUSH_CYCLES;
> > > +		else {
> > > +			max_pkts_num = param->max_flow_num *
> > > +				param->max_item_per_flow;
> > > +		}
> > > +		printf("Max packet number to GRO: %u\n", max_pkts_num);
> > > +		printf("Flushing cycles: %u\n", gro_flush_cycles);
> > > +	} else
> > > +		printf("Port %u doesn't enable GRO\n", port_id);
> > > +}
> > > +
> > >  char*
> > >  list_pkt_forwarding_modes(void)
> > >  {
> > > diff --git a/app/test-pmd/csumonly.c b/app/test-pmd/csumonly.c
> > > index 90c8119..ca50ab7 100644
> > > --- a/app/test-pmd/csumonly.c
> > > +++ b/app/test-pmd/csumonly.c
> > > @@ -631,6 +631,9 @@ pkt_burst_checksum_forward(struct fwd_stream
> > *fs)
> > >  	struct rte_mbuf *m, *p;
> > >  	struct ether_hdr *eth_hdr;
> > >  	void *l3_hdr = NULL, *outer_l3_hdr = NULL; /* can be IPv4 or IPv6 */
> > > +	void **gro_ctx;
> > > +	uint16_t gro_pkts_num;
> > > +	uint8_t gro_enable;
> > >  	uint16_t nb_rx;
> > >  	uint16_t nb_tx;
> > >  	uint16_t nb_prep;
> > > @@ -657,17 +660,13 @@ pkt_burst_checksum_forward(struct
> > fwd_stream
> > > *fs)
> > >  				 nb_pkt_per_burst);
> > >  	if (unlikely(nb_rx == 0))
> > >  		return;
> > > -	if (unlikely(gro_ports[fs->rx_port].enable))
> > > -		nb_rx = rte_gro_reassemble_burst(pkts_burst,
> > > -				nb_rx,
> > > -				&(gro_ports[fs->rx_port].param));
> > > -
> > >  #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
> > >  	fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
> > >  #endif
> > >  	fs->rx_packets += nb_rx;
> > >  	rx_bad_ip_csum = 0;
> > >  	rx_bad_l4_csum = 0;
> > > +	gro_enable = gro_ports[fs->rx_port].enable;
> > >
> > >  	txp = &ports[fs->tx_port];
> > >  	testpmd_ol_flags = txp->tx_ol_flags;
> > > @@ -851,6 +850,28 @@ pkt_burst_checksum_forward(struct
> fwd_stream
> > > *fs)
> > >  		}
> > >  	}
> > >
> > > +	if (unlikely(gro_enable)) {
> > > +		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
> > > +			nb_rx = rte_gro_reassemble_burst(pkts_burst,
> > > nb_rx,
> > > +					&(gro_ports[fs->rx_port].param));
> > > +		} else {
> > > +			gro_ctx = current_fwd_lcore()->gro_ctx;
> > > +			nb_rx = rte_gro_reassemble(pkts_burst, nb_rx,
> > > gro_ctx);
> > > +
> > > +			if (++fs->gro_times >= gro_flush_cycles) {
> > > +				gro_pkts_num =
> > > rte_gro_get_pkt_count(gro_ctx);
> > > +				if (gro_pkts_num > MAX_PKT_BURST - nb_rx)
> > > +					gro_pkts_num = MAX_PKT_BURST -
> > > nb_rx;
> > > +
> > > +				nb_rx += rte_gro_timeout_flush(gro_ctx, 0,
> > > +						RTE_GRO_TCP_IPV4,
> > > +						&pkts_burst[nb_rx],
> > > +						gro_pkts_num);
> > > +				fs->gro_times = 0;
> > > +			}
> > > +		}
> > > +	}
> > > +
> > >  	nb_prep = rte_eth_tx_prepare(fs->tx_port, fs->tx_queue,
> > >  			pkts_burst, nb_rx);
> > >  	if (nb_prep != nb_rx)
> > > diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
> > > index 7d40139..c1423d5 100644
> > > --- a/app/test-pmd/testpmd.c
> > > +++ b/app/test-pmd/testpmd.c
> > > @@ -90,7 +90,6 @@
> > >  #ifdef RTE_LIBRTE_LATENCY_STATS
> > >  #include <rte_latencystats.h>
> > >  #endif
> > > -#include <rte_gro.h>
> > >
> > >  #include "testpmd.h"
> > >
> > > @@ -386,6 +385,7 @@ uint8_t bitrate_enabled;
> > >  #endif
> > >
> > >  struct gro_status gro_ports[RTE_MAX_ETHPORTS];
> > > +uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
> > >
> > >  /* Forward function declarations */
> > >  static void map_port_queue_stats_mapping_registers(uint8_t pi, struct
> > > rte_port *port);
> > > @@ -570,6 +570,7 @@ init_config(void)
> > >  	unsigned int nb_mbuf_per_pool;
> > >  	lcoreid_t  lc_id;
> > >  	uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
> > > +	struct rte_gro_param gro_param;
> > >
> > >  	memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
> > >
> > > @@ -671,6 +672,20 @@ init_config(void)
> > >  		rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
> > >
> > >  	fwd_config_setup();
> > > +
> > > +	/* create a gro context for each lcore */
> > > +	gro_param.gro_types = RTE_GRO_TCP_IPV4;
> > > +	gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
> > > +	gro_param.max_item_per_flow = MAX_PKT_BURST;
> > > +	for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
> > > +		gro_param.socket_id = rte_lcore_to_socket_id(
> > > +				fwd_lcores_cpuids[lc_id]);
> > > +		fwd_lcores[lc_id]->gro_ctx =
> > > rte_gro_ctx_create(&gro_param);
> > > +		if (fwd_lcores[lc_id]->gro_ctx == NULL) {
> > > +			rte_exit(EXIT_FAILURE,
> > > +					"rte_gro_ctx_create() failed\n");
> > > +		}
> > > +	}
> > >  }
> > >
> > >
> > > @@ -1165,6 +1180,7 @@ start_packet_forwarding(int with_tx_first)
> > >  		fwd_streams[sm_id]->fwd_dropped = 0;
> > >  		fwd_streams[sm_id]->rx_bad_ip_csum = 0;
> > >  		fwd_streams[sm_id]->rx_bad_l4_csum = 0;
> > > +		fwd_streams[sm_id]->gro_times = 0;
> > >
> > >  #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
> > >  		memset(&fwd_streams[sm_id]->rx_burst_stats, 0,
> > > diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
> > > index c9d7739..e878bcb 100644
> > > --- a/app/test-pmd/testpmd.h
> > > +++ b/app/test-pmd/testpmd.h
> > > @@ -120,6 +120,7 @@ struct fwd_stream {
> > >  	unsigned int fwd_dropped; /**< received packets not forwarded */
> > >  	unsigned int rx_bad_ip_csum ; /**< received packets has bad ip
> > > checksum */
> > >  	unsigned int rx_bad_l4_csum ; /**< received packets has bad l4
> > > checksum */
> > > +	unsigned int gro_times;	/**< GRO operation times */
> > >  #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
> > >  	uint64_t     core_cycles; /**< used for RX and TX processing */
> > >  #endif
> > > @@ -206,6 +207,7 @@ struct rte_port {
> > >   */
> > >  struct fwd_lcore {
> > >  	struct rte_mempool *mbp; /**< The mbuf pool to use by this core */
> > > +	void *gro_ctx;		/**< GRO context */
> > >  	streamid_t stream_idx;   /**< index of 1st stream in "fwd_streams"
> > > */
> > >  	streamid_t stream_nb;    /**< number of streams in "fwd_streams"
> > > */
> > >  	lcoreid_t  cpuid_idx;    /**< index of logical core in CPU id table */
> > > @@ -434,13 +436,19 @@ extern struct ether_addr
> > > peer_eth_addrs[RTE_MAX_ETHPORTS];
> > >  extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for
> mac-
> > > retry. */
> > >  extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-
> > > retry. */
> > >
> > > -#define GRO_DEFAULT_FLOW_NUM 4
> > > -#define GRO_DEFAULT_ITEM_NUM_PER_FLOW DEF_PKT_BURST
> > > +#define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
> > > +#define GRO_DEFAULT_FLOW_NUM
> > (RTE_GRO_MAX_BURST_ITEM_NUM
> > > / \
> > > +		GRO_DEFAULT_ITEM_NUM_PER_FLOW)
> > > +
> > > +#define GRO_DEFAULT_FLUSH_CYCLES 1
> > > +#define GRO_MAX_FLUSH_CYCLES 4
> > > +
> > >  struct gro_status {
> > >  	struct rte_gro_param param;
> > >  	uint8_t enable;
> > >  };
> > >  extern struct gro_status gro_ports[RTE_MAX_ETHPORTS];
> > > +extern uint8_t gro_flush_cycles;
> > >
> > >  static inline unsigned int
> > >  lcore_num(void)
> > > @@ -640,7 +648,9 @@ void get_2tuple_filter(uint8_t port_id, uint16_t
> > > index);
> > >  void get_5tuple_filter(uint8_t port_id, uint16_t index);
> > >  int rx_queue_id_is_invalid(queueid_t rxq_id);
> > >  int tx_queue_id_is_invalid(queueid_t txq_id);
> > > -void setup_gro(const char *mode, uint8_t port_id);
> > > +void setup_gro(const char *onoff, uint8_t port_id);
> > > +void setup_gro_flush_cycles(uint8_t cycles);
> > > +void show_gro(uint8_t port_id);
> > >
> > >  /* Functions to manage the set of filtered Multicast MAC addresses */
> > >  void mcast_addr_add(uint8_t port_id, struct ether_addr *mc_addr);
> > > diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> > > b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> > > index 2ed62f5..7f21b7d 100644
> > > --- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> > > +++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> > > @@ -898,12 +898,12 @@ Display the status of TCP Segmentation
> Offload::
> > >
> > >     testpmd> tso show (port_id)
> > >
> > > -gro
> > > -~~~
> > > +set port - gro
> > > +~~~~~~~~~~~~~~
> > >
> > >  Enable or disable GRO in ``csum`` forwarding engine::
> > >
> > > -   testpmd> gro (on|off) (port_id)
> > > +   testpmd> set port <port_id> gro on|off
> > >
> > >  If enabled, the csum forwarding engine will perform GRO on the
> TCP/IPv4
> > >  packets received from the given port.
> > > @@ -914,23 +914,38 @@ GRO. By default, GRO is disabled for all ports.
> > >  .. note::
> > >
> > >     When enable GRO for a port, TCP/IPv4 packets received from the port
> > > -   will be performed GRO. After GRO, the merged packets are multi-
> > > segments.
> > > -   But csum forwarding engine doesn't support to calculate TCP
> checksum
> > > -   for multi-segment packets in SW. So please select TCP HW checksum
> > > -   calculation for the port which GROed packets are transmitted to.
> > > +   will be performed GRO. After GRO, all merged packets have bad
> > > +   checksums, since the GRO library doesn't re-calculate checksums for
> > > +   the merged packets. Therefore, if users want the merged packets to
> > > +   have correct checksums, please select IP and TCP HW checksum
> > > calculation
> > > +   for the port which the merged packets are transmitted to.
> > >
> > > -gro set
> > > -~~~~~~~
> > > +show port - gro
> > > +~~~~~~~~~~~~~~~
> > > +
> > > +Display GRO configuration for a given port::
> > > +
> > > +   testpmd> show port <port_id> gro
> > > +
> > > +set gro flush
> > > +~~~~~~~~~~~~~
> > > +
> > > +Set the cycle to flush the GROed packets from reassembly tables::
> > >
> > > -Set max flow number and max packet number per-flow for GRO::
> > > +   testpmd> set gro flush <cycles>
> > >
> > > -   testpmd> gro set (max_flow_num) (max_item_num_per_flow) (port_id)
> > > +When enable GRO, the csum forwarding engine performs GRO on
> > received
> > > +packets, and the GROed packets are stored in reassembly tables. Users
> > > +can use this command to determine when the GROed packets are
> flushed
> > > +from the reassembly tables.
> > >
> > > -The product of ``max_flow_num`` and ``max_item_num_per_flow`` is
> the
> > > max
> > > -number of packets a GRO table can store.
> > > +The ``cycles`` is measured in GRO operation times. The csum forwarding
> > > +engine flushes the GROed packets from the tables every ``cycles`` GRO
> > > +operations.
> > >
> > > -If current packet number is greater than or equal to the max value, GRO
> > > -will stop processing incoming packets.
> > > +By default, the value of ``cycles`` is 1, which means flush GROed packets
> > > +from the reassembly tables as soon as one GRO operation finishes. The
> > > value
> > > +of ``cycles`` should be in the range of 1 to ``GRO_MAX_FLUSH_CYCLES``.
> > >
> > >  mac_addr add
> > >  ~~~~~~~~~~~~
> > > --
> > > 2.7.4

[dpdk-dev] [PATCH v4] app/testpmd: enable the heavyweight mode TCP/IPv4 GRO

[Jiayu Hu]

The GRO library provides two modes to reassemble packets. Currently, the
csum forwarding engine has supported to use the lightweight mode to
reassemble TCP/IPv4 packets. This patch introduces the heavyweight mode
for TCP/IPv4 GRO in the csum forwarding engine.

With the command "set port <port_id> gro on|off", users can enable
TCP/IPv4 GRO for a given port. With the command "set gro flush <cycles>",
users can determine when the GROed TCP/IPv4 packets are flushed from
reassembly tables. With the command "show port <port_id> gro", users can
display GRO configuration.

The GRO library doesn't re-calculate checksums for merged packets. If
users want the merged packets to have correct IP and TCP checksums,
please select HW IP checksum calculation and HW TCP checksum calculation
for the port which the merged packets are transmitted to.

Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
Reviewed-by: Ferruh Yigit <ferruh.yigit@intel.com>
---
changes in v4:
- fix unchecking the min value of 'cycle' bug in setup_gro_flush_cycles
- update the context of the testpmd document and commit logs
changes in v3:
- remove "heavyweight mode" and "lightweight mode" from GRO commands
- combine two patches into one
- use consistent help string for GRO commands 
- remove the unnecessary command "gro set (max_flow_num)
	(max_item_num_per_flow) (port_id)"
changes in v2:
- use "set" and "show" as the root level command
- add a new command to show GRO configuration
- fix l2_len/l3_len/l4_len unset etc. bugs

 app/test-pmd/cmdline.c                      | 206 ++++++++++++++++------------
 app/test-pmd/config.c                       |  68 +++++++--
 app/test-pmd/csumonly.c                     |  31 ++++-
 app/test-pmd/testpmd.c                      |  19 ++-
 app/test-pmd/testpmd.h                      |  16 ++-
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  50 +++++--
 6 files changed, 270 insertions(+), 120 deletions(-)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index ccdf239..e44c02e 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -423,13 +423,16 @@ static void cmd_help_long_parsed(void *parsed_result,
 			"tso show (portid)"
 			"    Display the status of TCP Segmentation Offload.\n\n"
 
-			"gro (on|off) (port_id)"
+			"set port (port_id) gro on|off\n"
 			"    Enable or disable Generic Receive Offload in"
 			" csum forwarding engine.\n\n"
 
-			"gro set (max_flow_num) (max_item_num_per_flow) (port_id)\n"
-			"    Set max flow number and max packet number per-flow"
-			" for GRO.\n\n"
+			"show port (port_id) gro\n"
+			"    Display GRO configuration.\n\n"
+
+			"set gro flush (cycles)\n"
+			"    Set the cycle to flush GROed packets from"
+			" reassembly tables.\n\n"
 
 			"set fwd (%s)\n"
 			"    Set packet forwarding mode.\n\n"
@@ -3854,115 +3857,145 @@ cmdline_parse_inst_t cmd_tunnel_tso_show = {
 };
 
 /* *** SET GRO FOR A PORT *** */
-struct cmd_gro_result {
+struct cmd_gro_enable_result {
+	cmdline_fixed_string_t cmd_set;
+	cmdline_fixed_string_t cmd_port;
 	cmdline_fixed_string_t cmd_keyword;
-	cmdline_fixed_string_t mode;
-	uint8_t port_id;
+	cmdline_fixed_string_t cmd_onoff;
+	uint8_t cmd_pid;
 };
 
 static void
-cmd_enable_gro_parsed(void *parsed_result,
+cmd_gro_enable_parsed(void *parsed_result,
 		__attribute__((unused)) struct cmdline *cl,
 		__attribute__((unused)) void *data)
 {
-	struct cmd_gro_result *res;
+	struct cmd_gro_enable_result *res;
 
 	res = parsed_result;
-	setup_gro(res->mode, res->port_id);
-}
-
-cmdline_parse_token_string_t cmd_gro_keyword =
-	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
+	if (!strcmp(res->cmd_keyword, "gro"))
+		setup_gro(res->cmd_onoff, res->cmd_pid);
+}
+
+cmdline_parse_token_string_t cmd_gro_enable_set =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_set, "set");
+cmdline_parse_token_string_t cmd_gro_enable_port =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_keyword, "port");
+cmdline_parse_token_num_t cmd_gro_enable_pid =
+	TOKEN_NUM_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_pid, UINT8);
+cmdline_parse_token_string_t cmd_gro_enable_keyword =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
 			cmd_keyword, "gro");
-cmdline_parse_token_string_t cmd_gro_mode =
-	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
-			mode, "on#off");
-cmdline_parse_token_num_t cmd_gro_pid =
-	TOKEN_NUM_INITIALIZER(struct cmd_gro_result,
-			port_id, UINT8);
+cmdline_parse_token_string_t cmd_gro_enable_onoff =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_onoff, "on#off");
 
-cmdline_parse_inst_t cmd_enable_gro = {
-	.f = cmd_enable_gro_parsed,
+cmdline_parse_inst_t cmd_gro_enable = {
+	.f = cmd_gro_enable_parsed,
 	.data = NULL,
-	.help_str = "gro (on|off) (port_id)",
+	.help_str = "set port <port_id> gro on|off",
 	.tokens = {
-		(void *)&cmd_gro_keyword,
-		(void *)&cmd_gro_mode,
-		(void *)&cmd_gro_pid,
+		(void *)&cmd_gro_enable_set,
+		(void *)&cmd_gro_enable_port,
+		(void *)&cmd_gro_enable_pid,
+		(void *)&cmd_gro_enable_keyword,
+		(void *)&cmd_gro_enable_onoff,
 		NULL,
 	},
 };
 
-/* *** SET MAX FLOW NUMBER AND ITEM NUM PER FLOW FOR GRO *** */
-struct cmd_gro_set_result {
-	cmdline_fixed_string_t gro;
-	cmdline_fixed_string_t mode;
-	uint16_t flow_num;
-	uint16_t item_num_per_flow;
-	uint8_t port_id;
+/* *** DISPLAY GRO CONFIGURATION *** */
+struct cmd_gro_show_result {
+	cmdline_fixed_string_t cmd_show;
+	cmdline_fixed_string_t cmd_port;
+	cmdline_fixed_string_t cmd_keyword;
+	uint8_t cmd_pid;
 };
 
 static void
-cmd_gro_set_parsed(void *parsed_result,
-		       __attribute__((unused)) struct cmdline *cl,
-		       __attribute__((unused)) void *data)
+cmd_gro_show_parsed(void *parsed_result,
+		__attribute__((unused)) struct cmdline *cl,
+		__attribute__((unused)) void *data)
 {
-	struct cmd_gro_set_result *res = parsed_result;
+	struct cmd_gro_show_result *res;
 
-	if (port_id_is_invalid(res->port_id, ENABLED_WARN))
-		return;
-	if (test_done == 0) {
-		printf("Before set GRO flow_num and item_num_per_flow,"
-				" please stop forwarding first\n");
-		return;
-	}
+	res = parsed_result;
+	if (!strcmp(res->cmd_keyword, "gro"))
+		show_gro(res->cmd_pid);
+}
+
+cmdline_parse_token_string_t cmd_gro_show_show =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
+			cmd_show, "show");
+cmdline_parse_token_string_t cmd_gro_show_port =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
+			cmd_port, "port");
+cmdline_parse_token_num_t cmd_gro_show_pid =
+	TOKEN_NUM_INITIALIZER(struct cmd_gro_show_result,
+			cmd_pid, UINT8);
+cmdline_parse_token_string_t cmd_gro_show_keyword =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
+			cmd_keyword, "gro");
 
-	if (!strcmp(res->mode, "set")) {
-		if (res->flow_num == 0)
-			printf("Invalid flow number. Revert to default value:"
-					" %u.\n", GRO_DEFAULT_FLOW_NUM);
-		else
-			gro_ports[res->port_id].param.max_flow_num =
-				res->flow_num;
+cmdline_parse_inst_t cmd_gro_show = {
+	.f = cmd_gro_show_parsed,
+	.data = NULL,
+	.help_str = "show port <port_id> gro",
+	.tokens = {
+		(void *)&cmd_gro_show_show,
+		(void *)&cmd_gro_show_port,
+		(void *)&cmd_gro_show_pid,
+		(void *)&cmd_gro_show_keyword,
+		NULL,
+	},
+};
 
-		if (res->item_num_per_flow == 0)
-			printf("Invalid item number per-flow. Revert"
-					" to default value:%u.\n",
-					GRO_DEFAULT_ITEM_NUM_PER_FLOW);
-		else
-			gro_ports[res->port_id].param.max_item_per_flow =
-				res->item_num_per_flow;
-	}
+/* *** SET FLUSH CYCLES FOR GRO *** */
+struct cmd_gro_flush_result {
+	cmdline_fixed_string_t cmd_set;
+	cmdline_fixed_string_t cmd_keyword;
+	cmdline_fixed_string_t cmd_flush;
+	uint8_t cmd_cycles;
+};
+
+static void
+cmd_gro_flush_parsed(void *parsed_result,
+		__attribute__((unused)) struct cmdline *cl,
+		__attribute__((unused)) void *data)
+{
+	struct cmd_gro_flush_result *res;
+
+	res = parsed_result;
+	if ((!strcmp(res->cmd_keyword, "gro")) &&
+			(!strcmp(res->cmd_flush, "flush")))
+		setup_gro_flush_cycles(res->cmd_cycles);
 }
 
-cmdline_parse_token_string_t cmd_gro_set_gro =
-	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
-				gro, "gro");
-cmdline_parse_token_string_t cmd_gro_set_mode =
-	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
-				mode, "set");
-cmdline_parse_token_num_t cmd_gro_set_flow_num =
-	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
-				flow_num, UINT16);
-cmdline_parse_token_num_t cmd_gro_set_item_num_per_flow =
-	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
-				item_num_per_flow, UINT16);
-cmdline_parse_token_num_t cmd_gro_set_portid =
-	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
-				port_id, UINT8);
+cmdline_parse_token_string_t cmd_gro_flush_set =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_set, "set");
+cmdline_parse_token_string_t cmd_gro_flush_keyword =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_keyword, "gro");
+cmdline_parse_token_string_t cmd_gro_flush_flush =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_flush, "flush");
+cmdline_parse_token_num_t cmd_gro_flush_cycles =
+	TOKEN_NUM_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_cycles, UINT8);
 
-cmdline_parse_inst_t cmd_gro_set = {
-	.f = cmd_gro_set_parsed,
+cmdline_parse_inst_t cmd_gro_flush = {
+	.f = cmd_gro_flush_parsed,
 	.data = NULL,
-	.help_str = "gro set <max_flow_num> <max_item_num_per_flow> "
-		"<port_id>: set max flow number and max packet number per-flow "
-		"for GRO",
+	.help_str = "set gro flush <cycles>",
 	.tokens = {
-		(void *)&cmd_gro_set_gro,
-		(void *)&cmd_gro_set_mode,
-		(void *)&cmd_gro_set_flow_num,
-		(void *)&cmd_gro_set_item_num_per_flow,
-		(void *)&cmd_gro_set_portid,
+		(void *)&cmd_gro_flush_set,
+		(void *)&cmd_gro_flush_keyword,
+		(void *)&cmd_gro_flush_flush,
+		(void *)&cmd_gro_flush_cycles,
 		NULL,
 	},
 };
@@ -14253,8 +14286,9 @@ cmdline_parse_ctx_t main_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_tso_show,
 	(cmdline_parse_inst_t *)&cmd_tunnel_tso_set,
 	(cmdline_parse_inst_t *)&cmd_tunnel_tso_show,
-	(cmdline_parse_inst_t *)&cmd_enable_gro,
-	(cmdline_parse_inst_t *)&cmd_gro_set,
+	(cmdline_parse_inst_t *)&cmd_gro_enable,
+	(cmdline_parse_inst_t *)&cmd_gro_flush,
+	(cmdline_parse_inst_t *)&cmd_gro_show,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_rx,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_tx,
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index 3ae3e1c..92220b1 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -2420,7 +2420,7 @@ set_tx_pkt_segments(unsigned *seg_lengths, unsigned nb_segs)
 }
 
 void
-setup_gro(const char *mode, uint8_t port_id)
+setup_gro(const char *onoff, uint8_t port_id)
 {
 	if (!rte_eth_dev_is_valid_port(port_id)) {
 		printf("invalid port id %u\n", port_id);
@@ -2431,29 +2431,77 @@ setup_gro(const char *mode, uint8_t port_id)
 				" please stop forwarding first\n");
 		return;
 	}
-	if (strcmp(mode, "on") == 0) {
-		if (gro_ports[port_id].enable) {
-			printf("port %u has enabled GRO\n", port_id);
+	if (strcmp(onoff, "on") == 0) {
+		if (gro_ports[port_id].enable != 0) {
+			printf("Port %u has enabled GRO. Please"
+					" disable GRO first\n", port_id);
 			return;
 		}
-		gro_ports[port_id].enable = 1;
-		gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4;
-
-		if (gro_ports[port_id].param.max_flow_num == 0)
+		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
+			gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4;
 			gro_ports[port_id].param.max_flow_num =
 				GRO_DEFAULT_FLOW_NUM;
-		if (gro_ports[port_id].param.max_item_per_flow == 0)
 			gro_ports[port_id].param.max_item_per_flow =
 				GRO_DEFAULT_ITEM_NUM_PER_FLOW;
+		}
+		gro_ports[port_id].enable = 1;
 	} else {
 		if (gro_ports[port_id].enable == 0) {
-			printf("port %u has disabled GRO\n", port_id);
+			printf("Port %u has disabled GRO\n", port_id);
 			return;
 		}
 		gro_ports[port_id].enable = 0;
 	}
 }
 
+void
+setup_gro_flush_cycles(uint8_t cycles)
+{
+	if (test_done == 0) {
+		printf("Before change flush interval for GRO,"
+				" please stop forwarding first.\n");
+		return;
+	}
+
+	if (cycles > GRO_MAX_FLUSH_CYCLES || cycles <
+			GRO_DEFAULT_FLUSH_CYCLES) {
+		printf("The flushing cycle be in the range"
+				" of 1 to %u. Revert to the default"
+				" value %u.\n",
+				GRO_MAX_FLUSH_CYCLES,
+				GRO_DEFAULT_FLUSH_CYCLES);
+		cycles = GRO_DEFAULT_FLUSH_CYCLES;
+	}
+
+	gro_flush_cycles = cycles;
+}
+
+void
+show_gro(uint8_t port_id)
+{
+	struct rte_gro_param *param;
+	uint32_t max_pkts_num;
+
+	param = &gro_ports[port_id].param;
+
+	if (!rte_eth_dev_is_valid_port(port_id)) {
+		printf("Invalid port id %u.\n", port_id);
+		return;
+	}
+	if (gro_ports[port_id].enable) {
+		printf("GRO type: TCP/IPv4\n");
+		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
+			max_pkts_num = param->max_flow_num *
+				param->max_item_per_flow;
+		} else
+			max_pkts_num = MAX_PKT_BURST * GRO_MAX_FLUSH_CYCLES;
+		printf("Max number of packets to perform GRO: %u\n",
+				max_pkts_num);
+		printf("Flushing cycles: %u\n", gro_flush_cycles);
+	} else
+		printf("Port %u doesn't enable GRO.\n", port_id);
+}
+
 char*
 list_pkt_forwarding_modes(void)
 {
diff --git a/app/test-pmd/csumonly.c b/app/test-pmd/csumonly.c
index 90c8119..ca50ab7 100644
--- a/app/test-pmd/csumonly.c
+++ b/app/test-pmd/csumonly.c
@@ -631,6 +631,9 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 	struct rte_mbuf *m, *p;
 	struct ether_hdr *eth_hdr;
 	void *l3_hdr = NULL, *outer_l3_hdr = NULL; /* can be IPv4 or IPv6 */
+	void **gro_ctx;
+	uint16_t gro_pkts_num;
+	uint8_t gro_enable;
 	uint16_t nb_rx;
 	uint16_t nb_tx;
 	uint16_t nb_prep;
@@ -657,17 +660,13 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 				 nb_pkt_per_burst);
 	if (unlikely(nb_rx == 0))
 		return;
-	if (unlikely(gro_ports[fs->rx_port].enable))
-		nb_rx = rte_gro_reassemble_burst(pkts_burst,
-				nb_rx,
-				&(gro_ports[fs->rx_port].param));
-
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 	fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
 #endif
 	fs->rx_packets += nb_rx;
 	rx_bad_ip_csum = 0;
 	rx_bad_l4_csum = 0;
+	gro_enable = gro_ports[fs->rx_port].enable;
 
 	txp = &ports[fs->tx_port];
 	testpmd_ol_flags = txp->tx_ol_flags;
@@ -851,6 +850,28 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 		}
 	}
 
+	if (unlikely(gro_enable)) {
+		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
+			nb_rx = rte_gro_reassemble_burst(pkts_burst, nb_rx,
+					&(gro_ports[fs->rx_port].param));
+		} else {
+			gro_ctx = current_fwd_lcore()->gro_ctx;
+			nb_rx = rte_gro_reassemble(pkts_burst, nb_rx, gro_ctx);
+
+			if (++fs->gro_times >= gro_flush_cycles) {
+				gro_pkts_num = rte_gro_get_pkt_count(gro_ctx);
+				if (gro_pkts_num > MAX_PKT_BURST - nb_rx)
+					gro_pkts_num = MAX_PKT_BURST - nb_rx;
+
+				nb_rx += rte_gro_timeout_flush(gro_ctx, 0,
+						RTE_GRO_TCP_IPV4,
+						&pkts_burst[nb_rx],
+						gro_pkts_num);
+				fs->gro_times = 0;
+			}
+		}
+	}
+
 	nb_prep = rte_eth_tx_prepare(fs->tx_port, fs->tx_queue,
 			pkts_burst, nb_rx);
 	if (nb_prep != nb_rx)
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index e097ee0..c9d988e 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -90,7 +90,6 @@
 #ifdef RTE_LIBRTE_LATENCY_STATS
 #include <rte_latencystats.h>
 #endif
-#include <rte_gro.h>
 
 #include "testpmd.h"
 
@@ -386,6 +385,7 @@ uint8_t bitrate_enabled;
 #endif
 
 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
+uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 
 /* Forward function declarations */
 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port);
@@ -570,6 +570,7 @@ init_config(void)
 	unsigned int nb_mbuf_per_pool;
 	lcoreid_t  lc_id;
 	uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
+	struct rte_gro_param gro_param;
 
 	memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
 
@@ -671,6 +672,20 @@ init_config(void)
 		rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
 
 	fwd_config_setup();
+
+	/* create a gro context for each lcore */
+	gro_param.gro_types = RTE_GRO_TCP_IPV4;
+	gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
+	gro_param.max_item_per_flow = MAX_PKT_BURST;
+	for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
+		gro_param.socket_id = rte_lcore_to_socket_id(
+				fwd_lcores_cpuids[lc_id]);
+		fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
+		if (fwd_lcores[lc_id]->gro_ctx == NULL) {
+			rte_exit(EXIT_FAILURE,
+					"rte_gro_ctx_create() failed\n");
+		}
+	}
 }
 
 
@@ -1217,6 +1232,7 @@ stop_packet_forwarding(void)
 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 	uint64_t fwd_cycles;
 #endif
+
 	static const char *acc_stats_border = "+++++++++++++++";
 
 	if (test_done) {
@@ -1307,6 +1323,7 @@ stop_packet_forwarding(void)
 
 		fwd_port_stats_display(pt_id, &stats);
 	}
+
 	printf("\n  %s Accumulated forward statistics for all ports"
 	       "%s\n",
 	       acc_stats_border, acc_stats_border);
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 1d1ee75..9433eae 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -120,6 +120,7 @@ struct fwd_stream {
 	unsigned int fwd_dropped; /**< received packets not forwarded */
 	unsigned int rx_bad_ip_csum ; /**< received packets has bad ip checksum */
 	unsigned int rx_bad_l4_csum ; /**< received packets has bad l4 checksum */
+	unsigned int gro_times;	/**< GRO operation times */
 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 	uint64_t     core_cycles; /**< used for RX and TX processing */
 #endif
@@ -206,6 +207,7 @@ struct rte_port {
  */
 struct fwd_lcore {
 	struct rte_mempool *mbp; /**< The mbuf pool to use by this core */
+	void *gro_ctx;		/**< GRO context */
 	streamid_t stream_idx;   /**< index of 1st stream in "fwd_streams" */
 	streamid_t stream_nb;    /**< number of streams in "fwd_streams" */
 	lcoreid_t  cpuid_idx;    /**< index of logical core in CPU id table */
@@ -434,13 +436,19 @@ extern struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
 extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
 
-#define GRO_DEFAULT_FLOW_NUM 4
-#define GRO_DEFAULT_ITEM_NUM_PER_FLOW DEF_PKT_BURST
+#define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
+#define GRO_DEFAULT_FLOW_NUM (RTE_GRO_MAX_BURST_ITEM_NUM / \
+		GRO_DEFAULT_ITEM_NUM_PER_FLOW)
+
+#define GRO_DEFAULT_FLUSH_CYCLES 1
+#define GRO_MAX_FLUSH_CYCLES 4
+
 struct gro_status {
 	struct rte_gro_param param;
 	uint8_t enable;
 };
 extern struct gro_status gro_ports[RTE_MAX_ETHPORTS];
+extern uint8_t gro_flush_cycles;
 
 static inline unsigned int
 lcore_num(void)
@@ -641,7 +649,9 @@ void get_2tuple_filter(uint8_t port_id, uint16_t index);
 void get_5tuple_filter(uint8_t port_id, uint16_t index);
 int rx_queue_id_is_invalid(queueid_t rxq_id);
 int tx_queue_id_is_invalid(queueid_t txq_id);
-void setup_gro(const char *mode, uint8_t port_id);
+void setup_gro(const char *onoff, uint8_t port_id);
+void setup_gro_flush_cycles(uint8_t cycles);
+void show_gro(uint8_t port_id);
 
 /* Functions to manage the set of filtered Multicast MAC addresses */
 void mcast_addr_add(uint8_t port_id, struct ether_addr *mc_addr);
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 2ed62f5..74a1fb4 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -898,12 +898,12 @@ Display the status of TCP Segmentation Offload::
 
    testpmd> tso show (port_id)
 
-gro
-~~~
+set port - gro
+~~~~~~~~~~~~~~
 
 Enable or disable GRO in ``csum`` forwarding engine::
 
-   testpmd> gro (on|off) (port_id)
+   testpmd> set port <port_id> gro on|off
 
 If enabled, the csum forwarding engine will perform GRO on the TCP/IPv4
 packets received from the given port.
@@ -914,23 +914,43 @@ GRO. By default, GRO is disabled for all ports.
 .. note::
 
    When enable GRO for a port, TCP/IPv4 packets received from the port
-   will be performed GRO. After GRO, the merged packets are multi-segments.
-   But csum forwarding engine doesn't support to calculate TCP checksum
-   for multi-segment packets in SW. So please select TCP HW checksum
-   calculation for the port which GROed packets are transmitted to.
+   will be performed GRO. After GRO, all merged packets have bad
+   checksums, since the GRO library doesn't re-calculate checksums for
+   the merged packets. Therefore, if users want the merged packets to
+   have correct checksums, please select HW IP checksum calculation and
+   HW TCP checksum calculation for the port which the merged packets are
+   transmitted to.
+
+show port - gro
+~~~~~~~~~~~~~~~
 
-gro set
-~~~~~~~
+Display GRO configuration for a given port::
+
+   testpmd> show port <port_id> gro
+
+set gro flush
+~~~~~~~~~~~~~
+
+Set the cycle to flush the GROed packets from reassembly tables::
+
+   testpmd> set gro flush <cycles>
 
-Set max flow number and max packet number per-flow for GRO::
+When enable GRO, the csum forwarding engine performs GRO on received
+packets, and the GROed packets are stored in reassembly tables. Users
+can use this command to determine when the GROed packets are flushed
+from the reassembly tables.
 
-   testpmd> gro set (max_flow_num) (max_item_num_per_flow) (port_id)
+The ``cycles`` is measured in GRO operation times. The csum forwarding
+engine flushes the GROed packets from the tables every ``cycles`` GRO
+operations.
 
-The product of ``max_flow_num`` and ``max_item_num_per_flow`` is the max
-number of packets a GRO table can store.
+By default, the value of ``cycles`` is 1, which means flush GROed packets
+from the reassembly tables as soon as one GRO operation finishes. The value
+of ``cycles`` should be in the range of 1 to ``GRO_MAX_FLUSH_CYCLES``.
 
-If current packet number is greater than or equal to the max value, GRO
-will stop processing incoming packets.
+Please note that the large value of ``cycles`` may cause the poor TCP/IP
+stack performance. Because the GROed packets are delayed to arrive the
+stack, thus causing more duplicated ACKs and TCP retransmissions.
 
 mac_addr add
 ~~~~~~~~~~~~
-- 
2.7.4

Re: [dpdk-dev] [PATCH v4] app/testpmd: enable the heavyweight mode TCP/IPv4 GRO

[Yao, Lei A]

> -----Original Message-----
> From: Hu, Jiayu
> Sent: Tuesday, September 26, 2017 2:27 PM
> To: dev@dpdk.org
> Cc: Yigit, Ferruh <ferruh.yigit@intel.com>; Tan, Jianfeng
> <jianfeng.tan@intel.com>; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>; thomas@monjalon.net; Wu, Jingjing
> <jingjing.wu@intel.com>; Yao, Lei A <lei.a.yao@intel.com>; Hu, Jiayu
> <jiayu.hu@intel.com>
> Subject: [PATCH v4] app/testpmd: enable the heavyweight mode TCP/IPv4
> GRO
> 
> The GRO library provides two modes to reassemble packets. Currently, the
> csum forwarding engine has supported to use the lightweight mode to
> reassemble TCP/IPv4 packets. This patch introduces the heavyweight mode
> for TCP/IPv4 GRO in the csum forwarding engine.
> 
> With the command "set port <port_id> gro on|off", users can enable
> TCP/IPv4 GRO for a given port. With the command "set gro flush <cycles>",
> users can determine when the GROed TCP/IPv4 packets are flushed from
> reassembly tables. With the command "show port <port_id> gro", users can
> display GRO configuration.
> 
> The GRO library doesn't re-calculate checksums for merged packets. If
> users want the merged packets to have correct IP and TCP checksums,
> please select HW IP checksum calculation and HW TCP checksum calculation
> for the port which the merged packets are transmitted to.
> 
> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
> Reviewed-by: Ferruh Yigit <ferruh.yigit@intel.com>
Tested-by: Yao Lei<lei.a.yao@intel.com>

This patch has beed tested on my bench. The following
is the performance data got from iperf test with single flow
No GRO: 9.5 Gbps
Kernel GRO: 13.6 Gbps
DPDK GRO with flush cycle=1 : 25.9 Gbps
DPDK GRO with flush cycle=2 : 27.9 Gbps

Note: When use DPDK GRO with flush cycle=2, I set 
the vhost rx_queue_size to 1024, if use default number
256, sometimes I met the date stall. 
OS: Ubuntu 16.04
CPU: Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz  

> ---
> changes in v4:
> - fix unchecking the min value of 'cycle' bug in setup_gro_flush_cycles
> - update the context of the testpmd document and commit logs
> changes in v3:
> - remove "heavyweight mode" and "lightweight mode" from GRO
> commands
> - combine two patches into one
> - use consistent help string for GRO commands
> - remove the unnecessary command "gro set (max_flow_num)
> 	(max_item_num_per_flow) (port_id)"
> changes in v2:
> - use "set" and "show" as the root level command
> - add a new command to show GRO configuration
> - fix l2_len/l3_len/l4_len unset etc. bugs
> 
>  app/test-pmd/cmdline.c                      | 206 ++++++++++++++++------------
>  app/test-pmd/config.c                       |  68 +++++++--
>  app/test-pmd/csumonly.c                     |  31 ++++-
>  app/test-pmd/testpmd.c                      |  19 ++-
>  app/test-pmd/testpmd.h                      |  16 ++-
>  doc/guides/testpmd_app_ug/testpmd_funcs.rst |  50 +++++--
>  6 files changed, 270 insertions(+), 120 deletions(-)
> 
> diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
> index ccdf239..e44c02e 100644
> --- a/app/test-pmd/cmdline.c
> +++ b/app/test-pmd/cmdline.c
> @@ -423,13 +423,16 @@ static void cmd_help_long_parsed(void
> *parsed_result,
>  			"tso show (portid)"
>  			"    Display the status of TCP Segmentation
> Offload.\n\n"
> 
> -			"gro (on|off) (port_id)"
> +			"set port (port_id) gro on|off\n"
>  			"    Enable or disable Generic Receive Offload in"
>  			" csum forwarding engine.\n\n"
> 
> -			"gro set (max_flow_num)
> (max_item_num_per_flow) (port_id)\n"
> -			"    Set max flow number and max packet number
> per-flow"
> -			" for GRO.\n\n"
> +			"show port (port_id) gro\n"
> +			"    Display GRO configuration.\n\n"
> +
> +			"set gro flush (cycles)\n"
> +			"    Set the cycle to flush GROed packets from"
> +			" reassembly tables.\n\n"
> 
>  			"set fwd (%s)\n"
>  			"    Set packet forwarding mode.\n\n"
> @@ -3854,115 +3857,145 @@ cmdline_parse_inst_t cmd_tunnel_tso_show
> = {
>  };
> 
>  /* *** SET GRO FOR A PORT *** */
> -struct cmd_gro_result {
> +struct cmd_gro_enable_result {
> +	cmdline_fixed_string_t cmd_set;
> +	cmdline_fixed_string_t cmd_port;
>  	cmdline_fixed_string_t cmd_keyword;
> -	cmdline_fixed_string_t mode;
> -	uint8_t port_id;
> +	cmdline_fixed_string_t cmd_onoff;
> +	uint8_t cmd_pid;
>  };
> 
>  static void
> -cmd_enable_gro_parsed(void *parsed_result,
> +cmd_gro_enable_parsed(void *parsed_result,
>  		__attribute__((unused)) struct cmdline *cl,
>  		__attribute__((unused)) void *data)
>  {
> -	struct cmd_gro_result *res;
> +	struct cmd_gro_enable_result *res;
> 
>  	res = parsed_result;
> -	setup_gro(res->mode, res->port_id);
> -}
> -
> -cmdline_parse_token_string_t cmd_gro_keyword =
> -	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
> +	if (!strcmp(res->cmd_keyword, "gro"))
> +		setup_gro(res->cmd_onoff, res->cmd_pid);
> +}
> +
> +cmdline_parse_token_string_t cmd_gro_enable_set =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
> +			cmd_set, "set");
> +cmdline_parse_token_string_t cmd_gro_enable_port =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
> +			cmd_keyword, "port");
> +cmdline_parse_token_num_t cmd_gro_enable_pid =
> +	TOKEN_NUM_INITIALIZER(struct cmd_gro_enable_result,
> +			cmd_pid, UINT8);
> +cmdline_parse_token_string_t cmd_gro_enable_keyword =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
>  			cmd_keyword, "gro");
> -cmdline_parse_token_string_t cmd_gro_mode =
> -	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
> -			mode, "on#off");
> -cmdline_parse_token_num_t cmd_gro_pid =
> -	TOKEN_NUM_INITIALIZER(struct cmd_gro_result,
> -			port_id, UINT8);
> +cmdline_parse_token_string_t cmd_gro_enable_onoff =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
> +			cmd_onoff, "on#off");
> 
> -cmdline_parse_inst_t cmd_enable_gro = {
> -	.f = cmd_enable_gro_parsed,
> +cmdline_parse_inst_t cmd_gro_enable = {
> +	.f = cmd_gro_enable_parsed,
>  	.data = NULL,
> -	.help_str = "gro (on|off) (port_id)",
> +	.help_str = "set port <port_id> gro on|off",
>  	.tokens = {
> -		(void *)&cmd_gro_keyword,
> -		(void *)&cmd_gro_mode,
> -		(void *)&cmd_gro_pid,
> +		(void *)&cmd_gro_enable_set,
> +		(void *)&cmd_gro_enable_port,
> +		(void *)&cmd_gro_enable_pid,
> +		(void *)&cmd_gro_enable_keyword,
> +		(void *)&cmd_gro_enable_onoff,
>  		NULL,
>  	},
>  };
> 
> -/* *** SET MAX FLOW NUMBER AND ITEM NUM PER FLOW FOR GRO *** */
> -struct cmd_gro_set_result {
> -	cmdline_fixed_string_t gro;
> -	cmdline_fixed_string_t mode;
> -	uint16_t flow_num;
> -	uint16_t item_num_per_flow;
> -	uint8_t port_id;
> +/* *** DISPLAY GRO CONFIGURATION *** */
> +struct cmd_gro_show_result {
> +	cmdline_fixed_string_t cmd_show;
> +	cmdline_fixed_string_t cmd_port;
> +	cmdline_fixed_string_t cmd_keyword;
> +	uint8_t cmd_pid;
>  };
> 
>  static void
> -cmd_gro_set_parsed(void *parsed_result,
> -		       __attribute__((unused)) struct cmdline *cl,
> -		       __attribute__((unused)) void *data)
> +cmd_gro_show_parsed(void *parsed_result,
> +		__attribute__((unused)) struct cmdline *cl,
> +		__attribute__((unused)) void *data)
>  {
> -	struct cmd_gro_set_result *res = parsed_result;
> +	struct cmd_gro_show_result *res;
> 
> -	if (port_id_is_invalid(res->port_id, ENABLED_WARN))
> -		return;
> -	if (test_done == 0) {
> -		printf("Before set GRO flow_num and item_num_per_flow,"
> -				" please stop forwarding first\n");
> -		return;
> -	}
> +	res = parsed_result;
> +	if (!strcmp(res->cmd_keyword, "gro"))
> +		show_gro(res->cmd_pid);
> +}
> +
> +cmdline_parse_token_string_t cmd_gro_show_show =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
> +			cmd_show, "show");
> +cmdline_parse_token_string_t cmd_gro_show_port =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
> +			cmd_port, "port");
> +cmdline_parse_token_num_t cmd_gro_show_pid =
> +	TOKEN_NUM_INITIALIZER(struct cmd_gro_show_result,
> +			cmd_pid, UINT8);
> +cmdline_parse_token_string_t cmd_gro_show_keyword =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
> +			cmd_keyword, "gro");
> 
> -	if (!strcmp(res->mode, "set")) {
> -		if (res->flow_num == 0)
> -			printf("Invalid flow number. Revert to default value:"
> -					" %u.\n",
> GRO_DEFAULT_FLOW_NUM);
> -		else
> -			gro_ports[res->port_id].param.max_flow_num =
> -				res->flow_num;
> +cmdline_parse_inst_t cmd_gro_show = {
> +	.f = cmd_gro_show_parsed,
> +	.data = NULL,
> +	.help_str = "show port <port_id> gro",
> +	.tokens = {
> +		(void *)&cmd_gro_show_show,
> +		(void *)&cmd_gro_show_port,
> +		(void *)&cmd_gro_show_pid,
> +		(void *)&cmd_gro_show_keyword,
> +		NULL,
> +	},
> +};
> 
> -		if (res->item_num_per_flow == 0)
> -			printf("Invalid item number per-flow. Revert"
> -					" to default value:%u.\n",
> -
> 	GRO_DEFAULT_ITEM_NUM_PER_FLOW);
> -		else
> -			gro_ports[res->port_id].param.max_item_per_flow
> =
> -				res->item_num_per_flow;
> -	}
> +/* *** SET FLUSH CYCLES FOR GRO *** */
> +struct cmd_gro_flush_result {
> +	cmdline_fixed_string_t cmd_set;
> +	cmdline_fixed_string_t cmd_keyword;
> +	cmdline_fixed_string_t cmd_flush;
> +	uint8_t cmd_cycles;
> +};
> +
> +static void
> +cmd_gro_flush_parsed(void *parsed_result,
> +		__attribute__((unused)) struct cmdline *cl,
> +		__attribute__((unused)) void *data)
> +{
> +	struct cmd_gro_flush_result *res;
> +
> +	res = parsed_result;
> +	if ((!strcmp(res->cmd_keyword, "gro")) &&
> +			(!strcmp(res->cmd_flush, "flush")))
> +		setup_gro_flush_cycles(res->cmd_cycles);
>  }
> 
> -cmdline_parse_token_string_t cmd_gro_set_gro =
> -	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
> -				gro, "gro");
> -cmdline_parse_token_string_t cmd_gro_set_mode =
> -	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
> -				mode, "set");
> -cmdline_parse_token_num_t cmd_gro_set_flow_num =
> -	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> -				flow_num, UINT16);
> -cmdline_parse_token_num_t cmd_gro_set_item_num_per_flow =
> -	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> -				item_num_per_flow, UINT16);
> -cmdline_parse_token_num_t cmd_gro_set_portid =
> -	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> -				port_id, UINT8);
> +cmdline_parse_token_string_t cmd_gro_flush_set =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
> +			cmd_set, "set");
> +cmdline_parse_token_string_t cmd_gro_flush_keyword =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
> +			cmd_keyword, "gro");
> +cmdline_parse_token_string_t cmd_gro_flush_flush =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
> +			cmd_flush, "flush");
> +cmdline_parse_token_num_t cmd_gro_flush_cycles =
> +	TOKEN_NUM_INITIALIZER(struct cmd_gro_flush_result,
> +			cmd_cycles, UINT8);
> 
> -cmdline_parse_inst_t cmd_gro_set = {
> -	.f = cmd_gro_set_parsed,
> +cmdline_parse_inst_t cmd_gro_flush = {
> +	.f = cmd_gro_flush_parsed,
>  	.data = NULL,
> -	.help_str = "gro set <max_flow_num> <max_item_num_per_flow>
> "
> -		"<port_id>: set max flow number and max packet number
> per-flow "
> -		"for GRO",
> +	.help_str = "set gro flush <cycles>",
>  	.tokens = {
> -		(void *)&cmd_gro_set_gro,
> -		(void *)&cmd_gro_set_mode,
> -		(void *)&cmd_gro_set_flow_num,
> -		(void *)&cmd_gro_set_item_num_per_flow,
> -		(void *)&cmd_gro_set_portid,
> +		(void *)&cmd_gro_flush_set,
> +		(void *)&cmd_gro_flush_keyword,
> +		(void *)&cmd_gro_flush_flush,
> +		(void *)&cmd_gro_flush_cycles,
>  		NULL,
>  	},
>  };
> @@ -14253,8 +14286,9 @@ cmdline_parse_ctx_t main_ctx[] = {
>  	(cmdline_parse_inst_t *)&cmd_tso_show,
>  	(cmdline_parse_inst_t *)&cmd_tunnel_tso_set,
>  	(cmdline_parse_inst_t *)&cmd_tunnel_tso_show,
> -	(cmdline_parse_inst_t *)&cmd_enable_gro,
> -	(cmdline_parse_inst_t *)&cmd_gro_set,
> +	(cmdline_parse_inst_t *)&cmd_gro_enable,
> +	(cmdline_parse_inst_t *)&cmd_gro_flush,
> +	(cmdline_parse_inst_t *)&cmd_gro_show,
>  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set,
>  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_rx,
>  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_tx,
> diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
> index 3ae3e1c..92220b1 100644
> --- a/app/test-pmd/config.c
> +++ b/app/test-pmd/config.c
> @@ -2420,7 +2420,7 @@ set_tx_pkt_segments(unsigned *seg_lengths,
> unsigned nb_segs)
>  }
> 
>  void
> -setup_gro(const char *mode, uint8_t port_id)
> +setup_gro(const char *onoff, uint8_t port_id)
>  {
>  	if (!rte_eth_dev_is_valid_port(port_id)) {
>  		printf("invalid port id %u\n", port_id);
> @@ -2431,29 +2431,77 @@ setup_gro(const char *mode, uint8_t port_id)
>  				" please stop forwarding first\n");
>  		return;
>  	}
> -	if (strcmp(mode, "on") == 0) {
> -		if (gro_ports[port_id].enable) {
> -			printf("port %u has enabled GRO\n", port_id);
> +	if (strcmp(onoff, "on") == 0) {
> +		if (gro_ports[port_id].enable != 0) {
> +			printf("Port %u has enabled GRO. Please"
> +					" disable GRO first\n", port_id);
>  			return;
>  		}
> -		gro_ports[port_id].enable = 1;
> -		gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4;
> -
> -		if (gro_ports[port_id].param.max_flow_num == 0)
> +		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
> +			gro_ports[port_id].param.gro_types =
> RTE_GRO_TCP_IPV4;
>  			gro_ports[port_id].param.max_flow_num =
>  				GRO_DEFAULT_FLOW_NUM;
> -		if (gro_ports[port_id].param.max_item_per_flow == 0)
>  			gro_ports[port_id].param.max_item_per_flow =
>  				GRO_DEFAULT_ITEM_NUM_PER_FLOW;
> +		}
> +		gro_ports[port_id].enable = 1;
>  	} else {
>  		if (gro_ports[port_id].enable == 0) {
> -			printf("port %u has disabled GRO\n", port_id);
> +			printf("Port %u has disabled GRO\n", port_id);
>  			return;
>  		}
>  		gro_ports[port_id].enable = 0;
>  	}
>  }
> 
> +void
> +setup_gro_flush_cycles(uint8_t cycles)
> +{
> +	if (test_done == 0) {
> +		printf("Before change flush interval for GRO,"
> +				" please stop forwarding first.\n");
> +		return;
> +	}
> +
> +	if (cycles > GRO_MAX_FLUSH_CYCLES || cycles <
> +			GRO_DEFAULT_FLUSH_CYCLES) {
> +		printf("The flushing cycle be in the range"
> +				" of 1 to %u. Revert to the default"
> +				" value %u.\n",
> +				GRO_MAX_FLUSH_CYCLES,
> +				GRO_DEFAULT_FLUSH_CYCLES);
> +		cycles = GRO_DEFAULT_FLUSH_CYCLES;
> +	}
> +
> +	gro_flush_cycles = cycles;
> +}
> +
> +void
> +show_gro(uint8_t port_id)
> +{
> +	struct rte_gro_param *param;
> +	uint32_t max_pkts_num;
> +
> +	param = &gro_ports[port_id].param;
> +
> +	if (!rte_eth_dev_is_valid_port(port_id)) {
> +		printf("Invalid port id %u.\n", port_id);
> +		return;
> +	}
> +	if (gro_ports[port_id].enable) {
> +		printf("GRO type: TCP/IPv4\n");
> +		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
> +			max_pkts_num = param->max_flow_num *
> +				param->max_item_per_flow;
> +		} else
> +			max_pkts_num = MAX_PKT_BURST *
> GRO_MAX_FLUSH_CYCLES;
> +		printf("Max number of packets to perform GRO: %u\n",
> +				max_pkts_num);
> +		printf("Flushing cycles: %u\n", gro_flush_cycles);
> +	} else
> +		printf("Port %u doesn't enable GRO.\n", port_id);
> +}
> +
>  char*
>  list_pkt_forwarding_modes(void)
>  {
> diff --git a/app/test-pmd/csumonly.c b/app/test-pmd/csumonly.c
> index 90c8119..ca50ab7 100644
> --- a/app/test-pmd/csumonly.c
> +++ b/app/test-pmd/csumonly.c
> @@ -631,6 +631,9 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
>  	struct rte_mbuf *m, *p;
>  	struct ether_hdr *eth_hdr;
>  	void *l3_hdr = NULL, *outer_l3_hdr = NULL; /* can be IPv4 or IPv6 */
> +	void **gro_ctx;
> +	uint16_t gro_pkts_num;
> +	uint8_t gro_enable;
>  	uint16_t nb_rx;
>  	uint16_t nb_tx;
>  	uint16_t nb_prep;
> @@ -657,17 +660,13 @@ pkt_burst_checksum_forward(struct fwd_stream
> *fs)
>  				 nb_pkt_per_burst);
>  	if (unlikely(nb_rx == 0))
>  		return;
> -	if (unlikely(gro_ports[fs->rx_port].enable))
> -		nb_rx = rte_gro_reassemble_burst(pkts_burst,
> -				nb_rx,
> -				&(gro_ports[fs->rx_port].param));
> -
>  #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
>  	fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
>  #endif
>  	fs->rx_packets += nb_rx;
>  	rx_bad_ip_csum = 0;
>  	rx_bad_l4_csum = 0;
> +	gro_enable = gro_ports[fs->rx_port].enable;
> 
>  	txp = &ports[fs->tx_port];
>  	testpmd_ol_flags = txp->tx_ol_flags;
> @@ -851,6 +850,28 @@ pkt_burst_checksum_forward(struct fwd_stream
> *fs)
>  		}
>  	}
> 
> +	if (unlikely(gro_enable)) {
> +		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
> +			nb_rx = rte_gro_reassemble_burst(pkts_burst,
> nb_rx,
> +					&(gro_ports[fs->rx_port].param));
> +		} else {
> +			gro_ctx = current_fwd_lcore()->gro_ctx;
> +			nb_rx = rte_gro_reassemble(pkts_burst, nb_rx,
> gro_ctx);
> +
> +			if (++fs->gro_times >= gro_flush_cycles) {
> +				gro_pkts_num =
> rte_gro_get_pkt_count(gro_ctx);
> +				if (gro_pkts_num > MAX_PKT_BURST - nb_rx)
> +					gro_pkts_num = MAX_PKT_BURST -
> nb_rx;
> +
> +				nb_rx += rte_gro_timeout_flush(gro_ctx, 0,
> +						RTE_GRO_TCP_IPV4,
> +						&pkts_burst[nb_rx],
> +						gro_pkts_num);
> +				fs->gro_times = 0;
> +			}
> +		}
> +	}
> +
>  	nb_prep = rte_eth_tx_prepare(fs->tx_port, fs->tx_queue,
>  			pkts_burst, nb_rx);
>  	if (nb_prep != nb_rx)
> diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
> index e097ee0..c9d988e 100644
> --- a/app/test-pmd/testpmd.c
> +++ b/app/test-pmd/testpmd.c
> @@ -90,7 +90,6 @@
>  #ifdef RTE_LIBRTE_LATENCY_STATS
>  #include <rte_latencystats.h>
>  #endif
> -#include <rte_gro.h>
> 
>  #include "testpmd.h"
> 
> @@ -386,6 +385,7 @@ uint8_t bitrate_enabled;
>  #endif
> 
>  struct gro_status gro_ports[RTE_MAX_ETHPORTS];
> +uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
> 
>  /* Forward function declarations */
>  static void map_port_queue_stats_mapping_registers(uint8_t pi, struct
> rte_port *port);
> @@ -570,6 +570,7 @@ init_config(void)
>  	unsigned int nb_mbuf_per_pool;
>  	lcoreid_t  lc_id;
>  	uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
> +	struct rte_gro_param gro_param;
> 
>  	memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
> 
> @@ -671,6 +672,20 @@ init_config(void)
>  		rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
> 
>  	fwd_config_setup();
> +
> +	/* create a gro context for each lcore */
> +	gro_param.gro_types = RTE_GRO_TCP_IPV4;
> +	gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
> +	gro_param.max_item_per_flow = MAX_PKT_BURST;
> +	for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
> +		gro_param.socket_id = rte_lcore_to_socket_id(
> +				fwd_lcores_cpuids[lc_id]);
> +		fwd_lcores[lc_id]->gro_ctx =
> rte_gro_ctx_create(&gro_param);
> +		if (fwd_lcores[lc_id]->gro_ctx == NULL) {
> +			rte_exit(EXIT_FAILURE,
> +					"rte_gro_ctx_create() failed\n");
> +		}
> +	}
>  }
> 
> 
> @@ -1217,6 +1232,7 @@ stop_packet_forwarding(void)
>  #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
>  	uint64_t fwd_cycles;
>  #endif
> +
>  	static const char *acc_stats_border = "+++++++++++++++";
> 
>  	if (test_done) {
> @@ -1307,6 +1323,7 @@ stop_packet_forwarding(void)
> 
>  		fwd_port_stats_display(pt_id, &stats);
>  	}
> +
>  	printf("\n  %s Accumulated forward statistics for all ports"
>  	       "%s\n",
>  	       acc_stats_border, acc_stats_border);
> diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
> index 1d1ee75..9433eae 100644
> --- a/app/test-pmd/testpmd.h
> +++ b/app/test-pmd/testpmd.h
> @@ -120,6 +120,7 @@ struct fwd_stream {
>  	unsigned int fwd_dropped; /**< received packets not forwarded */
>  	unsigned int rx_bad_ip_csum ; /**< received packets has bad ip
> checksum */
>  	unsigned int rx_bad_l4_csum ; /**< received packets has bad l4
> checksum */
> +	unsigned int gro_times;	/**< GRO operation times */
>  #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
>  	uint64_t     core_cycles; /**< used for RX and TX processing */
>  #endif
> @@ -206,6 +207,7 @@ struct rte_port {
>   */
>  struct fwd_lcore {
>  	struct rte_mempool *mbp; /**< The mbuf pool to use by this core */
> +	void *gro_ctx;		/**< GRO context */
>  	streamid_t stream_idx;   /**< index of 1st stream in "fwd_streams"
> */
>  	streamid_t stream_nb;    /**< number of streams in "fwd_streams"
> */
>  	lcoreid_t  cpuid_idx;    /**< index of logical core in CPU id table */
> @@ -434,13 +436,19 @@ extern struct ether_addr
> peer_eth_addrs[RTE_MAX_ETHPORTS];
>  extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-
> retry. */
>  extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-
> retry. */
> 
> -#define GRO_DEFAULT_FLOW_NUM 4
> -#define GRO_DEFAULT_ITEM_NUM_PER_FLOW DEF_PKT_BURST
> +#define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
> +#define GRO_DEFAULT_FLOW_NUM (RTE_GRO_MAX_BURST_ITEM_NUM
> / \
> +		GRO_DEFAULT_ITEM_NUM_PER_FLOW)
> +
> +#define GRO_DEFAULT_FLUSH_CYCLES 1
> +#define GRO_MAX_FLUSH_CYCLES 4
> +
>  struct gro_status {
>  	struct rte_gro_param param;
>  	uint8_t enable;
>  };
>  extern struct gro_status gro_ports[RTE_MAX_ETHPORTS];
> +extern uint8_t gro_flush_cycles;
> 
>  static inline unsigned int
>  lcore_num(void)
> @@ -641,7 +649,9 @@ void get_2tuple_filter(uint8_t port_id, uint16_t
> index);
>  void get_5tuple_filter(uint8_t port_id, uint16_t index);
>  int rx_queue_id_is_invalid(queueid_t rxq_id);
>  int tx_queue_id_is_invalid(queueid_t txq_id);
> -void setup_gro(const char *mode, uint8_t port_id);
> +void setup_gro(const char *onoff, uint8_t port_id);
> +void setup_gro_flush_cycles(uint8_t cycles);
> +void show_gro(uint8_t port_id);
> 
>  /* Functions to manage the set of filtered Multicast MAC addresses */
>  void mcast_addr_add(uint8_t port_id, struct ether_addr *mc_addr);
> diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> index 2ed62f5..74a1fb4 100644
> --- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> +++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> @@ -898,12 +898,12 @@ Display the status of TCP Segmentation Offload::
> 
>     testpmd> tso show (port_id)
> 
> -gro
> -~~~
> +set port - gro
> +~~~~~~~~~~~~~~
> 
>  Enable or disable GRO in ``csum`` forwarding engine::
> 
> -   testpmd> gro (on|off) (port_id)
> +   testpmd> set port <port_id> gro on|off
> 
>  If enabled, the csum forwarding engine will perform GRO on the TCP/IPv4
>  packets received from the given port.
> @@ -914,23 +914,43 @@ GRO. By default, GRO is disabled for all ports.
>  .. note::
> 
>     When enable GRO for a port, TCP/IPv4 packets received from the port
> -   will be performed GRO. After GRO, the merged packets are multi-
> segments.
> -   But csum forwarding engine doesn't support to calculate TCP checksum
> -   for multi-segment packets in SW. So please select TCP HW checksum
> -   calculation for the port which GROed packets are transmitted to.
> +   will be performed GRO. After GRO, all merged packets have bad
> +   checksums, since the GRO library doesn't re-calculate checksums for
> +   the merged packets. Therefore, if users want the merged packets to
> +   have correct checksums, please select HW IP checksum calculation and
> +   HW TCP checksum calculation for the port which the merged packets are
> +   transmitted to.
> +
> +show port - gro
> +~~~~~~~~~~~~~~~
> 
> -gro set
> -~~~~~~~
> +Display GRO configuration for a given port::
> +
> +   testpmd> show port <port_id> gro
> +
> +set gro flush
> +~~~~~~~~~~~~~
> +
> +Set the cycle to flush the GROed packets from reassembly tables::
> +
> +   testpmd> set gro flush <cycles>
> 
> -Set max flow number and max packet number per-flow for GRO::
> +When enable GRO, the csum forwarding engine performs GRO on received
> +packets, and the GROed packets are stored in reassembly tables. Users
> +can use this command to determine when the GROed packets are flushed
> +from the reassembly tables.
> 
> -   testpmd> gro set (max_flow_num) (max_item_num_per_flow) (port_id)
> +The ``cycles`` is measured in GRO operation times. The csum forwarding
> +engine flushes the GROed packets from the tables every ``cycles`` GRO
> +operations.
> 
> -The product of ``max_flow_num`` and ``max_item_num_per_flow`` is the
> max
> -number of packets a GRO table can store.
> +By default, the value of ``cycles`` is 1, which means flush GROed packets
> +from the reassembly tables as soon as one GRO operation finishes. The
> value
> +of ``cycles`` should be in the range of 1 to ``GRO_MAX_FLUSH_CYCLES``.
> 
> -If current packet number is greater than or equal to the max value, GRO
> -will stop processing incoming packets.
> +Please note that the large value of ``cycles`` may cause the poor TCP/IP
> +stack performance. Because the GROed packets are delayed to arrive the
> +stack, thus causing more duplicated ACKs and TCP retransmissions.
> 
>  mac_addr add
>  ~~~~~~~~~~~~
> --
> 2.7.4

Re: [dpdk-dev] [PATCH v4] app/testpmd: enable the heavyweight mode TCP/IPv4 GRO

[Ferruh Yigit]

On 9/27/2017 8:23 AM, Yao, Lei A wrote:
<...>

>> The GRO library provides two modes to reassemble packets. Currently, the
>> csum forwarding engine has supported to use the lightweight mode to
>> reassemble TCP/IPv4 packets. This patch introduces the heavyweight mode
>> for TCP/IPv4 GRO in the csum forwarding engine.
>>
>> With the command "set port <port_id> gro on|off", users can enable
>> TCP/IPv4 GRO for a given port. With the command "set gro flush <cycles>",
>> users can determine when the GROed TCP/IPv4 packets are flushed from
>> reassembly tables. With the command "show port <port_id> gro", users can
>> display GRO configuration.
>>
>> The GRO library doesn't re-calculate checksums for merged packets. If
>> users want the merged packets to have correct IP and TCP checksums,
>> please select HW IP checksum calculation and HW TCP checksum calculation
>> for the port which the merged packets are transmitted to.
>>
>> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
>> Reviewed-by: Ferruh Yigit <ferruh.yigit@intel.com>

> Tested-by: Yao Lei<lei.a.yao@intel.com>

Applied to dpdk-next-net/master, thanks.

Re: [dpdk-dev] [PATCH v4] app/testpmd: enable the heavyweight mode TCP/IPv4 GRO

[Ferruh Yigit]

On 10/7/2017 12:04 AM, Ferruh Yigit wrote:
> On 9/27/2017 8:23 AM, Yao, Lei A wrote:
> <...>
> 
>>> The GRO library provides two modes to reassemble packets. Currently, the
>>> csum forwarding engine has supported to use the lightweight mode to
>>> reassemble TCP/IPv4 packets. This patch introduces the heavyweight mode
>>> for TCP/IPv4 GRO in the csum forwarding engine.
>>>
>>> With the command "set port <port_id> gro on|off", users can enable
>>> TCP/IPv4 GRO for a given port. With the command "set gro flush <cycles>",
>>> users can determine when the GROed TCP/IPv4 packets are flushed from
>>> reassembly tables. With the command "show port <port_id> gro", users can
>>> display GRO configuration.
>>>
>>> The GRO library doesn't re-calculate checksums for merged packets. If
>>> users want the merged packets to have correct IP and TCP checksums,
>>> please select HW IP checksum calculation and HW TCP checksum calculation
>>> for the port which the merged packets are transmitted to.
>>>
>>> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
>>> Reviewed-by: Ferruh Yigit <ferruh.yigit@intel.com>
> 
>> Tested-by: Yao Lei<lei.a.yao@intel.com>
> 
> Applied to dpdk-next-net/master, thanks.

nope, removing it out back.

There are "uint8_t port_id" usage a few places, now port_id is uint16_t,
this is new in the repo.

For testpmd you can prefer "portid_t" storage type as well.

Can you please send a new version?

Re: [dpdk-dev] [PATCH v4] app/testpmd: enable the heavyweight mode TCP/IPv4 GRO

[Hu, Jiayu]

Hi Ferruh,

> -----Original Message-----
> From: Yigit, Ferruh
> Sent: Saturday, October 7, 2017 7:26 AM
> To: Yao, Lei A <lei.a.yao@intel.com>; Hu, Jiayu <jiayu.hu@intel.com>;
> dev@dpdk.org
> Cc: Tan, Jianfeng <jianfeng.tan@intel.com>; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>; thomas@monjalon.net; Wu, Jingjing
> <jingjing.wu@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v4] app/testpmd: enable the heavyweight
> mode TCP/IPv4 GRO
> 
> On 10/7/2017 12:04 AM, Ferruh Yigit wrote:
> > On 9/27/2017 8:23 AM, Yao, Lei A wrote:
> > <...>
> >
> >>> The GRO library provides two modes to reassemble packets. Currently,
> the
> >>> csum forwarding engine has supported to use the lightweight mode to
> >>> reassemble TCP/IPv4 packets. This patch introduces the heavyweight
> mode
> >>> for TCP/IPv4 GRO in the csum forwarding engine.
> >>>
> >>> With the command "set port <port_id> gro on|off", users can enable
> >>> TCP/IPv4 GRO for a given port. With the command "set gro flush
> <cycles>",
> >>> users can determine when the GROed TCP/IPv4 packets are flushed from
> >>> reassembly tables. With the command "show port <port_id> gro", users
> can
> >>> display GRO configuration.
> >>>
> >>> The GRO library doesn't re-calculate checksums for merged packets. If
> >>> users want the merged packets to have correct IP and TCP checksums,
> >>> please select HW IP checksum calculation and HW TCP checksum
> calculation
> >>> for the port which the merged packets are transmitted to.
> >>>
> >>> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
> >>> Reviewed-by: Ferruh Yigit <ferruh.yigit@intel.com>
> >
> >> Tested-by: Yao Lei<lei.a.yao@intel.com>
> >
> > Applied to dpdk-next-net/master, thanks.
> 
> nope, removing it out back.
> 
> There are "uint8_t port_id" usage a few places, now port_id is uint16_t,
> this is new in the repo.
> 
> For testpmd you can prefer "portid_t" storage type as well.
> 
> Can you please send a new version?

No problem. I will send a new version.

Thanks,
Jiayu

[dpdk-dev] [PATCH v5] app/testpmd: enable the heavyweight mode TCP/IPv4 GRO

[Jiayu Hu]

The GRO library provides two modes to reassemble packets. Currently, the
csum forwarding engine has supported to use the lightweight mode to
reassemble TCP/IPv4 packets. This patch introduces the heavyweight mode
for TCP/IPv4 GRO in the csum forwarding engine.

With the command "set port <port_id> gro on|off", users can enable
TCP/IPv4 GRO for a given port. With the command "set gro flush <cycles>",
users can determine when the GROed TCP/IPv4 packets are flushed from
reassembly tables. With the command "show port <port_id> gro", users can
display GRO configuration.

The GRO library doesn't re-calculate checksums for merged packets. If
users want the merged packets to have correct IP and TCP checksums,
please select HW IP checksum calculation and HW TCP checksum calculation
for the port which the merged packets are transmitted to.

Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
Reviewed-by: Ferruh Yigit <ferruh.yigit@intel.com>
Tested-by: Yao Lei <lei.a.yao@intel.com>
---
changes in v5:
- fix port_id type conflict
changes in v4:
- fix unchecking the min value of 'cycle' bug in setup_gro_flush_cycles
- update the context of the testpmd document and commit logs
changes in v3:
- remove "heavyweight mode" and "lightweight mode" from GRO commands
- combine two patches into one
- use consistent help string for GRO commands 
- remove the unnecessary command "gro set (max_flow_num)
	(max_item_num_per_flow) (port_id)"
changes in v2:
- use "set" and "show" as the root level command
- add a new command to show GRO configuration
- fix l2_len/l3_len/l4_len unset etc. bugs

 app/test-pmd/cmdline.c                      | 206 ++++++++++++++++------------
 app/test-pmd/config.c                       |  68 +++++++--
 app/test-pmd/csumonly.c                     |  31 ++++-
 app/test-pmd/testpmd.c                      |  19 ++-
 app/test-pmd/testpmd.h                      |  16 ++-
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  50 +++++--
 6 files changed, 270 insertions(+), 120 deletions(-)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index 91766bc..516fc89 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -427,13 +427,16 @@ static void cmd_help_long_parsed(void *parsed_result,
 			"tso show (portid)"
 			"    Display the status of TCP Segmentation Offload.\n\n"
 
-			"gro (on|off) (port_id)"
+			"set port (port_id) gro on|off\n"
 			"    Enable or disable Generic Receive Offload in"
 			" csum forwarding engine.\n\n"
 
-			"gro set (max_flow_num) (max_item_num_per_flow) (port_id)\n"
-			"    Set max flow number and max packet number per-flow"
-			" for GRO.\n\n"
+			"show port (port_id) gro\n"
+			"    Display GRO configuration.\n\n"
+
+			"set gro flush (cycles)\n"
+			"    Set the cycle to flush GROed packets from"
+			" reassembly tables.\n\n"
 
 			"set fwd (%s)\n"
 			"    Set packet forwarding mode.\n\n"
@@ -3868,115 +3871,145 @@ cmdline_parse_inst_t cmd_tunnel_tso_show = {
 };
 
 /* *** SET GRO FOR A PORT *** */
-struct cmd_gro_result {
+struct cmd_gro_enable_result {
+	cmdline_fixed_string_t cmd_set;
+	cmdline_fixed_string_t cmd_port;
 	cmdline_fixed_string_t cmd_keyword;
-	cmdline_fixed_string_t mode;
-	uint8_t port_id;
+	cmdline_fixed_string_t cmd_onoff;
+	portid_t cmd_pid;
 };
 
 static void
-cmd_enable_gro_parsed(void *parsed_result,
+cmd_gro_enable_parsed(void *parsed_result,
 		__attribute__((unused)) struct cmdline *cl,
 		__attribute__((unused)) void *data)
 {
-	struct cmd_gro_result *res;
+	struct cmd_gro_enable_result *res;
 
 	res = parsed_result;
-	setup_gro(res->mode, res->port_id);
-}
-
-cmdline_parse_token_string_t cmd_gro_keyword =
-	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
+	if (!strcmp(res->cmd_keyword, "gro"))
+		setup_gro(res->cmd_onoff, res->cmd_pid);
+}
+
+cmdline_parse_token_string_t cmd_gro_enable_set =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_set, "set");
+cmdline_parse_token_string_t cmd_gro_enable_port =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_keyword, "port");
+cmdline_parse_token_num_t cmd_gro_enable_pid =
+	TOKEN_NUM_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_pid, UINT16);
+cmdline_parse_token_string_t cmd_gro_enable_keyword =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
 			cmd_keyword, "gro");
-cmdline_parse_token_string_t cmd_gro_mode =
-	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
-			mode, "on#off");
-cmdline_parse_token_num_t cmd_gro_pid =
-	TOKEN_NUM_INITIALIZER(struct cmd_gro_result,
-			port_id, UINT8);
+cmdline_parse_token_string_t cmd_gro_enable_onoff =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result,
+			cmd_onoff, "on#off");
 
-cmdline_parse_inst_t cmd_enable_gro = {
-	.f = cmd_enable_gro_parsed,
+cmdline_parse_inst_t cmd_gro_enable = {
+	.f = cmd_gro_enable_parsed,
 	.data = NULL,
-	.help_str = "gro (on|off) (port_id)",
+	.help_str = "set port <port_id> gro on|off",
 	.tokens = {
-		(void *)&cmd_gro_keyword,
-		(void *)&cmd_gro_mode,
-		(void *)&cmd_gro_pid,
+		(void *)&cmd_gro_enable_set,
+		(void *)&cmd_gro_enable_port,
+		(void *)&cmd_gro_enable_pid,
+		(void *)&cmd_gro_enable_keyword,
+		(void *)&cmd_gro_enable_onoff,
 		NULL,
 	},
 };
 
-/* *** SET MAX FLOW NUMBER AND ITEM NUM PER FLOW FOR GRO *** */
-struct cmd_gro_set_result {
-	cmdline_fixed_string_t gro;
-	cmdline_fixed_string_t mode;
-	uint16_t flow_num;
-	uint16_t item_num_per_flow;
-	uint8_t port_id;
+/* *** DISPLAY GRO CONFIGURATION *** */
+struct cmd_gro_show_result {
+	cmdline_fixed_string_t cmd_show;
+	cmdline_fixed_string_t cmd_port;
+	cmdline_fixed_string_t cmd_keyword;
+	portid_t cmd_pid;
 };
 
 static void
-cmd_gro_set_parsed(void *parsed_result,
-		       __attribute__((unused)) struct cmdline *cl,
-		       __attribute__((unused)) void *data)
+cmd_gro_show_parsed(void *parsed_result,
+		__attribute__((unused)) struct cmdline *cl,
+		__attribute__((unused)) void *data)
 {
-	struct cmd_gro_set_result *res = parsed_result;
+	struct cmd_gro_show_result *res;
 
-	if (port_id_is_invalid(res->port_id, ENABLED_WARN))
-		return;
-	if (test_done == 0) {
-		printf("Before set GRO flow_num and item_num_per_flow,"
-				" please stop forwarding first\n");
-		return;
-	}
+	res = parsed_result;
+	if (!strcmp(res->cmd_keyword, "gro"))
+		show_gro(res->cmd_pid);
+}
+
+cmdline_parse_token_string_t cmd_gro_show_show =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
+			cmd_show, "show");
+cmdline_parse_token_string_t cmd_gro_show_port =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
+			cmd_port, "port");
+cmdline_parse_token_num_t cmd_gro_show_pid =
+	TOKEN_NUM_INITIALIZER(struct cmd_gro_show_result,
+			cmd_pid, UINT16);
+cmdline_parse_token_string_t cmd_gro_show_keyword =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result,
+			cmd_keyword, "gro");
 
-	if (!strcmp(res->mode, "set")) {
-		if (res->flow_num == 0)
-			printf("Invalid flow number. Revert to default value:"
-					" %u.\n", GRO_DEFAULT_FLOW_NUM);
-		else
-			gro_ports[res->port_id].param.max_flow_num =
-				res->flow_num;
+cmdline_parse_inst_t cmd_gro_show = {
+	.f = cmd_gro_show_parsed,
+	.data = NULL,
+	.help_str = "show port <port_id> gro",
+	.tokens = {
+		(void *)&cmd_gro_show_show,
+		(void *)&cmd_gro_show_port,
+		(void *)&cmd_gro_show_pid,
+		(void *)&cmd_gro_show_keyword,
+		NULL,
+	},
+};
 
-		if (res->item_num_per_flow == 0)
-			printf("Invalid item number per-flow. Revert"
-					" to default value:%u.\n",
-					GRO_DEFAULT_ITEM_NUM_PER_FLOW);
-		else
-			gro_ports[res->port_id].param.max_item_per_flow =
-				res->item_num_per_flow;
-	}
+/* *** SET FLUSH CYCLES FOR GRO *** */
+struct cmd_gro_flush_result {
+	cmdline_fixed_string_t cmd_set;
+	cmdline_fixed_string_t cmd_keyword;
+	cmdline_fixed_string_t cmd_flush;
+	uint8_t cmd_cycles;
+};
+
+static void
+cmd_gro_flush_parsed(void *parsed_result,
+		__attribute__((unused)) struct cmdline *cl,
+		__attribute__((unused)) void *data)
+{
+	struct cmd_gro_flush_result *res;
+
+	res = parsed_result;
+	if ((!strcmp(res->cmd_keyword, "gro")) &&
+			(!strcmp(res->cmd_flush, "flush")))
+		setup_gro_flush_cycles(res->cmd_cycles);
 }
 
-cmdline_parse_token_string_t cmd_gro_set_gro =
-	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
-				gro, "gro");
-cmdline_parse_token_string_t cmd_gro_set_mode =
-	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
-				mode, "set");
-cmdline_parse_token_num_t cmd_gro_set_flow_num =
-	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
-				flow_num, UINT16);
-cmdline_parse_token_num_t cmd_gro_set_item_num_per_flow =
-	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
-				item_num_per_flow, UINT16);
-cmdline_parse_token_num_t cmd_gro_set_portid =
-	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
-				port_id, UINT8);
+cmdline_parse_token_string_t cmd_gro_flush_set =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_set, "set");
+cmdline_parse_token_string_t cmd_gro_flush_keyword =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_keyword, "gro");
+cmdline_parse_token_string_t cmd_gro_flush_flush =
+	TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_flush, "flush");
+cmdline_parse_token_num_t cmd_gro_flush_cycles =
+	TOKEN_NUM_INITIALIZER(struct cmd_gro_flush_result,
+			cmd_cycles, UINT8);
 
-cmdline_parse_inst_t cmd_gro_set = {
-	.f = cmd_gro_set_parsed,
+cmdline_parse_inst_t cmd_gro_flush = {
+	.f = cmd_gro_flush_parsed,
 	.data = NULL,
-	.help_str = "gro set <max_flow_num> <max_item_num_per_flow> "
-		"<port_id>: set max flow number and max packet number per-flow "
-		"for GRO",
+	.help_str = "set gro flush <cycles>",
 	.tokens = {
-		(void *)&cmd_gro_set_gro,
-		(void *)&cmd_gro_set_mode,
-		(void *)&cmd_gro_set_flow_num,
-		(void *)&cmd_gro_set_item_num_per_flow,
-		(void *)&cmd_gro_set_portid,
+		(void *)&cmd_gro_flush_set,
+		(void *)&cmd_gro_flush_keyword,
+		(void *)&cmd_gro_flush_flush,
+		(void *)&cmd_gro_flush_cycles,
 		NULL,
 	},
 };
@@ -14687,8 +14720,9 @@ cmdline_parse_ctx_t main_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_tso_show,
 	(cmdline_parse_inst_t *)&cmd_tunnel_tso_set,
 	(cmdline_parse_inst_t *)&cmd_tunnel_tso_show,
-	(cmdline_parse_inst_t *)&cmd_enable_gro,
-	(cmdline_parse_inst_t *)&cmd_gro_set,
+	(cmdline_parse_inst_t *)&cmd_gro_enable,
+	(cmdline_parse_inst_t *)&cmd_gro_flush,
+	(cmdline_parse_inst_t *)&cmd_gro_show,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_rx,
 	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_tx,
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index 0b2be2e..90e4f19 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -2428,7 +2428,7 @@ set_tx_pkt_segments(unsigned *seg_lengths, unsigned nb_segs)
 }
 
 void
-setup_gro(const char *mode, uint8_t port_id)
+setup_gro(const char *onoff, portid_t port_id)
 {
 	if (!rte_eth_dev_is_valid_port(port_id)) {
 		printf("invalid port id %u\n", port_id);
@@ -2439,29 +2439,77 @@ setup_gro(const char *mode, uint8_t port_id)
 				" please stop forwarding first\n");
 		return;
 	}
-	if (strcmp(mode, "on") == 0) {
-		if (gro_ports[port_id].enable) {
-			printf("port %u has enabled GRO\n", port_id);
+	if (strcmp(onoff, "on") == 0) {
+		if (gro_ports[port_id].enable != 0) {
+			printf("Port %u has enabled GRO. Please"
+					" disable GRO first\n", port_id);
 			return;
 		}
-		gro_ports[port_id].enable = 1;
-		gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4;
-
-		if (gro_ports[port_id].param.max_flow_num == 0)
+		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
+			gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4;
 			gro_ports[port_id].param.max_flow_num =
 				GRO_DEFAULT_FLOW_NUM;
-		if (gro_ports[port_id].param.max_item_per_flow == 0)
 			gro_ports[port_id].param.max_item_per_flow =
 				GRO_DEFAULT_ITEM_NUM_PER_FLOW;
+		}
+		gro_ports[port_id].enable = 1;
 	} else {
 		if (gro_ports[port_id].enable == 0) {
-			printf("port %u has disabled GRO\n", port_id);
+			printf("Port %u has disabled GRO\n", port_id);
 			return;
 		}
 		gro_ports[port_id].enable = 0;
 	}
 }
 
+void
+setup_gro_flush_cycles(uint8_t cycles)
+{
+	if (test_done == 0) {
+		printf("Before change flush interval for GRO,"
+				" please stop forwarding first.\n");
+		return;
+	}
+
+	if (cycles > GRO_MAX_FLUSH_CYCLES || cycles <
+			GRO_DEFAULT_FLUSH_CYCLES) {
+		printf("The flushing cycle be in the range"
+				" of 1 to %u. Revert to the default"
+				" value %u.\n",
+				GRO_MAX_FLUSH_CYCLES,
+				GRO_DEFAULT_FLUSH_CYCLES);
+		cycles = GRO_DEFAULT_FLUSH_CYCLES;
+	}
+
+	gro_flush_cycles = cycles;
+}
+
+void
+show_gro(portid_t port_id)
+{
+	struct rte_gro_param *param;
+	uint32_t max_pkts_num;
+
+	param = &gro_ports[port_id].param;
+
+	if (!rte_eth_dev_is_valid_port(port_id)) {
+		printf("Invalid port id %u.\n", port_id);
+		return;
+	}
+	if (gro_ports[port_id].enable) {
+		printf("GRO type: TCP/IPv4\n");
+		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
+			max_pkts_num = param->max_flow_num *
+				param->max_item_per_flow;
+		} else
+			max_pkts_num = MAX_PKT_BURST * GRO_MAX_FLUSH_CYCLES;
+		printf("Max number of packets to perform GRO: %u\n",
+				max_pkts_num);
+		printf("Flushing cycles: %u\n", gro_flush_cycles);
+	} else
+		printf("Port %u doesn't enable GRO.\n", port_id);
+}
+
 char*
 list_pkt_forwarding_modes(void)
 {
diff --git a/app/test-pmd/csumonly.c b/app/test-pmd/csumonly.c
index 90c8119..ca50ab7 100644
--- a/app/test-pmd/csumonly.c
+++ b/app/test-pmd/csumonly.c
@@ -631,6 +631,9 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 	struct rte_mbuf *m, *p;
 	struct ether_hdr *eth_hdr;
 	void *l3_hdr = NULL, *outer_l3_hdr = NULL; /* can be IPv4 or IPv6 */
+	void **gro_ctx;
+	uint16_t gro_pkts_num;
+	uint8_t gro_enable;
 	uint16_t nb_rx;
 	uint16_t nb_tx;
 	uint16_t nb_prep;
@@ -657,17 +660,13 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 				 nb_pkt_per_burst);
 	if (unlikely(nb_rx == 0))
 		return;
-	if (unlikely(gro_ports[fs->rx_port].enable))
-		nb_rx = rte_gro_reassemble_burst(pkts_burst,
-				nb_rx,
-				&(gro_ports[fs->rx_port].param));
-
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 	fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
 #endif
 	fs->rx_packets += nb_rx;
 	rx_bad_ip_csum = 0;
 	rx_bad_l4_csum = 0;
+	gro_enable = gro_ports[fs->rx_port].enable;
 
 	txp = &ports[fs->tx_port];
 	testpmd_ol_flags = txp->tx_ol_flags;
@@ -851,6 +850,28 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 		}
 	}
 
+	if (unlikely(gro_enable)) {
+		if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
+			nb_rx = rte_gro_reassemble_burst(pkts_burst, nb_rx,
+					&(gro_ports[fs->rx_port].param));
+		} else {
+			gro_ctx = current_fwd_lcore()->gro_ctx;
+			nb_rx = rte_gro_reassemble(pkts_burst, nb_rx, gro_ctx);
+
+			if (++fs->gro_times >= gro_flush_cycles) {
+				gro_pkts_num = rte_gro_get_pkt_count(gro_ctx);
+				if (gro_pkts_num > MAX_PKT_BURST - nb_rx)
+					gro_pkts_num = MAX_PKT_BURST - nb_rx;
+
+				nb_rx += rte_gro_timeout_flush(gro_ctx, 0,
+						RTE_GRO_TCP_IPV4,
+						&pkts_burst[nb_rx],
+						gro_pkts_num);
+				fs->gro_times = 0;
+			}
+		}
+	}
+
 	nb_prep = rte_eth_tx_prepare(fs->tx_port, fs->tx_queue,
 			pkts_burst, nb_rx);
 	if (nb_prep != nb_rx)
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index 49fd2ad..408db9f 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -90,7 +90,6 @@
 #ifdef RTE_LIBRTE_LATENCY_STATS
 #include <rte_latencystats.h>
 #endif
-#include <rte_gro.h>
 
 #include "testpmd.h"
 
@@ -386,6 +385,7 @@ uint8_t bitrate_enabled;
 #endif
 
 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
+uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 
 /* Forward function declarations */
 static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port);
@@ -570,6 +570,7 @@ init_config(void)
 	unsigned int nb_mbuf_per_pool;
 	lcoreid_t  lc_id;
 	uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
+	struct rte_gro_param gro_param;
 
 	memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
 
@@ -671,6 +672,20 @@ init_config(void)
 		rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
 
 	fwd_config_setup();
+
+	/* create a gro context for each lcore */
+	gro_param.gro_types = RTE_GRO_TCP_IPV4;
+	gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
+	gro_param.max_item_per_flow = MAX_PKT_BURST;
+	for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
+		gro_param.socket_id = rte_lcore_to_socket_id(
+				fwd_lcores_cpuids[lc_id]);
+		fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
+		if (fwd_lcores[lc_id]->gro_ctx == NULL) {
+			rte_exit(EXIT_FAILURE,
+					"rte_gro_ctx_create() failed\n");
+		}
+	}
 }
 
 
@@ -1217,6 +1232,7 @@ stop_packet_forwarding(void)
 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 	uint64_t fwd_cycles;
 #endif
+
 	static const char *acc_stats_border = "+++++++++++++++";
 
 	if (test_done) {
@@ -1307,6 +1323,7 @@ stop_packet_forwarding(void)
 
 		fwd_port_stats_display(pt_id, &stats);
 	}
+
 	printf("\n  %s Accumulated forward statistics for all ports"
 	       "%s\n",
 	       acc_stats_border, acc_stats_border);
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 657c123..2dc3b74 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -120,6 +120,7 @@ struct fwd_stream {
 	unsigned int fwd_dropped; /**< received packets not forwarded */
 	unsigned int rx_bad_ip_csum ; /**< received packets has bad ip checksum */
 	unsigned int rx_bad_l4_csum ; /**< received packets has bad l4 checksum */
+	unsigned int gro_times;	/**< GRO operation times */
 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 	uint64_t     core_cycles; /**< used for RX and TX processing */
 #endif
@@ -206,6 +207,7 @@ struct rte_port {
  */
 struct fwd_lcore {
 	struct rte_mempool *mbp; /**< The mbuf pool to use by this core */
+	void *gro_ctx;		/**< GRO context */
 	streamid_t stream_idx;   /**< index of 1st stream in "fwd_streams" */
 	streamid_t stream_nb;    /**< number of streams in "fwd_streams" */
 	lcoreid_t  cpuid_idx;    /**< index of logical core in CPU id table */
@@ -434,13 +436,19 @@ extern struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
 extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
 
-#define GRO_DEFAULT_FLOW_NUM 4
-#define GRO_DEFAULT_ITEM_NUM_PER_FLOW DEF_PKT_BURST
+#define GRO_DEFAULT_ITEM_NUM_PER_FLOW 32
+#define GRO_DEFAULT_FLOW_NUM (RTE_GRO_MAX_BURST_ITEM_NUM / \
+		GRO_DEFAULT_ITEM_NUM_PER_FLOW)
+
+#define GRO_DEFAULT_FLUSH_CYCLES 1
+#define GRO_MAX_FLUSH_CYCLES 4
+
 struct gro_status {
 	struct rte_gro_param param;
 	uint8_t enable;
 };
 extern struct gro_status gro_ports[RTE_MAX_ETHPORTS];
+extern uint8_t gro_flush_cycles;
 
 static inline unsigned int
 lcore_num(void)
@@ -641,7 +649,9 @@ void get_2tuple_filter(uint8_t port_id, uint16_t index);
 void get_5tuple_filter(uint8_t port_id, uint16_t index);
 int rx_queue_id_is_invalid(queueid_t rxq_id);
 int tx_queue_id_is_invalid(queueid_t txq_id);
-void setup_gro(const char *mode, uint8_t port_id);
+void setup_gro(const char *onoff, portid_t port_id);
+void setup_gro_flush_cycles(uint8_t cycles);
+void show_gro(portid_t port_id);
 
 /* Functions to manage the set of filtered Multicast MAC addresses */
 void mcast_addr_add(uint8_t port_id, struct ether_addr *mc_addr);
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 6615de8..0f45344 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -906,12 +906,12 @@ Display the status of TCP Segmentation Offload::
 
    testpmd> tso show (port_id)
 
-gro
-~~~
+set port - gro
+~~~~~~~~~~~~~~
 
 Enable or disable GRO in ``csum`` forwarding engine::
 
-   testpmd> gro (on|off) (port_id)
+   testpmd> set port <port_id> gro on|off
 
 If enabled, the csum forwarding engine will perform GRO on the TCP/IPv4
 packets received from the given port.
@@ -922,23 +922,43 @@ GRO. By default, GRO is disabled for all ports.
 .. note::
 
    When enable GRO for a port, TCP/IPv4 packets received from the port
-   will be performed GRO. After GRO, the merged packets are multi-segments.
-   But csum forwarding engine doesn't support to calculate TCP checksum
-   for multi-segment packets in SW. So please select TCP HW checksum
-   calculation for the port which GROed packets are transmitted to.
+   will be performed GRO. After GRO, all merged packets have bad
+   checksums, since the GRO library doesn't re-calculate checksums for
+   the merged packets. Therefore, if users want the merged packets to
+   have correct checksums, please select HW IP checksum calculation and
+   HW TCP checksum calculation for the port which the merged packets are
+   transmitted to.
+
+show port - gro
+~~~~~~~~~~~~~~~
 
-gro set
-~~~~~~~
+Display GRO configuration for a given port::
+
+   testpmd> show port <port_id> gro
+
+set gro flush
+~~~~~~~~~~~~~
+
+Set the cycle to flush the GROed packets from reassembly tables::
+
+   testpmd> set gro flush <cycles>
 
-Set max flow number and max packet number per-flow for GRO::
+When enable GRO, the csum forwarding engine performs GRO on received
+packets, and the GROed packets are stored in reassembly tables. Users
+can use this command to determine when the GROed packets are flushed
+from the reassembly tables.
 
-   testpmd> gro set (max_flow_num) (max_item_num_per_flow) (port_id)
+The ``cycles`` is measured in GRO operation times. The csum forwarding
+engine flushes the GROed packets from the tables every ``cycles`` GRO
+operations.
 
-The product of ``max_flow_num`` and ``max_item_num_per_flow`` is the max
-number of packets a GRO table can store.
+By default, the value of ``cycles`` is 1, which means flush GROed packets
+from the reassembly tables as soon as one GRO operation finishes. The value
+of ``cycles`` should be in the range of 1 to ``GRO_MAX_FLUSH_CYCLES``.
 
-If current packet number is greater than or equal to the max value, GRO
-will stop processing incoming packets.
+Please note that the large value of ``cycles`` may cause the poor TCP/IP
+stack performance. Because the GROed packets are delayed to arrive the
+stack, thus causing more duplicated ACKs and TCP retransmissions.
 
 mac_addr add
 ~~~~~~~~~~~~
-- 
2.7.4

Re: [dpdk-dev] [PATCH v5] app/testpmd: enable the heavyweight mode TCP/IPv4 GRO

[Ferruh Yigit]

On 10/7/2017 8:45 AM, Jiayu Hu wrote:
> The GRO library provides two modes to reassemble packets. Currently, the
> csum forwarding engine has supported to use the lightweight mode to
> reassemble TCP/IPv4 packets. This patch introduces the heavyweight mode
> for TCP/IPv4 GRO in the csum forwarding engine.
> 
> With the command "set port <port_id> gro on|off", users can enable
> TCP/IPv4 GRO for a given port. With the command "set gro flush <cycles>",
> users can determine when the GROed TCP/IPv4 packets are flushed from
> reassembly tables. With the command "show port <port_id> gro", users can
> display GRO configuration.
> 
> The GRO library doesn't re-calculate checksums for merged packets. If
> users want the merged packets to have correct IP and TCP checksums,
> please select HW IP checksum calculation and HW TCP checksum calculation
> for the port which the merged packets are transmitted to.
> 
> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
> Reviewed-by: Ferruh Yigit <ferruh.yigit@intel.com>
> Tested-by: Yao Lei <lei.a.yao@intel.com>

Applied to dpdk-next-net/master, thanks.