Re: [dpdk-dev] [PATCH v6 3/3] app/testpmd: enable TCP/IPv4 GRO

["Yao, Lei A" <lei.a.yao@intel.com>]

> -----Original Message-----
> From: Hu, Jiayu
> Sent: Friday, June 23, 2017 10:43 PM
> To: dev@dpdk.org
> Cc: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Tan, Jianfeng
> <jianfeng.tan@intel.com>; stephen@networkplumber.org;
> yliu@fridaylinux.org; Wiles, Keith <keith.wiles@intel.com>; Bie, Tiwei
> <tiwei.bie@intel.com>; Yao, Lei A <lei.a.yao@intel.com>; Hu, Jiayu
> <jiayu.hu@intel.com>
> Subject: [PATCH v6 3/3] app/testpmd: enable TCP/IPv4 GRO
> 
> This patch enables TCP/IPv4 GRO library in csum forwarding engine.
> By default, GRO is turned off. Users can use command "gro (on|off)
> (port_id)" to enable or disable GRO for a given port. If a port is
> enabled GRO, all TCP/IPv4 packets received from the port are performed
> GRO. Besides, users can set max flow number and packets number per-flow
> by command "gro set (max_flow_num) (max_item_num_per_flow)
> (port_id)".
> 
> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
Tested-By: Lei Yao<lei.a.yao@intel.com>
This patch is tested on the following test bench:
OS: Ubuntu 16.04
CPU: Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz
NIC: XXV710 25G
We can see the iperf result improve a lot after enable GRO. The data
flow is NIC1->NIC2->testpmd(GRO on/off)->vhost->virtio-net(in VM) 

> ---
>  app/test-pmd/cmdline.c                      | 125
> ++++++++++++++++++++++++++++
>  app/test-pmd/config.c                       |  37 ++++++++
>  app/test-pmd/csumonly.c                     |   5 ++
>  app/test-pmd/testpmd.c                      |   3 +
>  app/test-pmd/testpmd.h                      |  11 +++
>  doc/guides/testpmd_app_ug/testpmd_funcs.rst |  34 ++++++++
>  6 files changed, 215 insertions(+)
> 
> diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
> index ff8ffd2..cb359e1 100644
> --- a/app/test-pmd/cmdline.c
> +++ b/app/test-pmd/cmdline.c
> @@ -76,6 +76,7 @@
>  #include <rte_devargs.h>
>  #include <rte_eth_ctrl.h>
>  #include <rte_flow.h>
> +#include <rte_gro.h>
> 
>  #include <cmdline_rdline.h>
>  #include <cmdline_parse.h>
> @@ -419,6 +420,14 @@ static void cmd_help_long_parsed(void
> *parsed_result,
>  			"tso show (portid)"
>  			"    Display the status of TCP Segmentation
> Offload.\n\n"
> 
> +			"gro (on|off) (port_id)"
> +			"    Enable or disable Generic Receive Offload in io"
> +			" forward engine.\n\n"
> +
> +			"gro set (max_flow_num)
> (max_item_num_per_flow) (port_id)\n"
> +			"    Set max flow number and max packet number
> per-flow"
> +			" for GRO.\n\n"
> +
>  			"set fwd (%s)\n"
>  			"    Set packet forwarding mode.\n\n"
> 
> @@ -3827,6 +3836,120 @@ cmdline_parse_inst_t cmd_tunnel_tso_show = {
>  	},
>  };
> 
> +/* *** SET GRO FOR A PORT *** */
> +struct cmd_gro_result {
> +	cmdline_fixed_string_t cmd_keyword;
> +	cmdline_fixed_string_t mode;
> +	uint8_t port_id;
> +};
> +
> +static void
> +cmd_enable_gro_parsed(void *parsed_result,
> +		__attribute__((unused)) struct cmdline *cl,
> +		__attribute__((unused)) void *data)
> +{
> +	struct cmd_gro_result *res;
> +
> +	res = parsed_result;
> +	setup_gro(res->mode, res->port_id);
> +}
> +
> +cmdline_parse_token_string_t cmd_gro_keyword =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
> +			cmd_keyword, "gro");
> +cmdline_parse_token_string_t cmd_gro_mode =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_result,
> +			mode, "on#off");
> +cmdline_parse_token_num_t cmd_gro_pid =
> +	TOKEN_NUM_INITIALIZER(struct cmd_gro_result,
> +			port_id, UINT8);
> +
> +cmdline_parse_inst_t cmd_enable_gro = {
> +	.f = cmd_enable_gro_parsed,
> +	.data = NULL,
> +	.help_str = "gro (on|off) (port_id)",
> +	.tokens = {
> +		(void *)&cmd_gro_keyword,
> +		(void *)&cmd_gro_mode,
> +		(void *)&cmd_gro_pid,
> +		NULL,
> +	},
> +};
> +
> +/* *** SET MAX FLOW NUMBER AND ITEM NUM PER FLOW FOR GRO ***
> */
> +struct cmd_gro_set_result {
> +	cmdline_fixed_string_t gro;
> +	cmdline_fixed_string_t mode;
> +	uint16_t flow_num;
> +	uint16_t item_num_per_flow;
> +	uint8_t port_id;
> +};
> +
> +static void
> +cmd_gro_set_parsed(void *parsed_result,
> +		       __attribute__((unused)) struct cmdline *cl,
> +		       __attribute__((unused)) void *data)
> +{
> +	struct cmd_gro_set_result *res = parsed_result;
> +
> +	if (port_id_is_invalid(res->port_id, ENABLED_WARN))
> +		return;
> +	if (test_done == 0) {
> +		printf("Before set GRO flow_num and item_num_per_flow,"
> +				" please stop forwarding first\n");
> +		return;
> +	}
> +
> +	if (!strcmp(res->mode, "set")) {
> +		if (res->flow_num == 0)
> +			printf("Invalid flow number. Revert to default value:"
> +					" %u.\n",
> GRO_DEFAULT_FLOW_NUM);
> +		else
> +			gro_ports[res->port_id].param.max_flow_num =
> +				res->flow_num;
> +
> +		if (res->item_num_per_flow == 0)
> +			printf("Invalid item number per-flow. Revert"
> +					" to default value:%u.\n",
> +
> 	GRO_DEFAULT_ITEM_NUM_PER_FLOW);
> +		else
> +			gro_ports[res->port_id].param.max_item_per_flow
> =
> +				res->item_num_per_flow;
> +	}
> +}
> +
> +cmdline_parse_token_string_t cmd_gro_set_gro =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
> +				gro, "gro");
> +cmdline_parse_token_string_t cmd_gro_set_mode =
> +	TOKEN_STRING_INITIALIZER(struct cmd_gro_set_result,
> +				mode, "set");
> +cmdline_parse_token_num_t cmd_gro_set_flow_num =
> +	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> +				flow_num, UINT16);
> +cmdline_parse_token_num_t cmd_gro_set_item_num_per_flow =
> +	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> +				item_num_per_flow, UINT16);
> +cmdline_parse_token_num_t cmd_gro_set_portid =
> +	TOKEN_NUM_INITIALIZER(struct cmd_gro_set_result,
> +				port_id, UINT8);
> +
> +cmdline_parse_inst_t cmd_gro_set = {
> +	.f = cmd_gro_set_parsed,
> +	.data = NULL,
> +	.help_str = "gro set <max_flow_num> <max_item_num_per_flow>
> "
> +		"<port_id>: set max flow number and max packet number
> per-flow "
> +		"for GRO",
> +	.tokens = {
> +		(void *)&cmd_gro_set_gro,
> +		(void *)&cmd_gro_set_mode,
> +		(void *)&cmd_gro_set_flow_num,
> +		(void *)&cmd_gro_set_item_num_per_flow,
> +		(void *)&cmd_gro_set_portid,
> +		NULL,
> +	},
> +};
> +
>  /* *** ENABLE/DISABLE FLUSH ON RX STREAMS *** */
>  struct cmd_set_flush_rx {
>  	cmdline_fixed_string_t set;
> @@ -13732,6 +13855,8 @@ cmdline_parse_ctx_t main_ctx[] = {
>  	(cmdline_parse_inst_t *)&cmd_tso_show,
>  	(cmdline_parse_inst_t *)&cmd_tunnel_tso_set,
>  	(cmdline_parse_inst_t *)&cmd_tunnel_tso_show,
> +	(cmdline_parse_inst_t *)&cmd_enable_gro,
> +	(cmdline_parse_inst_t *)&cmd_gro_set,
>  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set,
>  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_rx,
>  	(cmdline_parse_inst_t *)&cmd_link_flow_control_set_tx,
> diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
> index b0b340e..2a33a63 100644
> --- a/app/test-pmd/config.c
> +++ b/app/test-pmd/config.c
> @@ -71,6 +71,7 @@
>  #ifdef RTE_LIBRTE_BNXT_PMD
>  #include <rte_pmd_bnxt.h>
>  #endif
> +#include <rte_gro.h>
> 
>  #include "testpmd.h"
> 
> @@ -2414,6 +2415,42 @@ set_tx_pkt_segments(unsigned *seg_lengths,
> unsigned nb_segs)
>  	tx_pkt_nb_segs = (uint8_t) nb_segs;
>  }
> 
> +void
> +setup_gro(const char *mode, uint8_t port_id)
> +{
> +	if (!rte_eth_dev_is_valid_port(port_id)) {
> +		printf("invalid port id %u\n", port_id);
> +		return;
> +	}
> +	if (test_done == 0) {
> +		printf("Before enable/disable GRO,"
> +				" please stop forwarding first\n");
> +		return;
> +	}
> +	if (strcmp(mode, "on") == 0) {
> +		if (gro_ports[port_id].enable) {
> +			printf("port %u has enabled GRO\n", port_id);
> +			return;
> +		}
> +		gro_ports[port_id].enable = 1;
> +		gro_ports[port_id].param.desired_gro_types =
> GRO_TCP_IPV4;
> +		gro_ports[port_id].param.max_packet_size = UINT16_MAX;
> +
> +		if (gro_ports[port_id].param.max_flow_num == 0)
> +			gro_ports[port_id].param.max_flow_num =
> +				GRO_DEFAULT_FLOW_NUM;
> +		if (gro_ports[port_id].param.max_item_per_flow == 0)
> +			gro_ports[port_id].param.max_item_per_flow =
> +				GRO_DEFAULT_ITEM_NUM_PER_FLOW;
> +	} else {
> +		if (gro_ports[port_id].enable == 0) {
> +			printf("port %u has disabled GRO\n", port_id);
> +			return;
> +		}
> +		gro_ports[port_id].enable = 0;
> +	}
> +}
> +
>  char*
>  list_pkt_forwarding_modes(void)
>  {
> diff --git a/app/test-pmd/csumonly.c b/app/test-pmd/csumonly.c
> index 66fc9a0..430bd8b 100644
> --- a/app/test-pmd/csumonly.c
> +++ b/app/test-pmd/csumonly.c
> @@ -71,6 +71,7 @@
>  #include <rte_prefetch.h>
>  #include <rte_string_fns.h>
>  #include <rte_flow.h>
> +#include <rte_gro.h>
>  #include "testpmd.h"
> 
>  #define IP_DEFTTL  64   /* from RFC 1340. */
> @@ -658,6 +659,10 @@ pkt_burst_checksum_forward(struct fwd_stream
> *fs)
>  				 nb_pkt_per_burst);
>  	if (unlikely(nb_rx == 0))
>  		return;
> +	if (unlikely(gro_ports[fs->rx_port].enable))
> +		nb_rx = rte_gro_reassemble_burst(pkts_burst,
> +				nb_rx,
> +				gro_ports[fs->rx_port].param);
> 
>  #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
>  	fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
> diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
> index b29328a..ed27c7a 100644
> --- a/app/test-pmd/testpmd.c
> +++ b/app/test-pmd/testpmd.c
> @@ -90,6 +90,7 @@
>  #ifdef RTE_LIBRTE_LATENCY_STATS
>  #include <rte_latencystats.h>
>  #endif
> +#include <rte_gro.h>
> 
>  #include "testpmd.h"
> 
> @@ -378,6 +379,8 @@ lcoreid_t bitrate_lcore_id;
>  uint8_t bitrate_enabled;
>  #endif
> 
> +struct gro_status gro_ports[RTE_MAX_ETHPORTS];
> +
>  /* Forward function declarations */
>  static void map_port_queue_stats_mapping_registers(uint8_t pi, struct
> rte_port *port);
>  static void check_all_ports_link_status(uint32_t port_mask);
> diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
> index 364502d..377d933 100644
> --- a/app/test-pmd/testpmd.h
> +++ b/app/test-pmd/testpmd.h
> @@ -34,6 +34,8 @@
>  #ifndef _TESTPMD_H_
>  #define _TESTPMD_H_
> 
> +#include <rte_gro.h>
> +
>  #define RTE_PORT_ALL            (~(portid_t)0x0)
> 
>  #define RTE_TEST_RX_DESC_MAX    2048
> @@ -428,6 +430,14 @@ extern struct ether_addr
> peer_eth_addrs[RTE_MAX_ETHPORTS];
>  extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-
> retry. */
>  extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-
> retry. */
> 
> +#define GRO_DEFAULT_FLOW_NUM 4
> +#define GRO_DEFAULT_ITEM_NUM_PER_FLOW DEF_PKT_BURST
> +struct gro_status {
> +	struct rte_gro_param param;
> +	uint8_t enable;
> +};
> +extern struct gro_status gro_ports[RTE_MAX_ETHPORTS];
> +
>  static inline unsigned int
>  lcore_num(void)
>  {
> @@ -626,6 +636,7 @@ void get_2tuple_filter(uint8_t port_id, uint16_t
> index);
>  void get_5tuple_filter(uint8_t port_id, uint16_t index);
>  int rx_queue_id_is_invalid(queueid_t rxq_id);
>  int tx_queue_id_is_invalid(queueid_t txq_id);
> +void setup_gro(const char *mode, uint8_t port_id);
> 
>  /* Functions to manage the set of filtered Multicast MAC addresses */
>  void mcast_addr_add(uint8_t port_id, struct ether_addr *mc_addr);
> diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> index 2b9a1ea..528c833 100644
> --- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> +++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> @@ -884,6 +884,40 @@ Display the status of TCP Segmentation Offload::
> 
>     testpmd> tso show (port_id)
> 
> +gro
> +~~~~~~~~
> +
> +Enable or disable GRO in ``csum`` forwarding engine::
> +
> +   testpmd> gro (on|off) (port_id)
> +
> +If enabled, the csum forwarding engine will perform GRO on the TCP/IPv4
> +packets received from the given port.
> +
> +If disabled, packets received from the given port won't be performed
> +GRO. By default, GRO is disabled for all ports.
> +
> +.. note::
> +
> +   When enable GRO for a port, TCP/IPv4 packets received from the port
> +   will be performed GRO. After GRO, the merged packets are multi-
> segments.
> +   But csum forwarding engine doesn't support to calculate TCP checksum
> +   for multi-segment packets in SW. So please select TCP HW checksum
> +   calculation for the port which GROed packets are transmitted to.
> +
> +gro set
> +~~~~~~~~
> +
> +Set max flow number and max packet number per-flow for GRO::
> +
> +   testpmd> gro set (max_flow_num) (max_item_num_per_flow) (port_id)
> +
> +The product of ``max_flow_num`` and ``max_item_num_per_flow`` is the
> max
> +number of packets a GRO table can store.
> +
> +If current packet number is greater than or equal to the max value, GRO
> +will stop processing incoming packets.
> +
>  mac_addr add
>  ~~~~~~~~~~~~
> 
> --
> 2.7.4