Re: [dpdk-dev] [PATCH v2 10/12] examples/ipsec-secgw: add driver mode worker

["Ananyev, Konstantin" <konstantin.ananyev@intel.com>]

> Add driver inbound and outbound worker thread for ipsec-secgw. In driver
> mode application does as little as possible. It simply forwards packets
> back to port from which traffic was received instructing HW to apply
> inline security processing using first outbound SA configured for
> a given port. If a port does not have SA configured outbound traffic
> on that port will be silently dropped. The aim of this mode is to
> measure HW capabilities. Driver mode is selected with single-sa option.
> The single-sa option accepts SA index however in event mode the SA
> index is ignored.
> 
> Example command to run ipsec-secgw in driver mode:
> ipsec-secgw -w 0002:02:00.0,ipsec_in_max_spi=128
> -w 0002:03:00.0,ipsec_in_max_spi=128 -w 0002:0e:00.0 -w 0002:10:00.1
> --log-level=8 -c 0x1 -- -P -p 0x3 -u 0x1 --config "(1,0,0),(0,0,0)"
> -f aes-gcm.cfg --transfer-mode event --schedule-type parallel
> --single-sa 0
> 
> Signed-off-by: Anoob Joseph <anoobj@marvell.com>
> Signed-off-by: Ankur Dwivedi <adwivedi@marvell.com>
> Signed-off-by: Lukasz Bartosik <lbartosik@marvell.com>
> ---
>  examples/ipsec-secgw/Makefile       |   1 +
>  examples/ipsec-secgw/ipsec-secgw.c  |  36 +++---
>  examples/ipsec-secgw/ipsec-secgw.h  |  17 +++
>  examples/ipsec-secgw/ipsec.h        |  11 ++
>  examples/ipsec-secgw/ipsec_worker.c | 240 ++++++++++++++++++++++++++++++++++++
>  examples/ipsec-secgw/meson.build    |   2 +-
>  6 files changed, 291 insertions(+), 16 deletions(-)
>  create mode 100644 examples/ipsec-secgw/ipsec-secgw.h
>  create mode 100644 examples/ipsec-secgw/ipsec_worker.c
> 
> diff --git a/examples/ipsec-secgw/Makefile b/examples/ipsec-secgw/Makefile
> index 09e3c5a..f6fd94c 100644
> --- a/examples/ipsec-secgw/Makefile
> +++ b/examples/ipsec-secgw/Makefile
> @@ -15,6 +15,7 @@ SRCS-y += sa.c
>  SRCS-y += rt.c
>  SRCS-y += ipsec_process.c
>  SRCS-y += ipsec-secgw.c
> +SRCS-y += ipsec_worker.c
>  SRCS-y += event_helper.c
> 
>  CFLAGS += -gdwarf-2
> diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
> index f1cc3fb..86215fb 100644
> --- a/examples/ipsec-secgw/ipsec-secgw.c
> +++ b/examples/ipsec-secgw/ipsec-secgw.c
> @@ -70,8 +70,6 @@ volatile bool force_quit;
> 
>  #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
> 
> -#define NB_SOCKETS 4
> -
>  /* Configure how many packets ahead to prefetch, when reading packets */
>  #define PREFETCH_OFFSET	3
> 
> @@ -79,8 +77,6 @@ volatile bool force_quit;
> 
>  #define MAX_LCORE_PARAMS 1024
> 
> -#define UNPROTECTED_PORT(port) (unprotected_port_mask & (1 << portid))
> -
>  /*
>   * Configurable number of RX/TX ring descriptors
>   */
> @@ -190,12 +186,10 @@ static const struct option lgopts[] = {
>  /* mask of enabled ports */
>  static uint32_t enabled_port_mask;
>  static uint64_t enabled_cryptodev_mask = UINT64_MAX;
> -static uint32_t unprotected_port_mask;
>  static int32_t promiscuous_on = 1;
>  static int32_t numa_on = 1; /**< NUMA is enabled by default. */
>  static uint32_t nb_lcores;
>  static uint32_t single_sa;
> -static uint32_t single_sa_idx;
>  static uint32_t schedule_type;
> 
>  /*
> @@ -279,8 +273,6 @@ static struct rte_eth_conf port_conf = {
>  	},
>  };
> 
> -static struct socket_ctx socket_ctx[NB_SOCKETS];
> -
>  /*
>   * Determine is multi-segment support required:
>   *  - either frame buffer size is smaller then mtu
> @@ -1114,8 +1106,8 @@ drain_outbound_crypto_queues(const struct lcore_conf *qconf,
>  }
> 
>  /* main processing loop */
> -static int32_t
> -main_loop(__attribute__((unused)) void *dummy)
> +void
> +ipsec_poll_mode_worker(void)
>  {
>  	struct rte_mbuf *pkts[MAX_PKT_BURST];
>  	uint32_t lcore_id;
> @@ -1157,7 +1149,7 @@ main_loop(__attribute__((unused)) void *dummy)
>  	if (qconf->nb_rx_queue == 0) {
>  		RTE_LOG(DEBUG, IPSEC, "lcore %u has nothing to do\n",
>  			lcore_id);
> -		return 0;
> +		return;
>  	}
> 
>  	RTE_LOG(INFO, IPSEC, "entering main loop on lcore %u\n", lcore_id);
> @@ -1170,7 +1162,7 @@ main_loop(__attribute__((unused)) void *dummy)
>  			lcore_id, portid, queueid);
>  	}
> 
> -	while (1) {
> +	while (!force_quit) {
>  		cur_tsc = rte_rdtsc();
> 
>  		/* TX queue buffer drain */
> @@ -1324,8 +1316,10 @@ print_usage(const char *prgname)
>  		"  -a enables SA SQN atomic behaviour\n"
>  		"  -f CONFIG_FILE: Configuration file\n"
>  		"  --config (port,queue,lcore): Rx queue configuration\n"
> -		"  --single-sa SAIDX: Use single SA index for outbound traffic,\n"
> -		"                     bypassing the SP\n"
> +		"  --single-sa SAIDX: In poll mode use single SA index for\n"
> +		"                     outbound traffic, bypassing the SP\n"
> +		"                     In event mode selects driver mode,\n"
> +		"                     SA index value is ignored\n"
>  		"  --cryptodev_mask MASK: Hexadecimal bitmask of the crypto\n"
>  		"                         devices to configure\n"
>  		"  --transfer-mode MODE\n"
> @@ -1980,6 +1974,18 @@ cryptodevs_init(void)
>  			i++;
>  		}
> 
> +		/*
> +		 * Set the queue pair to at least the number of ethernet
> +		 * devices for inline outbound.
> +		 */
> +		qp = RTE_MAX(rte_eth_dev_count_avail(), qp);
> +
> +		/*
> +		 * The requested number of queues should never exceed
> +		 * the max available
> +		 */
> +		qp = RTE_MIN(qp, max_nb_qps);
> +

Same comment as for v1:
I still don't understand why we have to do it for unconditionally.
For poll mode it seems to bring nothing but waste of resources.
Konstantin

>  		if (qp == 0)
>  			continue;
> 
> @@ -2871,7 +2877,7 @@ main(int32_t argc, char **argv)
>  	check_all_ports_link_status(enabled_port_mask);
> 
>  	/* launch per-lcore init on every lcore */
> -	rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER);
> +	rte_eal_mp_remote_launch(ipsec_launch_one_lcore, eh_conf, CALL_MASTER);
> 
>  	RTE_LCORE_FOREACH_SLAVE(lcore_id) {
>  		if (rte_eal_wait_lcore(lcore_id) < 0)
> diff --git a/examples/ipsec-secgw/ipsec-secgw.h b/examples/ipsec-secgw/ipsec-secgw.h
> new file mode 100644
> index 0000000..5b19e29
> --- /dev/null
> +++ b/examples/ipsec-secgw/ipsec-secgw.h
> @@ -0,0 +1,17 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright (C) 2020 Marvell International Ltd.
> + */
> +#ifndef _IPSEC_SECGW_H_
> +#define _IPSEC_SECGW_H_
> +
> +#define NB_SOCKETS 4
> +
> +#define UNPROTECTED_PORT(portid) (unprotected_port_mask & (1 << portid))
> +
> +/* Port mask to identify the unprotected ports */
> +uint32_t unprotected_port_mask;
> +
> +/* Index of SA in single mode */
> +uint32_t single_sa_idx;
> +
> +#endif /* _IPSEC_SECGW_H_ */
> diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
> index 0539aec..65be2ac 100644
> --- a/examples/ipsec-secgw/ipsec.h
> +++ b/examples/ipsec-secgw/ipsec.h
> @@ -13,6 +13,8 @@
>  #include <rte_flow.h>
>  #include <rte_ipsec.h>
> 
> +#include "ipsec-secgw.h"
> +
>  #define RTE_LOGTYPE_IPSEC       RTE_LOGTYPE_USER1
>  #define RTE_LOGTYPE_IPSEC_ESP   RTE_LOGTYPE_USER2
>  #define RTE_LOGTYPE_IPSEC_IPIP  RTE_LOGTYPE_USER3
> @@ -258,6 +260,15 @@ struct ipsec_traffic {
>  	struct traffic_type ip6;
>  };
> 
> +/* Socket ctx */
> +struct socket_ctx socket_ctx[NB_SOCKETS];
> +
> +void
> +ipsec_poll_mode_worker(void);
> +
> +int
> +ipsec_launch_one_lcore(void *args);
> +
>  uint16_t
>  ipsec_inbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
>  		uint16_t nb_pkts, uint16_t len);