Re: [dpdk-dev] [PATCH v4 1/2] testpmd: add forwarding mode to simulate a noisy neighbour

[Shahaf Shuler <shahafs@mellanox.com>]

HI Maxime, Jens

Saturday, June 23, 2018 11:09 AM, Maxime Coquelin:
> Subject: [dpdk-dev] [PATCH v4 1/2] testpmd: add forwarding mode to
> simulate a noisy neighbour
> 
> From: Jens Freimann <jfreimann@redhat.com>
> 
> This adds a new forwarding mode to testpmd to simulate more realistic
> behavior of a guest machine engaged in receiving and sending packets
> performing Virtual Network Function (VNF).

I really like the idea of adding application w/ more realistic behavior to the DPDK tree in order to use them in the non-regression cycle. 

I am just wondering if putting it as another forward engine in testpmd is the right place for it.
>From one side it reuses testpmd command line options which is good.
On the other hand it tends to have a lot of options specific only for the VNF forward use case. And I am quite sure that w/ the time goes by the number of configuration options will increase. 

I think this code needs to be as a dedicated example under examples/. 
Moreover I would refine from directly associate it with VNF. VNF can be one example but for sure there are more. 
We can call the example noisy_neigbour or any other name you like.

One more comment below. 

> 
> The goal is to enable a simple way of measuring performance impact on
> cache and memory footprint utilization from various VNF co-located on the
> same host machine. For this it does:
> 
> * Buffer packets in a FIFO:
> 
> Create a fifo to buffer received packets. Once it flows over put those packets
> into the actual tx queue. The fifo is created per tx queue and its size can be
> set with the --buffersize-before-sending commandline parameter.
> 
> A second commandline parameter is used to set a timeout in milliseconds
> after which the fifo is flushed.
> 
> --noisy-buffersize-before-sending [packet numbers] Keep the mbuf in a
> FIFO and forward the over flooding packets from the FIFO. This queue is per
> TX-queue (after all other packet processing).
> 
> --noisy-flush-timeout [delay]
> Flush the packet queue if no packets have been seen during [delay]. As long
> as packets are seen, the timer is reset.
> 
> Add several options to simulate route lookups (memory reads) in tables that
> can be quite large, as well as route hit statistics update.
> These options simulates the while stack traversal and will trash the cache.
> Memory access is random.
> 
> * simulate route lookups:
> 
> Allocate a buffer and perform reads and writes on it as specified by
> commandline options:
> 
> --noisy-memory-footprint [size]
> Size of the VNF internal memory (MB), in which the random read/write will
> be done, allocated by rte_malloc (hugepages).
> 
> --noisy-nb-rnd-write [num]
> Number of random writes in memory per packet should be performed,
> simulating hit-flags update. 64 bits per write, all write in different cache lines.
> 
> --noisy-nb-rnd-read [num]
> Number of random reads in memory per packet should be performed,
> simulating FIB/table lookups. 64 bits per read, all write in different cache
> lines.
> 
> --noisy-nb-rnd-read-write [num]
> Number of random reads and writes in memory per packet should be
> performed, simulating stats update. 64 bits per read-write, all reads and
> writes in different cache lines.
> 
> Signed-off-by: Jens Freimann <jfreimann@redhat.com>
> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> ---
>  app/test-pmd/Makefile     |   1 +
>  app/test-pmd/meson.build  |   1 +
>  app/test-pmd/noisy_vnf.c  | 204
> ++++++++++++++++++++++++++++++++++++++++++++++
>  app/test-pmd/noisy_vnf.h  |  41 ++++++++++  app/test-pmd/parameters.c
> |  54 ++++++++++++
>  app/test-pmd/testpmd.c    |  52 ++++++++++++
>  app/test-pmd/testpmd.h    |  17 ++++
>  7 files changed, 370 insertions(+)
>  create mode 100644 app/test-pmd/noisy_vnf.c  create mode 100644
> app/test-pmd/noisy_vnf.h
> 
> diff --git a/app/test-pmd/Makefile b/app/test-pmd/Makefile index
> a5a827bbd..2a1030120 100644
> --- a/app/test-pmd/Makefile
> +++ b/app/test-pmd/Makefile
> @@ -32,6 +32,7 @@ SRCS-y += rxonly.c
>  SRCS-y += txonly.c
>  SRCS-y += csumonly.c
>  SRCS-y += icmpecho.c
> +SRCS-y += noisy_vnf.c
>  SRCS-$(CONFIG_RTE_LIBRTE_IEEE1588) += ieee1588fwd.c
>  SRCS-$(CONFIG_RTE_LIBRTE_BPF) += bpf_cmd.c
> 
> diff --git a/app/test-pmd/meson.build b/app/test-pmd/meson.build index
> 4c9fbbfda..b3abc6e24 100644
> --- a/app/test-pmd/meson.build
> +++ b/app/test-pmd/meson.build
> @@ -16,6 +16,7 @@ sources = files('cmdline.c',
>  	'iofwd.c',
>  	'macfwd.c',
>  	'macswap.c',
> +	'noisy_vnf.c',
>  	'parameters.c',
>  	'rxonly.c',
>  	'testpmd.c',
> diff --git a/app/test-pmd/noisy_vnf.c b/app/test-pmd/noisy_vnf.c new file
> mode 100644 index 000000000..ff205495f
> --- /dev/null
> +++ b/app/test-pmd/noisy_vnf.c
> @@ -0,0 +1,204 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2018 Red Hat Corp.
> + */
> +
> +#include <stdarg.h>
> +#include <stdio.h>
> +#include <string.h>
> +#include <errno.h>
> +#include <stdint.h>
> +#include <unistd.h>
> +#include <inttypes.h>
> +
> +#include <sys/queue.h>
> +#include <sys/stat.h>
> +
> +#include <rte_common.h>
> +#include <rte_log.h>
> +#include <rte_debug.h>
> +#include <rte_cycles.h>
> +#include <rte_memory.h>
> +#include <rte_launch.h>
> +#include <rte_eal.h>
> +#include <rte_per_lcore.h>
> +#include <rte_lcore.h>
> +#include <rte_memcpy.h>
> +#include <rte_mempool.h>
> +#include <rte_mbuf.h>
> +#include <rte_ethdev.h>
> +#include <rte_flow.h>
> +#include <rte_malloc.h>
> +
> +#include "testpmd.h"
> +#include "noisy_vnf.h"
> +
> +static inline void
> +do_write(char *vnf_mem)
> +{
> +	uint64_t i = rte_rand();
> +	uint64_t w = rte_rand();
> +
> +	vnf_mem[i % ((noisy_vnf_memory_footprint * 1024 * 1024) /
> +			RTE_CACHE_LINE_SIZE)] = w;
> +}
> +
> +	static inline void
> +do_read(char *vnf_mem)
> +{
> +	uint64_t i = rte_rand();
> +	uint64_t r;
> +
> +	r = vnf_mem[i % ((noisy_vnf_memory_footprint * 1024 * 1024) /
> +			RTE_CACHE_LINE_SIZE)];
> +	r++;
> +}
> +
> +	static inline void
> +do_rw(char *vnf_mem)
> +{
> +	do_read(vnf_mem);
> +	do_write(vnf_mem);
> +}
> +
> +/*
> + * Simulate route lookups as defined by commandline parameters  */
> +	static void
> +sim_memory_lookups(struct noisy_config *ncf, uint16_t nb_pkts) {
> +	uint16_t i, j;
> +
> +	for (i = 0; i < nb_pkts; i++) {
> +		for (j = 0; j < noisy_nb_rnd_write; j++)
> +			do_write(ncf->vnf_mem);
> +		for (j = 0; j < noisy_nb_rnd_read; j++)
> +			do_read(ncf->vnf_mem);
> +		for (j = 0; j < noisy_nb_rnd_read_write; j++)
> +			do_rw(ncf->vnf_mem);
> +	}
> +}
> +
> +/*
> + * Forwarding of packets in I/O mode.
> + * Forward packets "as-is".
> + * This is the fastest possible forwarding operation, as it does not
> +access
> + * to packets data.
> + */
> +static void
> +pkt_burst_noisy_vnf(struct fwd_stream *fs) {
> +	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
> +	uint16_t nb_rx;
> +	uint16_t nb_tx = 0;
> +	uint32_t retry;
> +	const uint64_t freq_khz = rte_get_timer_hz() / 1000;
> +	struct noisy_config *ncf = &noisy_cfg[fs->tx_queue];
> +	struct rte_mbuf *tmp_pkts[MAX_PKT_BURST];
> +	uint16_t nb_enqd;
> +	uint16_t nb_deqd = 0;
> +	uint64_t delta_ms;
> +	uint64_t now;
> +
> +	/*
> +	 * Receive a burst of packets and forward them.
> +	 */
> +	nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue,
> +			pkts_burst, nb_pkt_per_burst);
> +	if (unlikely(nb_rx == 0))
> +		return;
> +	fs->rx_packets += nb_rx;
> +
> +	if (noisy_bsize_before_send > 0) {
> +		if (rte_ring_free_count(ncf->f) >= nb_rx) {
> +			/* enqueue into fifo */
> +			nb_enqd = fifo_put(ncf->f, pkts_burst, nb_rx);
> +			if (nb_enqd < nb_rx)
> +				nb_rx = nb_enqd;

Am not sure I understand this part.
The simulation of the memory lookups should happen for every packet **before** deciding on the output queue. Isn't it more realistic to simulate the memory access after the rx_burst?

> +		} else {
> +			/* fifo is full, dequeue first */
> +			nb_deqd = fifo_get(ncf->f, tmp_pkts, nb_rx);
> +			/* enqueue into fifo */
> +			nb_enqd = fifo_put(ncf->f, pkts_burst, nb_deqd);
> +			sim_memory_lookups(ncf, nb_rx);
> +			if (nb_enqd < nb_rx)
> +				nb_rx = nb_enqd;
> +			if (nb_deqd > 0)
> +				nb_tx = rte_eth_tx_burst(fs->tx_port,
> +						fs->tx_queue, tmp_pkts,
> +						nb_deqd);
> +		}
> +	} else {
> +		sim_memory_lookups(ncf, nb_rx);
> +		nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
> +				pkts_burst, nb_rx);
> +	}
> +
> +	/*
> +	 * TX burst queue drain
> +	 */
> +	if (ncf->prev_time == 0)
> +		now = ncf->prev_time = rte_get_timer_cycles();
> +	else
> +		now = rte_get_timer_cycles();
> +	delta_ms = (now - ncf->prev_time) / freq_khz;
> +	if (unlikely(delta_ms >= noisy_flush_timer) && noisy_flush_timer > 0
> &&
> +		     (nb_tx == 0)) {
> +		while (fifo_count(ncf->f) > 0) {
> +			nb_deqd = fifo_get(ncf->f, tmp_pkts, nb_rx);
> +			nb_tx = rte_eth_tx_burst(fs->tx_port, fs-
> >tx_queue,
> +						 tmp_pkts, nb_deqd);
> +			if (rte_ring_empty(ncf->f))
> +				break;
> +		}
> +		ncf->prev_time = now;
> +	}
> +	if (nb_tx < nb_rx && fs->retry_enabled)
> +		*pkts_burst = *tmp_pkts;
> +
> +	/*
> +	 * Retry if necessary
> +	 */
> +	if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) {
> +		retry = 0;
> +		while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) {
> +			rte_delay_us(burst_tx_delay_time);
> +			sim_memory_lookups(ncf, nb_rx);
> +			nb_tx += rte_eth_tx_burst(fs->tx_port, fs-
> >tx_queue,
> +					&pkts_burst[nb_tx], nb_rx - nb_tx);
> +		}
> +	}
> +	fs->tx_packets += nb_tx;
> +	if (unlikely(nb_tx < nb_rx)) {
> +		fs->fwd_dropped += (nb_rx - nb_tx);
> +		do {
> +			rte_pktmbuf_free(pkts_burst[nb_tx]);
> +		} while (++nb_tx < nb_rx);
> +	}
> +}
> +
> +struct fwd_engine noisy_vnf_engine = {
> +	.fwd_mode_name  = "noisy",
> +	.port_fwd_begin = NULL,
> +	.port_fwd_end   = NULL,
> +	.packet_fwd     = pkt_burst_noisy_vnf,
> +};
> +
> +#define NOISY_STRSIZE 256
> +#define NOISY_RING "noisy_ring_%d:%d\n"
> +struct rte_ring *
> +noisy_init(uint32_t qi, uint32_t pi)
> +{
> +	struct noisy_config *n = &noisy_cfg[qi];
> +	char name[NOISY_STRSIZE];
> +
> +	snprintf(name, NOISY_STRSIZE, NOISY_RING, pi, qi);
> +	n->f = rte_ring_create(name, noisy_bsize_before_send,
> +			rte_socket_id(), 0);
> +	n->vnf_mem = (char *) rte_zmalloc("vnf sim memory",
> +			 noisy_vnf_memory_footprint * 1024 * 1024,
> +			 RTE_CACHE_LINE_SIZE);
> +	if (n->vnf_mem == NULL)
> +		printf("allocating vnf memory failed\n");
> +
> +	return n->f;
> +}
> diff --git a/app/test-pmd/noisy_vnf.h b/app/test-pmd/noisy_vnf.h new file
> mode 100644 index 000000000..6c6df7a9e
> --- /dev/null
> +++ b/app/test-pmd/noisy_vnf.h
> @@ -0,0 +1,41 @@
> +#ifndef __NOISY_VNF_H
> +#define __NOISY_VNF_H
> +#include <stdint.h>
> +#include <rte_mbuf.h>
> +#include <rte_ring.h>
> +#include "testpmd.h"
> +
> +/**
> + * Add elements to fifo. Return number of written elements  */ static
> +inline unsigned int fifo_put(struct rte_ring *r, struct rte_mbuf
> +**data, unsigned num) {
> +
> +	return rte_ring_enqueue_burst(r, (void **)data, num, NULL); }
> +
> +/**
> + * Get elements from fifo. Return number of read elements  */ static
> +inline unsigned int fifo_get(struct rte_ring *r, struct rte_mbuf
> +**data, unsigned num) {
> +	return rte_ring_dequeue_burst(r, (void **) data, num, NULL); }
> +
> +static inline unsigned int
> +fifo_count(struct rte_ring *r)
> +{
> +	return rte_ring_count(r);
> +}
> +
> +static inline int
> +fifo_full(struct rte_ring *r)
> +{
> +	return rte_ring_full(r);
> +}
> +
> +struct rte_ring *noisy_init(uint32_t qi, uint32_t pi);
> +
> +#endif
> diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c index
> 75807623c..e6d8b3ef9 100644
> --- a/app/test-pmd/parameters.c
> +++ b/app/test-pmd/parameters.c
> @@ -625,6 +625,12 @@ launch_args_parse(int argc, char** argv)
>  		{ "vxlan-gpe-port",		1, 0, 0 },
>  		{ "mlockall",			0, 0, 0 },
>  		{ "no-mlockall",		0, 0, 0 },
> +		{ "noisy-buffersize-before-sending", 1, 0, 0 },
> +		{ "noisy-flush-timeout",        1, 0, 0 },
> +		{ "noisy-memory-footprint",     1, 0, 0 },
> +		{ "noisy-nb-rnd-write",         1, 0, 0 },
> +		{ "noisy-nb-rnd-read",          1, 0, 0 },
> +		{ "noisy-nb-rnd-read-write",    1, 0, 0 },
>  		{ 0, 0, 0, 0 },
>  	};
> 
> @@ -1145,6 +1151,54 @@ launch_args_parse(int argc, char** argv)
>  				do_mlockall = 1;
>  			if (!strcmp(lgopts[opt_idx].name, "no-mlockall"))
>  				do_mlockall = 0;
> +			if (!strcmp(lgopts[opt_idx].name, "noisy-buffersize-
> before-sending")) {
> +				n = atoi(optarg);
> +				if (n > 0)
> +					noisy_bsize_before_send =
> (uint16_t) n;
> +				else
> +					rte_exit(EXIT_FAILURE,
> +						 "noisy-buffersize-before-
> sending must be > 0\n");
> +			}
> +			if (!strcmp(lgopts[opt_idx].name, "noisy-flush-
> timeout")) {
> +				n = atoi(optarg);
> +				if (n >= 0)
> +					noisy_flush_timer = (uint16_t) n;
> +				else
> +					rte_exit(EXIT_FAILURE,
> +						 "noisy-flush-timeout must
> be > 0\n");
> +			}
> +			if (!strcmp(lgopts[opt_idx].name, "noisy-memory-
> footprint")) {
> +				n = atoi(optarg);
> +				if (n > 0)
> +					noisy_vnf_memory_footprint =
> (uint16_t) n;
> +				else
> +					rte_exit(EXIT_FAILURE,
> +						 "noisy-memory-footprint
> must be > 0\n");
> +			}
> +			if (!strcmp(lgopts[opt_idx].name, "noisy-nb-rnd-
> write")) {
> +				n = atoi(optarg);
> +				if (n > 0)
> +					noisy_nb_rnd_write = (uint16_t) n;
> +				else
> +					rte_exit(EXIT_FAILURE,
> +						 "noisy-nb-rnd-write must be
> > 0\n");
> +			}
> +			if (!strcmp(lgopts[opt_idx].name, "noisy-nb-rnd-
> read")) {
> +				n = atoi(optarg);
> +				if (n > 0)
> +					noisy_nb_rnd_read = (uint16_t) n;
> +				else
> +					rte_exit(EXIT_FAILURE,
> +						 "noisy-nb-rnd-read must be
> > 0\n");
> +			}
> +			if (!strcmp(lgopts[opt_idx].name, "noisy-nb-rnd-
> read-write")) {
> +				n = atoi(optarg);
> +				if (n > 0)
> +					noisy_nb_rnd_read_write =
> (uint16_t) n;
> +				else
> +					rte_exit(EXIT_FAILURE,
> +						 "noisy-nb-rnd-read-write
> must be > 0\n");
> +			}
>  			break;
>  		case 'h':
>  			usage(argv[0]);
> diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c index
> 24c199844..7f3dc6f1c 100644
> --- a/app/test-pmd/testpmd.c
> +++ b/app/test-pmd/testpmd.c
> @@ -61,6 +61,7 @@
>  #include <rte_latencystats.h>
>  #endif
> 
> +#include "noisy_vnf.h"
>  #include "testpmd.h"
> 
>  uint16_t verbose_level = 0; /**< Silent by default. */ @@ -155,6 +156,7 @@
> struct fwd_engine * fwd_engines[] = {
>  	&tx_only_engine,
>  	&csum_fwd_engine,
>  	&icmp_echo_engine,
> +	&noisy_vnf_engine,
>  #if defined RTE_LIBRTE_PMD_SOFTNIC && defined RTE_LIBRTE_SCHED
>  	&softnic_tm_engine,
>  	&softnic_tm_bypass_engine,
> @@ -251,6 +253,40 @@ int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
>   */
>  int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
> 
> +/*
> + * Configurable value of buffered packed before sending.
> + */
> +uint16_t noisy_bsize_before_send;
> +
> +/*
> + * Configurable value of packet buffer timeout.
> + */
> +uint16_t noisy_flush_timer;
> +
> +/*
> + * Configurable value for size of VNF internal memory area
> + * used for simulating noisy neighbour behaviour  */ uint64_t
> +noisy_vnf_memory_footprint;
> +
> +/*
> + * Configurable value of number of random writes done in
> + * VNF simulation memory area.
> + */
> +uint64_t noisy_nb_rnd_write;
> +
> +/*
> + * Configurable value of number of random reads done in
> + * VNF simulation memory area.
> + */
> +uint64_t noisy_nb_rnd_read;
> +
> +/*
> + * Configurable value of number of random reads/wirtes done in
> + * VNF simulation memory area.
> + */
> +uint64_t noisy_nb_rnd_read_write;
> +
>  /*
>   * Receive Side Scaling (RSS) configuration.
>   */
> @@ -1641,12 +1677,26 @@ start_port(portid_t pid)
>  				return -1;
>  			}
>  		}
> +		noisy_cfg = (struct noisy_config *) rte_zmalloc(
> +				"testpmd noisy fifo and timers",
> +				nb_txq * sizeof(struct noisy_config),
> +				RTE_CACHE_LINE_SIZE);
> +		if (noisy_cfg == NULL) {
> +			rte_exit(EXIT_FAILURE,
> +					"rte_zmalloc(%d) struct noisy_config)
> failed\n",
> +					(int)(nb_txq *
> +					sizeof(struct noisy_config)));
> +		}
>  		if (port->need_reconfig_queues > 0) {
>  			port->need_reconfig_queues = 0;
>  			/* setup tx queues */
>  			for (qi = 0; qi < nb_txq; qi++) {
>  				port->tx_conf[qi].txq_flags =
>  					ETH_TXQ_FLAGS_IGNORE;
> +				if (!noisy_init(qi, pi) &&
> +						noisy_bsize_before_send >
> 0)
> +					rte_exit(EXIT_FAILURE, "%s\n",
> +						 rte_strerror(rte_errno));
>  				if ((numa_support) &&
>  					(txring_numa[pi] !=
> NUMA_NO_CONFIG))
>  					diag = rte_eth_tx_queue_setup(pi,
> qi, @@ -1817,6 +1867,8 @@ stop_port(portid_t pid)
>  			RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
>  			printf("Port %d can not be set into stopped\n", pi);
>  		need_check_link_status = 1;
> +
> +		rte_free(noisy_cfg);
>  	}
>  	if (need_check_link_status && !no_link_check)
>  		check_all_ports_link_status(RTE_PORT_ALL);
> diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h index
> f51cd9dd9..75fb18a25 100644
> --- a/app/test-pmd/testpmd.h
> +++ b/app/test-pmd/testpmd.h
> @@ -9,6 +9,7 @@
>  #include <rte_bus_pci.h>
>  #include <rte_gro.h>
>  #include <rte_gso.h>
> +#include "noisy_vnf.h"
> 
>  #define RTE_PORT_ALL            (~(portid_t)0x0)
> 
> @@ -122,6 +123,14 @@ struct fwd_stream {  #endif  };
> 
> +struct noisy_config {
> +	struct rte_ring *f;
> +	uint64_t prev_time;
> +	char *vnf_mem;
> +};
> +struct noisy_config *noisy_cfg;
> +
> +
>  /** Descriptor for a single flow. */
>  struct port_flow {
>  	size_t size; /**< Allocated space including data[]. */ @@ -266,6
> +275,7 @@ extern struct fwd_engine rx_only_engine;  extern struct
> fwd_engine tx_only_engine;  extern struct fwd_engine csum_fwd_engine;
> extern struct fwd_engine icmp_echo_engine;
> +extern struct fwd_engine noisy_vnf_engine;
>  #ifdef TM_MODE
>  extern struct fwd_engine softnic_tm_engine;  extern struct fwd_engine
> softnic_tm_bypass_engine; @@ -399,6 +409,13 @@ extern int8_t
> rx_drop_en;  extern int16_t tx_free_thresh;  extern int16_t tx_rs_thresh;
> 
> +extern uint16_t noisy_bsize_before_send; extern uint16_t
> +noisy_flush_timer; extern uint64_t noisy_vnf_memory_footprint; extern
> +uint64_t noisy_nb_rnd_write; extern uint64_t noisy_nb_rnd_read; extern
> +uint64_t noisy_nb_rnd_read_write;
> +
>  extern uint8_t dcb_config;
>  extern uint8_t dcb_test;
> 
> --
> 2.14.4