DPDK patches and discussions
 help / color / mirror / Atom feed
From: <skori@marvell.com>
To: Sunil Kumar Kori <skori@marvell.com>,
	Rakesh Kudurumalla <rkudurumalla@marvell.com>
Cc: <dev@dpdk.org>
Subject: [PATCH v8 10/12] app/graph: add graph command line interfaces
Date: Fri, 29 Sep 2023 15:28:12 +0530	[thread overview]
Message-ID: <20230929095814.692890-11-skori@marvell.com> (raw)
In-Reply-To: <20230929095814.692890-1-skori@marvell.com>

From: Rakesh Kudurumalla <rkudurumalla@marvell.com>

It adds graph module to create a graph for a given use case like
l3fwd.

Following commands are exposed:
 - graph <usecases> [bsz <size>] [tmo <ns>] [coremask <bitmask>] \
	model <rtc | mcd | default> pcap_enable <0 | 1> num_pcap_pkts <num> \
	pcap_file <output_capture_file>
 - graph start
 - graph stats show
 - help graph

Signed-off-by: Sunil Kumar Kori <skori@marvell.com>
Signed-off-by: Rakesh Kudurumalla <rkudurumalla@marvell.com>
---
 app/graph/cli.c            |   4 +
 app/graph/ethdev_rx.c      |   2 +-
 app/graph/graph.c          | 550 +++++++++++++++++++++++++++++++++++++
 app/graph/graph.h          |  21 ++
 app/graph/graph_priv.h     |  70 +++++
 app/graph/ip4_route.c      |   5 +-
 app/graph/ip6_route.c      |   5 +-
 app/graph/meson.build      |   1 +
 app/graph/module_api.h     |   1 +
 app/graph/neigh.c          |  10 +-
 doc/guides/tools/graph.rst |  19 +-
 11 files changed, 681 insertions(+), 7 deletions(-)
 create mode 100644 app/graph/graph.c
 create mode 100644 app/graph/graph.h
 create mode 100644 app/graph/graph_priv.h

diff --git a/app/graph/cli.c b/app/graph/cli.c
index e947f61ee4..c43af5925c 100644
--- a/app/graph/cli.c
+++ b/app/graph/cli.c
@@ -20,6 +20,10 @@
 #define MAX_LINE_SIZE 2048
 
 cmdline_parse_ctx_t modules_ctx[] = {
+	(cmdline_parse_inst_t *)&graph_config_cmd_ctx,
+	(cmdline_parse_inst_t *)&graph_start_cmd_ctx,
+	(cmdline_parse_inst_t *)&graph_stats_cmd_ctx,
+	(cmdline_parse_inst_t *)&graph_help_cmd_ctx,
 	(cmdline_parse_inst_t *)&mempool_config_cmd_ctx,
 	(cmdline_parse_inst_t *)&mempool_help_cmd_ctx,
 	(cmdline_parse_inst_t *)&ethdev_show_cmd_ctx,
diff --git a/app/graph/ethdev_rx.c b/app/graph/ethdev_rx.c
index f2cb8cf9a5..03f8effcca 100644
--- a/app/graph/ethdev_rx.c
+++ b/app/graph/ethdev_rx.c
@@ -69,7 +69,7 @@ ethdev_rx_map_add(char *name, uint32_t queue, uint32_t core)
 	if (rc)
 		return -EINVAL;
 
-	coremask = 0xff; /* FIXME: Read from graph configuration */
+	coremask = graph_coremask_get();
 
 	if (!(coremask & (1 << core)))
 		return -EINVAL;
diff --git a/app/graph/graph.c b/app/graph/graph.c
new file mode 100644
index 0000000000..b27abcf1f9
--- /dev/null
+++ b/app/graph/graph.c
@@ -0,0 +1,550 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Marvell.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <cmdline_parse.h>
+#include <cmdline_parse_num.h>
+#include <cmdline_parse_string.h>
+#include <cmdline_socket.h>
+#include <rte_ethdev.h>
+#include <rte_graph_worker.h>
+#include <rte_log.h>
+
+#include "graph_priv.h"
+#include "module_api.h"
+
+#define RTE_LOGTYPE_APP_GRAPH RTE_LOGTYPE_USER1
+
+static const char
+cmd_graph_help[] = "graph <usecases> bsz <size> tmo <ns> coremask <bitmask> "
+		   "model <rtc | mcd | default> pcap_enable <0 | 1> num_pcap_pkts <num>"
+		   "pcap_file <output_capture_file>";
+
+static const char * const supported_usecases[] = {"l3fwd"};
+struct graph_config graph_config;
+bool graph_started;
+
+/* Check the link rc of all ports in up to 9s, and print them finally */
+static void
+check_all_ports_link_status(uint32_t port_mask)
+{
+#define CHECK_INTERVAL 100 /* 100ms */
+#define MAX_CHECK_TIME 90  /* 9s (90 * 100ms) in total */
+	char link_rc_text[RTE_ETH_LINK_MAX_STR_LEN];
+	uint8_t count, all_ports_up, print_flag = 0;
+	struct rte_eth_link link;
+	uint16_t portid;
+	int rc;
+
+	printf("\nChecking link rc");
+	fflush(stdout);
+	for (count = 0; count <= MAX_CHECK_TIME; count++) {
+		if (force_quit)
+			return;
+
+		all_ports_up = 1;
+		RTE_ETH_FOREACH_DEV(portid)
+		{
+			if (force_quit)
+				return;
+
+			if ((port_mask & (1 << portid)) == 0)
+				continue;
+
+			memset(&link, 0, sizeof(link));
+			rc = rte_eth_link_get_nowait(portid, &link);
+			if (rc < 0) {
+				all_ports_up = 0;
+				if (print_flag == 1)
+					printf("Port %u link get failed: %s\n",
+					       portid, rte_strerror(-rc));
+				continue;
+			}
+
+			/* Print link rc if flag set */
+			if (print_flag == 1) {
+				rte_eth_link_to_str(link_rc_text, sizeof(link_rc_text),
+					&link);
+				printf("Port %d %s\n", portid, link_rc_text);
+				continue;
+			}
+
+			/* Clear all_ports_up flag if any link down */
+			if (link.link_status == RTE_ETH_LINK_DOWN) {
+				all_ports_up = 0;
+				break;
+			}
+		}
+
+		/* After finally printing all link rc, get out */
+		if (print_flag == 1)
+			break;
+
+		if (all_ports_up == 0) {
+			printf(".");
+			fflush(stdout);
+			rte_delay_ms(CHECK_INTERVAL);
+		}
+
+		/* Set the print_flag if all ports up or timeout */
+		if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
+			print_flag = 1;
+			printf("Done\n");
+		}
+	}
+}
+
+static bool
+parser_usecases_read(char *usecases)
+{
+	bool valid = false;
+	uint32_t i, j = 0;
+	char *token;
+
+	token = strtok(usecases, ",");
+	while (token != NULL) {
+		for (i = 0; i < RTE_DIM(supported_usecases); i++) {
+			if (strcmp(supported_usecases[i], token) == 0) {
+				graph_config.usecases[j].enabled = true;
+				strncpy(graph_config.usecases[j].name, token, 31);
+				valid = true;
+				j++;
+				break;
+			}
+		}
+		token = strtok(NULL, ",");
+	}
+
+	return valid;
+}
+
+static uint64_t
+graph_worker_count_get(void)
+{
+	uint64_t nb_worker = 0;
+	uint64_t coremask;
+
+	coremask = graph_config.params.coremask;
+	while (coremask) {
+		if (coremask & 0x1)
+			nb_worker++;
+
+		coremask = (coremask >> 1);
+	}
+
+	return nb_worker;
+}
+
+static struct rte_node_ethdev_config *
+graph_rxtx_node_config_get(uint32_t *num_conf, uint32_t *num_graphs)
+{
+	uint32_t n_tx_queue, nb_conf = 0, lcore_id;
+	uint16_t queueid, portid, nb_graphs = 0;
+	uint8_t nb_rx_queue, queue;
+	struct lcore_conf *qconf;
+
+	n_tx_queue = graph_worker_count_get();
+	if (n_tx_queue > RTE_MAX_ETHPORTS)
+		n_tx_queue = RTE_MAX_ETHPORTS;
+
+	RTE_ETH_FOREACH_DEV(portid) {
+		/* Skip ports that are not enabled */
+		if ((enabled_port_mask & (1 << portid)) == 0) {
+			printf("\nSkipping disabled port %d\n", portid);
+			continue;
+		}
+
+		nb_rx_queue = ethdev_rx_num_rx_queues_get(portid);
+
+		/* Setup ethdev node config */
+		ethdev_conf[nb_conf].port_id = portid;
+		ethdev_conf[nb_conf].num_rx_queues = nb_rx_queue;
+		ethdev_conf[nb_conf].num_tx_queues = n_tx_queue;
+		ethdev_conf[nb_conf].mp = ethdev_mempool_list_by_portid(portid);
+		ethdev_conf[nb_conf].mp_count = 1; /* Check with pools */
+
+		nb_conf++;
+	}
+
+	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
+		if (rte_lcore_is_enabled(lcore_id) == 0)
+			continue;
+
+		qconf = &lcore_conf[lcore_id];
+		printf("\nInitializing rx queues on lcore %u ... ", lcore_id);
+		fflush(stdout);
+
+		/* Init RX queues */
+		for (queue = 0; queue < qconf->n_rx_queue; ++queue) {
+			portid = qconf->rx_queue_list[queue].port_id;
+			queueid = qconf->rx_queue_list[queue].queue_id;
+
+			/* Add this queue node to its graph */
+			snprintf(qconf->rx_queue_list[queue].node_name, RTE_NODE_NAMESIZE,
+				 "ethdev_rx-%u-%u", portid, queueid);
+		}
+		if (qconf->n_rx_queue)
+			nb_graphs++;
+	}
+
+	printf("\n");
+
+	ethdev_start();
+	check_all_ports_link_status(enabled_port_mask);
+
+	*num_conf = nb_conf;
+	*num_graphs = nb_graphs;
+	return ethdev_conf;
+}
+
+static void
+graph_stats_print_to_file(void)
+{
+	struct rte_graph_cluster_stats_param s_param;
+	struct rte_graph_cluster_stats *stats;
+	const char *pattern = "worker_*";
+	FILE *fp = NULL;
+	size_t sz, len;
+
+	/* Prepare stats object */
+	fp = fopen("/tmp/graph_stats.txt", "w+");
+	if (fp == NULL)
+		rte_exit(EXIT_FAILURE, "Errot in opening stats file\n");
+
+	memset(&s_param, 0, sizeof(s_param));
+	s_param.f = fp;
+	s_param.socket_id = SOCKET_ID_ANY;
+	s_param.graph_patterns = &pattern;
+	s_param.nb_graph_patterns = 1;
+
+	stats = rte_graph_cluster_stats_create(&s_param);
+	if (stats == NULL)
+		rte_exit(EXIT_FAILURE, "Unable to create stats object\n");
+
+	/* Clear screen and move to top left */
+	rte_graph_cluster_stats_get(stats, 0);
+	rte_delay_ms(1E3);
+
+	fseek(fp, 0L, SEEK_END);
+	sz = ftell(fp);
+	fseek(fp, 0L, SEEK_SET);
+
+	len = strlen(conn->msg_out);
+	conn->msg_out += len;
+
+	sz = fread(conn->msg_out, sizeof(char), sz, fp);
+	len = strlen(conn->msg_out);
+	conn->msg_out_len_max -= len;
+	rte_graph_cluster_stats_destroy(stats);
+
+	fclose(fp);
+}
+
+static void
+cli_graph_stats(__rte_unused void *parsed_result, __rte_unused struct cmdline *cl,
+		__rte_unused void *data)
+{
+	graph_stats_print_to_file();
+}
+
+bool
+graph_status_get(void)
+{
+	return graph_started;
+}
+
+static void
+cli_graph_start(__rte_unused void *parsed_result, __rte_unused struct cmdline *cl,
+		__rte_unused void *data)
+{
+	struct rte_node_ethdev_config *conf;
+	uint32_t nb_graphs = 0, nb_conf, i;
+	int rc = -EINVAL;
+
+	conf = graph_rxtx_node_config_get(&nb_conf, &nb_graphs);
+	for (i = 0; i < MAX_GRAPH_USECASES; i++) {
+		if (!strcmp(graph_config.usecases[i].name, "l3fwd")) {
+			if (graph_config.usecases[i].enabled) {
+				RTE_SET_USED(conf);
+				break;
+			}
+		}
+	}
+
+	if (!rc)
+		graph_started = true;
+}
+
+static int
+graph_config_add(char *usecases, struct graph_config *config)
+{
+	uint64_t lcore_id, core_num;
+	uint64_t eal_coremask = 0;
+
+	if (!parser_usecases_read(usecases))
+		return -EINVAL;
+
+	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
+		if (rte_lcore_is_enabled(lcore_id))
+			eal_coremask |= RTE_BIT64(lcore_id);
+	}
+
+	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
+		core_num = 1 << lcore_id;
+		if (config->params.coremask & core_num) {
+			if (eal_coremask & core_num)
+				continue;
+			else
+				return -EINVAL;
+		}
+	}
+
+	graph_config.params.bsz = config->params.bsz;
+	graph_config.params.tmo = config->params.tmo;
+	graph_config.params.coremask = config->params.coremask;
+	graph_config.model = config->model;
+	graph_config.pcap_ena = config->pcap_ena;
+	graph_config.num_pcap_pkts = config->num_pcap_pkts;
+	graph_config.pcap_file = strdup(config->pcap_file);
+
+	return 0;
+}
+
+void
+graph_pcap_config_get(uint8_t *pcap_ena, uint64_t *num_pkts, char **file)
+{
+
+	*pcap_ena = graph_config.pcap_ena;
+	*num_pkts = graph_config.num_pcap_pkts;
+	*file = graph_config.pcap_file;
+}
+
+int
+graph_walk_start(void *conf)
+{
+	struct lcore_conf *qconf;
+	struct rte_graph *graph;
+	uint32_t lcore_id;
+
+	RTE_SET_USED(conf);
+
+	lcore_id = rte_lcore_id();
+	qconf = &lcore_conf[lcore_id];
+	graph = qconf->graph;
+
+	if (!graph) {
+		RTE_LOG(INFO, APP_GRAPH, "Lcore %u has nothing to do\n", lcore_id);
+		return 0;
+	}
+
+	RTE_LOG(INFO, APP_GRAPH, "Entering main loop on lcore %u, graph %s(%p)\n", lcore_id,
+		qconf->name, graph);
+
+	while (likely(!force_quit))
+		rte_graph_walk(graph);
+
+	return 0;
+}
+
+void
+graph_stats_print(void)
+{
+	const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'};
+	const char clr[] = {27, '[', '2', 'J', '\0'};
+	struct rte_graph_cluster_stats_param s_param;
+	struct rte_graph_cluster_stats *stats;
+	const char *pattern = "worker_*";
+
+	/* Prepare stats object */
+	memset(&s_param, 0, sizeof(s_param));
+	s_param.f = stdout;
+	s_param.socket_id = SOCKET_ID_ANY;
+	s_param.graph_patterns = &pattern;
+	s_param.nb_graph_patterns = 1;
+
+	stats = rte_graph_cluster_stats_create(&s_param);
+	if (stats == NULL)
+		rte_exit(EXIT_FAILURE, "Unable to create stats object\n");
+
+	while (!force_quit) {
+		/* Clear screen and move to top left */
+		printf("%s%s", clr, topLeft);
+		rte_graph_cluster_stats_get(stats, 0);
+		rte_delay_ms(1E3);
+		if (app_graph_exit())
+			force_quit = true;
+	}
+
+	rte_graph_cluster_stats_destroy(stats);
+}
+
+uint64_t
+graph_coremask_get(void)
+{
+	return graph_config.params.coremask;
+}
+
+static void
+cli_graph(void *parsed_result, __rte_unused struct cmdline *cl, __rte_unused void *data)
+{
+	struct graph_config_cmd_tokens *res = parsed_result;
+	struct graph_config config;
+	char *model_name;
+	uint8_t model;
+	int rc;
+
+	model_name = res->model_name;
+	if (strcmp(model_name, "default") == 0) {
+		model = GRAPH_MODEL_RTC;
+	} else if (strcmp(model_name, "rtc") == 0) {
+		model = GRAPH_MODEL_RTC;
+	} else if (strcmp(model_name, "mcd") == 0) {
+		model = GRAPH_MODEL_MCD;
+	} else {
+		printf(MSG_ARG_NOT_FOUND, "model arguments");
+		return;
+	}
+
+	config.params.bsz = res->size;
+	config.params.tmo = res->ns;
+	config.params.coremask = res->mask;
+	config.model = model;
+	config.pcap_ena = res->pcap_ena;
+	config.num_pcap_pkts = res->num_pcap_pkts;
+	config.pcap_file = res->pcap_file;
+	rc = graph_config_add(res->usecase, &config);
+	if (rc < 0) {
+		cli_exit();
+		printf(MSG_CMD_FAIL, res->graph);
+		rte_exit(EXIT_FAILURE, "coremask is Invalid\n");
+	}
+}
+
+static void
+cli_graph_help(__rte_unused void *parsed_result, __rte_unused struct cmdline *cl,
+	       __rte_unused void *data)
+{
+	size_t len;
+
+	len = strlen(conn->msg_out);
+	conn->msg_out += len;
+	snprintf(conn->msg_out, conn->msg_out_len_max, "\n%s\n%s\n%s\n",
+		 "----------------------------- graph command help -----------------------------",
+		 cmd_graph_help, "graph start");
+
+	len = strlen(conn->msg_out);
+	conn->msg_out_len_max -= len;
+}
+
+cmdline_parse_token_string_t graph_display_graph =
+	TOKEN_STRING_INITIALIZER(struct graph_stats_cmd_tokens, graph, "graph");
+cmdline_parse_token_string_t graph_display_stats =
+	TOKEN_STRING_INITIALIZER(struct graph_stats_cmd_tokens, stats, "stats");
+cmdline_parse_token_string_t graph_display_show =
+	TOKEN_STRING_INITIALIZER(struct graph_stats_cmd_tokens, show, "show");
+
+cmdline_parse_inst_t graph_stats_cmd_ctx = {
+	.f = cli_graph_stats,
+	.data = NULL,
+	.help_str = "graph stats show",
+	.tokens = {
+		(void *)&graph_display_graph,
+		(void *)&graph_display_stats,
+		(void *)&graph_display_show,
+		NULL,
+	},
+};
+
+cmdline_parse_token_string_t graph_config_start_graph =
+	TOKEN_STRING_INITIALIZER(struct graph_start_cmd_tokens, graph, "graph");
+cmdline_parse_token_string_t graph_config_start =
+	TOKEN_STRING_INITIALIZER(struct graph_start_cmd_tokens, start, "start");
+
+cmdline_parse_inst_t graph_start_cmd_ctx = {
+	.f = cli_graph_start,
+	.data = NULL,
+	.help_str = "graph start",
+	.tokens = {
+		(void *)&graph_config_start_graph,
+		(void *)&graph_config_start,
+		NULL,
+	},
+};
+
+cmdline_parse_token_string_t graph_config_add_graph =
+	TOKEN_STRING_INITIALIZER(struct graph_config_cmd_tokens, graph, "graph");
+cmdline_parse_token_string_t graph_config_add_usecase =
+	TOKEN_STRING_INITIALIZER(struct graph_config_cmd_tokens, usecase, NULL);
+cmdline_parse_token_string_t graph_config_add_coremask =
+	TOKEN_STRING_INITIALIZER(struct graph_config_cmd_tokens, coremask, "coremask");
+cmdline_parse_token_num_t graph_config_add_mask =
+	TOKEN_NUM_INITIALIZER(struct graph_config_cmd_tokens, mask, RTE_UINT64);
+cmdline_parse_token_string_t graph_config_add_bsz =
+	TOKEN_STRING_INITIALIZER(struct graph_config_cmd_tokens, bsz, "bsz");
+cmdline_parse_token_num_t graph_config_add_size =
+	TOKEN_NUM_INITIALIZER(struct graph_config_cmd_tokens, size, RTE_UINT16);
+cmdline_parse_token_string_t graph_config_add_tmo =
+	TOKEN_STRING_INITIALIZER(struct graph_config_cmd_tokens, tmo, "tmo");
+cmdline_parse_token_num_t graph_config_add_ns =
+	TOKEN_NUM_INITIALIZER(struct graph_config_cmd_tokens, ns, RTE_UINT64);
+cmdline_parse_token_string_t graph_config_add_model =
+	TOKEN_STRING_INITIALIZER(struct graph_config_cmd_tokens, model, "model");
+cmdline_parse_token_string_t graph_config_add_model_name =
+	TOKEN_STRING_INITIALIZER(struct graph_config_cmd_tokens, model_name, "rtc#mcd#default");
+cmdline_parse_token_string_t graph_config_add_capt_ena =
+	TOKEN_STRING_INITIALIZER(struct graph_config_cmd_tokens, capt_ena, "pcap_enable");
+cmdline_parse_token_num_t graph_config_add_pcap_ena =
+	TOKEN_NUM_INITIALIZER(struct graph_config_cmd_tokens, pcap_ena, RTE_UINT8);
+cmdline_parse_token_string_t graph_config_add_capt_pkts_count =
+	TOKEN_STRING_INITIALIZER(struct graph_config_cmd_tokens, capt_pkts_count, "num_pcap_pkts");
+cmdline_parse_token_num_t graph_config_add_num_pcap_pkts =
+	TOKEN_NUM_INITIALIZER(struct graph_config_cmd_tokens, num_pcap_pkts, RTE_UINT64);
+cmdline_parse_token_string_t graph_config_add_capt_file =
+	TOKEN_STRING_INITIALIZER(struct graph_config_cmd_tokens, capt_file, "pcap_file");
+cmdline_parse_token_string_t graph_config_add_pcap_file =
+	TOKEN_STRING_INITIALIZER(struct graph_config_cmd_tokens, pcap_file, NULL);
+
+cmdline_parse_inst_t graph_config_cmd_ctx = {
+	.f = cli_graph,
+	.data = NULL,
+	.help_str = cmd_graph_help,
+	.tokens = {
+		(void *)&graph_config_add_graph,
+		(void *)&graph_config_add_usecase,
+		(void *)&graph_config_add_coremask,
+		(void *)&graph_config_add_mask,
+		(void *)&graph_config_add_bsz,
+		(void *)&graph_config_add_size,
+		(void *)&graph_config_add_tmo,
+		(void *)&graph_config_add_ns,
+		(void *)&graph_config_add_model,
+		(void *)&graph_config_add_model_name,
+		(void *)&graph_config_add_capt_ena,
+		(void *)&graph_config_add_pcap_ena,
+		(void *)&graph_config_add_capt_pkts_count,
+		(void *)&graph_config_add_num_pcap_pkts,
+		(void *)&graph_config_add_capt_file,
+		(void *)&graph_config_add_pcap_file,
+		NULL,
+	},
+};
+
+cmdline_parse_token_string_t graph_help_cmd =
+	TOKEN_STRING_INITIALIZER(struct graph_help_cmd_tokens, help, "help");
+cmdline_parse_token_string_t graph_help_graph =
+	TOKEN_STRING_INITIALIZER(struct graph_help_cmd_tokens, graph, "graph");
+
+cmdline_parse_inst_t graph_help_cmd_ctx = {
+	.f = cli_graph_help,
+	.data = NULL,
+	.help_str = "",
+	.tokens = {
+		(void *)&graph_help_cmd,
+		(void *)&graph_help_graph,
+		NULL,
+	},
+};
diff --git a/app/graph/graph.h b/app/graph/graph.h
new file mode 100644
index 0000000000..a14fa37ccd
--- /dev/null
+++ b/app/graph/graph.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Marvell.
+ */
+
+#ifndef APP_GRAPH_H
+#define APP_GRAPH_H
+
+#include <cmdline_parse.h>
+
+extern cmdline_parse_inst_t graph_config_cmd_ctx;
+extern cmdline_parse_inst_t graph_start_cmd_ctx;
+extern cmdline_parse_inst_t graph_stats_cmd_ctx;
+extern cmdline_parse_inst_t graph_help_cmd_ctx;
+
+int graph_walk_start(void *conf);
+void graph_stats_print(void);
+void graph_pcap_config_get(uint8_t *pcap_ena, uint64_t *num_pkts, char **file);
+uint64_t graph_coremask_get(void);
+bool graph_status_get(void);
+
+#endif
diff --git a/app/graph/graph_priv.h b/app/graph/graph_priv.h
new file mode 100644
index 0000000000..a48a35daa3
--- /dev/null
+++ b/app/graph/graph_priv.h
@@ -0,0 +1,70 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Marvell.
+ */
+
+#ifndef APP_GRAPH_PRIV_H
+#define APP_GRAPH_PRIV_H
+
+#define MAX_GRAPH_USECASES 32
+
+struct graph_help_cmd_tokens {
+	cmdline_fixed_string_t help;
+	cmdline_fixed_string_t graph;
+};
+
+struct graph_start_cmd_tokens {
+	cmdline_fixed_string_t graph;
+	cmdline_fixed_string_t start;
+};
+
+struct graph_stats_cmd_tokens {
+	cmdline_fixed_string_t show;
+	cmdline_fixed_string_t graph;
+	cmdline_fixed_string_t stats;
+};
+
+struct graph_config_cmd_tokens {
+	cmdline_fixed_string_t graph;
+	cmdline_fixed_string_t usecase;
+	cmdline_fixed_string_t bsz;
+	cmdline_fixed_string_t tmo;
+	cmdline_fixed_string_t coremask;
+	cmdline_fixed_string_t model;
+	cmdline_fixed_string_t capt_ena;
+	cmdline_fixed_string_t capt_pkts_count;
+	cmdline_fixed_string_t capt_file;
+	cmdline_fixed_string_t model_name;
+	cmdline_fixed_string_t pcap_file;
+	uint16_t size;
+	uint64_t ns;
+	uint64_t mask;
+	uint64_t num_pcap_pkts;
+	uint8_t pcap_ena;
+};
+
+enum graph_model {
+	GRAPH_MODEL_RTC = 0x01,
+	GRAPH_MODEL_MCD = 0x02,
+};
+
+struct usecases {
+	char name[32];
+	bool enabled;
+};
+
+struct usecase_params {
+	uint64_t coremask;
+	uint32_t bsz;
+	uint32_t tmo;
+};
+
+struct graph_config {
+	struct usecases usecases[MAX_GRAPH_USECASES];
+	struct usecase_params params;
+	enum graph_model model;
+	uint64_t num_pcap_pkts;
+	char *pcap_file;
+	uint8_t pcap_ena;
+};
+
+#endif
diff --git a/app/graph/ip4_route.c b/app/graph/ip4_route.c
index db3354c270..fc83586427 100644
--- a/app/graph/ip4_route.c
+++ b/app/graph/ip4_route.c
@@ -97,11 +97,14 @@ route_ip4_add(struct route_ipv4_config *route)
 	ipv4route->via = route->via;
 	ipv4route->is_used = true;
 
-	/* FIXME: Get graph status here and then update table */
+	if (!graph_status_get())
+		goto exit;
+
 	rc = route4_rewirte_table_update(ipv4route);
 	if (rc)
 		goto free;
 
+exit:
 	TAILQ_INSERT_TAIL(&route4, ipv4route, next);
 	return 0;
 free:
diff --git a/app/graph/ip6_route.c b/app/graph/ip6_route.c
index e793cde830..1fa4865220 100644
--- a/app/graph/ip6_route.c
+++ b/app/graph/ip6_route.c
@@ -102,11 +102,14 @@ route_ip6_add(struct route_ipv6_config *route)
 	}
 	ipv6route->is_used = true;
 
-	/* FIXME: Get graph status here and then update table */
+	if (!graph_status_get())
+		goto exit;
+
 	rc = route6_rewirte_table_update(ipv6route);
 	if (rc)
 		goto free;
 
+exit:
 	TAILQ_INSERT_TAIL(&route6, ipv6route, next);
 	return 0;
 free:
diff --git a/app/graph/meson.build b/app/graph/meson.build
index cafab04f70..c0b59b9a92 100644
--- a/app/graph/meson.build
+++ b/app/graph/meson.build
@@ -14,6 +14,7 @@ sources = files(
         'conn.c',
         'ethdev_rx.c',
         'ethdev.c',
+        'graph.c',
         'ip4_route.c',
         'ip6_route.c',
         'main.c',
diff --git a/app/graph/module_api.h b/app/graph/module_api.h
index 56b7c94ecc..392dcfb222 100644
--- a/app/graph/module_api.h
+++ b/app/graph/module_api.h
@@ -12,6 +12,7 @@
 #include "conn.h"
 #include "ethdev.h"
 #include "ethdev_rx.h"
+#include "graph.h"
 #include "mempool.h"
 #include "neigh.h"
 #include "route.h"
diff --git a/app/graph/neigh.c b/app/graph/neigh.c
index af69fc8ade..706be3840f 100644
--- a/app/graph/neigh.c
+++ b/app/graph/neigh.c
@@ -156,11 +156,14 @@ neigh_ip4_add(uint32_t ip, uint64_t mac)
 	v4_config->mac = mac;
 	v4_config->is_used = true;
 
-	/* FIXME: Get graph status here and then update table */
+	if (!graph_status_get())
+		goto exit;
+
 	rc = ip4_rewrite_node_add(v4_config);
 	if (rc)
 		goto free;
 
+exit:
 	TAILQ_INSERT_TAIL(&neigh4, v4_config, next);
 	return 0;
 free:
@@ -189,11 +192,14 @@ neigh_ip6_add(uint8_t *ip, uint64_t mac)
 	v6_config->mac = mac;
 	v6_config->is_used = true;
 
-	/* FIXME: Get graph status here and then update table */
+	if (!graph_status_get())
+		goto exit;
+
 	rc =  ip6_rewrite_node_add(v6_config);
 	if (rc)
 		goto free;
 
+exit:
 	TAILQ_INSERT_TAIL(&neigh6, v6_config, next);
 	return 0;
 free:
diff --git a/doc/guides/tools/graph.rst b/doc/guides/tools/graph.rst
index f2920ff136..7d2aa95c95 100644
--- a/doc/guides/tools/graph.rst
+++ b/doc/guides/tools/graph.rst
@@ -76,10 +76,25 @@ file to express the requested use case configuration.
      - Description
      - Dynamic
      - Optional
-   * - Dummy command
-     - Dummy command description
+   * - graph <usecases> [bsz <size>] [tmo <ns>] [coremask <bitmask>] model <rtc | mcd | default>
+       pcap_enable <0 | 1> num_pcap_pkts <num> pcap_file <output_capture_file>
+     - Command to express the desired use case. Also enables/disable pcap capturing
      - No
      - No
+   * - graph start
+     - Command to start the graph.
+       This command triggers that no more commands are left to be parsed and graph
+       initialization can be started now. It must be the last command in ``<usecase>.cli``
+     - No
+     - No
+   * - graph stats show
+     - Command to dump current graph statistics
+     - Yes
+     - Yes
+   * - help graph
+     - Command to dump graph help message
+     - Yes
+     - Yes
    * - mempool <mempool_name> size <mbuf_size> buffers <number_of_buffers> cache <cache_size> numa <numa_id>
      - Command to create mempool which will be further associated to RxQ to dequeue the packets
      - No
-- 
2.25.1


  parent reply	other threads:[~2023-09-29  9:59 UTC|newest]

Thread overview: 182+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2023-04-21  6:02 [PATCH 0/4] app: introduce testgraph application Vamsi Attunuru
2023-04-21  6:02 ` [PATCH 1/4] node: add pkt punt to kernel node Vamsi Attunuru
2023-05-29 17:29   ` Jerin Jacob
2023-05-31 12:36     ` [EXT] " Vamsi Krishna Attunuru
2023-04-21  6:02 ` [PATCH 2/4] node: add a node to receive pkts from kernel Vamsi Attunuru
2023-05-29 17:39   ` Jerin Jacob
2023-06-01  2:40     ` [EXT] " Vamsi Krishna Attunuru
2023-04-21  6:02 ` [PATCH 3/4] node: remove hardcoded node next details Vamsi Attunuru
2023-04-21  6:02 ` [PATCH 4/4] app: add testgraph application Vamsi Attunuru
2023-05-09  6:09   ` Jerin Jacob
2023-04-25 13:15 ` [PATCH v2 0/4] app: introduce " Vamsi Attunuru
2023-04-25 13:15   ` [PATCH v2 1/4] node: add pkt punt to kernel node Vamsi Attunuru
2023-05-30  8:16     ` Nithin Dabilpuram
2023-05-31 12:35       ` [EXT] " Vamsi Krishna Attunuru
2023-04-25 13:15   ` [PATCH v2 2/4] node: add a node to receive pkts from kernel Vamsi Attunuru
2023-04-25 13:15   ` [PATCH v2 3/4] node: remove hardcoded node next details Vamsi Attunuru
2023-04-25 13:15   ` [PATCH v2 4/4] app: add testgraph application Vamsi Attunuru
2023-05-09  2:34     ` [EXT] " Sunil Kumar Kori
2023-05-09  3:39       ` Vamsi Krishna Attunuru
2023-05-09  8:53         ` Sunil Kumar Kori
2023-05-09 12:24           ` Vamsi Krishna Attunuru
2023-05-12 11:08     ` Yan, Zhirun
2023-05-22  7:07       ` Vamsi Krishna Attunuru
2023-05-30  7:34         ` Jerin Jacob
2023-06-01  2:44           ` [EXT] " Vamsi Krishna Attunuru
2023-07-20 15:52             ` Rakesh Kudurumalla
2023-07-21  6:48               ` Jerin Jacob
2023-07-21  7:01                 ` Sunil Kumar Kori
2023-07-21  7:39                   ` Rakesh Kudurumalla
2023-09-08 11:00                     ` Sunil Kumar Kori
2023-09-08 10:49     ` [PATCH v3 1/1] app/graph: add example for different usecases skori
2023-09-09  1:18       ` [EXT] " Nithin Kumar Dabilpuram
2023-09-19 16:04       ` [PATCH v4 01/14] app/graph: add application framework to read CLI skori
2023-09-19 16:04         ` [PATCH v4 02/14] app/graph: add telnet connectivity framework skori
2023-09-20  4:34           ` Jerin Jacob
2023-09-20  8:14             ` Bruce Richardson
2023-09-19 16:04         ` [PATCH v4 03/14] app/graph: add parser utility APIs skori
2023-09-19 16:04         ` [PATCH v4 04/14] app/graph: add mempool command line interfaces skori
2023-09-19 16:04         ` [PATCH v4 05/14] app/graph: add ethdev " skori
2023-09-19 16:04         ` [PATCH v4 06/14] app/graph: add ipv4_lookup " skori
2023-09-19 16:04         ` [PATCH v4 07/14] app/graph: add ipv6_lookup " skori
2023-09-19 16:04         ` [PATCH v4 08/14] app/graph: add neigh " skori
2023-09-19 16:04         ` [PATCH v4 09/14] app/graph: add ethdev_rx " skori
2023-09-19 16:04         ` [PATCH v4 10/14] app/graph: add graph " skori
2023-09-19 16:04         ` [PATCH v4 11/14] app/graph: add CLI option to enable graph stats skori
2023-09-19 16:04         ` [PATCH v4 12/14] app/graph: add l3fwd usecase skori
2023-09-19 16:04         ` [PATCH v4 13/14] doc: add graph application user guide skori
2023-09-20  4:28           ` Jerin Jacob
2023-09-19 16:04         ` [PATCH v4 14/14] maintainers: add maintainers for graph app skori
2023-09-20  4:40         ` [PATCH v4 01/14] app/graph: add application framework to read CLI Jerin Jacob
2023-09-21 10:08         ` [PATCH v5 00/12] add CLI based graph application skori
2023-09-21 10:08           ` [PATCH v5 01/12] app/graph: add application framework to read CLI skori
2023-09-21 10:08           ` [PATCH v5 02/12] app/graph: add telnet connectivity framework skori
2023-09-21 10:08           ` [PATCH v5 03/12] app/graph: add parser utility APIs skori
2023-09-21 10:08           ` [PATCH v5 04/12] app/graph: add mempool command line interfaces skori
2023-09-21 10:08           ` [PATCH v5 05/12] app/graph: add ethdev " skori
2023-09-21 10:08           ` [PATCH v5 06/12] app/graph: add ipv4_lookup " skori
2023-09-21 10:08           ` [PATCH v5 07/12] app/graph: add ipv6_lookup " skori
2023-09-21 10:08           ` [PATCH v5 08/12] app/graph: add neigh " skori
2023-09-21 10:08           ` [PATCH v5 09/12] app/graph: add ethdev_rx " skori
2023-09-21 10:08           ` [PATCH v5 10/12] app/graph: add graph " skori
2023-09-21 10:08           ` [PATCH v5 11/12] app/graph: add CLI option to enable graph stats skori
2023-09-21 10:08           ` [PATCH v5 12/12] app/graph: add l3fwd usecase skori
2023-09-26 10:57             ` [PATCH v6 00/12] add CLI based graph application skori
2023-09-26 10:57               ` [PATCH v6 01/12] app/graph: add application framework to read CLI skori
2023-09-26 10:57               ` [PATCH v6 02/12] app/graph: add telnet connectivity framework skori
2023-09-26 10:57               ` [PATCH v6 03/12] app/graph: add parser utility APIs skori
2023-09-26 10:57               ` [PATCH v6 04/12] app/graph: add mempool command line interfaces skori
2023-09-26 10:57               ` [PATCH v6 05/12] app/graph: add ethdev " skori
2023-09-26 10:57               ` [PATCH v6 06/12] app/graph: add ipv4_lookup " skori
2023-09-26 10:57               ` [PATCH v6 07/12] app/graph: add ipv6_lookup " skori
2023-09-26 10:57               ` [PATCH v6 08/12] app/graph: add neigh " skori
2023-09-26 10:57               ` [PATCH v6 09/12] app/graph: add ethdev_rx " skori
2023-09-26 10:57               ` [PATCH v6 10/12] app/graph: add graph " skori
2023-09-26 10:57               ` [PATCH v6 11/12] app/graph: add CLI option to enable graph stats skori
2023-09-26 10:57               ` [PATCH v6 12/12] app/graph: add l3fwd use case skori
2023-09-27 11:54                 ` [PATCH v7 00/12] add CLI based graph application skori
2023-09-27 11:54                   ` [PATCH v7 01/12] app/graph: add application framework to read CLI skori
2023-09-27 11:54                   ` [PATCH v7 02/12] app/graph: add telnet connectivity framework skori
2023-09-27 11:54                   ` [PATCH v7 03/12] app/graph: add parser utility APIs skori
2023-09-27 11:54                   ` [PATCH v7 04/12] app/graph: add mempool command line interfaces skori
2023-09-27 11:54                   ` [PATCH v7 05/12] app/graph: add ethdev " skori
2023-09-27 11:54                   ` [PATCH v7 06/12] app/graph: add ipv4_lookup " skori
2023-09-27 11:54                   ` [PATCH v7 07/12] app/graph: add ipv6_lookup " skori
2023-09-27 11:54                   ` [PATCH v7 08/12] app/graph: add neigh " skori
2023-09-27 11:54                   ` [PATCH v7 09/12] app/graph: add ethdev_rx " skori
2023-09-27 11:54                   ` [PATCH v7 10/12] app/graph: add graph " skori
2023-09-27 11:54                   ` [PATCH v7 11/12] app/graph: add CLI option to enable graph stats skori
2023-09-27 11:54                   ` [PATCH v7 12/12] app/graph: add l3fwd use case skori
2023-09-29  9:58                     ` [PATCH v8 00/12] add CLI based graph application skori
2023-09-29  9:58                       ` [PATCH v8 01/12] app/graph: add application framework to read CLI skori
2023-10-16  9:00                         ` Jerin Jacob
2023-10-17  6:19                           ` [EXT] " Sunil Kumar Kori
2023-10-18  6:33                         ` [PATCH v9 00/12] add CLI based graph application skori
2023-10-18  6:33                           ` [PATCH v9 01/12] app/graph: support application CLI framework skori
2023-10-18  6:33                           ` [PATCH v9 02/12] app/graph: support telnet connectivity framework skori
2023-10-18  6:33                           ` [PATCH v9 03/12] app/graph: support parser utility APIs skori
2023-10-18  6:33                           ` [PATCH v9 04/12] app/graph: support mempool command line interfaces skori
2023-10-18  6:33                           ` [PATCH v9 05/12] app/graph: support ethdev " skori
2023-10-18  6:33                           ` [PATCH v9 06/12] app/graph: support IPv4 lookup " skori
2023-10-18  6:33                           ` [PATCH v9 07/12] app/graph: support IPv6 " skori
2023-10-18  6:33                           ` [PATCH v9 08/12] app/graph: support neigh " skori
2023-10-18  6:33                           ` [PATCH v9 09/12] app/graph: support ethdev Rx " skori
2023-10-18  6:33                           ` [PATCH v9 10/12] app/graph: support graph " skori
2023-10-18  6:33                           ` [PATCH v9 11/12] app/graph: support CLI option to enable graph stats skori
2023-10-18  6:33                           ` [PATCH v9 12/12] app/graph: support l3fwd use case skori
2023-10-18 10:38                             ` Jerin Jacob
2023-10-19 10:49                             ` [PATCH v10 00/12] add CLI based graph application skori
2023-10-19 10:49                               ` [PATCH v10 01/12] app/graph: support application CLI framework skori
2023-10-19 10:49                               ` [PATCH v10 02/12] app/graph: support telnet connectivity framework skori
2023-10-19 10:49                               ` [PATCH v10 03/12] app/graph: support parser utility APIs skori
2023-10-19 10:49                               ` [PATCH v10 04/12] app/graph: support mempool command line interfaces skori
2023-10-19 10:49                               ` [PATCH v10 05/12] app/graph: support ethdev " skori
2023-10-19 10:49                               ` [PATCH v10 06/12] app/graph: support IPv4 lookup " skori
2023-10-19 10:49                               ` [PATCH v10 07/12] app/graph: support IPv6 " skori
2023-10-19 10:49                               ` [PATCH v10 08/12] app/graph: support neigh " skori
2023-10-19 10:49                               ` [PATCH v10 09/12] app/graph: support ethdev Rx " skori
2023-10-19 10:49                               ` [PATCH v10 10/12] app/graph: support graph " skori
2023-10-19 10:49                               ` [PATCH v10 11/12] app/graph: support CLI option to enable graph stats skori
2023-10-19 10:50                               ` [PATCH v10 12/12] app/graph: support l3fwd use case skori
2023-10-19 12:28                                 ` [EXT] " Jerin Jacob Kollanukkaran
2023-10-23  7:06                                   ` Nithin Dabilpuram
2023-10-19 17:29                                 ` [PATCH v11 00/12] add CLI based graph application skori
2023-10-19 17:30                                   ` [PATCH v11 01/12] app/graph: support application CLI framework skori
2023-10-23  7:03                                     ` Nithin Dabilpuram
2023-10-19 17:30                                   ` [PATCH v11 02/12] app/graph: support telnet connectivity framework skori
2023-10-23  7:03                                     ` Nithin Dabilpuram
2023-10-19 17:30                                   ` [PATCH v11 03/12] app/graph: support parser utility APIs skori
2023-10-23  7:03                                     ` Nithin Dabilpuram
2023-10-19 17:30                                   ` [PATCH v11 04/12] app/graph: support mempool command line interfaces skori
2023-10-23  7:04                                     ` Nithin Dabilpuram
2023-10-19 17:30                                   ` [PATCH v11 05/12] app/graph: support ethdev " skori
2023-10-23  7:04                                     ` Nithin Dabilpuram
2023-10-19 17:30                                   ` [PATCH v11 06/12] app/graph: support IPv4 lookup " skori
2023-10-23  7:04                                     ` Nithin Dabilpuram
2023-10-19 17:30                                   ` [PATCH v11 07/12] app/graph: support IPv6 " skori
2023-10-23  7:04                                     ` Nithin Dabilpuram
2023-10-19 17:30                                   ` [PATCH v11 08/12] app/graph: support neigh " skori
2023-10-23  7:05                                     ` Nithin Dabilpuram
2023-10-19 17:30                                   ` [PATCH v11 09/12] app/graph: support ethdev Rx " skori
2023-10-23  7:05                                     ` Nithin Dabilpuram
2023-10-19 17:30                                   ` [PATCH v11 10/12] app/graph: support graph " skori
2023-10-23  7:06                                     ` Nithin Dabilpuram
2023-10-19 17:30                                   ` [PATCH v11 11/12] app/graph: support CLI option to enable graph stats skori
2023-10-23  7:06                                     ` Nithin Dabilpuram
2023-10-19 17:30                                   ` [PATCH v11 12/12] app/graph: support l3fwd use case skori
2023-10-23 13:03                                   ` [PATCH v11 00/12] add CLI based graph application Sunil Kumar Kori
2023-11-03 16:44                                   ` Thomas Monjalon
2023-09-29  9:58                       ` [PATCH v8 02/12] app/graph: add telnet connectivity framework skori
2023-10-16  9:04                         ` Jerin Jacob
2023-10-17  6:21                           ` [EXT] " Sunil Kumar Kori
2023-09-29  9:58                       ` [PATCH v8 03/12] app/graph: add parser utility APIs skori
2023-10-16  9:05                         ` Jerin Jacob
2023-09-29  9:58                       ` [PATCH v8 04/12] app/graph: add mempool command line interfaces skori
2023-10-16  9:09                         ` Jerin Jacob
2023-10-17  6:22                           ` [EXT] " Sunil Kumar Kori
2023-09-29  9:58                       ` [PATCH v8 05/12] app/graph: add ethdev " skori
2023-10-16 13:20                         ` Jerin Jacob
2023-10-16 14:10                           ` Bruce Richardson
2023-10-17  6:26                             ` [EXT] " Sunil Kumar Kori
2023-09-29  9:58                       ` [PATCH v8 06/12] app/graph: add ipv4_lookup " skori
2023-10-16 15:47                         ` Jerin Jacob
2023-10-17  6:30                           ` [EXT] " Sunil Kumar Kori
2023-09-29  9:58                       ` [PATCH v8 07/12] app/graph: add ipv6_lookup " skori
2023-09-29  9:58                       ` [PATCH v8 08/12] app/graph: add neigh " skori
2023-09-29  9:58                       ` [PATCH v8 09/12] app/graph: add ethdev_rx " skori
2023-09-29  9:58                       ` skori [this message]
2023-09-29  9:58                       ` [PATCH v8 11/12] app/graph: add CLI option to enable graph stats skori
2023-10-16 15:55                         ` Jerin Jacob
2023-09-29  9:58                       ` [PATCH v8 12/12] app/graph: add l3fwd use case skori
2023-10-16 16:01                       ` [PATCH v8 00/12] add CLI based graph application Jerin Jacob
2023-10-16 16:27                         ` Stephen Hemminger
2023-06-02 16:22   ` [PATCH v3 0/3] node: Introduce kernel_rx & kernel_tx nodes Vamsi Attunuru
2023-06-02 16:22     ` [PATCH v3 1/3] node/kernel_tx: support packet transmit to kernel Vamsi Attunuru
2023-06-05 12:47       ` Nithin Dabilpuram
2023-06-12 19:32         ` Thomas Monjalon
2023-06-02 16:22     ` [PATCH v3 2/3] node/kernel_rx: support receiving packets from kernel Vamsi Attunuru
2023-06-05 12:50       ` Nithin Dabilpuram
2023-06-02 16:22     ` [PATCH v3 3/3] node/ethdev_rx: remove hardcoded node next details Vamsi Attunuru
2023-06-05 12:51       ` Nithin Dabilpuram
2023-06-12 16:12     ` [PATCH v3 0/3] node: Introduce kernel_rx & kernel_tx nodes Vamsi Krishna Attunuru
2023-06-12 19:31     ` Thomas Monjalon

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20230929095814.692890-11-skori@marvell.com \
    --to=skori@marvell.com \
    --cc=dev@dpdk.org \
    --cc=rkudurumalla@marvell.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).