[PATCH v3 2/3] graph: pcap capture for graph nodes

[Amit Prakash Shukla <amitprakashs@marvell.com>]

Implementation adds support to capture packets at each node with
packet metadata and node name.

Signed-off-by: Amit Prakash Shukla <amitprakashs@marvell.com>
---
v2:
 - Fixed code style issue
 - Fixed CI compilation issue on github-robot

v3:
 - Code review suggestion from Stephen
 - Fixed potential memory leak

 lib/graph/graph_pcap_trace.c     | 155 +++++++++++++++++++++++++++++++
 lib/graph/graph_populate.c       |   6 +-
 lib/graph/graph_private.h        |  21 +++++
 lib/graph/meson.build            |   5 +-
 lib/graph/rte_graph_pcap_trace.h |  96 +++++++++++++++++++
 lib/graph/rte_graph_worker.h     |   3 +
 lib/graph/version.map            |   7 ++
 7 files changed, 290 insertions(+), 3 deletions(-)
 create mode 100644 lib/graph/graph_pcap_trace.c
 create mode 100644 lib/graph/rte_graph_pcap_trace.h

diff --git a/lib/graph/graph_pcap_trace.c b/lib/graph/graph_pcap_trace.c
new file mode 100644
index 0000000000..f7b81a7ad1
--- /dev/null
+++ b/lib/graph/graph_pcap_trace.c
@@ -0,0 +1,155 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2022 Marvell International Ltd.
+ */
+
+#include <errno.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <fcntl.h>
+
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+#include <rte_pcapng.h>
+
+#include "rte_graph_worker.h"
+#include "graph_private.h"
+
+#define GRAPH_PCAP_BUF_SZ	128
+#define GRAPH_PCAP_NUM_PACKETS	1024
+#define GRAPH_PCAP_FILE_NAME_SZ	128
+#define GRAPH_PCAP_FILE_NAME	"/tmp/graph_pcap_capture.pcapng"
+
+static char file_name[GRAPH_PCAP_FILE_NAME_SZ];
+static uint32_t pkt_buf_sz = RTE_MBUF_DEFAULT_BUF_SIZE;
+static uint64_t packet_to_capture = GRAPH_PCAP_NUM_PACKETS;
+static rte_pcapng_t *pcapng_fd;
+static struct rte_mempool *mp;
+static uint16_t port_id;
+static uint64_t packet_captured[RTE_MAX_LCORE];
+static int pcap_trace_enable;
+
+void
+rte_num_pkt_to_capture(uint64_t val)
+{
+	packet_to_capture = val;
+}
+
+void
+rte_pcap_trace_enable(int val)
+{
+	pcap_trace_enable = val;
+}
+
+int
+rte_pcap_trace_is_enable(void)
+{
+	return pcap_trace_enable;
+}
+
+void
+rte_filename_to_capture_pkt(const char *filename)
+{
+	if (filename[0] == '\0')
+		rte_strscpy(file_name, GRAPH_PCAP_FILE_NAME,
+			    GRAPH_PCAP_FILE_NAME_SZ);
+	else
+		rte_strscpy(file_name, filename, GRAPH_PCAP_FILE_NAME_SZ);
+}
+
+void
+rte_graph_pcap_trace_exit(void)
+{
+	rte_pcapng_close(pcapng_fd);
+}
+
+int
+rte_graph_pcap_trace_init(void)
+{
+	int fd;
+
+	port_id = rte_eth_find_next(0);
+	if (port_id >= RTE_MAX_ETHPORTS) {
+		fprintf(stderr, "No valid Ether port\n");
+		return -1;
+	}
+
+	if (file_name[0] == '\0')
+		rte_strscpy(file_name, GRAPH_PCAP_FILE_NAME,
+			    GRAPH_PCAP_FILE_NAME_SZ);
+
+	fd = open(file_name, O_CREAT | O_TRUNC | O_WRONLY, 0664);
+	if (fd < 0) {
+		perror("pcap file open failure");
+		return -1;
+	}
+
+	fprintf(stdout, "pcapng: output file %s\n", file_name);
+
+	/* open a test capture file */
+	pcapng_fd = rte_pcapng_fdopen(fd, NULL, NULL, "Graph pcap tracer", NULL);
+	if (pcapng_fd == NULL) {
+		fprintf(stderr, "Graph rte_pcapng_fdopen failed\n");
+		close(fd);
+		return -1;
+	}
+
+	/* Make a pool for cloned packets */
+	mp = rte_pktmbuf_pool_create_by_ops("pcapng_graph_pool",
+			IOV_MAX + RTE_GRAPH_BURST_SIZE,
+			0, 0, rte_pcapng_mbuf_size(pkt_buf_sz),
+			SOCKET_ID_ANY, "ring_mp_sc");
+	if (mp == NULL) {
+		fprintf(stderr, "Cannot create mempool for graph pcap capture\n");
+		rte_pcapng_close(pcapng_fd);
+		return -1;
+	}
+
+	return 0;
+}
+
+uint16_t
+rte_graph_pcap_trace_dispatch(struct rte_graph *graph,
+			      struct rte_node *node, void **objs,
+			      uint16_t nb_objs)
+{
+	uint64_t i, num_packets;
+	struct rte_mbuf *mbuf_clones[RTE_GRAPH_BURST_SIZE];
+	char buffer[GRAPH_PCAP_BUF_SZ];
+	struct rte_mbuf *mbuf;
+	ssize_t len;
+	uint16_t gid = graph->id;
+
+	if (!nb_objs || (packet_captured[gid] >= packet_to_capture))
+		goto done;
+
+	num_packets = packet_to_capture - packet_captured[gid];
+	/* nb_objs will never be greater than RTE_GRAPH_BURST_SIZE */
+	if (num_packets > nb_objs)
+		num_packets = nb_objs;
+
+	rte_strlcpy(buffer, node->name, GRAPH_PCAP_BUF_SZ);
+
+	for (i = 0; i < num_packets; i++) {
+		struct rte_mbuf *mc;
+		mbuf = (struct rte_mbuf *)objs[i];
+
+		mc = rte_pcapng_copy(port_id, 0, mbuf, mp, mbuf->pkt_len,
+				     rte_get_tsc_cycles(), 0, buffer);
+		if (mc == NULL)
+			break;
+
+		mbuf_clones[i] = mc;
+	}
+
+	/* write it to capture file */
+	len = rte_pcapng_write_packets(pcapng_fd, mbuf_clones, i);
+	rte_pktmbuf_free_bulk(mbuf_clones, i);
+	if (len <= 0)
+		goto done;
+
+	packet_captured[gid] += i;
+
+done:
+	return node->original_process(graph, node, objs, nb_objs);
+}
diff --git a/lib/graph/graph_populate.c b/lib/graph/graph_populate.c
index 102fd6c29b..aa26b1c51d 100644
--- a/lib/graph/graph_populate.c
+++ b/lib/graph/graph_populate.c
@@ -75,7 +75,11 @@ graph_nodes_populate(struct graph *_graph)
 		memset(node, 0, sizeof(*node));
 		node->fence = RTE_GRAPH_FENCE;
 		node->off = off;
-		node->process = graph_node->node->process;
+		if (rte_pcap_trace_is_enable()) {
+			node->process = rte_graph_pcap_trace_dispatch;
+			node->original_process = graph_node->node->process;
+		} else
+			node->process = graph_node->node->process;
 		memcpy(node->name, graph_node->node->name, RTE_GRAPH_NAMESIZE);
 		pid = graph_node->node->parent_id;
 		if (pid != RTE_NODE_ID_INVALID) { /* Cloned node */
diff --git a/lib/graph/graph_private.h b/lib/graph/graph_private.h
index f9a85c8926..3edbb42692 100644
--- a/lib/graph/graph_private.h
+++ b/lib/graph/graph_private.h
@@ -319,6 +319,27 @@ struct rte_node *graph_node_id_to_ptr(const struct rte_graph *graph,
 struct rte_node *graph_node_name_to_ptr(const struct rte_graph *graph,
 					const char *node_name);
 
+/**
+ * @internal
+ *
+ * Capture mbuf metadata and node metadata to a pcap file.
+ *
+ * When graph pcap trace enabled, this function is invoked prior to each node
+ * and mbuf, node metadata is parsed and captured in a pcap file.
+ *
+ * @param graph
+ *   Pointer to the graph object.
+ * @param node
+ *   Pointer to the node object.
+ * @param objs
+ *   Pointer to an array of objects to be processed.
+ * @param nb_objs
+ *   Number of objects in the array.
+ */
+uint16_t rte_graph_pcap_trace_dispatch(struct rte_graph *graph __rte_unused,
+				       struct rte_node *node, void **objs,
+				       uint16_t nb_objs);
+
 /* Debug functions */
 /**
  * @internal
diff --git a/lib/graph/meson.build b/lib/graph/meson.build
index c7327549e8..955c051a20 100644
--- a/lib/graph/meson.build
+++ b/lib/graph/meson.build
@@ -14,7 +14,8 @@ sources = files(
         'graph_debug.c',
         'graph_stats.c',
         'graph_populate.c',
+        'graph_pcap_trace.c',
 )
-headers = files('rte_graph.h', 'rte_graph_worker.h')
+headers = files('rte_graph.h', 'rte_graph_worker.h', 'rte_graph_pcap_trace.h')
 
-deps += ['eal']
+deps += ['eal', 'mbuf', 'ethdev', 'pcapng']
diff --git a/lib/graph/rte_graph_pcap_trace.h b/lib/graph/rte_graph_pcap_trace.h
new file mode 100644
index 0000000000..2af807c95b
--- /dev/null
+++ b/lib/graph/rte_graph_pcap_trace.h
@@ -0,0 +1,96 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2022 Marvell International Ltd.
+ */
+
+#ifndef _RTE_GRAPH_PCAP_TRACE_H_
+#define _RTE_GRAPH_PCAP_TRACE_H_
+
+/**
+ * @file rte_graph_pcap_trace.h
+ *
+ * @warning
+ * @b EXPERIMENTAL:
+ * All functions in this file may be changed or removed without prior notice.
+ *
+ * This API enables to capture packet at each node with mbuf and node metadata.
+ *
+ */
+
+#include <stdint.h>
+#include <sys/types.h>
+#include <rte_compat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Pcap trace enable/disable function.
+ *
+ * The function is called to enable/disable graph pcap trace functionality.
+ *
+ * @param val
+ *   Value to be set to enable/disable graph pcap trace.
+ */
+__rte_experimental
+void rte_pcap_trace_enable(int val);
+
+/**
+ * Check graph pcap trace is enable/disable.
+ *
+ * The function is called to check if the graph pcap trace is enabled/disabled.
+ *
+ * @return
+ *   - 1: Enable
+ *   - 0: Disable
+ */
+__rte_experimental
+int rte_pcap_trace_is_enable(void);
+
+/**
+ * Initialise graph pcap trace functionality.
+ *
+ * The function invoked when the graph pcap trace is enabled from the
+ * application.
+ *
+ * @return
+ *   0 on success and -1 on failure.
+ */
+__rte_experimental
+int rte_graph_pcap_trace_init(void);
+
+/**
+ * Pcap trace set number of packets to capture.
+ *
+ * The function is called to configure total number of packets to be captured.
+ *
+ * @param val
+ *   Number of packets to capture.
+ */
+__rte_experimental
+void rte_num_pkt_to_capture(uint64_t val);
+
+/**
+ * Pcap trace file name to capture packets.
+ *
+ * The function is called to configure file name to capture packets in.
+ *
+ * @param filename
+ *   Number of packets to capture.
+ */
+__rte_experimental
+void rte_filename_to_capture_pkt(const char *filename);
+
+/**
+ * Exit graph pcap trace functionality.
+ *
+ * The function is called to exit graph pcap trace and close open fd's.
+ */
+__rte_experimental
+void rte_graph_pcap_trace_exit(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_GRAPH_PCAP_TRACE_H_ */
diff --git a/lib/graph/rte_graph_worker.h b/lib/graph/rte_graph_worker.h
index fc6fee48c8..fcaedbdb12 100644
--- a/lib/graph/rte_graph_worker.h
+++ b/lib/graph/rte_graph_worker.h
@@ -24,6 +24,7 @@
 #include <rte_memory.h>
 
 #include "rte_graph.h"
+#include "rte_graph_pcap_trace.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -64,6 +65,8 @@ struct rte_node {
 	char parent[RTE_NODE_NAMESIZE];	/**< Parent node name. */
 	char name[RTE_NODE_NAMESIZE];	/**< Name of the node. */
 
+	rte_node_process_t original_process; /**< Pcap enabled node callback */
+
 	/* Fast path area  */
 #define RTE_NODE_CTX_SZ 16
 	uint8_t ctx[RTE_NODE_CTX_SZ] __rte_cache_aligned; /**< Node Context. */
diff --git a/lib/graph/version.map b/lib/graph/version.map
index 13b838752d..36153a75b2 100644
--- a/lib/graph/version.map
+++ b/lib/graph/version.map
@@ -43,5 +43,12 @@ EXPERIMENTAL {
 	rte_node_next_stream_put;
 	rte_node_next_stream_move;
 
+	rte_pcap_trace_is_enable;
+	rte_pcap_trace_enable;
+	rte_graph_pcap_trace_init;
+	rte_num_pkt_to_capture;
+	rte_filename_to_capture_pkt;
+	rte_graph_pcap_trace_exit;
+
 	local: *;
 };
-- 
2.25.1