[dpdk-dev] [PATCH v1] app/test-regex: add RegEx test application

[dpdk-dev] [PATCH v1] app/test-regex: add RegEx test application

[Ori Kam]

From: Yuval Avnery <yuvalav@mellanox.com>

Following the new RegEx class.
There is a need to create a dedicated test application in order to
validate this class and PMD.

Unlike net device this application loads data from a file.

This commit introduce the new RegEx test app.

The basic app flow:
1. Configure the RegEx device to use one queue, and set the rule
   database, using pre compiled file.
2. Allocate mbufs based on the requested number of jobs, each job will
   get one mbuf.
3. Enqueue as much as possible jobs.
4. Dequeue jobs.
5. if number of dequeue jobs < requested number of jobs job to step 3.

Signed-off-by: Ori Kam <orika@mellanox.com>
Signed-off-by: Yuval Avnery <yuvalav@mellanox.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>

---
 app/Makefile                         |   1 +
 app/meson.build                      |   1 +
 app/test-regex/Makefile              |  24 ++
 app/test-regex/generate_data_file.py |  29 +++
 app/test-regex/hello_world.rof2      |  45 ++++
 app/test-regex/main.c                | 429 +++++++++++++++++++++++++++++++++++
 app/test-regex/meson.build           |   5 +
 doc/guides/tools/index.rst           |   1 +
 doc/guides/tools/testregex.rst       |  84 +++++++
 9 files changed, 619 insertions(+)
 create mode 100644 app/test-regex/Makefile
 create mode 100644 app/test-regex/generate_data_file.py
 create mode 100644 app/test-regex/hello_world.rof2
 create mode 100644 app/test-regex/main.c
 create mode 100644 app/test-regex/meson.build
 create mode 100644 doc/guides/tools/testregex.rst

diff --git a/app/Makefile b/app/Makefile
index 0392a7d..453c4fe 100644
--- a/app/Makefile
+++ b/app/Makefile
@@ -13,6 +13,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_FIB) += test-fib
 DIRS-$(CONFIG_RTE_TEST_FLOW_PERF) += test-flow-perf
 DIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += test-pipeline
 DIRS-$(CONFIG_RTE_LIBRTE_IPSEC) += test-sad
+DIRS-$(CONFIG_RTE_LIBRTE_REGEXDEV) += test-regex
 
 ifeq ($(CONFIG_RTE_LIBRTE_BBDEV),y)
 DIRS-$(CONFIG_RTE_TEST_BBDEV) += test-bbdev
diff --git a/app/meson.build b/app/meson.build
index 585b908..8cd854e 100644
--- a/app/meson.build
+++ b/app/meson.build
@@ -12,6 +12,7 @@ apps = [
 	'test-bbdev',
 	'test-cmdline',
 	'test-compress-perf',
+	'test-regex',
 	'test-crypto-perf',
 	'test-eventdev',
 	'test-fib',
diff --git a/app/test-regex/Makefile b/app/test-regex/Makefile
new file mode 100644
index 0000000..d73e776
--- /dev/null
+++ b/app/test-regex/Makefile
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2020 Mellanox Corporation
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+ifeq ($(CONFIG_RTE_LIBRTE_REGEXDEV),y)
+
+#
+# library name
+#
+APP = testregex
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -Wno-deprecated-declarations
+
+#
+# all source are stored in SRCS-y
+#
+SRCS-y := main.c
+
+include $(RTE_SDK)/mk/rte.app.mk
+
+endif
diff --git a/app/test-regex/generate_data_file.py b/app/test-regex/generate_data_file.py
new file mode 100644
index 0000000..d3a98de
--- /dev/null
+++ b/app/test-regex/generate_data_file.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2020 Mellanox Technologies, Ltd
+
+import random
+
+KEYWORD = 'hello world'
+MAX_COUNT = 10
+MIN_COUNT = 5
+MAX_LEN = 1024
+REPEAT_COUNT = random.randrange(MIN_COUNT, MAX_COUNT)
+
+current_pos = 0;
+match_pos = []
+
+fd_input = open('input.txt','w')
+fd_res = open('res.txt','w')
+
+for i in range(REPEAT_COUNT):
+    rand = random.randrange(MAX_LEN)
+    fd_input.write(' ' * rand)
+    current_pos += rand
+    fd_input.write(KEYWORD)
+    match_pos.append(current_pos)
+    fd_res.write('{}\n'.format(str(current_pos)))
+    current_pos += len(KEYWORD)
+
+fd_input.close()
+fd_res.close()
diff --git a/app/test-regex/hello_world.rof2 b/app/test-regex/hello_world.rof2
new file mode 100644
index 0000000..fb7db75
--- /dev/null
+++ b/app/test-regex/hello_world.rof2
@@ -0,0 +1,45 @@
+#
+# rof_version: 2
+#
+# date:20200210_164643
+#
+# rxp_compiler:5.7.18007
+#
+# rof_revision:0
+#
+2, 0x00000000, 0x0000000004055254
+3, 0x00000000, 0x0000000007055254
+1, 0x00000005, 0x0000000000aa0008
+1, 0x00000006, 0x00000000dddd00aa
+4, 0x00010010, 0x0000000000000000
+4, 0x00010011, 0x0000000000000000
+4, 0x00010012, 0x0000000000000000
+5, 0x00010010, 0x0000000000000000
+5, 0x00010011, 0x0000000000000000
+5, 0x00010012, 0x0000000000000000
+6, 0x00000000, 0x6c6c65680040bd82
+6, 0x00000001, 0x6f77206f0040bd82
+6, 0x00000002, 0x00646c720040bd62
+6, 0x00000003, 0x0000000100403a48
+6, 0x00000004, 0x00000001003b3c1f
+6, 0x00000005, 0x0000000000000000
+6, 0x00000006, 0x0000000000000000
+6, 0x00000007, 0x0000000000000000
+6, 0x00d30000, 0x0000000000000000
+6, 0x00e401e5, 0xbdb9180002000000
+6, 0x00e80000, 0x31ac75f69abc779c
+6, 0x00ec0000, 0xabe575e975011cf4
+6, 0x00ed0000, 0x38bf9e2967d22ac4
+6, 0x00f00000, 0x34f501ce6aa8a034
+6, 0x00f10000, 0x85e40fd9e10676d7
+6, 0x00f40000, 0x1258c0a059dc9464
+6, 0x00f401e5, 0x680262622a400800
+6, 0x00f50000, 0x000d32453675686d
+6, 0x00f60000, 0x4136d69d550adc7a
+6, 0x00f70000, 0x6db55b9daab1c703
+4, 0x00010010, 0x0000000087a5837c
+4, 0x00010011, 0x00000000d75b7cab
+4, 0x00010012, 0x000000000884a03a
+5, 0x00010010, 0x0000000087a5837c
+5, 0x00010011, 0x00000000d75b7cab
+5, 0x00010012, 0x000000000884a03a
diff --git a/app/test-regex/main.c b/app/test-regex/main.c
new file mode 100644
index 0000000..f0740a3
--- /dev/null
+++ b/app/test-regex/main.c
@@ -0,0 +1,429 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ *
+ * This file contain the application main file
+ * This application provides a way to test the RegEx class.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdarg.h>
+#include <ctype.h>
+#include <errno.h>
+#include <getopt.h>
+#include <signal.h>
+
+#include <rte_eal.h>
+#include <rte_common.h>
+#include <rte_malloc.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_cycles.h>
+#include <rte_regexdev.h>
+
+#define HELP_VAL 0
+#define RULES_VAL 1
+#define DATA_VAL 2
+#define JOB_VAL 3
+#define PERF_VAL 4
+#define ITER_VAL 5
+#define MAX_FILE_NAME 255
+
+static char rules_file[MAX_FILE_NAME];
+static char data_file[MAX_FILE_NAME];
+static uint32_t jobs;
+static struct rte_mempool *mbuf_mp;
+static uint8_t nb_max_matches;
+static uint16_t nb_max_payload;
+static int perf_test;
+static uint32_t iter;
+
+static void
+usage(const char *prog_name)
+{
+	printf("%s [EAL options] --\n"
+		" --rules NAME: precompiled rules file\n"
+		" --data NAME: data file to use\n"
+		" --nb_jobs: number of jobs to use\n"
+		" --perf N: only outputs the performance data\n"
+		" --nb_iter N: number of iteration to run\n",
+		prog_name);
+}
+
+static void
+args_parse(int argc, char **argv)
+{
+	char **argvopt;
+	int opt;
+	int opt_idx;
+	size_t len;
+	static struct option lgopts[] = {
+		{ "help",  0, 0, HELP_VAL },
+		{ "rules",  1, 0, RULES_VAL },
+		/* Rules database file to load. */
+		{ "data",  1, 0, DATA_VAL },
+		/* Data file to load. */
+		{ "nb_jobs",  1, 0, JOB_VAL },
+		/* Number of jobs to create. */
+		{ "perf", 0, 0, PERF_VAL},
+		/* Perf test only */
+		{ "nb_iter", 1, 0, ITER_VAL}
+		/* Number of iterations to run with perf test */
+	};
+
+	argvopt = argv;
+
+	while ((opt = getopt_long(argc, argvopt, "",
+				lgopts, &opt_idx)) != EOF) {
+		switch (opt) {
+		case RULES_VAL:
+			len = strnlen(optarg, MAX_FILE_NAME - 1);
+			if (len == MAX_FILE_NAME)
+				rte_exit(EXIT_FAILURE,
+					 "Rule file name to long max %d\n",
+					 MAX_FILE_NAME - 1);
+			strncpy(rules_file, optarg, MAX_FILE_NAME - 1);
+			break;
+		case DATA_VAL:
+			len = strnlen(optarg, MAX_FILE_NAME - 1);
+			if (len == MAX_FILE_NAME)
+				rte_exit(EXIT_FAILURE,
+					 "Data file name to long max %d\n",
+					 MAX_FILE_NAME - 1);
+			strncpy(data_file, optarg, MAX_FILE_NAME - 1);
+			break;
+		case JOB_VAL:
+			jobs = atoi(optarg);
+			break;
+		case PERF_VAL:
+			perf_test = 1;
+			break;
+		case ITER_VAL:
+			iter = atoi(optarg);
+			break;
+		case HELP_VAL:
+			usage("RegEx test app");
+			break;
+		default:
+			fprintf(stderr, "Invalid option: %s\n", argv[optind]);
+			usage("RegEx test app");
+			rte_exit(EXIT_FAILURE, "Invalid option\n");
+			break;
+		}
+	}
+
+	if (!perf_test)
+		iter = 1;
+}
+
+static long
+read_file(char *file, char **buf)
+{
+	FILE *fp;
+	long buf_len = 0;
+	size_t read_len;
+	int res = 0;
+
+	fp = fopen(file, "r");
+	if (!fp)
+		return -EIO;
+	if (fseek(fp, 0L, SEEK_END) == 0) {
+		buf_len = ftell(fp);
+		if (buf_len == -1) {
+			res = EIO;
+			goto error;
+		}
+		*buf = rte_malloc(NULL, sizeof(char) * (buf_len + 1), 4096);
+		if (!*buf) {
+			res = ENOMEM;
+			goto error;
+		}
+		if (fseek(fp, 0L, SEEK_SET) != 0) {
+			res = EIO;
+			goto error;
+		}
+		read_len = fread(*buf, sizeof(char), buf_len, fp);
+		if (read_len != (unsigned long)buf_len) {
+			res = EIO;
+			goto error;
+		}
+	}
+	fclose(fp);
+	return buf_len;
+error:
+	printf("Error, can't open file %s\n, err = %d", file, res);
+	if (fp)
+		fclose(fp);
+	if (*buf)
+		rte_free(*buf);
+	return -res;
+}
+#define MBUF_CACHE_SIZE 256
+#define MBUF_SIZE (1 << 14)
+
+
+static int
+init_port(void)
+{
+	uint16_t id;
+	uint16_t num_devs;
+	char *rules = NULL;
+	long rules_len;
+	struct rte_regexdev_info info;
+	struct rte_regexdev_config dev_conf = {
+		.nb_queue_pairs = 1,
+		.nb_groups = 1,
+	};
+	struct rte_regexdev_qp_conf qp_conf = {
+		.nb_desc = 1024,
+		.qp_conf_flags = RTE_REGEX_QUEUE_PAIR_CFG_OOS_F,
+	};
+	int res = 0;
+
+	num_devs = rte_regexdev_count();
+	if (num_devs == 0) {
+		printf("Error, no devices detected.\n");
+		return -EINVAL;
+	}
+
+	mbuf_mp = rte_pktmbuf_pool_create("mbuf_pool", jobs, 0,
+					  0, MBUF_SIZE, rte_socket_id());
+	if (mbuf_mp == NULL) {
+		printf("Error, can't create memory pool\n");
+		res = -ENOMEM;
+		goto error;
+	}
+
+	rules_len = read_file(rules_file, &rules);
+	if (rules_len < 0) {
+		printf("Error, can't read rules files.\n");
+		res = -EIO;
+		goto error;
+	}
+
+	for (id = 0; id < num_devs; id++) {
+		res = rte_regexdev_info_get(id, &info);
+		if (res != 0) {
+			printf("Error, can't get device info.\n");
+			goto error;
+		}
+		printf(":: initializing dev: %d\n", id);
+		nb_max_matches = info.max_matches;
+		nb_max_payload = info.max_payload_size;
+		if (info.regexdev_capa & RTE_REGEXDEV_SUPP_MATCH_AS_END_F)
+			dev_conf.dev_cfg_flags |= RTE_REGEXDEV_CFG_MATCH_AS_END_F;
+		dev_conf.nb_max_matches = info.max_matches;
+		dev_conf.nb_rules_per_group = info.max_rules_per_group;
+		dev_conf.rule_db_len = rules_len;
+		dev_conf.rule_db = rules;
+		res = rte_regexdev_configure(id, &dev_conf);
+		if (res < 0) {
+			printf("Error, can't configure device %d.\n", id);
+			goto error;
+		}
+		res = rte_regexdev_queue_pair_setup(id, 0, &qp_conf);
+		if (res < 0) {
+			printf("Error, can't setup queue pair for device %d.\n",
+			       id);
+			goto error;
+		}
+		printf(":: initializing device: %d done\n", id);
+	}
+	rte_free(rules);
+	return 0;
+error:
+	if (rules)
+		rte_free(rules);
+	if (mbuf_mp)
+		rte_mempool_free(mbuf_mp);
+	return res;
+
+}
+
+static void
+extbuf_free_cb(void *addr __rte_unused, void *fcb_opaque __rte_unused)
+{
+
+}
+
+#define START_BURST_SIZE 32u
+
+static int
+run_regex(void)
+{
+	char *buf;
+	long buf_len;
+	long job_len;
+	uint32_t actual_jobs = 0;
+	uint32_t i;
+	struct rte_regex_ops **ops;
+	uint16_t dev_id = 0;
+	uint16_t qp_id = 0;
+	uint8_t nb_matches;
+	struct rte_regexdev_match *match;
+	long pos = 0;
+	unsigned long d_ind = 0;
+	struct rte_mbuf_ext_shared_info shinfo;
+	uint32_t total_enqueue = 0;
+	uint32_t total_dequeue = 0;
+	uint32_t total_matches = 0;
+	int res = 0;
+	time_t start;
+	time_t end;
+	double time;
+
+	shinfo.free_cb = extbuf_free_cb;
+
+	ops = rte_malloc(NULL, sizeof(*ops) * jobs, 0);
+	if (!ops) {
+		printf("Error, can't allocate memory for ops.\n");
+		return -ENOMEM;
+	}
+
+	/* Allocate the jobs and assign each job with an mbuf. */
+	for (i = 0; i < jobs; i++) {
+		ops[i] = rte_malloc(NULL, sizeof(*ops[0]) + nb_max_matches *
+				    sizeof(struct rte_regexdev_match), 0);
+		if (!ops[i]) {
+			printf("Error, can't allocate memory for op.\n");
+			res = -ENOMEM;
+			goto end;
+		}
+		ops[i]->mbuf = rte_pktmbuf_alloc(mbuf_mp);
+		if (!ops[i]->mbuf) {
+			printf("Error, can't attach mbuf.\n");
+			res = -ENOMEM;
+			goto end;
+		}
+	}
+
+	buf_len = read_file(data_file, &buf);
+	if (buf_len <= 0) {
+		printf("Error, can't read file, or file is empty.\n");
+		res = -EXIT_FAILURE;
+		goto end;
+	}
+
+	job_len = buf_len / jobs;
+	if (job_len == 0) {
+		printf("Error, To many jobs, for the given input.\n");
+		res = -EXIT_FAILURE;
+		goto end;
+	}
+
+	if (job_len > nb_max_payload) {
+		printf("Error, not enough jobs to cover input.\n");
+		res = -EXIT_FAILURE;
+		goto end;
+	}
+
+	/* Assign each mbuf with the data to handle. */
+	for (i = 0; (pos < buf_len) && (i < jobs) ; i++) {
+		long act_job_len = RTE_MIN(job_len, buf_len - pos);
+		rte_pktmbuf_attach_extbuf(ops[i]->mbuf, &buf[pos], 0,
+					  act_job_len, &shinfo);
+		ops[i]->mbuf->data_len = job_len;
+		ops[i]->mbuf->pkt_len = act_job_len;
+		ops[i]->user_id = i;
+		ops[i]->group_id0 = 1;
+		pos += act_job_len;
+		actual_jobs++;
+	}
+
+	start = clock();
+	for (i = 0; i < iter; i++) {
+		total_enqueue = 0;
+		total_dequeue = 0;
+		while (total_dequeue < actual_jobs) {
+			struct rte_regex_ops **cur_ops_to_enqueue = ops +
+				total_enqueue;
+			struct rte_regex_ops **cur_ops_to_dequeue = ops +
+				total_dequeue;
+
+			if (actual_jobs - total_enqueue)
+				total_enqueue += rte_regexdev_enqueue_burst
+					(dev_id, qp_id, cur_ops_to_enqueue,
+					 actual_jobs - total_enqueue);
+
+			total_dequeue += rte_regexdev_dequeue_burst
+				(dev_id, qp_id, cur_ops_to_dequeue,
+				 total_enqueue - total_dequeue);
+		}
+	}
+	end = clock();
+	time = ((double)end - start) / CLOCKS_PER_SEC;
+	printf("Job len = %ld Bytes\n",  job_len);
+	printf("Time = %lf sec\n",  time);
+	printf("Perf = %lf Gbps\n",
+	       (((double)buf_len * iter * 8) / time)/1000000000.0);
+
+	if (!perf_test) {
+		/* Log results per job. */
+		for (d_ind = 0; d_ind < total_dequeue; d_ind++) {
+			nb_matches = ops[d_ind % actual_jobs]->nb_matches;
+			printf("Job id %ld number of matches = %d\n",
+			       ops[d_ind]->user_id, nb_matches);
+			total_matches += nb_matches;
+			match = ops[d_ind % actual_jobs]->matches;
+			for (i = 0; i < nb_matches; i++) {
+				printf("match %d, rule = %d, start = %d,len = %d\n",
+				       i, match->rule_id, match->start_offset,
+				       match->len);
+				match++;
+			}
+		}
+		printf("Total matches = %d\n", total_matches);
+		printf("All Matches:\n");
+
+		/* Log absolute results. */
+		for (d_ind = 0; d_ind < total_dequeue; d_ind++) {
+			nb_matches = ops[d_ind % actual_jobs]->nb_matches;
+			total_matches += nb_matches;
+			match = ops[d_ind % actual_jobs]->matches;
+			for (i = 0; i < nb_matches; i++) {
+				printf("start = %ld, len = %d, rule = %d\n",
+				       match->start_offset + d_ind * job_len,
+				       match->len, match->rule_id);
+				match++;
+			}
+		}
+	}
+end:
+	for (i = 0; i < actual_jobs; i++) {
+		if (ops[i]) {
+			if (ops[i]->mbuf)
+				rte_pktmbuf_free(ops[i]->mbuf);
+			rte_free(ops[i]);
+		}
+	}
+	rte_free(ops);
+	rte_free(buf);
+	return res;
+}
+
+int
+main(int argc, char **argv)
+{
+	int ret;
+
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "EAL init failed\n");
+	argc -= ret;
+	argv += ret;
+	if (argc > 1)
+		args_parse(argc, argv);
+
+	ret = init_port();
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "init port failed\n");
+	ret = run_regex();
+	if (ret < 0) {
+		rte_mempool_free(mbuf_mp);
+		rte_exit(EXIT_FAILURE, "RegEx function failed\n");
+	}
+	rte_mempool_free(mbuf_mp);
+	return 0;
+}
diff --git a/app/test-regex/meson.build b/app/test-regex/meson.build
new file mode 100644
index 0000000..7c9357f
--- /dev/null
+++ b/app/test-regex/meson.build
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+sources = files('main.c')
+deps = ['regexdev']
diff --git a/doc/guides/tools/index.rst b/doc/guides/tools/index.rst
index 4840cf47..c721943 100644
--- a/doc/guides/tools/index.rst
+++ b/doc/guides/tools/index.rst
@@ -17,3 +17,4 @@ DPDK Tools User Guides
     cryptoperf
     comp_perf
     testeventdev
+    testregex
diff --git a/doc/guides/tools/testregex.rst b/doc/guides/tools/testregex.rst
new file mode 100644
index 0000000..7420937
--- /dev/null
+++ b/doc/guides/tools/testregex.rst
@@ -0,0 +1,84 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright 2020 Mellanox Technologies, Ltd
+
+dpdk-test-regex Tool
+====================
+
+The ``dpdk-test-regex`` tool is a Data Plane Development Kit (DPDK)
+application that allows functional testing and performance measurement for
+the RegEx PMDs.
+The test supports only one core and one PMD.
+It is based on precomplied rule file, and an input file, both of them can
+be selected using command-line options.
+
+In general case, each PMD has it's own rule file.
+
+The test outputs the performance, the results matching (rule id, position, len)
+for each job and also a list of matches (rule id, position , len) in absulote
+position.
+
+
+Limitations
+~~~~~~~~~~~
+
+* Only one queue is supported.
+
+* Supports only precompiled rules.
+
+EAL Options
+~~~~~~~~~~~
+
+The following are the EAL command-line options that can be used in conjunction
+with the ``dpdk-test-regex`` application.
+See the DPDK Getting Started Guides for more information on these options.
+
+
+*   ``-w <PCI>``
+
+	Add a PCI device in white list.
+
+
+Application Options
+~~~~~~~~~~~~~~~~~~~
+
+ ``--rules NAME``: precompiled rule file
+
+ ``--data NAME``: data file to use
+
+ ``--nb_jobs N``: number of jobs to use
+
+ ``--perf N``: only outputs the performance data
+
+ ``--nb_iter N``: number of iteration to run
+
+ ``--help``: prints this help
+
+
+Compiling the Tool
+------------------
+
+The ``dpdk-test-regex`` application depends on RegEx lib ``rte_regexdev``.
+
+
+Generating the data
+-------------------
+
+In the current version, the compiled rule file is loaded with a rule that
+matches 'hello world'. To create the data file,
+it is possible to use the included python script ``generate_data_file.py``
+ which generates two files,
+``input.txt`` which holds the input buffer. An input buffer is a random number
+of spaces chars followed by the phrase 'hello world'.
+This sequence is repeated a random number of times.
+The second file is ``res.txt`` which holds the position of each
+of the 'hello world' in the input file.
+
+
+Running the Tool
+----------------
+
+The tool has a number of command line options. Here is the sample command line:
+
+.. code-block:: console
+
+   ./build/app/testregex -w 83:00.0 -- --rules app/test-regex/hello_world.rof2 --data app/test-regex/input.txt --job 100
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v1] app/test-regex: add RegEx test application

[Jerin Jacob]

On Mon, Jul 27, 2020 at 1:28 AM Ori Kam <orika@mellanox.com> wrote:
>

> diff --git a/app/test-regex/hello_world.rof2 b/app/test-regex/hello_world.rof2
> new file mode 100644
> index 0000000..fb7db75
> --- /dev/null
> +++ b/app/test-regex/hello_world.rof2
> @@ -0,0 +1,45 @@
> +#
> +# rof_version: 2
> +#
> +# date:20200210_164643
> +#
> +# rxp_compiler:5.7.18007

Please don't check-in vendor/driver specific file formats in the main
repository.
See below.

> +#
> +# rof_revision:0
> +#
> +2, 0x00000000, 0x0000000004055254
> +3, 0x00000000, 0x0000000007055254
> +1, 0x00000005, 0x0000000000aa0008
> +1, 0x00000006, 0x00000000dddd00aa
> +4, 0x00010010, 0x0000000000000000
> +4, 0x00010011, 0x0000000000000000
> +4, 0x00010012, 0x0000000000000000
> +5, 0x00010010, 0x0000000000000000
> +5, 0x00010011, 0x0000000000000000
> +5, 0x00010012, 0x0000000000000000
> +6, 0x00000000, 0x6c6c65680040bd82
> +6, 0x00000001, 0x6f77206f0040bd82
> +6, 0x00000002, 0x00646c720040bd62
> +6, 0x00000003, 0x0000000100403a48
> +6, 0x00000004, 0x00000001003b3c1f
> +6, 0x00000005, 0x0000000000000000
> +6, 0x00000006, 0x0000000000000000
> +6, 0x00000007, 0x0000000000000000
> +6, 0x00d30000, 0x0000000000000000
> +6, 0x00e401e5, 0xbdb9180002000000
> +6, 0x00e80000, 0x31ac75f69abc779c
> +6, 0x00ec0000, 0xabe575e975011cf4
> +6, 0x00ed0000, 0x38bf9e2967d22ac4
> +6, 0x00f00000, 0x34f501ce6aa8a034
> +6, 0x00f10000, 0x85e40fd9e10676d7
> +6, 0x00f40000, 0x1258c0a059dc9464
> +6, 0x00f401e5, 0x680262622a400800
> +6, 0x00f50000, 0x000d32453675686d
> +6, 0x00f60000, 0x4136d69d550adc7a
> +6, 0x00f70000, 0x6db55b9daab1c703
> +4, 0x00010010, 0x0000000087a5837c
> +4, 0x00010011, 0x00000000d75b7cab
> +4, 0x00010012, 0x000000000884a03a
> +5, 0x00010010, 0x0000000087a5837c
> +5, 0x00010011, 0x00000000d75b7cab
> +5, 0x00010012, 0x000000000884a03a
> +
> +Running the Tool
> +----------------
> +
> +The tool has a number of command line options. Here is the sample command line:
> +
> +.. code-block:: console
> +
> +   ./build/app/testregex -w 83:00.0 -- --rules app/test-regex/hello_world.rof2 --data app/test-regex/input.txt --job 100

Instead of giving the binary rule format, Primary option could to be
compile the rule by the application itself.
If the driver does not have such capability then the application can
look for binary rule file in such case please don't host
binary rules in dpdk.org.


> --
> 1.8.3.1
>

Re: [dpdk-dev] [PATCH v1] app/test-regex: add RegEx test application

[Ori Kam]

Hi Jerin,

> -----Original Message-----
> From: Jerin Jacob <jerinjacobk@gmail.com>
> 
> On Mon, Jul 27, 2020 at 1:28 AM Ori Kam <orika@mellanox.com> wrote:
> >
> 
> > diff --git a/app/test-regex/hello_world.rof2 b/app/test-
> regex/hello_world.rof2
> > new file mode 100644
> > index 0000000..fb7db75
> > --- /dev/null
> > +++ b/app/test-regex/hello_world.rof2
> > @@ -0,0 +1,45 @@
> > +#
> > +# rof_version: 2
> > +#
> > +# date:20200210_164643
> > +#
> > +# rxp_compiler:5.7.18007
> 
> Please don't check-in vendor/driver specific file formats in the main
> repository.
> See below.

I fully agree with you that in normal cases such files should not be part of
the main repository, but this case is different from the regard that this test
can't be run without vendor specific files. I expect that each vendor will
add its own file for this repo.

> 
> > +#
> > +# rof_revision:0
> > +#
> > +2, 0x00000000, 0x0000000004055254
> > +3, 0x00000000, 0x0000000007055254
> > +1, 0x00000005, 0x0000000000aa0008
> > +1, 0x00000006, 0x00000000dddd00aa
> > +4, 0x00010010, 0x0000000000000000
> > +4, 0x00010011, 0x0000000000000000
> > +4, 0x00010012, 0x0000000000000000
> > +5, 0x00010010, 0x0000000000000000
> > +5, 0x00010011, 0x0000000000000000
> > +5, 0x00010012, 0x0000000000000000
> > +6, 0x00000000, 0x6c6c65680040bd82
> > +6, 0x00000001, 0x6f77206f0040bd82
> > +6, 0x00000002, 0x00646c720040bd62
> > +6, 0x00000003, 0x0000000100403a48
> > +6, 0x00000004, 0x00000001003b3c1f
> > +6, 0x00000005, 0x0000000000000000
> > +6, 0x00000006, 0x0000000000000000
> > +6, 0x00000007, 0x0000000000000000
> > +6, 0x00d30000, 0x0000000000000000
> > +6, 0x00e401e5, 0xbdb9180002000000
> > +6, 0x00e80000, 0x31ac75f69abc779c
> > +6, 0x00ec0000, 0xabe575e975011cf4
> > +6, 0x00ed0000, 0x38bf9e2967d22ac4
> > +6, 0x00f00000, 0x34f501ce6aa8a034
> > +6, 0x00f10000, 0x85e40fd9e10676d7
> > +6, 0x00f40000, 0x1258c0a059dc9464
> > +6, 0x00f401e5, 0x680262622a400800
> > +6, 0x00f50000, 0x000d32453675686d
> > +6, 0x00f60000, 0x4136d69d550adc7a
> > +6, 0x00f70000, 0x6db55b9daab1c703
> > +4, 0x00010010, 0x0000000087a5837c
> > +4, 0x00010011, 0x00000000d75b7cab
> > +4, 0x00010012, 0x000000000884a03a
> > +5, 0x00010010, 0x0000000087a5837c
> > +5, 0x00010011, 0x00000000d75b7cab
> > +5, 0x00010012, 0x000000000884a03a
> > +
> > +Running the Tool
> > +----------------
> > +
> > +The tool has a number of command line options. Here is the sample
> command line:
> > +
> > +.. code-block:: console
> > +
> > +   ./build/app/testregex -w 83:00.0 -- --rules app/test-regex/hello_world.rof2
> --data app/test-regex/input.txt --job 100
> 
> Instead of giving the binary rule format, Primary option could to be
> compile the rule by the application itself.
> If the driver does not have such capability then the application can
> look for binary rule file in such case please don't host
> binary rules in dpdk.org.
> 
Like I said above, in Mellanox case the rule must be precompiled, (I think the same goes for 
number of other vendors) and it doesn’t look
complete code, if we will just give the code, and the user will have to download files
from other places.

I think for this specific example and since the file is small we should allow it.

What do you think? 
> 
> > --
> > 1.8.3.1
> >

Re: [dpdk-dev] [PATCH v1] app/test-regex: add RegEx test application

[Thomas Monjalon]

27/07/2020 07:12, Ori Kam:
> From: Jerin Jacob <jerinjacobk@gmail.com>
> > On Mon, Jul 27, 2020 at 1:28 AM Ori Kam <orika@mellanox.com> wrote:
> > > --- /dev/null
> > > +++ b/app/test-regex/hello_world.rof2
> > > @@ -0,0 +1,45 @@
> > > +#
> > > +# rof_version: 2
> > > +#
> > > +# date:20200210_164643
> > > +#
> > > +# rxp_compiler:5.7.18007
> > 
> > Please don't check-in vendor/driver specific file formats in the main
> > repository.
> > See below.
> 
> I fully agree with you that in normal cases such files should not be part of
> the main repository, but this case is different from the regard that this test
> can't be run without vendor specific files. I expect that each vendor will
> add its own file for this repo.

I don't think it's "different".
As an open source project, we prefer dealing only with source files.
We must provide the tools to generate required binaries.

[...]
> > > +Running the Tool
> > > +----------------
> > > +
> > > +The tool has a number of command line options. Here is the sample
> > > command line:

In docs, it's better to have each sentence on its own line.

> > > +
> > > +.. code-block:: console

If you end previous line with "::" instead of ":" then you can drop
the code-block line.

> > > +
> > > +   ./build/app/testregex -w 83:00.0 -- --rules app/test-regex/hello_world.rof2
> > > --data app/test-regex/input.txt --job 100

Don't write too much long lines in verbatim blocks.

> > 
> > Instead of giving the binary rule format, Primary option could to be
> > compile the rule by the application itself.
> > If the driver does not have such capability then the application can
> > look for binary rule file in such case please don't host
> > binary rules in dpdk.org.
> 
> Like I said above, in Mellanox case the rule must be precompiled,
> (I think the same goes for number of other vendors) and it doesn’t look
> complete code, if we will just give the code,
> and the user will have to download files from other places.
> 
> I think for this specific example and since the file is small we should allow it.

No

> What do you think? 

We should provide all the tools.
If the tool is external, it must become a requirement to run the app.

Re: [dpdk-dev] [PATCH v1] app/test-regex: add RegEx test application

[Thomas Monjalon]

26/07/2020 21:58, Ori Kam:
> --- a/app/meson.build
> +++ b/app/meson.build
> @@ -12,6 +12,7 @@ apps = [
>  	'test-bbdev',
>  	'test-cmdline',
>  	'test-compress-perf',
> +	'test-regex',
>  	'test-crypto-perf',
>  	'test-eventdev',
>  	'test-fib',

In this list, I think the alphabetical order was chosen.

> diff --git a/app/test-regex/Makefile b/app/test-regex/Makefile
> new file mode 100644
> index 0000000..d73e776
> --- /dev/null
> +++ b/app/test-regex/Makefile
> @@ -0,0 +1,24 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2020 Mellanox Corporation

It does not comply with Mellanox copyright syntax.
Note: I already did this comment in recent past.

> +
> +include $(RTE_SDK)/mk/rte.vars.mk
> +
> +ifeq ($(CONFIG_RTE_LIBRTE_REGEXDEV),y)
> +
> +#
> +# library name
> +#
> +APP = testregex
> +
> +CFLAGS += -O3
> +CFLAGS += $(WERROR_FLAGS)
> +CFLAGS += -Wno-deprecated-declarations

This flag is not acceptable.

> +
> +#
> +# all source are stored in SRCS-y
> +#
> +SRCS-y := main.c
> +
> +include $(RTE_SDK)/mk/rte.app.mk
> +
> +endif
[...]
> --- /dev/null
> +++ b/app/test-regex/generate_data_file.py
> @@ -0,0 +1,29 @@
> +#!/usr/bin/env python
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright 2020 Mellanox Technologies, Ltd
> +
> +import random
> +
> +KEYWORD = 'hello world'
> +MAX_COUNT = 10
> +MIN_COUNT = 5
> +MAX_LEN = 1024
> +REPEAT_COUNT = random.randrange(MIN_COUNT, MAX_COUNT)
> +
> +current_pos = 0;
> +match_pos = []
> +
> +fd_input = open('input.txt','w')
> +fd_res = open('res.txt','w')

space missing

> +
> +for i in range(REPEAT_COUNT):
> +    rand = random.randrange(MAX_LEN)
> +    fd_input.write(' ' * rand)
> +    current_pos += rand
> +    fd_input.write(KEYWORD)
> +    match_pos.append(current_pos)
> +    fd_res.write('{}\n'.format(str(current_pos)))
> +    current_pos += len(KEYWORD)
> +
> +fd_input.close()
> +fd_res.close()

I think there is a more pythonic way of writing in a file.
At least, you can use "with".

> --- /dev/null
> +++ b/app/test-regex/hello_world.rof2

Already discussed in a separate thread.
This file should be generated.

> --- /dev/null
> +++ b/app/test-regex/main.c
> @@ -0,0 +1,429 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2020 Mellanox Technologies, Ltd
> + *
> + * This file contain the application main file
> + * This application provides a way to test the RegEx class.

In general I like comments explaining what is a file for.
But here it looks useless.

> + */
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include <stdint.h>
> +#include <stdarg.h>
> +#include <ctype.h>
> +#include <errno.h>
> +#include <getopt.h>
> +#include <signal.h>
> +
> +#include <rte_eal.h>
> +#include <rte_common.h>
> +#include <rte_malloc.h>
> +#include <rte_mempool.h>
> +#include <rte_mbuf.h>
> +#include <rte_cycles.h>
> +#include <rte_regexdev.h>
> +
> +#define HELP_VAL 0
> +#define RULES_VAL 1
> +#define DATA_VAL 2
> +#define JOB_VAL 3
> +#define PERF_VAL 4
> +#define ITER_VAL 5

Please add comments to explain what are these constants for.
I think an enum, and a common prefix would be better.

> +#define MAX_FILE_NAME 255
> +
> +static char rules_file[MAX_FILE_NAME];
> +static char data_file[MAX_FILE_NAME];
> +static uint32_t jobs;
> +static struct rte_mempool *mbuf_mp;
> +static uint8_t nb_max_matches;
> +static uint16_t nb_max_payload;
> +static int perf_test;
> +static uint32_t iter;

Please avoid global variables.

> +
> +static void
> +usage(const char *prog_name)
> +{
> +	printf("%s [EAL options] --\n"
> +		" --rules NAME: precompiled rules file\n"
> +		" --data NAME: data file to use\n"
> +		" --nb_jobs: number of jobs to use\n"
> +		" --perf N: only outputs the performance data\n"
> +		" --nb_iter N: number of iteration to run\n",
> +		prog_name);
> +}
> +
> +static void
> +args_parse(int argc, char **argv)
> +{
> +	char **argvopt;
> +	int opt;
> +	int opt_idx;
> +	size_t len;
> +	static struct option lgopts[] = {
> +		{ "help",  0, 0, HELP_VAL },
> +		{ "rules",  1, 0, RULES_VAL },
> +		/* Rules database file to load. */
> +		{ "data",  1, 0, DATA_VAL },
> +		/* Data file to load. */
> +		{ "nb_jobs",  1, 0, JOB_VAL },
> +		/* Number of jobs to create. */
> +		{ "perf", 0, 0, PERF_VAL},
> +		/* Perf test only */
> +		{ "nb_iter", 1, 0, ITER_VAL}
> +		/* Number of iterations to run with perf test */
> +	};
> +
> +	argvopt = argv;
> +

Useless newline.

> +	while ((opt = getopt_long(argc, argvopt, "",
> +				lgopts, &opt_idx)) != EOF) {
> +		switch (opt) {

[...]
> +#define MBUF_SIZE (1 << 14)

Why this size?
Add a comment?

> +static void
> +extbuf_free_cb(void *addr __rte_unused, void *fcb_opaque __rte_unused)
> +{
> +
> +}

Empty function can be dropped.

[...]
> +It is based on precomplied rule file, and an input file, both of them can

precompiled

> +be selected using command-line options.
> +
> +In general case, each PMD has it's own rule file.

its

> +
> +The test outputs the performance, the results matching (rule id, position, len)

length

A list will look better.

> +for each job and also a list of matches (rule id, position , len) in absulote

absolute

> +position.
> +
> +
> +Limitations
> +~~~~~~~~~~~
> +
> +* Only one queue is supported.
> +
> +* Supports only precompiled rules.
> +
> +EAL Options
> +~~~~~~~~~~~
> +
> +The following are the EAL command-line options that can be used in conjunction
> +with the ``dpdk-test-regex`` application.
> +See the DPDK Getting Started Guides for more information on these options.
> +
> +
> +*   ``-w <PCI>``
> +
> +	Add a PCI device in white list.

Please drop "EAL options" chapter.
It is not specific to the app.

> +Application Options
> +~~~~~~~~~~~~~~~~~~~
> +
> + ``--rules NAME``: precompiled rule file
> +
> + ``--data NAME``: data file to use
> +
> + ``--nb_jobs N``: number of jobs to use
> +
> + ``--perf N``: only outputs the performance data
> +
> + ``--nb_iter N``: number of iteration to run
> +
> + ``--help``: prints this help

Please use definition list.

> +Compiling the Tool
> +------------------
> +
> +The ``dpdk-test-regex`` application depends on RegEx lib ``rte_regexdev``.

Useless

> +
> +
> +Generating the data
> +-------------------
> +
> +In the current version, the compiled rule file is loaded with a rule that
> +matches 'hello world'. To create the data file,
> +it is possible to use the included python script ``generate_data_file.py``
> + which generates two files,
> +``input.txt`` which holds the input buffer. An input buffer is a random number
> +of spaces chars followed by the phrase 'hello world'.
> +This sequence is repeated a random number of times.
> +The second file is ``res.txt`` which holds the position of each
> +of the 'hello world' in the input file.

A script is missing to generate a default set of input data.


> +Running the Tool
> +----------------
> +
> +The tool has a number of command line options. Here is the sample command line:
> +
> +.. code-block:: console
> +
> +   ./build/app/testregex -w 83:00.0 -- --rules app/test-regex/hello_world.rof2 --data app/test-regex/input.txt --job 100


Bottom line, I think this application is not ready for DPDK 20.08.
It's good to have it available as a patch for first users who
want to play with the new regex library.
However, I propose waiting 20.11 to integrate a better version of it.

Re: [dpdk-dev] [PATCH v1] app/test-regex: add RegEx test application

[Ori Kam]

Hi Thomas,

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> 
> 26/07/2020 21:58, Ori Kam:
> > --- a/app/meson.build
> > +++ b/app/meson.build
> > @@ -12,6 +12,7 @@ apps = [
> >  	'test-bbdev',
> >  	'test-cmdline',
> >  	'test-compress-perf',
> > +	'test-regex',
> >  	'test-crypto-perf',
> >  	'test-eventdev',
> >  	'test-fib',
> 
> In this list, I think the alphabetical order was chosen.
> 
Will change, I did that since in all other cases the regex is after the compress.

> > diff --git a/app/test-regex/Makefile b/app/test-regex/Makefile
> > new file mode 100644
> > index 0000000..d73e776
> > --- /dev/null
> > +++ b/app/test-regex/Makefile
> > @@ -0,0 +1,24 @@
> > +# SPDX-License-Identifier: BSD-3-Clause
> > +# Copyright(c) 2020 Mellanox Corporation
> 
> It does not comply with Mellanox copyright syntax.
> Note: I already did this comment in recent past.
> 
Will fix.

> > +
> > +include $(RTE_SDK)/mk/rte.vars.mk
> > +
> > +ifeq ($(CONFIG_RTE_LIBRTE_REGEXDEV),y)
> > +
> > +#
> > +# library name
> > +#
> > +APP = testregex
> > +
> > +CFLAGS += -O3
> > +CFLAGS += $(WERROR_FLAGS)
> > +CFLAGS += -Wno-deprecated-declarations
> 
> This flag is not acceptable.
>
Will remove.
 
> > +
> > +#
> > +# all source are stored in SRCS-y
> > +#
> > +SRCS-y := main.c
> > +
> > +include $(RTE_SDK)/mk/rte.app.mk
> > +
> > +endif
> [...]
> > --- /dev/null
> > +++ b/app/test-regex/generate_data_file.py
> > @@ -0,0 +1,29 @@
> > +#!/usr/bin/env python
> > +# SPDX-License-Identifier: BSD-3-Clause
> > +# Copyright 2020 Mellanox Technologies, Ltd
> > +
> > +import random
> > +
> > +KEYWORD = 'hello world'
> > +MAX_COUNT = 10
> > +MIN_COUNT = 5
> > +MAX_LEN = 1024
> > +REPEAT_COUNT = random.randrange(MIN_COUNT, MAX_COUNT)
> > +
> > +current_pos = 0;
> > +match_pos = []
> > +
> > +fd_input = open('input.txt','w')
> > +fd_res = open('res.txt','w')
> 
> space missing
> 
Will remove this file.

> > +
> > +for i in range(REPEAT_COUNT):
> > +    rand = random.randrange(MAX_LEN)
> > +    fd_input.write(' ' * rand)
> > +    current_pos += rand
> > +    fd_input.write(KEYWORD)
> > +    match_pos.append(current_pos)
> > +    fd_res.write('{}\n'.format(str(current_pos)))
> > +    current_pos += len(KEYWORD)
> > +
> > +fd_input.close()
> > +fd_res.close()
> 
> I think there is a more pythonic way of writing in a file.
> At least, you can use "with".
>

Will remove this file.
Since this file use used to generate the data for the given
rule file and the rule file will be removed I will also remove this file. 

> > --- /dev/null
> > +++ b/app/test-regex/hello_world.rof2
> 
> Already discussed in a separate thread.
> This file should be generated.
> 
> > --- /dev/null
> > +++ b/app/test-regex/main.c
> > @@ -0,0 +1,429 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright 2020 Mellanox Technologies, Ltd
> > + *
> > + * This file contain the application main file
> > + * This application provides a way to test the RegEx class.
> 
> In general I like comments explaining what is a file for.
> But here it looks useless.
> 
Will remove.

> > + */
> > +
> > +#include <stdio.h>
> > +#include <stdlib.h>
> > +#include <string.h>
> > +#include <stdint.h>
> > +#include <stdarg.h>
> > +#include <ctype.h>
> > +#include <errno.h>
> > +#include <getopt.h>
> > +#include <signal.h>
> > +
> > +#include <rte_eal.h>
> > +#include <rte_common.h>
> > +#include <rte_malloc.h>
> > +#include <rte_mempool.h>
> > +#include <rte_mbuf.h>
> > +#include <rte_cycles.h>
> > +#include <rte_regexdev.h>
> > +
> > +#define HELP_VAL 0
> > +#define RULES_VAL 1
> > +#define DATA_VAL 2
> > +#define JOB_VAL 3
> > +#define PERF_VAL 4
> > +#define ITER_VAL 5
> 
> Please add comments to explain what are these constants for.
> I think an enum, and a common prefix would be better.
>
Will fix.
 
> > +#define MAX_FILE_NAME 255
> > +
> > +static char rules_file[MAX_FILE_NAME];
> > +static char data_file[MAX_FILE_NAME];
> > +static uint32_t jobs;
> > +static struct rte_mempool *mbuf_mp;
> > +static uint8_t nb_max_matches;
> > +static uint16_t nb_max_payload;
> > +static int perf_test;
> > +static uint32_t iter;
> 
> Please avoid global variables.
> 
Will change.

> > +
> > +static void
> > +usage(const char *prog_name)
> > +{
> > +	printf("%s [EAL options] --\n"
> > +		" --rules NAME: precompiled rules file\n"
> > +		" --data NAME: data file to use\n"
> > +		" --nb_jobs: number of jobs to use\n"
> > +		" --perf N: only outputs the performance data\n"
> > +		" --nb_iter N: number of iteration to run\n",
> > +		prog_name);
> > +}
> > +
> > +static void
> > +args_parse(int argc, char **argv)
> > +{
> > +	char **argvopt;
> > +	int opt;
> > +	int opt_idx;
> > +	size_t len;
> > +	static struct option lgopts[] = {
> > +		{ "help",  0, 0, HELP_VAL },
> > +		{ "rules",  1, 0, RULES_VAL },
> > +		/* Rules database file to load. */
> > +		{ "data",  1, 0, DATA_VAL },
> > +		/* Data file to load. */
> > +		{ "nb_jobs",  1, 0, JOB_VAL },
> > +		/* Number of jobs to create. */
> > +		{ "perf", 0, 0, PERF_VAL},
> > +		/* Perf test only */
> > +		{ "nb_iter", 1, 0, ITER_VAL}
> > +		/* Number of iterations to run with perf test */
> > +	};
> > +
> > +	argvopt = argv;
> > +
> 
> Useless newline.
> 
Will remove.

> > +	while ((opt = getopt_long(argc, argvopt, "",
> > +				lgopts, &opt_idx)) != EOF) {
> > +		switch (opt) {
> 
> [...]
> > +#define MBUF_SIZE (1 << 14)
> 
> Why this size?
> Add a comment?
>
No reason will change it to smaller size in any case.
 
> > +static void
> > +extbuf_free_cb(void *addr __rte_unused, void *fcb_opaque __rte_unused)
> > +{
> > +
> > +}
> 
> Empty function can be dropped.
> 
It must exist since I need to pass it to function call.

> [...]
> > +It is based on precomplied rule file, and an input file, both of them can
> 
> precompiled
> 
> > +be selected using command-line options.
> > +
> > +In general case, each PMD has it's own rule file.
> 
> its
> 
Will fix.

> > +
> > +The test outputs the performance, the results matching (rule id, position,
> len)
> 
> length
> 
> A list will look better.
> 
Will change.

> > +for each job and also a list of matches (rule id, position , len) in absulote
> 
> absolute
> 
Will fix.

> > +position.
> > +
> > +
> > +Limitations
> > +~~~~~~~~~~~
> > +
> > +* Only one queue is supported.
> > +
> > +* Supports only precompiled rules.
> > +
> > +EAL Options
> > +~~~~~~~~~~~
> > +
> > +The following are the EAL command-line options that can be used in
> conjunction
> > +with the ``dpdk-test-regex`` application.
> > +See the DPDK Getting Started Guides for more information on these options.
> > +
> > +
> > +*   ``-w <PCI>``
> > +
> > +	Add a PCI device in white list.
> 
> Please drop "EAL options" chapter.
> It is not specific to the app.
> 
Sure.

> > +Application Options
> > +~~~~~~~~~~~~~~~~~~~
> > +
> > + ``--rules NAME``: precompiled rule file
> > +
> > + ``--data NAME``: data file to use
> > +
> > + ``--nb_jobs N``: number of jobs to use
> > +
> > + ``--perf N``: only outputs the performance data
> > +
> > + ``--nb_iter N``: number of iteration to run
> > +
> > + ``--help``: prints this help
> 
> Please use definition list.
>
Will change.
 
> > +Compiling the Tool
> > +------------------
> > +
> > +The ``dpdk-test-regex`` application depends on RegEx lib ``rte_regexdev``.
> 
> Useless
> 
Will remove,

> > +
> > +
> > +Generating the data
> > +-------------------
> > +
> > +In the current version, the compiled rule file is loaded with a rule that
> > +matches 'hello world'. To create the data file,
> > +it is possible to use the included python script ``generate_data_file.py``
> > + which generates two files,
> > +``input.txt`` which holds the input buffer. An input buffer is a random
> number
> > +of spaces chars followed by the phrase 'hello world'.
> > +This sequence is repeated a random number of times.
> > +The second file is ``res.txt`` which holds the position of each
> > +of the 'hello world' in the input file.
> 
> A script is missing to generate a default set of input data.
> 
Why? It has the python script that generate this input.
In any case I'm going to remove it. The generation of data will be done outside DPDK just like the
rule compilation.
 
> 
> > +Running the Tool
> > +----------------
> > +
> > +The tool has a number of command line options. Here is the sample
> command line:
> > +
> > +.. code-block:: console
> > +
> > +   ./build/app/testregex -w 83:00.0 -- --rules app/test-regex/hello_world.rof2
> --data app/test-regex/input.txt --job 100
> 
> 
> Bottom line, I think this application is not ready for DPDK 20.08.
> It's good to have it available as a patch for first users who
> want to play with the new regex library.
> However, I propose waiting 20.11 to integrate a better version of it.
> 
I think the change are no major remarks that can't be fixed in a day.
can it target rc4?

Re: [dpdk-dev] [PATCH v1] app/test-regex: add RegEx test application

[Ori Kam]

Hi Thomas,

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> 
> 27/07/2020 07:12, Ori Kam:
> > From: Jerin Jacob <jerinjacobk@gmail.com>
> > > On Mon, Jul 27, 2020 at 1:28 AM Ori Kam <orika@mellanox.com> wrote:
> > > > --- /dev/null
> > > > +++ b/app/test-regex/hello_world.rof2
> > > > @@ -0,0 +1,45 @@
> > > > +#
> > > > +# rof_version: 2
> > > > +#
> > > > +# date:20200210_164643
> > > > +#
> > > > +# rxp_compiler:5.7.18007
> > >
> > > Please don't check-in vendor/driver specific file formats in the main
> > > repository.
> > > See below.
> >
> > I fully agree with you that in normal cases such files should not be part of
> > the main repository, but this case is different from the regard that this test
> > can't be run without vendor specific files. I expect that each vendor will
> > add its own file for this repo.
> 
> I don't think it's "different".
> As an open source project, we prefer dealing only with source files.
> We must provide the tools to generate required binaries.
> 
This "tool" is private and large, so I can't share it. I will remove this file.

> [...]
> > > > +Running the Tool
> > > > +----------------
> > > > +
> > > > +The tool has a number of command line options. Here is the sample
> > > > command line:
> 
> In docs, it's better to have each sentence on its own line.
> 
Will change.

> > > > +
> > > > +.. code-block:: console
> 
> If you end previous line with "::" instead of ":" then you can drop
> the code-block line.
> 
Do I still need to keep the same indentation?

> > > > +
> > > > +   ./build/app/testregex -w 83:00.0 -- --rules app/test-
> regex/hello_world.rof2
> > > > --data app/test-regex/input.txt --job 100
> 
> Don't write too much long lines in verbatim blocks.
> 
Will see how to shorten it.

> > >
> > > Instead of giving the binary rule format, Primary option could to be
> > > compile the rule by the application itself.
> > > If the driver does not have such capability then the application can
> > > look for binary rule file in such case please don't host
> > > binary rules in dpdk.org.
> >
> > Like I said above, in Mellanox case the rule must be precompiled,
> > (I think the same goes for number of other vendors) and it doesn’t look
> > complete code, if we will just give the code,
> > and the user will have to download files from other places.
> >
> > I think for this specific example and since the file is small we should allow it.
> 
> No
> 
Will remove this file.

> > What do you think?
> 
> We should provide all the tools.
> If the tool is external, it must become a requirement to run the app.
>
I will modify the doc to set this file as a requirement.
 
> 

Thanks,
Ori

[dpdk-dev] [PATCH v2] app/test-regex: add RegEx test application

[Ori Kam]

From: Yuval Avnery <yuvalav@mellanox.com>

Following the new RegEx class.
There is a need to create a dedicated test application in order to
validate this class and PMD.

Unlike net device this application loads data from a file.

This commit introduces the new RegEx test app.

The basic app flow:
1. Configure the RegEx device to use one queue, and set the rule database,
   using precompiled file.
2. Allocate mbufs based on the requested number of jobs,
   each job will get one mbuf.
3. Enqueue as much as possible jobs.
4. Dequeue jobs.
5. if the number of dequeue jobs < requested number of jobs job to step 3

Signed-off-by: Ori Kam <orika@mellanox.com>
Signed-off-by: Yuval Avnery <yuvalav@mellanox.com>
---
v2:
* Remove rule file.
* Remove data generation script.
* Address ML comments.

---
 app/Makefile                   |   1 +
 app/meson.build                |   1 +
 app/test-regex/Makefile        |  21 ++
 app/test-regex/main.c          | 446 +++++++++++++++++++++++++++++++++++++++++
 app/test-regex/meson.build     |   5 +
 doc/guides/tools/index.rst     |   1 +
 doc/guides/tools/testregex.rst |  73 +++++++
 7 files changed, 548 insertions(+)
 create mode 100644 app/test-regex/Makefile
 create mode 100644 app/test-regex/main.c
 create mode 100644 app/test-regex/meson.build
 create mode 100644 doc/guides/tools/testregex.rst

diff --git a/app/Makefile b/app/Makefile
index 0392a7d..453c4fe 100644
--- a/app/Makefile
+++ b/app/Makefile
@@ -13,6 +13,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_FIB) += test-fib
 DIRS-$(CONFIG_RTE_TEST_FLOW_PERF) += test-flow-perf
 DIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += test-pipeline
 DIRS-$(CONFIG_RTE_LIBRTE_IPSEC) += test-sad
+DIRS-$(CONFIG_RTE_LIBRTE_REGEXDEV) += test-regex
 
 ifeq ($(CONFIG_RTE_LIBRTE_BBDEV),y)
 DIRS-$(CONFIG_RTE_TEST_BBDEV) += test-bbdev
diff --git a/app/meson.build b/app/meson.build
index 585b908..eb74f21 100644
--- a/app/meson.build
+++ b/app/meson.build
@@ -18,6 +18,7 @@ apps = [
 	'test-flow-perf',
 	'test-pipeline',
 	'test-pmd',
+	'test-regex',
 	'test-sad']
 
 # for BSD only
diff --git a/app/test-regex/Makefile b/app/test-regex/Makefile
new file mode 100644
index 0000000..550f997
--- /dev/null
+++ b/app/test-regex/Makefile
@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2020 Mellanox Technologies, Ltd
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+APP = testregex
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+#
+# all source are stored in SRCS-y
+#
+SRCS-y := main.c
+
+include $(RTE_SDK)/mk/rte.app.mk
+
diff --git a/app/test-regex/main.c b/app/test-regex/main.c
new file mode 100644
index 0000000..70ec44a
--- /dev/null
+++ b/app/test-regex/main.c
@@ -0,0 +1,446 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdarg.h>
+#include <ctype.h>
+#include <errno.h>
+#include <getopt.h>
+#include <signal.h>
+
+#include <rte_eal.h>
+#include <rte_common.h>
+#include <rte_malloc.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_cycles.h>
+#include <rte_regexdev.h>
+
+#define HELP_VAL 0
+#define RULES_VAL 1
+#define DATA_VAL 2
+#define JOB_VAL 3
+#define PERF_VAL 4
+#define ITER_VAL 5
+
+#define MAX_FILE_NAME 255
+
+/* enum that holds the value for the application arguments. */
+enum app_arg_values {
+	ARG_HELP,
+	ARG_RULES_FILE_NAME,
+	ARG_DATA_FILE_NAME,
+	ARG_NUM_OF_JOBS,
+	ARG_PERF_MODE,
+	ARG_NUM_OF_ITERATIONS,
+
+};
+
+static void
+usage(const char *prog_name)
+{
+	printf("%s [EAL options] --\n"
+		" --rules NAME: precompiled rules file\n"
+		" --data NAME: data file to use\n"
+		" --nb_jobs: number of jobs to use\n"
+		" --perf N: only outputs the performance data\n"
+		" --nb_iter N: number of iteration to run\n",
+		prog_name);
+}
+
+static void
+args_parse(int argc, char **argv, char *rules_file, char *data_file,
+	   uint32_t *nb_jobs, bool *perf_mode, uint32_t *nb_iterations)
+{
+	char **argvopt;
+	int opt;
+	int opt_idx;
+	size_t len;
+	static struct option lgopts[] = {
+		{ "help",  0, 0, ARG_HELP},
+		{ "rules",  1, 0, ARG_RULES_FILE_NAME},
+		/* Rules database file to load. */
+		{ "data",  1, 0, ARG_DATA_FILE_NAME},
+		/* Data file to load. */
+		{ "nb_jobs",  1, 0, ARG_NUM_OF_JOBS},
+		/* Number of jobs to create. */
+		{ "perf", 0, 0, ARG_PERF_MODE},
+		/* Perf test only */
+		{ "nb_iter", 1, 0, ARG_NUM_OF_ITERATIONS}
+		/* Number of iterations to run with perf test */
+	};
+
+	argvopt = argv;
+	while ((opt = getopt_long(argc, argvopt, "",
+				lgopts, &opt_idx)) != EOF) {
+		switch (opt) {
+		case ARG_RULES_FILE_NAME:
+			len = strnlen(optarg, MAX_FILE_NAME - 1);
+			if (len == MAX_FILE_NAME)
+				rte_exit(EXIT_FAILURE,
+					 "Rule file name to long max %d\n",
+					 MAX_FILE_NAME - 1);
+			strncpy(rules_file, optarg, MAX_FILE_NAME - 1);
+			break;
+		case ARG_DATA_FILE_NAME:
+			len = strnlen(optarg, MAX_FILE_NAME - 1);
+			if (len == MAX_FILE_NAME)
+				rte_exit(EXIT_FAILURE,
+					 "Data file name to long max %d\n",
+					 MAX_FILE_NAME - 1);
+			strncpy(data_file, optarg, MAX_FILE_NAME - 1);
+			break;
+		case ARG_NUM_OF_JOBS:
+			*nb_jobs = atoi(optarg);
+			break;
+		case ARG_PERF_MODE:
+			*perf_mode = true;
+			break;
+		case ARG_NUM_OF_ITERATIONS:
+			*nb_iterations = atoi(optarg);
+			break;
+		case ARG_HELP:
+			usage("RegEx test app");
+			break;
+		default:
+			fprintf(stderr, "Invalid option: %s\n", argv[optind]);
+			usage("RegEx test app");
+			rte_exit(EXIT_FAILURE, "Invalid option\n");
+			break;
+		}
+	}
+
+	if (!perf_mode)
+		*nb_iterations = 1;
+}
+
+static long
+read_file(char *file, char **buf)
+{
+	FILE *fp;
+	long buf_len = 0;
+	size_t read_len;
+	int res = 0;
+
+	fp = fopen(file, "r");
+	if (!fp)
+		return -EIO;
+	if (fseek(fp, 0L, SEEK_END) == 0) {
+		buf_len = ftell(fp);
+		if (buf_len == -1) {
+			res = EIO;
+			goto error;
+		}
+		*buf = rte_malloc(NULL, sizeof(char) * (buf_len + 1), 4096);
+		if (!*buf) {
+			res = ENOMEM;
+			goto error;
+		}
+		if (fseek(fp, 0L, SEEK_SET) != 0) {
+			res = EIO;
+			goto error;
+		}
+		read_len = fread(*buf, sizeof(char), buf_len, fp);
+		if (read_len != (unsigned long)buf_len) {
+			res = EIO;
+			goto error;
+		}
+	}
+	fclose(fp);
+	return buf_len;
+error:
+	printf("Error, can't open file %s\n, err = %d", file, res);
+	if (fp)
+		fclose(fp);
+	if (*buf)
+		rte_free(*buf);
+	return -res;
+}
+
+#define MBUF_CACHE_SIZE 256
+#define MBUF_SIZE (1 << 8)
+
+
+static int
+init_port(struct rte_mempool **mbuf_mp, uint32_t nb_jobs,
+	  uint16_t *nb_max_payload, char *rules_file, uint8_t *nb_max_matches)
+{
+	uint16_t id;
+	uint16_t num_devs;
+	char *rules = NULL;
+	long rules_len;
+	struct rte_regexdev_info info;
+	struct rte_regexdev_config dev_conf = {
+		.nb_queue_pairs = 1,
+		.nb_groups = 1,
+	};
+	struct rte_regexdev_qp_conf qp_conf = {
+		.nb_desc = 1024,
+		.qp_conf_flags = RTE_REGEX_QUEUE_PAIR_CFG_OOS_F,
+	};
+	int res = 0;
+
+	num_devs = rte_regexdev_count();
+	if (num_devs == 0) {
+		printf("Error, no devices detected.\n");
+		return -EINVAL;
+	}
+
+	*mbuf_mp = rte_pktmbuf_pool_create("mbuf_pool", nb_jobs, 0,
+					  0, MBUF_SIZE, rte_socket_id());
+	if (*mbuf_mp == NULL) {
+		printf("Error, can't create memory pool\n");
+		res = -ENOMEM;
+		goto error;
+	}
+
+	rules_len = read_file(rules_file, &rules);
+	if (rules_len < 0) {
+		printf("Error, can't read rules files.\n");
+		res = -EIO;
+		goto error;
+	}
+
+	for (id = 0; id < num_devs; id++) {
+		res = rte_regexdev_info_get(id, &info);
+		if (res != 0) {
+			printf("Error, can't get device info.\n");
+			goto error;
+		}
+		printf(":: initializing dev: %d\n", id);
+		*nb_max_matches = info.max_matches;
+		*nb_max_payload = info.max_payload_size;
+		if (info.regexdev_capa & RTE_REGEXDEV_SUPP_MATCH_AS_END_F)
+			dev_conf.dev_cfg_flags |= RTE_REGEXDEV_CFG_MATCH_AS_END_F;
+		dev_conf.nb_max_matches = info.max_matches;
+		dev_conf.nb_rules_per_group = info.max_rules_per_group;
+		dev_conf.rule_db_len = rules_len;
+		dev_conf.rule_db = rules;
+		res = rte_regexdev_configure(id, &dev_conf);
+		if (res < 0) {
+			printf("Error, can't configure device %d.\n", id);
+			goto error;
+		}
+		res = rte_regexdev_queue_pair_setup(id, 0, &qp_conf);
+		if (res < 0) {
+			printf("Error, can't setup queue pair for device %d.\n",
+			       id);
+			goto error;
+		}
+		printf(":: initializing device: %d done\n", id);
+	}
+	rte_free(rules);
+	return 0;
+error:
+	if (rules)
+		rte_free(rules);
+	if (*mbuf_mp)
+		rte_mempool_free(*mbuf_mp);
+	return res;
+
+}
+
+static void
+extbuf_free_cb(void *addr __rte_unused, void *fcb_opaque __rte_unused)
+{
+
+}
+
+#define START_BURST_SIZE 32u
+
+static int
+run_regex(struct rte_mempool *mbuf_mp, uint32_t nb_jobs,
+	  uint16_t nb_max_payload, bool perf_mode, uint32_t nb_iterations,
+	  char *data_file, uint8_t nb_max_matches)
+{
+	char *buf;
+	long buf_len;
+	long job_len;
+	uint32_t actual_jobs = 0;
+	uint32_t i;
+	struct rte_regex_ops **ops;
+	uint16_t dev_id = 0;
+	uint16_t qp_id = 0;
+	uint8_t nb_matches;
+	struct rte_regexdev_match *match;
+	long pos = 0;
+	unsigned long d_ind = 0;
+	struct rte_mbuf_ext_shared_info shinfo;
+	uint32_t total_enqueue = 0;
+	uint32_t total_dequeue = 0;
+	uint32_t total_matches = 0;
+	int res = 0;
+	time_t start;
+	time_t end;
+	double time;
+
+	shinfo.free_cb = extbuf_free_cb;
+
+	ops = rte_malloc(NULL, sizeof(*ops) * nb_jobs, 0);
+	if (!ops) {
+		printf("Error, can't allocate memory for ops.\n");
+		return -ENOMEM;
+	}
+
+	/* Allocate the jobs and assign each job with an mbuf. */
+	for (i = 0; i < nb_jobs; i++) {
+		ops[i] = rte_malloc(NULL, sizeof(*ops[0]) + nb_max_matches *
+				    sizeof(struct rte_regexdev_match), 0);
+		if (!ops[i]) {
+			printf("Error, can't allocate memory for op.\n");
+			res = -ENOMEM;
+			goto end;
+		}
+		ops[i]->mbuf = rte_pktmbuf_alloc(mbuf_mp);
+		if (!ops[i]->mbuf) {
+			printf("Error, can't attach mbuf.\n");
+			res = -ENOMEM;
+			goto end;
+		}
+	}
+
+	buf_len = read_file(data_file, &buf);
+	if (buf_len <= 0) {
+		printf("Error, can't read file, or file is empty.\n");
+		res = -EXIT_FAILURE;
+		goto end;
+	}
+
+	job_len = buf_len / nb_jobs;
+	if (job_len == 0) {
+		printf("Error, To many jobs, for the given input.\n");
+		res = -EXIT_FAILURE;
+		goto end;
+	}
+
+	if (job_len > nb_max_payload) {
+		printf("Error, not enough jobs to cover input.\n");
+		res = -EXIT_FAILURE;
+		goto end;
+	}
+
+	/* Assign each mbuf with the data to handle. */
+	for (i = 0; (pos < buf_len) && (i < nb_jobs) ; i++) {
+		long act_job_len = RTE_MIN(job_len, buf_len - pos);
+		rte_pktmbuf_attach_extbuf(ops[i]->mbuf, &buf[pos], 0,
+					  act_job_len, &shinfo);
+		ops[i]->mbuf->data_len = job_len;
+		ops[i]->mbuf->pkt_len = act_job_len;
+		ops[i]->user_id = i;
+		ops[i]->group_id0 = 1;
+		pos += act_job_len;
+		actual_jobs++;
+	}
+
+	start = clock();
+	for (i = 0; i < nb_iterations; i++) {
+		total_enqueue = 0;
+		total_dequeue = 0;
+		while (total_dequeue < actual_jobs) {
+			struct rte_regex_ops **cur_ops_to_enqueue = ops +
+				total_enqueue;
+			struct rte_regex_ops **cur_ops_to_dequeue = ops +
+				total_dequeue;
+
+			if (actual_jobs - total_enqueue)
+				total_enqueue += rte_regexdev_enqueue_burst
+					(dev_id, qp_id, cur_ops_to_enqueue,
+					 actual_jobs - total_enqueue);
+
+			total_dequeue += rte_regexdev_dequeue_burst
+				(dev_id, qp_id, cur_ops_to_dequeue,
+				 total_enqueue - total_dequeue);
+		}
+	}
+	end = clock();
+	time = ((double)end - start) / CLOCKS_PER_SEC;
+	printf("Job len = %ld Bytes\n",  job_len);
+	printf("Time = %lf sec\n",  time);
+	printf("Perf = %lf Gbps\n",
+	       (((double)actual_jobs * job_len * nb_iterations * 8) / time) /
+		1000000000.0);
+
+	if (!perf_mode) {
+		/* Log results per job. */
+		for (d_ind = 0; d_ind < total_dequeue; d_ind++) {
+			nb_matches = ops[d_ind % actual_jobs]->nb_matches;
+			printf("Job id %ld number of matches = %d\n",
+			       ops[d_ind]->user_id, nb_matches);
+			total_matches += nb_matches;
+			match = ops[d_ind % actual_jobs]->matches;
+			for (i = 0; i < nb_matches; i++) {
+				printf("match %d, rule = %d, start = %d,len = %d\n",
+				       i, match->rule_id, match->start_offset,
+				       match->len);
+				match++;
+			}
+		}
+		printf("Total matches = %d\n", total_matches);
+		printf("All Matches:\n");
+
+		/* Log absolute results. */
+		for (d_ind = 0; d_ind < total_dequeue; d_ind++) {
+			nb_matches = ops[d_ind % actual_jobs]->nb_matches;
+			total_matches += nb_matches;
+			match = ops[d_ind % actual_jobs]->matches;
+			for (i = 0; i < nb_matches; i++) {
+				printf("start = %ld, len = %d, rule = %d\n",
+				       match->start_offset + d_ind * job_len,
+				       match->len, match->rule_id);
+				match++;
+			}
+		}
+	}
+end:
+	for (i = 0; i < actual_jobs; i++) {
+		if (ops[i]) {
+			if (ops[i]->mbuf)
+				rte_pktmbuf_free(ops[i]->mbuf);
+			rte_free(ops[i]);
+		}
+	}
+	rte_free(ops);
+	rte_free(buf);
+	return res;
+}
+
+int
+main(int argc, char **argv)
+{
+	char rules_file[MAX_FILE_NAME];
+	char data_file[MAX_FILE_NAME];
+	struct rte_mempool *mbuf_mp = NULL;
+	uint32_t nb_jobs = 0;
+	uint16_t nb_max_payload = 0;
+	bool perf_mode = 0;
+	uint32_t nb_iterations = 0;
+	uint8_t nb_max_matches = 0;
+	int ret;
+
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "EAL init failed\n");
+	argc -= ret;
+	argv += ret;
+	if (argc > 1)
+		args_parse(argc, argv, rules_file, data_file, &nb_jobs,
+			   &perf_mode, &nb_iterations);
+
+	ret = init_port(&mbuf_mp, nb_jobs, &nb_max_payload, rules_file,
+			&nb_max_matches);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "init port failed\n");
+	ret = run_regex(mbuf_mp, nb_jobs, nb_max_payload, perf_mode,
+			nb_iterations, data_file, nb_max_matches);
+	if (ret < 0) {
+		rte_mempool_free(mbuf_mp);
+		rte_exit(EXIT_FAILURE, "RegEx function failed\n");
+	}
+	rte_mempool_free(mbuf_mp);
+	return 0;
+}
diff --git a/app/test-regex/meson.build b/app/test-regex/meson.build
new file mode 100644
index 0000000..7c9357f
--- /dev/null
+++ b/app/test-regex/meson.build
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+sources = files('main.c')
+deps = ['regexdev']
diff --git a/doc/guides/tools/index.rst b/doc/guides/tools/index.rst
index 4840cf47..c721943 100644
--- a/doc/guides/tools/index.rst
+++ b/doc/guides/tools/index.rst
@@ -17,3 +17,4 @@ DPDK Tools User Guides
     cryptoperf
     comp_perf
     testeventdev
+    testregex
diff --git a/doc/guides/tools/testregex.rst b/doc/guides/tools/testregex.rst
new file mode 100644
index 0000000..24a1e3e
--- /dev/null
+++ b/doc/guides/tools/testregex.rst
@@ -0,0 +1,73 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright 2020 Mellanox Technologies, Ltd
+
+dpdk-test-regex Tool
+====================
+
+The ``dpdk-test-regex`` tool is a Data Plane Development Kit (DPDK)
+application that allows functional testing and performance measurement for
+the RegEx PMDs.
+The test supports only one core and one PMD.
+It is based on precompiled rule file, and an input file, both of them can
+be selected using command-line options.
+
+In general case, each PMD has its own rule file.
+
+The test outputs the following data:
+
+* Performance, in gigabit per second.
+
+* Matching results (rule id, position, length), for each job.
+
+* Matching results in absolute location (rule id, position , length),
+  relative to the start of the input data.
+
+
+Limitations
+~~~~~~~~~~~
+
+* Only one queue is supported.
+
+* Supports only precompiled rules.
+
+
+Application Options
+~~~~~~~~~~~~~~~~~~~
+
+* ``--rules NAME``: precompiled rule file
+
+* ``--data NAME``: data file to use
+
+* ``--nb_jobs N``: number of jobs to use
+
+* ``--perf N``: only outputs the performance data
+
+* ``--nb_iter N``: number of iteration to run
+
+* ``--help``: prints this help
+
+
+Compiling the Tool
+------------------
+
+The ``dpdk-test-regex`` application depends on RegEx lib ``rte_regexdev``.
+
+
+Running the Tool
+----------------
+
+**Step 1: Compile a rule file**
+
+In order for the RegEx to work it must have a precompiled rule file.
+to generate this file there is a need to use a RegEx compiler that matches the
+RegEx PMD.
+
+**Step 2: Generate a data file**
+
+The data file, will be used as a source data for the RegEx to work on.
+
+**Step 3: Run the tool**
+
+The tool has a number of command line options. Here is the sample command line::
+
+   .testregex -w 83:00.0 -- --rules rule_file.rof2 --data data_file.txt --job 100
-- 
1.8.3.1

[dpdk-dev] [PATCH v3] app/test-regex: add RegEx test application

[Ori Kam]

From: Yuval Avnery <yuvalav@mellanox.com>

Following the new RegEx class.
There is a need to create a dedicated test application in order to
validate this class and PMD.

Unlike net device this application loads data from a file.

This commit introduces the new RegEx test app.

The basic app flow:
1. Configure the RegEx device to use one queue, and set the rule
   database, using precompiled file.
2. Allocate mbufs based on the requested number of jobs, each job will
i  get one mbuf.
3. Enqueue as much as possible jobs.
4. Dequeue jobs.
5. if the number of dequeue jobs < requested number of jobs job to step

Signed-off-by: Ori Kam <orika@mellanox.com>
Signed-off-by: Yuval Avnery <yuvalav@mellanox.com>
---
v3:
 * Fix possible issue in case of error.
v2:
 * Remove rule file and data generation data.
 * Address ML comments. 
---
 app/Makefile                   |   1 +
 app/meson.build                |   1 +
 app/test-regex/Makefile        |  21 ++
 app/test-regex/main.c          | 447 +++++++++++++++++++++++++++++++++++++++++
 app/test-regex/meson.build     |   5 +
 doc/guides/tools/index.rst     |   1 +
 doc/guides/tools/testregex.rst |  73 +++++++
 7 files changed, 549 insertions(+)
 create mode 100644 app/test-regex/Makefile
 create mode 100644 app/test-regex/main.c
 create mode 100644 app/test-regex/meson.build
 create mode 100644 doc/guides/tools/testregex.rst

diff --git a/app/Makefile b/app/Makefile
index 0392a7d..453c4fe 100644
--- a/app/Makefile
+++ b/app/Makefile
@@ -13,6 +13,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_FIB) += test-fib
 DIRS-$(CONFIG_RTE_TEST_FLOW_PERF) += test-flow-perf
 DIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += test-pipeline
 DIRS-$(CONFIG_RTE_LIBRTE_IPSEC) += test-sad
+DIRS-$(CONFIG_RTE_LIBRTE_REGEXDEV) += test-regex
 
 ifeq ($(CONFIG_RTE_LIBRTE_BBDEV),y)
 DIRS-$(CONFIG_RTE_TEST_BBDEV) += test-bbdev
diff --git a/app/meson.build b/app/meson.build
index 585b908..eb74f21 100644
--- a/app/meson.build
+++ b/app/meson.build
@@ -18,6 +18,7 @@ apps = [
 	'test-flow-perf',
 	'test-pipeline',
 	'test-pmd',
+	'test-regex',
 	'test-sad']
 
 # for BSD only
diff --git a/app/test-regex/Makefile b/app/test-regex/Makefile
new file mode 100644
index 0000000..550f997
--- /dev/null
+++ b/app/test-regex/Makefile
@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2020 Mellanox Technologies, Ltd
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+APP = testregex
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+#
+# all source are stored in SRCS-y
+#
+SRCS-y := main.c
+
+include $(RTE_SDK)/mk/rte.app.mk
+
diff --git a/app/test-regex/main.c b/app/test-regex/main.c
new file mode 100644
index 0000000..789d9ec
--- /dev/null
+++ b/app/test-regex/main.c
@@ -0,0 +1,447 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdarg.h>
+#include <ctype.h>
+#include <errno.h>
+#include <getopt.h>
+#include <signal.h>
+
+#include <rte_eal.h>
+#include <rte_common.h>
+#include <rte_malloc.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_cycles.h>
+#include <rte_regexdev.h>
+
+#define HELP_VAL 0
+#define RULES_VAL 1
+#define DATA_VAL 2
+#define JOB_VAL 3
+#define PERF_VAL 4
+#define ITER_VAL 5
+
+#define MAX_FILE_NAME 255
+
+/* enum that holds the value for the application arguments. */
+enum app_arg_values {
+	ARG_HELP,
+	ARG_RULES_FILE_NAME,
+	ARG_DATA_FILE_NAME,
+	ARG_NUM_OF_JOBS,
+	ARG_PERF_MODE,
+	ARG_NUM_OF_ITERATIONS,
+
+};
+
+static void
+usage(const char *prog_name)
+{
+	printf("%s [EAL options] --\n"
+		" --rules NAME: precompiled rules file\n"
+		" --data NAME: data file to use\n"
+		" --nb_jobs: number of jobs to use\n"
+		" --perf N: only outputs the performance data\n"
+		" --nb_iter N: number of iteration to run\n",
+		prog_name);
+}
+
+static void
+args_parse(int argc, char **argv, char *rules_file, char *data_file,
+	   uint32_t *nb_jobs, bool *perf_mode, uint32_t *nb_iterations)
+{
+	char **argvopt;
+	int opt;
+	int opt_idx;
+	size_t len;
+	static struct option lgopts[] = {
+		{ "help",  0, 0, ARG_HELP},
+		{ "rules",  1, 0, ARG_RULES_FILE_NAME},
+		/* Rules database file to load. */
+		{ "data",  1, 0, ARG_DATA_FILE_NAME},
+		/* Data file to load. */
+		{ "nb_jobs",  1, 0, ARG_NUM_OF_JOBS},
+		/* Number of jobs to create. */
+		{ "perf", 0, 0, ARG_PERF_MODE},
+		/* Perf test only */
+		{ "nb_iter", 1, 0, ARG_NUM_OF_ITERATIONS}
+		/* Number of iterations to run with perf test */
+	};
+
+	argvopt = argv;
+	while ((opt = getopt_long(argc, argvopt, "",
+				lgopts, &opt_idx)) != EOF) {
+		switch (opt) {
+		case ARG_RULES_FILE_NAME:
+			len = strnlen(optarg, MAX_FILE_NAME - 1);
+			if (len == MAX_FILE_NAME)
+				rte_exit(EXIT_FAILURE,
+					 "Rule file name to long max %d\n",
+					 MAX_FILE_NAME - 1);
+			strncpy(rules_file, optarg, MAX_FILE_NAME - 1);
+			break;
+		case ARG_DATA_FILE_NAME:
+			len = strnlen(optarg, MAX_FILE_NAME - 1);
+			if (len == MAX_FILE_NAME)
+				rte_exit(EXIT_FAILURE,
+					 "Data file name to long max %d\n",
+					 MAX_FILE_NAME - 1);
+			strncpy(data_file, optarg, MAX_FILE_NAME - 1);
+			break;
+		case ARG_NUM_OF_JOBS:
+			*nb_jobs = atoi(optarg);
+			break;
+		case ARG_PERF_MODE:
+			*perf_mode = true;
+			break;
+		case ARG_NUM_OF_ITERATIONS:
+			*nb_iterations = atoi(optarg);
+			break;
+		case ARG_HELP:
+			usage("RegEx test app");
+			break;
+		default:
+			fprintf(stderr, "Invalid option: %s\n", argv[optind]);
+			usage("RegEx test app");
+			rte_exit(EXIT_FAILURE, "Invalid option\n");
+			break;
+		}
+	}
+
+	if (!perf_mode)
+		*nb_iterations = 1;
+}
+
+static long
+read_file(char *file, char **buf)
+{
+	FILE *fp;
+	long buf_len = 0;
+	size_t read_len;
+	int res = 0;
+
+	fp = fopen(file, "r");
+	if (!fp)
+		return -EIO;
+	if (fseek(fp, 0L, SEEK_END) == 0) {
+		buf_len = ftell(fp);
+		if (buf_len == -1) {
+			res = EIO;
+			goto error;
+		}
+		*buf = rte_malloc(NULL, sizeof(char) * (buf_len + 1), 4096);
+		if (!*buf) {
+			res = ENOMEM;
+			goto error;
+		}
+		if (fseek(fp, 0L, SEEK_SET) != 0) {
+			res = EIO;
+			goto error;
+		}
+		read_len = fread(*buf, sizeof(char), buf_len, fp);
+		if (read_len != (unsigned long)buf_len) {
+			res = EIO;
+			goto error;
+		}
+	}
+	fclose(fp);
+	return buf_len;
+error:
+	printf("Error, can't open file %s\n, err = %d", file, res);
+	if (fp)
+		fclose(fp);
+	if (*buf)
+		rte_free(*buf);
+	return -res;
+}
+
+#define MBUF_CACHE_SIZE 256
+#define MBUF_SIZE (1 << 8)
+
+
+static int
+init_port(struct rte_mempool **mbuf_mp, uint32_t nb_jobs,
+	  uint16_t *nb_max_payload, char *rules_file, uint8_t *nb_max_matches)
+{
+	uint16_t id;
+	uint16_t num_devs;
+	char *rules = NULL;
+	long rules_len;
+	struct rte_regexdev_info info;
+	struct rte_regexdev_config dev_conf = {
+		.nb_queue_pairs = 1,
+		.nb_groups = 1,
+	};
+	struct rte_regexdev_qp_conf qp_conf = {
+		.nb_desc = 1024,
+		.qp_conf_flags = RTE_REGEX_QUEUE_PAIR_CFG_OOS_F,
+	};
+	int res = 0;
+
+	num_devs = rte_regexdev_count();
+	if (num_devs == 0) {
+		printf("Error, no devices detected.\n");
+		return -EINVAL;
+	}
+
+	*mbuf_mp = rte_pktmbuf_pool_create("mbuf_pool", nb_jobs, 0,
+					  0, MBUF_SIZE, rte_socket_id());
+	if (*mbuf_mp == NULL) {
+		printf("Error, can't create memory pool\n");
+		res = -ENOMEM;
+		goto error;
+	}
+
+	rules_len = read_file(rules_file, &rules);
+	if (rules_len < 0) {
+		printf("Error, can't read rules files.\n");
+		res = -EIO;
+		goto error;
+	}
+
+	for (id = 0; id < num_devs; id++) {
+		res = rte_regexdev_info_get(id, &info);
+		if (res != 0) {
+			printf("Error, can't get device info.\n");
+			goto error;
+		}
+		printf(":: initializing dev: %d\n", id);
+		*nb_max_matches = info.max_matches;
+		*nb_max_payload = info.max_payload_size;
+		if (info.regexdev_capa & RTE_REGEXDEV_SUPP_MATCH_AS_END_F)
+			dev_conf.dev_cfg_flags |= RTE_REGEXDEV_CFG_MATCH_AS_END_F;
+		dev_conf.nb_max_matches = info.max_matches;
+		dev_conf.nb_rules_per_group = info.max_rules_per_group;
+		dev_conf.rule_db_len = rules_len;
+		dev_conf.rule_db = rules;
+		res = rte_regexdev_configure(id, &dev_conf);
+		if (res < 0) {
+			printf("Error, can't configure device %d.\n", id);
+			goto error;
+		}
+		res = rte_regexdev_queue_pair_setup(id, 0, &qp_conf);
+		if (res < 0) {
+			printf("Error, can't setup queue pair for device %d.\n",
+			       id);
+			goto error;
+		}
+		printf(":: initializing device: %d done\n", id);
+	}
+	rte_free(rules);
+	return 0;
+error:
+	if (rules)
+		rte_free(rules);
+	if (*mbuf_mp)
+		rte_mempool_free(*mbuf_mp);
+	return res;
+
+}
+
+static void
+extbuf_free_cb(void *addr __rte_unused, void *fcb_opaque __rte_unused)
+{
+
+}
+
+#define START_BURST_SIZE 32u
+
+static int
+run_regex(struct rte_mempool *mbuf_mp, uint32_t nb_jobs,
+	  uint16_t nb_max_payload, bool perf_mode, uint32_t nb_iterations,
+	  char *data_file, uint8_t nb_max_matches)
+{
+	char *buf = NULL;
+	long buf_len;
+	long job_len;
+	uint32_t actual_jobs = 0;
+	uint32_t i;
+	struct rte_regex_ops **ops;
+	uint16_t dev_id = 0;
+	uint16_t qp_id = 0;
+	uint8_t nb_matches;
+	struct rte_regexdev_match *match;
+	long pos = 0;
+	unsigned long d_ind = 0;
+	struct rte_mbuf_ext_shared_info shinfo;
+	uint32_t total_enqueue = 0;
+	uint32_t total_dequeue = 0;
+	uint32_t total_matches = 0;
+	int res = 0;
+	time_t start;
+	time_t end;
+	double time;
+
+	shinfo.free_cb = extbuf_free_cb;
+
+	ops = rte_malloc(NULL, sizeof(*ops) * nb_jobs, 0);
+	if (!ops) {
+		printf("Error, can't allocate memory for ops.\n");
+		return -ENOMEM;
+	}
+
+	/* Allocate the jobs and assign each job with an mbuf. */
+	for (i = 0; i < nb_jobs; i++) {
+		ops[i] = rte_malloc(NULL, sizeof(*ops[0]) + nb_max_matches *
+				    sizeof(struct rte_regexdev_match), 0);
+		if (!ops[i]) {
+			printf("Error, can't allocate memory for op.\n");
+			res = -ENOMEM;
+			goto end;
+		}
+		ops[i]->mbuf = rte_pktmbuf_alloc(mbuf_mp);
+		if (!ops[i]->mbuf) {
+			printf("Error, can't attach mbuf.\n");
+			res = -ENOMEM;
+			goto end;
+		}
+	}
+
+	buf_len = read_file(data_file, &buf);
+	if (buf_len <= 0) {
+		printf("Error, can't read file, or file is empty.\n");
+		res = -EXIT_FAILURE;
+		goto end;
+	}
+
+	job_len = buf_len / nb_jobs;
+	if (job_len == 0) {
+		printf("Error, To many jobs, for the given input.\n");
+		res = -EXIT_FAILURE;
+		goto end;
+	}
+
+	if (job_len > nb_max_payload) {
+		printf("Error, not enough jobs to cover input.\n");
+		res = -EXIT_FAILURE;
+		goto end;
+	}
+
+	/* Assign each mbuf with the data to handle. */
+	for (i = 0; (pos < buf_len) && (i < nb_jobs) ; i++) {
+		long act_job_len = RTE_MIN(job_len, buf_len - pos);
+		rte_pktmbuf_attach_extbuf(ops[i]->mbuf, &buf[pos], 0,
+					  act_job_len, &shinfo);
+		ops[i]->mbuf->data_len = job_len;
+		ops[i]->mbuf->pkt_len = act_job_len;
+		ops[i]->user_id = i;
+		ops[i]->group_id0 = 1;
+		pos += act_job_len;
+		actual_jobs++;
+	}
+
+	start = clock();
+	for (i = 0; i < nb_iterations; i++) {
+		total_enqueue = 0;
+		total_dequeue = 0;
+		while (total_dequeue < actual_jobs) {
+			struct rte_regex_ops **cur_ops_to_enqueue = ops +
+				total_enqueue;
+			struct rte_regex_ops **cur_ops_to_dequeue = ops +
+				total_dequeue;
+
+			if (actual_jobs - total_enqueue)
+				total_enqueue += rte_regexdev_enqueue_burst
+					(dev_id, qp_id, cur_ops_to_enqueue,
+					 actual_jobs - total_enqueue);
+
+			total_dequeue += rte_regexdev_dequeue_burst
+				(dev_id, qp_id, cur_ops_to_dequeue,
+				 total_enqueue - total_dequeue);
+		}
+	}
+	end = clock();
+	time = ((double)end - start) / CLOCKS_PER_SEC;
+	printf("Job len = %ld Bytes\n",  job_len);
+	printf("Time = %lf sec\n",  time);
+	printf("Perf = %lf Gbps\n",
+	       (((double)actual_jobs * job_len * nb_iterations * 8) / time) /
+		1000000000.0);
+
+	if (!perf_mode) {
+		/* Log results per job. */
+		for (d_ind = 0; d_ind < total_dequeue; d_ind++) {
+			nb_matches = ops[d_ind % actual_jobs]->nb_matches;
+			printf("Job id %"PRIu64" number of matches = %d\n",
+			       ops[d_ind]->user_id, nb_matches);
+			total_matches += nb_matches;
+			match = ops[d_ind % actual_jobs]->matches;
+			for (i = 0; i < nb_matches; i++) {
+				printf("match %d, rule = %d, start = %d,len = %d\n",
+				       i, match->rule_id, match->start_offset,
+				       match->len);
+				match++;
+			}
+		}
+		printf("Total matches = %d\n", total_matches);
+		printf("All Matches:\n");
+
+		/* Log absolute results. */
+		for (d_ind = 0; d_ind < total_dequeue; d_ind++) {
+			nb_matches = ops[d_ind % actual_jobs]->nb_matches;
+			total_matches += nb_matches;
+			match = ops[d_ind % actual_jobs]->matches;
+			for (i = 0; i < nb_matches; i++) {
+				printf("start = %ld, len = %d, rule = %d\n",
+				       match->start_offset + d_ind * job_len,
+				       match->len, match->rule_id);
+				match++;
+			}
+		}
+	}
+end:
+	for (i = 0; i < actual_jobs; i++) {
+		if (ops[i]) {
+			if (ops[i]->mbuf)
+				rte_pktmbuf_free(ops[i]->mbuf);
+			rte_free(ops[i]);
+		}
+	}
+	rte_free(ops);
+	if (buf)
+		rte_free(buf);
+	return res;
+}
+
+int
+main(int argc, char **argv)
+{
+	char rules_file[MAX_FILE_NAME];
+	char data_file[MAX_FILE_NAME];
+	struct rte_mempool *mbuf_mp = NULL;
+	uint32_t nb_jobs = 0;
+	uint16_t nb_max_payload = 0;
+	bool perf_mode = 0;
+	uint32_t nb_iterations = 0;
+	uint8_t nb_max_matches = 0;
+	int ret;
+
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "EAL init failed\n");
+	argc -= ret;
+	argv += ret;
+	if (argc > 1)
+		args_parse(argc, argv, rules_file, data_file, &nb_jobs,
+			   &perf_mode, &nb_iterations);
+
+	ret = init_port(&mbuf_mp, nb_jobs, &nb_max_payload, rules_file,
+			&nb_max_matches);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "init port failed\n");
+	ret = run_regex(mbuf_mp, nb_jobs, nb_max_payload, perf_mode,
+			nb_iterations, data_file, nb_max_matches);
+	if (ret < 0) {
+		rte_mempool_free(mbuf_mp);
+		rte_exit(EXIT_FAILURE, "RegEx function failed\n");
+	}
+	rte_mempool_free(mbuf_mp);
+	return 0;
+}
diff --git a/app/test-regex/meson.build b/app/test-regex/meson.build
new file mode 100644
index 0000000..7c9357f
--- /dev/null
+++ b/app/test-regex/meson.build
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+sources = files('main.c')
+deps = ['regexdev']
diff --git a/doc/guides/tools/index.rst b/doc/guides/tools/index.rst
index 4840cf47..c721943 100644
--- a/doc/guides/tools/index.rst
+++ b/doc/guides/tools/index.rst
@@ -17,3 +17,4 @@ DPDK Tools User Guides
     cryptoperf
     comp_perf
     testeventdev
+    testregex
diff --git a/doc/guides/tools/testregex.rst b/doc/guides/tools/testregex.rst
new file mode 100644
index 0000000..24a1e3e
--- /dev/null
+++ b/doc/guides/tools/testregex.rst
@@ -0,0 +1,73 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright 2020 Mellanox Technologies, Ltd
+
+dpdk-test-regex Tool
+====================
+
+The ``dpdk-test-regex`` tool is a Data Plane Development Kit (DPDK)
+application that allows functional testing and performance measurement for
+the RegEx PMDs.
+The test supports only one core and one PMD.
+It is based on precompiled rule file, and an input file, both of them can
+be selected using command-line options.
+
+In general case, each PMD has its own rule file.
+
+The test outputs the following data:
+
+* Performance, in gigabit per second.
+
+* Matching results (rule id, position, length), for each job.
+
+* Matching results in absolute location (rule id, position , length),
+  relative to the start of the input data.
+
+
+Limitations
+~~~~~~~~~~~
+
+* Only one queue is supported.
+
+* Supports only precompiled rules.
+
+
+Application Options
+~~~~~~~~~~~~~~~~~~~
+
+* ``--rules NAME``: precompiled rule file
+
+* ``--data NAME``: data file to use
+
+* ``--nb_jobs N``: number of jobs to use
+
+* ``--perf N``: only outputs the performance data
+
+* ``--nb_iter N``: number of iteration to run
+
+* ``--help``: prints this help
+
+
+Compiling the Tool
+------------------
+
+The ``dpdk-test-regex`` application depends on RegEx lib ``rte_regexdev``.
+
+
+Running the Tool
+----------------
+
+**Step 1: Compile a rule file**
+
+In order for the RegEx to work it must have a precompiled rule file.
+to generate this file there is a need to use a RegEx compiler that matches the
+RegEx PMD.
+
+**Step 2: Generate a data file**
+
+The data file, will be used as a source data for the RegEx to work on.
+
+**Step 3: Run the tool**
+
+The tool has a number of command line options. Here is the sample command line::
+
+   .testregex -w 83:00.0 -- --rules rule_file.rof2 --data data_file.txt --job 100
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v3] app/test-regex: add RegEx test application

[Thomas Monjalon]

29/07/2020 13:26, Ori Kam:
> --- /dev/null
> +++ b/app/test-regex/Makefile
> @@ -0,0 +1,21 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright 2020 Mellanox Technologies, Ltd
> +
> +include $(RTE_SDK)/mk/rte.vars.mk
> +
> +#
> +# library name

It's not a library.
You can completely drop this useless comment.

> +#
> +APP = testregex
> +
> +CFLAGS += -O3
> +CFLAGS += $(WERROR_FLAGS)
> +CFLAGS += -DALLOW_EXPERIMENTAL_API
> +
> +#
> +# all source are stored in SRCS-y
> +#
> +SRCS-y := main.c
> +
> +include $(RTE_SDK)/mk/rte.app.mk
> +

extra blank line at EOF

> diff --git a/app/test-regex/main.c b/app/test-regex/main.c
> new file mode 100644
> index 0000000..789d9ec
> --- /dev/null
> +++ b/app/test-regex/main.c
> @@ -0,0 +1,447 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2020 Mellanox Technologies, Ltd
> + */
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include <stdint.h>
> +#include <stdbool.h>
> +#include <stdarg.h>
> +#include <ctype.h>
> +#include <errno.h>
> +#include <getopt.h>
> +#include <signal.h>
> +
> +#include <rte_eal.h>
> +#include <rte_common.h>
> +#include <rte_malloc.h>
> +#include <rte_mempool.h>
> +#include <rte_mbuf.h>
> +#include <rte_cycles.h>
> +#include <rte_regexdev.h>
> +
> +#define HELP_VAL 0
> +#define RULES_VAL 1
> +#define DATA_VAL 2
> +#define JOB_VAL 3
> +#define PERF_VAL 4
> +#define ITER_VAL 5

These macros are not used anymore.

> +
> +#define MAX_FILE_NAME 255
> +
> +/* enum that holds the value for the application arguments. */

There is no value in "enum that holds the value for the".
You can just keep "application arguments",
but the real info is to say it is not a value but an argument index.
What about "arguments parsed with getopt_long"?

> +enum app_arg_values {

_values suffix look wrong

> +	ARG_HELP,
> +	ARG_RULES_FILE_NAME,
> +	ARG_DATA_FILE_NAME,
> +	ARG_NUM_OF_JOBS,
> +	ARG_PERF_MODE,
> +	ARG_NUM_OF_ITERATIONS,
> +
> +};
> +
[...]
> +
> +#define MBUF_CACHE_SIZE 256
> +#define MBUF_SIZE (1 << 8)

I expect such definitions at the beginning of the file.

[...]
> +static void
> +extbuf_free_cb(void *addr __rte_unused, void *fcb_opaque __rte_unused)
> +{
> +
> +}

extra blank line

> +
> +#define START_BURST_SIZE 32u

could be at beginning also

> --- /dev/null
> +++ b/app/test-regex/meson.build
> @@ -0,0 +1,5 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2018 Intel Corporation

Please don't assign copyright to someone not involved.

> +
> +sources = files('main.c')
> +deps = ['regexdev']

[...]
> +Application Options
> +~~~~~~~~~~~~~~~~~~~
> +
> +* ``--rules NAME``: precompiled rule file
> +
> +* ``--data NAME``: data file to use
> +
> +* ``--nb_jobs N``: number of jobs to use
> +
> +* ``--perf N``: only outputs the performance data
> +
> +* ``--nb_iter N``: number of iteration to run
> +
> +* ``--help``: prints this help

Same comment as v1, definition list is better.

> +Compiling the Tool
> +------------------
> +
> +The ``dpdk-test-regex`` application depends on RegEx lib ``rte_regexdev``.

It is obvious.

> +Running the Tool
> +----------------
> +
> +**Step 1: Compile a rule file**
> +
> +In order for the RegEx to work it must have a precompiled rule file.
> +to generate this file there is a need to use a RegEx compiler that matches the
> +RegEx PMD.
> +
> +**Step 2: Generate a data file**
> +
> +The data file, will be used as a source data for the RegEx to work on.
> +
> +**Step 3: Run the tool**
> +
> +The tool has a number of command line options. Here is the sample command line::
> +
> +   .testregex -w 83:00.0 -- --rules rule_file.rof2 --data data_file.txt --job 100

What is .testregex?

OK, these steps are important to understand well.

[dpdk-dev] [PATCH v4] app/test-regex: add RegEx test application

[Ori Kam]

From: Yuval Avnery <yuvalav@mellanox.com>

Following the new RegEx class.
There is a need to create a dedicated test application in order to
validate this class and PMD.

Unlike net device this application loads data from a file.

This commit introduces the new RegEx test app.

The basic app flow:
1. Configure the RegEx device to use one queue, and set the rule
   database, using precompiled file.
2. Allocate mbufs based on the requested number of jobs, each job will
i  get one mbuf.
3. Enqueue as much as possible jobs.
4. Dequeue jobs.
5. if the number of dequeue jobs < requested number of jobs job to step

Signed-off-by: Ori Kam <orika@mellanox.com>
Signed-off-by: Yuval Avnery <yuvalav@mellanox.com>
---
v4:
 * Address ML comments.
v3:
 * Fix possible issue in case of error.
v2:
 * Remove rule file and data generation data.
 * Address ML comments. 
---
 app/Makefile                   |   1 +
 app/meson.build                |   1 +
 app/test-regex/Makefile        |  17 ++
 app/test-regex/main.c          | 436 +++++++++++++++++++++++++++++++++++++++++
 app/test-regex/meson.build     |   5 +
 doc/guides/tools/index.rst     |   1 +
 doc/guides/tools/testregex.rst |  73 +++++++
 7 files changed, 534 insertions(+)
 create mode 100644 app/test-regex/Makefile
 create mode 100644 app/test-regex/main.c
 create mode 100644 app/test-regex/meson.build
 create mode 100644 doc/guides/tools/testregex.rst

diff --git a/app/Makefile b/app/Makefile
index 0392a7d..453c4fe 100644
--- a/app/Makefile
+++ b/app/Makefile
@@ -13,6 +13,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_FIB) += test-fib
 DIRS-$(CONFIG_RTE_TEST_FLOW_PERF) += test-flow-perf
 DIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += test-pipeline
 DIRS-$(CONFIG_RTE_LIBRTE_IPSEC) += test-sad
+DIRS-$(CONFIG_RTE_LIBRTE_REGEXDEV) += test-regex
 
 ifeq ($(CONFIG_RTE_LIBRTE_BBDEV),y)
 DIRS-$(CONFIG_RTE_TEST_BBDEV) += test-bbdev
diff --git a/app/meson.build b/app/meson.build
index 585b908..eb74f21 100644
--- a/app/meson.build
+++ b/app/meson.build
@@ -18,6 +18,7 @@ apps = [
 	'test-flow-perf',
 	'test-pipeline',
 	'test-pmd',
+	'test-regex',
 	'test-sad']
 
 # for BSD only
diff --git a/app/test-regex/Makefile b/app/test-regex/Makefile
new file mode 100644
index 0000000..a059a8c
--- /dev/null
+++ b/app/test-regex/Makefile
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2020 Mellanox Technologies, Ltd
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+APP = testregex
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+#
+# all source are stored in SRCS-y
+#
+SRCS-y := main.c
+
+include $(RTE_SDK)/mk/rte.app.mk
diff --git a/app/test-regex/main.c b/app/test-regex/main.c
new file mode 100644
index 0000000..9cdf1da
--- /dev/null
+++ b/app/test-regex/main.c
@@ -0,0 +1,436 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdarg.h>
+#include <ctype.h>
+#include <errno.h>
+#include <getopt.h>
+#include <signal.h>
+
+#include <rte_eal.h>
+#include <rte_common.h>
+#include <rte_malloc.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_cycles.h>
+#include <rte_regexdev.h>
+
+#define MAX_FILE_NAME 255
+#define MBUF_CACHE_SIZE 256
+#define MBUF_SIZE (1 << 8)
+#define START_BURST_SIZE 32u
+
+enum app_args {
+	ARG_HELP,
+	ARG_RULES_FILE_NAME,
+	ARG_DATA_FILE_NAME,
+	ARG_NUM_OF_JOBS,
+	ARG_PERF_MODE,
+	ARG_NUM_OF_ITERATIONS,
+
+};
+
+static void
+usage(const char *prog_name)
+{
+	printf("%s [EAL options] --\n"
+		" --rules NAME: precompiled rules file\n"
+		" --data NAME: data file to use\n"
+		" --nb_jobs: number of jobs to use\n"
+		" --perf N: only outputs the performance data\n"
+		" --nb_iter N: number of iteration to run\n",
+		prog_name);
+}
+
+static void
+args_parse(int argc, char **argv, char *rules_file, char *data_file,
+	   uint32_t *nb_jobs, bool *perf_mode, uint32_t *nb_iterations)
+{
+	char **argvopt;
+	int opt;
+	int opt_idx;
+	size_t len;
+	static struct option lgopts[] = {
+		{ "help",  0, 0, ARG_HELP},
+		{ "rules",  1, 0, ARG_RULES_FILE_NAME},
+		/* Rules database file to load. */
+		{ "data",  1, 0, ARG_DATA_FILE_NAME},
+		/* Data file to load. */
+		{ "nb_jobs",  1, 0, ARG_NUM_OF_JOBS},
+		/* Number of jobs to create. */
+		{ "perf", 0, 0, ARG_PERF_MODE},
+		/* Perf test only */
+		{ "nb_iter", 1, 0, ARG_NUM_OF_ITERATIONS}
+		/* Number of iterations to run with perf test */
+	};
+
+	argvopt = argv;
+	while ((opt = getopt_long(argc, argvopt, "",
+				lgopts, &opt_idx)) != EOF) {
+		switch (opt) {
+		case ARG_RULES_FILE_NAME:
+			len = strnlen(optarg, MAX_FILE_NAME - 1);
+			if (len == MAX_FILE_NAME)
+				rte_exit(EXIT_FAILURE,
+					 "Rule file name to long max %d\n",
+					 MAX_FILE_NAME - 1);
+			strncpy(rules_file, optarg, MAX_FILE_NAME - 1);
+			break;
+		case ARG_DATA_FILE_NAME:
+			len = strnlen(optarg, MAX_FILE_NAME - 1);
+			if (len == MAX_FILE_NAME)
+				rte_exit(EXIT_FAILURE,
+					 "Data file name to long max %d\n",
+					 MAX_FILE_NAME - 1);
+			strncpy(data_file, optarg, MAX_FILE_NAME - 1);
+			break;
+		case ARG_NUM_OF_JOBS:
+			*nb_jobs = atoi(optarg);
+			break;
+		case ARG_PERF_MODE:
+			*perf_mode = true;
+			break;
+		case ARG_NUM_OF_ITERATIONS:
+			*nb_iterations = atoi(optarg);
+			break;
+		case ARG_HELP:
+			usage("RegEx test app");
+			break;
+		default:
+			fprintf(stderr, "Invalid option: %s\n", argv[optind]);
+			usage("RegEx test app");
+			rte_exit(EXIT_FAILURE, "Invalid option\n");
+			break;
+		}
+	}
+
+	if (!perf_mode)
+		*nb_iterations = 1;
+}
+
+static long
+read_file(char *file, char **buf)
+{
+	FILE *fp;
+	long buf_len = 0;
+	size_t read_len;
+	int res = 0;
+
+	fp = fopen(file, "r");
+	if (!fp)
+		return -EIO;
+	if (fseek(fp, 0L, SEEK_END) == 0) {
+		buf_len = ftell(fp);
+		if (buf_len == -1) {
+			res = EIO;
+			goto error;
+		}
+		*buf = rte_malloc(NULL, sizeof(char) * (buf_len + 1), 4096);
+		if (!*buf) {
+			res = ENOMEM;
+			goto error;
+		}
+		if (fseek(fp, 0L, SEEK_SET) != 0) {
+			res = EIO;
+			goto error;
+		}
+		read_len = fread(*buf, sizeof(char), buf_len, fp);
+		if (read_len != (unsigned long)buf_len) {
+			res = EIO;
+			goto error;
+		}
+	}
+	fclose(fp);
+	return buf_len;
+error:
+	printf("Error, can't open file %s\n, err = %d", file, res);
+	if (fp)
+		fclose(fp);
+	if (*buf)
+		rte_free(*buf);
+	return -res;
+}
+
+static int
+init_port(struct rte_mempool **mbuf_mp, uint32_t nb_jobs,
+	  uint16_t *nb_max_payload, char *rules_file, uint8_t *nb_max_matches)
+{
+	uint16_t id;
+	uint16_t num_devs;
+	char *rules = NULL;
+	long rules_len;
+	struct rte_regexdev_info info;
+	struct rte_regexdev_config dev_conf = {
+		.nb_queue_pairs = 1,
+		.nb_groups = 1,
+	};
+	struct rte_regexdev_qp_conf qp_conf = {
+		.nb_desc = 1024,
+		.qp_conf_flags = RTE_REGEX_QUEUE_PAIR_CFG_OOS_F,
+	};
+	int res = 0;
+
+	num_devs = rte_regexdev_count();
+	if (num_devs == 0) {
+		printf("Error, no devices detected.\n");
+		return -EINVAL;
+	}
+
+	*mbuf_mp = rte_pktmbuf_pool_create("mbuf_pool", nb_jobs, 0,
+					  0, MBUF_SIZE, rte_socket_id());
+	if (*mbuf_mp == NULL) {
+		printf("Error, can't create memory pool\n");
+		res = -ENOMEM;
+		goto error;
+	}
+
+	rules_len = read_file(rules_file, &rules);
+	if (rules_len < 0) {
+		printf("Error, can't read rules files.\n");
+		res = -EIO;
+		goto error;
+	}
+
+	for (id = 0; id < num_devs; id++) {
+		res = rte_regexdev_info_get(id, &info);
+		if (res != 0) {
+			printf("Error, can't get device info.\n");
+			goto error;
+		}
+		printf(":: initializing dev: %d\n", id);
+		*nb_max_matches = info.max_matches;
+		*nb_max_payload = info.max_payload_size;
+		if (info.regexdev_capa & RTE_REGEXDEV_SUPP_MATCH_AS_END_F)
+			dev_conf.dev_cfg_flags |= RTE_REGEXDEV_CFG_MATCH_AS_END_F;
+		dev_conf.nb_max_matches = info.max_matches;
+		dev_conf.nb_rules_per_group = info.max_rules_per_group;
+		dev_conf.rule_db_len = rules_len;
+		dev_conf.rule_db = rules;
+		res = rte_regexdev_configure(id, &dev_conf);
+		if (res < 0) {
+			printf("Error, can't configure device %d.\n", id);
+			goto error;
+		}
+		res = rte_regexdev_queue_pair_setup(id, 0, &qp_conf);
+		if (res < 0) {
+			printf("Error, can't setup queue pair for device %d.\n",
+			       id);
+			goto error;
+		}
+		printf(":: initializing device: %d done\n", id);
+	}
+	rte_free(rules);
+	return 0;
+error:
+	if (rules)
+		rte_free(rules);
+	if (*mbuf_mp)
+		rte_mempool_free(*mbuf_mp);
+	return res;
+
+}
+
+static void
+extbuf_free_cb(void *addr __rte_unused, void *fcb_opaque __rte_unused)
+{
+
+}
+
+static int
+run_regex(struct rte_mempool *mbuf_mp, uint32_t nb_jobs,
+	  uint16_t nb_max_payload, bool perf_mode, uint32_t nb_iterations,
+	  char *data_file, uint8_t nb_max_matches)
+{
+	char *buf = NULL;
+	long buf_len;
+	long job_len;
+	uint32_t actual_jobs = 0;
+	uint32_t i;
+	struct rte_regex_ops **ops;
+	uint16_t dev_id = 0;
+	uint16_t qp_id = 0;
+	uint8_t nb_matches;
+	struct rte_regexdev_match *match;
+	long pos = 0;
+	unsigned long d_ind = 0;
+	struct rte_mbuf_ext_shared_info shinfo;
+	uint32_t total_enqueue = 0;
+	uint32_t total_dequeue = 0;
+	uint32_t total_matches = 0;
+	int res = 0;
+	time_t start;
+	time_t end;
+	double time;
+
+	shinfo.free_cb = extbuf_free_cb;
+
+	ops = rte_malloc(NULL, sizeof(*ops) * nb_jobs, 0);
+	if (!ops) {
+		printf("Error, can't allocate memory for ops.\n");
+		return -ENOMEM;
+	}
+
+	/* Allocate the jobs and assign each job with an mbuf. */
+	for (i = 0; i < nb_jobs; i++) {
+		ops[i] = rte_malloc(NULL, sizeof(*ops[0]) + nb_max_matches *
+				    sizeof(struct rte_regexdev_match), 0);
+		if (!ops[i]) {
+			printf("Error, can't allocate memory for op.\n");
+			res = -ENOMEM;
+			goto end;
+		}
+		ops[i]->mbuf = rte_pktmbuf_alloc(mbuf_mp);
+		if (!ops[i]->mbuf) {
+			printf("Error, can't attach mbuf.\n");
+			res = -ENOMEM;
+			goto end;
+		}
+	}
+
+	buf_len = read_file(data_file, &buf);
+	if (buf_len <= 0) {
+		printf("Error, can't read file, or file is empty.\n");
+		res = -EXIT_FAILURE;
+		goto end;
+	}
+
+	job_len = buf_len / nb_jobs;
+	if (job_len == 0) {
+		printf("Error, To many jobs, for the given input.\n");
+		res = -EXIT_FAILURE;
+		goto end;
+	}
+
+	if (job_len > nb_max_payload) {
+		printf("Error, not enough jobs to cover input.\n");
+		res = -EXIT_FAILURE;
+		goto end;
+	}
+
+	/* Assign each mbuf with the data to handle. */
+	for (i = 0; (pos < buf_len) && (i < nb_jobs) ; i++) {
+		long act_job_len = RTE_MIN(job_len, buf_len - pos);
+		rte_pktmbuf_attach_extbuf(ops[i]->mbuf, &buf[pos], 0,
+					  act_job_len, &shinfo);
+		ops[i]->mbuf->data_len = job_len;
+		ops[i]->mbuf->pkt_len = act_job_len;
+		ops[i]->user_id = i;
+		ops[i]->group_id0 = 1;
+		pos += act_job_len;
+		actual_jobs++;
+	}
+
+	start = clock();
+	for (i = 0; i < nb_iterations; i++) {
+		total_enqueue = 0;
+		total_dequeue = 0;
+		while (total_dequeue < actual_jobs) {
+			struct rte_regex_ops **cur_ops_to_enqueue = ops +
+				total_enqueue;
+			struct rte_regex_ops **cur_ops_to_dequeue = ops +
+				total_dequeue;
+
+			if (actual_jobs - total_enqueue)
+				total_enqueue += rte_regexdev_enqueue_burst
+					(dev_id, qp_id, cur_ops_to_enqueue,
+					 actual_jobs - total_enqueue);
+
+			total_dequeue += rte_regexdev_dequeue_burst
+				(dev_id, qp_id, cur_ops_to_dequeue,
+				 total_enqueue - total_dequeue);
+		}
+	}
+	end = clock();
+	time = ((double)end - start) / CLOCKS_PER_SEC;
+	printf("Job len = %ld Bytes\n",  job_len);
+	printf("Time = %lf sec\n",  time);
+	printf("Perf = %lf Gbps\n",
+	       (((double)actual_jobs * job_len * nb_iterations * 8) / time) /
+		1000000000.0);
+
+	if (!perf_mode) {
+		/* Log results per job. */
+		for (d_ind = 0; d_ind < total_dequeue; d_ind++) {
+			nb_matches = ops[d_ind % actual_jobs]->nb_matches;
+			printf("Job id %"PRIu64" number of matches = %d\n",
+			       ops[d_ind]->user_id, nb_matches);
+			total_matches += nb_matches;
+			match = ops[d_ind % actual_jobs]->matches;
+			for (i = 0; i < nb_matches; i++) {
+				printf("match %d, rule = %d, start = %d,len = %d\n",
+				       i, match->rule_id, match->start_offset,
+				       match->len);
+				match++;
+			}
+		}
+		printf("Total matches = %d\n", total_matches);
+		printf("All Matches:\n");
+
+		/* Log absolute results. */
+		for (d_ind = 0; d_ind < total_dequeue; d_ind++) {
+			nb_matches = ops[d_ind % actual_jobs]->nb_matches;
+			total_matches += nb_matches;
+			match = ops[d_ind % actual_jobs]->matches;
+			for (i = 0; i < nb_matches; i++) {
+				printf("start = %ld, len = %d, rule = %d\n",
+				       match->start_offset + d_ind * job_len,
+				       match->len, match->rule_id);
+				match++;
+			}
+		}
+	}
+end:
+	for (i = 0; i < actual_jobs; i++) {
+		if (ops[i]) {
+			if (ops[i]->mbuf)
+				rte_pktmbuf_free(ops[i]->mbuf);
+			rte_free(ops[i]);
+		}
+	}
+	rte_free(ops);
+	if (buf)
+		rte_free(buf);
+	return res;
+}
+
+int
+main(int argc, char **argv)
+{
+	char rules_file[MAX_FILE_NAME];
+	char data_file[MAX_FILE_NAME];
+	struct rte_mempool *mbuf_mp = NULL;
+	uint32_t nb_jobs = 0;
+	uint16_t nb_max_payload = 0;
+	bool perf_mode = 0;
+	uint32_t nb_iterations = 0;
+	uint8_t nb_max_matches = 0;
+	int ret;
+
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "EAL init failed\n");
+	argc -= ret;
+	argv += ret;
+	if (argc > 1)
+		args_parse(argc, argv, rules_file, data_file, &nb_jobs,
+			   &perf_mode, &nb_iterations);
+
+	ret = init_port(&mbuf_mp, nb_jobs, &nb_max_payload, rules_file,
+			&nb_max_matches);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "init port failed\n");
+	ret = run_regex(mbuf_mp, nb_jobs, nb_max_payload, perf_mode,
+			nb_iterations, data_file, nb_max_matches);
+	if (ret < 0) {
+		rte_mempool_free(mbuf_mp);
+		rte_exit(EXIT_FAILURE, "RegEx function failed\n");
+	}
+	rte_mempool_free(mbuf_mp);
+	return 0;
+}
diff --git a/app/test-regex/meson.build b/app/test-regex/meson.build
new file mode 100644
index 0000000..472677f
--- /dev/null
+++ b/app/test-regex/meson.build
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2020 Mellanox Technologies, Ltd
+
+sources = files('main.c')
+deps = ['regexdev']
diff --git a/doc/guides/tools/index.rst b/doc/guides/tools/index.rst
index 4840cf47..c721943 100644
--- a/doc/guides/tools/index.rst
+++ b/doc/guides/tools/index.rst
@@ -17,3 +17,4 @@ DPDK Tools User Guides
     cryptoperf
     comp_perf
     testeventdev
+    testregex
diff --git a/doc/guides/tools/testregex.rst b/doc/guides/tools/testregex.rst
new file mode 100644
index 0000000..347d607
--- /dev/null
+++ b/doc/guides/tools/testregex.rst
@@ -0,0 +1,73 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright 2020 Mellanox Technologies, Ltd
+
+dpdk-test-regex Tool
+====================
+
+The ``dpdk-test-regex`` tool is a Data Plane Development Kit (DPDK)
+application that allows functional testing and performance measurement for
+the RegEx PMDs.
+The test supports only one core and one PMD.
+It is based on precompiled rule file, and an input file, both of them can
+be selected using command-line options.
+
+In general case, each PMD has its own rule file.
+
+The test outputs the following data:
+
+* Performance, in gigabit per second.
+
+* Matching results (rule id, position, length), for each job.
+
+* Matching results in absolute location (rule id, position , length),
+  relative to the start of the input data.
+
+
+Limitations
+~~~~~~~~~~~
+
+* Only one queue is supported.
+
+* Supports only precompiled rules.
+
+
+Application Options
+~~~~~~~~~~~~~~~~~~~
+
+``--rules NAME``
+  precompiled rule file
+
+``--data NAME``
+  data file to use
+
+``--nb_jobs N``
+  number of jobs to use
+
+``--perf N``
+  only outputs the performance data
+
+``--nb_iter N``
+  number of iteration to run
+
+``--help``
+  prints this help
+
+
+Running the Tool
+----------------
+
+**Step 1: Compile a rule file**
+
+In order for the RegEx to work it must have a precompiled rule file.
+to generate this file there is a need to use a RegEx compiler that matches the
+RegEx PMD.
+
+**Step 2: Generate a data file**
+
+The data file, will be used as a source data for the RegEx to work on.
+
+**Step 3: Run the tool**
+
+The tool has a number of command line options. Here is the sample command line::
+
+   ./dpdk-test-regex -w 83:00.0 -- --rules rule_file.rof2 --data data_file.txt --job 100
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v4] app/test-regex: add RegEx test application

[Thomas Monjalon]

29/07/2020 20:09, Ori Kam:
> From: Yuval Avnery <yuvalav@mellanox.com>
> 
> Following the new RegEx class.
> There is a need to create a dedicated test application in order to
> validate this class and PMD.
> 
> Unlike net device this application loads data from a file.
> 
> This commit introduces the new RegEx test app.
> 
> The basic app flow:
> 1. Configure the RegEx device to use one queue, and set the rule
>    database, using precompiled file.
> 2. Allocate mbufs based on the requested number of jobs, each job will
> i  get one mbuf.
> 3. Enqueue as much as possible jobs.
> 4. Dequeue jobs.
> 5. if the number of dequeue jobs < requested number of jobs job to step
> 
> Signed-off-by: Ori Kam <orika@mellanox.com>
> Signed-off-by: Yuval Avnery <yuvalav@mellanox.com>
[...]
> +++ b/app/test-regex/Makefile
> @@ -0,0 +1,17 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright 2020 Mellanox Technologies, Ltd
> +
> +include $(RTE_SDK)/mk/rte.vars.mk
> +
> +APP = testregex

In order to match meson and the doc,
I rename to dpdk-test-regex.

> +
> +CFLAGS += -O3
> +CFLAGS += $(WERROR_FLAGS)
> +CFLAGS += -DALLOW_EXPERIMENTAL_API

Experimental API is already enabled via rte.vars.mk.
I remove this line.

[...]
> +enum app_args {
> +	ARG_HELP,
> +	ARG_RULES_FILE_NAME,
> +	ARG_DATA_FILE_NAME,
> +	ARG_NUM_OF_JOBS,
> +	ARG_PERF_MODE,
> +	ARG_NUM_OF_ITERATIONS,
> +
> +};

Extra blank line

[...]
> +error:
> +	if (rules)
> +		rte_free(rules);
> +	if (*mbuf_mp)
> +		rte_mempool_free(*mbuf_mp);
> +	return res;
> +
> +}
> +
> +static void
> +extbuf_free_cb(void *addr __rte_unused, void *fcb_opaque __rte_unused)
> +{
> +
> +}

Extra blank lines

[...]
> +       if (ret < 0)
> +               rte_exit(EXIT_FAILURE, "init port failed\n");
> +       ret = run_regex(mbuf_mp, nb_jobs, nb_max_payload, perf_mode,
> +                       nb_iterations, data_file, nb_max_matches);
> +       if (ret < 0) {
> +               rte_mempool_free(mbuf_mp);
> +               rte_exit(EXIT_FAILURE, "RegEx function failed\n");
> +       }
> +       rte_mempool_free(mbuf_mp);
> +       return 0;

Should return EXIT_SUCCESS

[...]
> +``--help``
> +  prints this help

In the doc, it should be better to reword to "print application options".

The MAINTAINERS file must be updated for this new directory.

Applied with above minor changes, thanks.