[dpdk-dev] [RFC] app/test-regex: add RegEx test application.

[dpdk-dev] [RFC] app/test-regex: add RegEx test application.

[Ori Kam]

Following the new RegEx class [1].
There is a need to create a dedecated test application in order to
validate this class and PMD.

Unlike net device this application loads data from a file.

This commit introduce the new RegEx test app.

The basic app flow:
1. query probed RegEx device.
2. configure the probed device accoding to the device caps.
3. read a rule database file.
4. read a data file to be searched.
5. allocate one mbuf.
6. select the request job len, based on the number of jobs requsted.
7. create one RegEx op which uses the mbuf defined in step 5.
8. attach the data buffer at offset 0 with len accoding to request
   len to the mbuf as external buffer.
9. enqueue one job.
10. wait for completion of the job.
11. check job results and print them.
12. move the mbuf to point to the next part of the buffer.
13. while there are more parts jump to 9.

Future improvments to this application may include:
1. submiting more then one job at a time.
2. working with more then one queue.
3. support out of order.
4. allow runtime rule updating.

This commit is only at RFC level, which means that
the code is only basic reference and can't be compiled and doesn't
include error or edge case handling.

[1]https://patches.dpdk.org/cover/69919/

Signed-off-by: Ori Kam <orika@mellanox.com>
---
 app/test-regex/main.c | 217 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 217 insertions(+)
 create mode 100644 app/test-regex/main.c

diff --git a/app/test-regex/main.c b/app/test-regex/main.c
new file mode 100644
index 0000000..ca90855
--- /dev/null
+++ b/app/test-regex/main.c
@@ -0,0 +1,217 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ *
+ * This file contain the application main file
+ * This application provides a way to test the RegEx class.
+ */
+
+#define HELP_VAL 0
+#define RULES_VAL 1
+#define DATA_VAL 2
+#define JOB_VAL 3
+#define MAX_FILE_NAME 255
+static char rules_file[MAX_FILE_NAME];
+static char data_file[MAX_FILE_NAME];
+static int jobs;
+static struct rte_mempool *mbuf_mp;
+static uint16_t nb_max_matches;
+static uint8_t nb_max_matches;
+
+static void
+args_parse(int argc, char **argv)
+{
+	char **argvopt;
+	int opt;
+	int opt_idx;
+	static struct option lgopts[] = {
+		{ "help",  0, 0, HELP_VAL },
+		{ "rules",  1, 0, RULES_VAL },
+		/* Rules database file to load. */
+		{ "data",  1, 0, DATA_VAL },
+		/* Data file to load. */
+		{ "job",  1, 0, JOB_VAL },
+		/* Number of jobs to create. */
+	};
+
+	argvopt = argv;
+
+	while ((opt = getopt_long(argc, argvopt, "",
+				lgopts, &opt_idx)) != EOF) {
+		switch (opt) {
+		case RULES_VAL:
+			strncpy(rules_file, optarg, MAX_FILE_NAME);
+			break;
+		case DATA_VAL:
+			strncpy(data_file, optarg, MAX_FILE_NAME);
+			break;
+		case JOB_VAL:
+			jobs = atoi(optarg);
+			break;
+		default:
+			fprintf(stderr, "Invalid option: %s\n", argv[optind]);
+			usage(argv[0]);
+			rte_exit(EXIT_SUCCESS, "Invalid option\n");
+			break;
+		}
+	}
+}
+
+static long
+read_file(char *file, char **buf)
+{
+	FILE *fp;
+	long buf_len;
+	size_t read_len;
+
+	fp = fopen(file, "r");
+	if (fp != NULL)
+		return -EIO;
+	if (fseek(fp, 0L, SEEK_END) == 0) {
+		buf_len = ftell(fp);
+		if (buf_len == -1)
+			goto error;
+		*buf = rte_malloc(sizeof(char) * (buf_len + 1));
+		if (fseek(fp, 0L, SEEK_SET) != 0)
+			goto error;
+		newLen = fread(source, sizeof(char), buf_len, fp);
+		if (new_len != buf_len)
+			goto error;
+	}
+	fclose(fp);
+	return buf_size;
+error:
+	if (fp)
+		fclose(fp);
+	if (*buf)
+		rte_free(*buf);
+	return -EIO;
+}
+
+static void
+init_port(void)
+{
+	int ret;
+	uint16_t id;
+	uint16_t num_devs;
+	char *rules;
+	long rules_len;
+	char *data = NULL;
+	long data_size;
+	struct rte_regexdev_info info;
+	struct rte_regexdev_config dev_conf = {
+		.nb_queue_pairs = 1,
+		.nb_groups = 1,
+	};
+	struct rte_regexdev_qp_conf qp_conf = {
+		.nb_desc = 1024,
+	};
+
+	num_devs = rte_regexdev_count();
+	if (nr_ports == 0)
+		rte_exit(EXIT_FAILURE, "Error: no devices detected\n");
+
+	mbuf_mp = rte_pktmbuf_pool_create("mbuf_pool",
+					TOTAL_MBUF_NUM, MBUF_CACHE_SIZE,
+					0, MBUF_SIZE,
+					rte_socket_id());
+	if (mbuf_mp == NULL)
+		rte_exit(EXIT_FAILURE, "Error: can't init mbuf pool\n");
+	rules_len = read_file(rule_file, &rules);
+	if (rule_size < 0)
+		rte_exit(EXIT_FAILURE, "Error: can't read rule file\n");
+	for (id = 0; id < num_devs; id++) {
+		ret = rte_regexdev_info_get(id, &info);
+		if (ret != 0)
+			rte_exit(EXIT_FAILURE,
+				"Error during getting device"
+				" (port %u) info: %s\n", id, strerror(-ret));
+		printf(":: initializing dev: %d\n", id);
+		nb_max_matches = info.max_matches;
+		if (info.regexdev_capa & RTE_REGEXDEV_SUPP_MATCH_AS_END_F)
+			conf->dev_cfg_flags |= RTE_REGEXDEV_CFG_MATCH_AS_END_F;
+		dev_conf->nb_max_matches = info.max_matches;
+		dev_conf->nb_rules_per_group = info.max_rules_per_group;
+		dev_conf->db_len = rules_len;
+		dev_conf->rule_db = rules;
+		ret = rte_regexdev_configure(id, &dev_conf);
+		if (ret < 0)
+			rte_exit(EXIT_FAILURE,
+				":: cannot configure device: err=%d, dev=%u\n",
+				ret, port_id);
+		ret = rte_regexdev_queue_pair_setup(id, 0, qp_conf);
+		if (ret < 0)
+			rte_exit(EXIT_FAILURE,
+				":: cannot configure qp %d: err=%d, dev=%u\n",
+				0, ret, port_id);
+		ret = rte_regexdev_start(id);
+		if (ret < 0)
+			rte_exit(EXIT_FAILURE, "can't start dev %d\n", id);
+		printf(":: initializing device: %d done\n", id);
+	}
+}
+
+void
+main_loop(void)
+{
+	char *buf;
+	long buf_len;
+	long job_len;
+	int i;
+	struct rte_regex_ops *ops[1];
+	uint16_t dev_id = 0;
+	uint16_t qp_id = 0;
+	uint32_t job_ind = 0;
+	uint8_t nb_matches;
+	struct rte_regexdev_match *match;
+	struct rte_mbuf *mbuf_job[1];
+	long pos;
+
+	ops[0] = rte_malloc(NULL, sizeof(*ops[0]) + nb_max_matches *
+				    sizeof(struct rte_regexdev_match), 0);
+	if (!ops[0])
+		goto main_loop_err;
+	ops[0]->group_id0 = 0;
+	buf_len = read_file(data_file, &buf);
+	if (buf_len <= 0)
+		return;
+	job_len = buf_len / jobs;
+	mbuf_job = rte_pktmbuf_allocate(mbuf_mp);
+	ops[0]->mbuf = mbuf_job;
+	pos = 0;
+	while (pos < buf_len) {
+		rte_pktmbuf_attach_extbuf(mbuf_job, pos, 0, job_len, NULL);
+		rte_regexdev_enqueue_burst(dev_id, qp_id, ops, 1);
+		while(rte_regexdev_dequeue_burst(dev_id, qp_id, ops, 1));
+		nb_matches = ops[0]->nb_matches;
+		printf("Job id %d number of matches = %d\n", job_ind,
+		       nb_matches);
+		match = ops[0]->matches;
+		for (i = 0; i < nb_maches; i++) {
+			printf("match %d, rule = %d, start = %d, len = %d\n",
+			       i, match->rule_id, match->start_offet,
+			       match->len);
+			match++;
+		}
+		pos += job_len;
+	}
+main_loop_err:
+	if (ops[0])
+		rte_free(ops[0]);
+}
+
+int
+main(int argc, char **argv)
+{
+	int ret;
+
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "EAL init failed\n");
+	argc -= ret;
+	argv += ret;
+	if (argc > 1)
+		args_parse(argc, argv);
+	init_port();
+	main_loop();
+	return 0;
+}
-- 
1.8.3.1

Re: [dpdk-dev] [RFC] app/test-regex: add RegEx test application.

[Jerin Jacob]

On Thu, Jun 4, 2020 at 2:59 AM Ori Kam <orika@mellanox.com> wrote:
>
> Following the new RegEx class [1].
> There is a need to create a dedecated test application in order to
> validate this class and PMD.
>
> Unlike net device this application loads data from a file.
>
> This commit introduce the new RegEx test app.
>
> The basic app flow:
> 1. query probed RegEx device.
> 2. configure the probed device accoding to the device caps.
> 3. read a rule database file.
> 4. read a data file to be searched.
> 5. allocate one mbuf.
> 6. select the request job len, based on the number of jobs requsted.
> 7. create one RegEx op which uses the mbuf defined in step 5.
> 8. attach the data buffer at offset 0 with len accoding to request
>    len to the mbuf as external buffer.
> 9. enqueue one job.
> 10. wait for completion of the job.
> 11. check job results and print them.
> 12. move the mbuf to point to the next part of the buffer.
> 13. while there are more parts jump to 9.
>
> Future improvments to this application may include:
> 1. submiting more then one job at a time.
> 2. working with more then one queue.
> 3. support out of order.
> 4. allow runtime rule updating.
>
> This commit is only at RFC level, which means that
> the code is only basic reference and can't be compiled and doesn't
> include error or edge case handling.
>
> [1]https://patches.dpdk.org/cover/69919/
>
> Signed-off-by: Ori Kam <orika@mellanox.com>

IMO, this kind simile application with fixed-function can go in the
example folder.
I think, for app/test-regex, it better to have a framework to add new
tests, modes, parse the modes, etc.
Please see git log app/test-eventdev/ initial check-ins for the
skeleton as a reference

Re: [dpdk-dev] [RFC] app/test-regex: add RegEx test application.

[Ori Kam]

Hi Jerin,

> -----Original Message-----
> From: Jerin Jacob <jerinjacobk@gmail.com>
> Sent: Thursday, June 4, 2020 7:56 AM
> Subject: Re: [dpdk-dev] [RFC] app/test-regex: add RegEx test application.
> 
> On Thu, Jun 4, 2020 at 2:59 AM Ori Kam <orika@mellanox.com> wrote:
> >
> > Following the new RegEx class [1].
> > There is a need to create a dedecated test application in order to
> > validate this class and PMD.
> >
> > Unlike net device this application loads data from a file.
> >
> > This commit introduce the new RegEx test app.
> >
> > The basic app flow:
> > 1. query probed RegEx device.
> > 2. configure the probed device accoding to the device caps.
> > 3. read a rule database file.
> > 4. read a data file to be searched.
> > 5. allocate one mbuf.
> > 6. select the request job len, based on the number of jobs requsted.
> > 7. create one RegEx op which uses the mbuf defined in step 5.
> > 8. attach the data buffer at offset 0 with len accoding to request
> >    len to the mbuf as external buffer.
> > 9. enqueue one job.
> > 10. wait for completion of the job.
> > 11. check job results and print them.
> > 12. move the mbuf to point to the next part of the buffer.
> > 13. while there are more parts jump to 9.
> >
> > Future improvments to this application may include:
> > 1. submiting more then one job at a time.
> > 2. working with more then one queue.
> > 3. support out of order.
> > 4. allow runtime rule updating.
> >
> > This commit is only at RFC level, which means that
> > the code is only basic reference and can't be compiled and doesn't
> > include error or edge case handling.
> >
> >
> [1]https://eur03.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpatche
> s.dpdk.org%2Fcover%2F69919%2F&amp;data=02%7C01%7Corika%40mellanox.
> com%7C777869b8df174845154008d80843aa33%7Ca652971c7d2e4d9ba6a4d1
> 49256f461b%7C0%7C0%7C637268433993878386&amp;sdata=GzSEh6O0rqPHBl
> OtkTGx3j6VGOdVykga2zYsuyK%2BxtM%3D&amp;reserved=0
> >
> > Signed-off-by: Ori Kam <orika@mellanox.com>
> 
> IMO, this kind simile application with fixed-function can go in the
> example folder.
> I think, for app/test-regex, it better to have a framework to add new
> tests, modes, parse the modes, etc.
> Please see git log app/test-eventdev/ initial check-ins for the
> skeleton as a reference

Currently all dedicateded tests for example, the crypto are located in this folder.
As I see it the example folder is used for simple programs to be used as a reference, while this app should be used to test the
RegEx class, I assume it will get much more functionality to support all test modes.

Best,
Ori

Re: [dpdk-dev] [RFC] app/test-regex: add RegEx test application.

[Jerin Jacob]

On Thu, Jun 4, 2020 at 11:21 AM Ori Kam <orika@mellanox.com> wrote:
>
> Hi Jerin,

HI Ori,

>
> > -----Original Message-----
> > From: Jerin Jacob <jerinjacobk@gmail.com>
> > Sent: Thursday, June 4, 2020 7:56 AM
> > Subject: Re: [dpdk-dev] [RFC] app/test-regex: add RegEx test application.
> >
> > On Thu, Jun 4, 2020 at 2:59 AM Ori Kam <orika@mellanox.com> wrote:
> > >
> > > Following the new RegEx class [1].
> > > There is a need to create a dedecated test application in order to
> > > validate this class and PMD.
> > >
> > > Unlike net device this application loads data from a file.
> > >
> > > This commit introduce the new RegEx test app.
> > >
> > > The basic app flow:
> > > 1. query probed RegEx device.
> > > 2. configure the probed device accoding to the device caps.
> > > 3. read a rule database file.
> > > 4. read a data file to be searched.
> > > 5. allocate one mbuf.
> > > 6. select the request job len, based on the number of jobs requsted.
> > > 7. create one RegEx op which uses the mbuf defined in step 5.
> > > 8. attach the data buffer at offset 0 with len accoding to request
> > >    len to the mbuf as external buffer.
> > > 9. enqueue one job.
> > > 10. wait for completion of the job.
> > > 11. check job results and print them.
> > > 12. move the mbuf to point to the next part of the buffer.
> > > 13. while there are more parts jump to 9.
> > >
> > > Future improvments to this application may include:
> > > 1. submiting more then one job at a time.
> > > 2. working with more then one queue.
> > > 3. support out of order.
> > > 4. allow runtime rule updating.
> > >
> > > This commit is only at RFC level, which means that
> > > the code is only basic reference and can't be compiled and doesn't
> > > include error or edge case handling.
> > >
> > >
> > [1]https://eur03.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpatche
> > s.dpdk.org%2Fcover%2F69919%2F&amp;data=02%7C01%7Corika%40mellanox.
> > com%7C777869b8df174845154008d80843aa33%7Ca652971c7d2e4d9ba6a4d1
> > 49256f461b%7C0%7C0%7C637268433993878386&amp;sdata=GzSEh6O0rqPHBl
> > OtkTGx3j6VGOdVykga2zYsuyK%2BxtM%3D&amp;reserved=0
> > >
> > > Signed-off-by: Ori Kam <orika@mellanox.com>
> >
> > IMO, this kind simile application with fixed-function can go in the
> > example folder.
> > I think, for app/test-regex, it better to have a framework to add new
> > tests, modes, parse the modes, etc.
> > Please see git log app/test-eventdev/ initial check-ins for the
> > skeleton as a reference
>
> Currently all dedicateded tests for example, the crypto are located in this folder.
> As I see it the example folder is used for simple programs to be used as a reference, while this app should be used to test the
> RegEx class, I assume it will get much more functionality to support all test modes.

Yes. We are on the same, What I meant is to create  a framework for
adding testcase and options
first before addingthe test so that it can scale.
see as just reference
app/test-eventdev/evt_test.h
app/test-eventdev/evt_options.h



>
> Best,
> Ori
>
>

Re: [dpdk-dev] [RFC] app/test-regex: add RegEx test application.

[Ori Kam]

Hi Jerin

> -----Original Message-----
> From: Jerin Jacob <jerinjacobk@gmail.com>
> Sent: Thursday, June 4, 2020 9:31 AM
> Subject: Re: [dpdk-dev] [RFC] app/test-regex: add RegEx test application.
> 
> On Thu, Jun 4, 2020 at 11:21 AM Ori Kam <orika@mellanox.com> wrote:
> >
> > Hi Jerin,
> 
> HI Ori,
> 
> >
> > > -----Original Message-----
> > > From: Jerin Jacob <jerinjacobk@gmail.com>
> > > Sent: Thursday, June 4, 2020 7:56 AM
> > > Subject: Re: [dpdk-dev] [RFC] app/test-regex: add RegEx test application.
> > >
> > > On Thu, Jun 4, 2020 at 2:59 AM Ori Kam <orika@mellanox.com> wrote:
> > > >
> > > > Following the new RegEx class [1].
> > > > There is a need to create a dedecated test application in order to
> > > > validate this class and PMD.
> > > >
> > > > Unlike net device this application loads data from a file.
> > > >
> > > > This commit introduce the new RegEx test app.
> > > >
> > > > The basic app flow:
> > > > 1. query probed RegEx device.
> > > > 2. configure the probed device accoding to the device caps.
> > > > 3. read a rule database file.
> > > > 4. read a data file to be searched.
> > > > 5. allocate one mbuf.
> > > > 6. select the request job len, based on the number of jobs requsted.
> > > > 7. create one RegEx op which uses the mbuf defined in step 5.
> > > > 8. attach the data buffer at offset 0 with len accoding to request
> > > >    len to the mbuf as external buffer.
> > > > 9. enqueue one job.
> > > > 10. wait for completion of the job.
> > > > 11. check job results and print them.
> > > > 12. move the mbuf to point to the next part of the buffer.
> > > > 13. while there are more parts jump to 9.
> > > >
> > > > Future improvments to this application may include:
> > > > 1. submiting more then one job at a time.
> > > > 2. working with more then one queue.
> > > > 3. support out of order.
> > > > 4. allow runtime rule updating.
> > > >
> > > > This commit is only at RFC level, which means that
> > > > the code is only basic reference and can't be compiled and doesn't
> > > > include error or edge case handling.
> > > >
> > > >
> > >
> [1]https://eur03.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpatche
> > >
> s.dpdk.org%2Fcover%2F69919%2F&amp;data=02%7C01%7Corika%40mellanox.
> > >
> com%7C777869b8df174845154008d80843aa33%7Ca652971c7d2e4d9ba6a4d1
> > >
> 49256f461b%7C0%7C0%7C637268433993878386&amp;sdata=GzSEh6O0rqPHBl
> > > OtkTGx3j6VGOdVykga2zYsuyK%2BxtM%3D&amp;reserved=0
> > > >
> > > > Signed-off-by: Ori Kam <orika@mellanox.com>
> > >
> > > IMO, this kind simile application with fixed-function can go in the
> > > example folder.
> > > I think, for app/test-regex, it better to have a framework to add new
> > > tests, modes, parse the modes, etc.
> > > Please see git log app/test-eventdev/ initial check-ins for the
> > > skeleton as a reference
> >
> > Currently all dedicateded tests for example, the crypto are located in this
> folder.
> > As I see it the example folder is used for simple programs to be used as a
> reference, while this app should be used to test the
> > RegEx class, I assume it will get much more functionality to support all test
> modes.
> 
> Yes. We are on the same, What I meant is to create  a framework for
> adding testcase and options
> first before addingthe test so that it can scale.
> see as just reference
> app/test-eventdev/evt_test.h
> app/test-eventdev/evt_options.h
> 
> 
I guess some of the new features will be in different files, while others will be just command
line option. The beauty  is that we don't need to decide everything now, and the desition will be based 
on the scale of the feature.

> 
> >
> > Best,
> > Ori
> >
> >