[dpdk-dev] [RFC 3/3] test/tqs: Add API and functional tests

[Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>]

From: Dharmik Thakkar <dharmik.thakkar@arm.com>

Add API positive/negative test cases and functional tests.

Signed-off-by: Malvika Gupta <malvika.gupta@arm.com>
Signed-off-by: Dharmik Thakkar <dharmik.thakkar@arm.com>
Signed-off-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
Reviewed-by: Gavin Hu <gavin.hu@arm.com>
---
 test/test/Makefile         |   2 +
 test/test/autotest_data.py |   6 +
 test/test/meson.build      |   5 +-
 test/test/test_tqs.c       | 540 +++++++++++++++++++++++++++++++++++++
 4 files changed, 552 insertions(+), 1 deletion(-)
 create mode 100644 test/test/test_tqs.c

diff --git a/test/test/Makefile b/test/test/Makefile
index ab4fec34a..7a07039e7 100644
--- a/test/test/Makefile
+++ b/test/test/Makefile
@@ -207,6 +207,8 @@ SRCS-$(CONFIG_RTE_LIBRTE_KVARGS) += test_kvargs.c
 
 SRCS-$(CONFIG_RTE_LIBRTE_BPF) += test_bpf.c
 
+SRCS-$(CONFIG_RTE_LIBRTE_TQS) += test_tqs.c
+
 CFLAGS += -DALLOW_EXPERIMENTAL_API
 
 CFLAGS += -O3
diff --git a/test/test/autotest_data.py b/test/test/autotest_data.py
index 0fb7866db..e676757cd 100644
--- a/test/test/autotest_data.py
+++ b/test/test/autotest_data.py
@@ -676,6 +676,12 @@
         "Func":    default_autotest,
         "Report":  None,
     },
+    {
+        "Name":    "Thread Quiescent State autotest",
+        "Command": "tqs_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
     #
     # Please always make sure that ring_perf is the last test!
     #
diff --git a/test/test/meson.build b/test/test/meson.build
index 554e9945f..b80a449ad 100644
--- a/test/test/meson.build
+++ b/test/test/meson.build
@@ -100,6 +100,7 @@ test_sources = files('commands.c',
 	'test_timer.c',
 	'test_timer_perf.c',
 	'test_timer_racecond.c',
+	'tet_tqs.c',
 	'test_version.c',
 	'virtual_pmd.c'
 )
@@ -122,7 +123,8 @@ test_deps = ['acl',
 	'port',
 	'reorder',
 	'ring',
-	'timer'
+	'timer',
+	'tqs'
 ]
 
 test_names = [
@@ -228,6 +230,7 @@ test_names = [
 	'timer_autotest',
 	'timer_perf__autotest',
 	'timer_racecond_autotest',
+	'tqs_autotest',
 	'user_delay_us',
 	'version_autotest',
 ]
diff --git a/test/test/test_tqs.c b/test/test/test_tqs.c
new file mode 100644
index 000000000..8633a80a6
--- /dev/null
+++ b/test/test/test_tqs.c
@@ -0,0 +1,540 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018 Arm Limited
+ */
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <rte_pause.h>
+#include <rte_tqs.h>
+#include <rte_hash.h>
+#include <rte_hash_crc.h>
+#include <rte_malloc.h>
+
+#include "test.h"
+
+/* Check condition and return an error if true. */
+#define TQS_RETURN_IF_ERROR(tqs, cond, str, ...) do { \
+	if (cond) { \
+		printf("ERROR file %s, line %d: " str "\n", __FILE__, \
+			__LINE__, ##__VA_ARGS__); \
+		if (tqs) \
+			rte_tqs_free(tqs); \
+		return -1; \
+	} \
+} while (0)
+
+#define RTE_TQS_MAX_LCORE 64
+uint16_t enabled_core_ids[RTE_TQS_MAX_LCORE];
+uint64_t core_mask;
+uint32_t sw_token;
+uint16_t num_1qs = 1; /* Number of quiescent states = 1 */
+
+struct node {
+	int data;
+	struct node *next;
+};
+
+struct node *head = NULL, *lastNode;
+
+static inline int
+get_enabled_cores_mask(void)
+{
+	uint32_t i;
+	uint16_t core_id;
+	uint32_t max_cores = rte_lcore_count();
+
+	if (max_cores > RTE_TQS_MAX_LCORE) {
+		printf("Number of cores exceed %d\n", RTE_TQS_MAX_LCORE);
+		return -1;
+	}
+
+	core_id = 0;
+	core_mask = 0;
+	i = 0;
+	RTE_LCORE_FOREACH_SLAVE(core_id) {
+		enabled_core_ids[i] = core_id;
+		i++;
+		core_mask |= 1UL << core_id;
+	}
+
+	return 0;
+}
+
+/*
+ * rte_tqs_alloc: Allocate a TQS variable
+ */
+static int
+test_tqs_alloc(void)
+{
+	const char name32[] = "xyzxyzxyzxyzxyzxyzxyzxyzxyzx123";
+	const char name33[] = "xyzxyizxyzxyzxyzxyzxyzxyzxyzxyzxyz";
+	const char name3[] = "xyz";
+	struct rte_tqs *t;
+
+	printf("Test rte_tqs_alloc()\n");
+
+	t = rte_tqs_alloc(NULL, SOCKET_ID_ANY, core_mask);
+	TQS_RETURN_IF_ERROR(t, (t != NULL), "NULL TQS variable");
+
+	t = rte_tqs_alloc(name3, SOCKET_ID_ANY, core_mask);
+	TQS_RETURN_IF_ERROR(t, (t == NULL), "Variable name < %d",
+				RTE_TQS_NAMESIZE);
+	rte_tqs_free(t);
+
+	t = rte_tqs_alloc(name32, SOCKET_ID_ANY, core_mask);
+	TQS_RETURN_IF_ERROR(t, (t == NULL), "Variable name < %d",
+				RTE_TQS_NAMESIZE);
+	rte_tqs_free(t);
+
+	t = rte_tqs_alloc(name33, SOCKET_ID_ANY, core_mask);
+	TQS_RETURN_IF_ERROR(t, (t != NULL), "Variable name > %d",
+				RTE_TQS_NAMESIZE);
+
+	t = rte_tqs_alloc(name3, 0, core_mask);
+	TQS_RETURN_IF_ERROR(t, (t == NULL), "Valid socket ID");
+	rte_tqs_free(t);
+
+	t = rte_tqs_alloc(name3, 10000, core_mask);
+	TQS_RETURN_IF_ERROR(t, (t != NULL), "Invalid socket ID");
+
+	t = rte_tqs_alloc(name3, SOCKET_ID_ANY, 0);
+	TQS_RETURN_IF_ERROR(t, (t == NULL), "0 core mask");
+	rte_tqs_free(t);
+
+	return 0;
+}
+
+/*
+ * rte_tqs_register_lcore: Register threads
+ */
+static int
+test_tqs_register_lcore(void)
+{
+	struct rte_tqs *t;
+	const char *name = "TQS";
+	int ret;
+
+	printf("\nTest rte_tqs_register_lcore()\n");
+
+	t = rte_tqs_alloc(name, SOCKET_ID_ANY, 0);
+	TQS_RETURN_IF_ERROR(t, (t == NULL), "Failed to alloc TQS variable");
+
+	ret = rte_tqs_register_lcore(t, enabled_core_ids[0]);
+	TQS_RETURN_IF_ERROR(t, (ret == -EINVAL),
+			    "lcore_id < RTE_TQS_MAX_LCORE");
+	rte_tqs_free(t);
+	return 0;
+}
+
+/*
+ * rte_tqs_unregister_lcore: Unregister threads
+ */
+static int
+test_tqs_unregister_lcore(void)
+{
+	struct rte_tqs *t;
+	const char *name = "TQS";
+	int i, ret;
+
+	printf("\nTest rte_tqs_unregister_lcore()\n");
+
+	t = rte_tqs_alloc(name, SOCKET_ID_ANY, 0);
+	TQS_RETURN_IF_ERROR(t, (t == NULL), "Failed to alloc TQS variable");
+
+	ret = rte_tqs_register_lcore(t, enabled_core_ids[0]);
+	TQS_RETURN_IF_ERROR(t, (ret == -EINVAL), "Register lcore failed");
+
+	/* Find first disabled core */
+	for (i = 0; i < RTE_TQS_MAX_LCORE; i++) {
+		if (enabled_core_ids[i] == 0)
+			break;
+	}
+	ret = rte_tqs_unregister_lcore(t, i);
+	TQS_RETURN_IF_ERROR(t, (ret != 0), "lcore disabled");
+
+	ret = rte_tqs_unregister_lcore(t, enabled_core_ids[0]);
+	TQS_RETURN_IF_ERROR(t, (ret == -EINVAL), "Valid lcore_id");
+
+	rte_tqs_free(t);
+	return 0;
+}
+
+/*
+ * rte_tqs_start: Trigger reader threads to count the number of times they enter
+ * the quiescent state
+ */
+static int
+test_tqs_start(void)
+{
+	struct rte_tqs *t;
+	const char *name = "TQS";
+	uint32_t token;
+	int i, ret;
+
+	printf("\nTest rte_tqs_start()\n");
+
+	t = rte_tqs_alloc(name, SOCKET_ID_ANY, 0);
+	TQS_RETURN_IF_ERROR(t, (t == NULL), "Failed to alloc TQS variable");
+
+	for (i = 0; i < 3; i++) {
+		ret = rte_tqs_register_lcore(t, enabled_core_ids[i]);
+		TQS_RETURN_IF_ERROR(t, (ret == -EINVAL),
+				    "Register lcore failed");
+	}
+
+	ret = rte_tqs_start(t, 1, &token);
+	TQS_RETURN_IF_ERROR(t, (ret == -EINVAL), "1 quiescent states");
+
+	rte_tqs_free(t);
+	return 0;
+}
+
+/*
+ * rte_tqs_check: Check if all reader threads have completed entering the
+ * quiescent state 'n' times
+ */
+static int
+test_tqs_check(void)
+{
+
+	struct rte_tqs *t;
+	const char *name = "TQS";
+	int i, ret;
+	uint32_t token;
+
+	printf("\nTest rte_tqs_check()\n");
+
+	t = rte_tqs_alloc(name, SOCKET_ID_ANY, 0);
+	TQS_RETURN_IF_ERROR(t, (t == NULL), "Failed to alloc TQS variable");
+
+	ret = rte_tqs_start(t, 1, &token);
+	TQS_RETURN_IF_ERROR(t, (ret == -EINVAL), "TQS Start failed");
+
+	ret = rte_tqs_check(t, 0, true);
+	TQS_RETURN_IF_ERROR(t, (ret == -EINVAL), "Token = 0");
+
+	ret = rte_tqs_check(t, token, true);
+	TQS_RETURN_IF_ERROR(t, (ret == -EINVAL), "Blocking TQS check");
+
+	for (i = 0; i < 3; i++) {
+		ret = rte_tqs_register_lcore(t, enabled_core_ids[i]);
+		TQS_RETURN_IF_ERROR(t, (ret == -EINVAL),
+				    "Register lcore failed");
+	}
+
+	ret = rte_tqs_check(t, token, false);
+	TQS_RETURN_IF_ERROR(t, (ret == -EINVAL), "Non-blocking TQS check");
+
+	rte_tqs_free(t);
+	return 0;
+}
+
+/*
+ * rte_tqs_lookup: Lookup a TQS variable by its name
+ */
+static int
+test_tqs_lookup(void)
+{
+	struct rte_tqs *t, *ret;
+	const char *name1 = "TQS";
+	const char *name2 = "NO_TQS";
+
+	printf("\nTest rte_tqs_lookup()\n");
+
+	t = rte_tqs_alloc(name1, SOCKET_ID_ANY, 0);
+	TQS_RETURN_IF_ERROR(t, (t == NULL), "Failed to alloc TQS variable");
+
+	ret = rte_tqs_lookup(name1);
+	TQS_RETURN_IF_ERROR(t, (ret != t), "Allocated TQS variable name");
+
+	ret = rte_tqs_lookup(name2);
+	TQS_RETURN_IF_ERROR(t, (ret != NULL), "Unallocated TQS variable name");
+
+	rte_tqs_free(t);
+	return 0;
+}
+
+/*
+ * rte_tqs_list_dump: Dump the status of all TQS variables to a file
+ */
+static int
+test_tqs_list_dump(void)
+{
+	struct rte_tqs *t1, *t2;
+	const char name1[] = "TQS_1";
+	const char name2[] = "TQS_2";
+	int i, ret;
+
+	printf("\nTest rte_tqs_list_dump()\n");
+
+	/* File pointer NULL */
+	rte_tqs_list_dump(NULL);
+
+	/* Dump an empty list */
+	rte_tqs_list_dump(stdout);
+
+	t1 = rte_tqs_alloc(name1, SOCKET_ID_ANY, 0);
+	TQS_RETURN_IF_ERROR(t1, (t1 == NULL), "Failed to alloc TQS variable");
+
+	/* Dump a list with TQS variable that has no readers */
+	rte_tqs_list_dump(stdout);
+
+	t2 = rte_tqs_alloc(name2, SOCKET_ID_ANY, 0);
+	TQS_RETURN_IF_ERROR(t2, (t2 == NULL), "Failed to alloc TQS variable");
+
+	ret = rte_tqs_register_lcore(t1, enabled_core_ids[0]);
+	if (ret != 0) {
+		printf("ERROR file %s, line %d: Failed to register lcore\n",
+			__FILE__, __LINE__);
+		return -1;
+	}
+
+	for (i = 1; i < 3; i++) {
+		ret = rte_tqs_register_lcore(t2, enabled_core_ids[i]);
+		if (ret != 0) {
+			printf("ERROR file %s, line %d: Failed to register lcore\n",
+				__FILE__, __LINE__);
+			return -1;
+		}
+	}
+
+	rte_tqs_list_dump(stdout);
+
+	rte_tqs_free(t1);
+	rte_tqs_free(t2);
+
+	return 0;
+}
+
+/*
+ * rte_tqs_dump: Dump status of a single TQS variable to a file
+ */
+static int
+test_tqs_dump(void)
+{
+	struct rte_tqs *t1, *t2;
+	const char name1[] = "TQS_1";
+	const char name2[] = "TQS_2";
+	int i, ret;
+
+	printf("\nTest rte_tqs_dump()\n");
+
+	/* NULL TQS variable */
+	rte_tqs_dump(stdout, NULL);
+
+	t1 = rte_tqs_alloc(name1, SOCKET_ID_ANY, 0);
+	TQS_RETURN_IF_ERROR(t1, (t1 == NULL), "Failed to alloc TQS variable");
+
+	/* NULL file pointer */
+	rte_tqs_dump(NULL, t1);
+
+	/* TQS variable with 0 core mask */
+	rte_tqs_dump(stdout, t1);
+
+	t2 = rte_tqs_alloc(name2, SOCKET_ID_ANY, 0);
+	TQS_RETURN_IF_ERROR(t2, (t2 == NULL), "Failed to alloc TQS variable");
+
+	ret = rte_tqs_register_lcore(t1, enabled_core_ids[0]);
+	if (ret != 0) {
+		printf("ERROR file %s, line %d: Failed to register lcore\n",
+			__FILE__, __LINE__);
+		return -1;
+	}
+
+	for (i = 1; i < 3; i++) {
+		ret = rte_tqs_register_lcore(t2, enabled_core_ids[i]);
+		if (ret != 0) {
+			printf("ERROR file %s, line %d: Failed to register lcore\n",
+				__FILE__, __LINE__);
+			return -1;
+		}
+	}
+
+	rte_tqs_dump(stdout, t1);
+	rte_tqs_dump(stdout, t2);
+
+	return 0;
+}
+
+struct rte_hash *handle;
+static uint32_t *keys;
+#define TOTAL_ENTRY (1 * 8)
+#define COUNTER_VALUE 4096
+uint32_t *hash_data[TOTAL_ENTRY];
+uint8_t writer_done;
+static int
+
+test_tqs_reader(__attribute__((unused)) void *arg)
+{
+	struct rte_tqs *t;
+	int i, ret;
+	uint32_t lcore_id = rte_lcore_id();
+
+	t = rte_tqs_lookup("TQS");
+	TQS_RETURN_IF_ERROR(t, (t == NULL), "TQS variable lookup failed");
+
+	do {
+		for (i = 0; i < TOTAL_ENTRY; i++) {
+			uint32_t *pdata;
+			ret = rte_hash_lookup_data(handle, keys+i,
+						   (void **)&pdata);
+			if (ret != -ENOENT)
+				while (*pdata < COUNTER_VALUE)
+					++*pdata;
+		}
+
+		/* Update quiescent state counter */
+		rte_tqs_update(t, lcore_id);
+	} while (!writer_done);
+
+	return 0;
+}
+
+static int
+init_hash(void)
+{
+	int i, ret;
+	struct rte_hash_parameters hash_params = {
+		.entries = TOTAL_ENTRY,
+		.key_len = sizeof(uint32_t),
+		.hash_func_init_val = 0,
+		.socket_id = rte_socket_id(),
+		.hash_func = rte_hash_crc,
+		.extra_flag =
+			RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF,
+		.name = "tests",
+	};
+
+	handle = rte_hash_create(&hash_params);
+	TQS_RETURN_IF_ERROR(NULL, (handle == NULL), "Hash create Failed");
+
+	for (i = 0; i < TOTAL_ENTRY; i++) {
+		hash_data[i] = rte_zmalloc(NULL, sizeof(uint32_t), 0);
+		TQS_RETURN_IF_ERROR(NULL, (hash_data[i] == NULL), "No memory");
+	}
+	keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
+	TQS_RETURN_IF_ERROR(NULL, (keys == NULL), "No memory");
+
+	for (i = 0; i < TOTAL_ENTRY; i++)
+		keys[i] = i;
+
+	for (i = 0; i < TOTAL_ENTRY; i++) {
+		ret = rte_hash_add_key_data(handle, keys + i,
+					    (void *)((uintptr_t)hash_data[i]));
+		TQS_RETURN_IF_ERROR(NULL, (ret < 0),
+				    "Hash key add Failed #%d\n", i);
+	}
+	return 0;
+}
+
+/*
+ * Single writer, 1 TQS variable, 1 quiescent state
+ */
+static int
+test_tqs_sw_sv_1qs(void)
+{
+	struct rte_tqs *t;
+	const char *name = "TQS";
+	static uint32_t token;
+	int i, ret;
+	int32_t pos;
+	writer_done = 0;
+
+	printf("\nTest Single writer, 1 TQS variable, pass 1 quiescent state\n");
+
+	/* TQS variable is allocated */
+	t = rte_tqs_alloc(name, SOCKET_ID_ANY, 0);
+	TQS_RETURN_IF_ERROR(t, (t == NULL), "Failed to alloc TQS variable");
+
+	/* Register worker threads on 4 cores */
+	for (i = 0; i < 4; i++) {
+		ret = rte_tqs_register_lcore(t, enabled_core_ids[i]);
+		TQS_RETURN_IF_ERROR(t, (ret == -EINVAL),
+				    "Register lcore failed");
+	}
+
+	/* Shared data structure created */
+	ret = init_hash();
+	TQS_RETURN_IF_ERROR(t, (ret != 0), "Hash init failed");
+
+	/* Reader threads are launched */
+	for (i = 0; i < 4; i++)
+		rte_eal_remote_launch(test_tqs_reader, NULL,
+					enabled_core_ids[i]);
+
+	for (i = 0; i < TOTAL_ENTRY; i++) {
+		/* Delete elements from the shared data structure */
+		pos = rte_hash_del_key(handle, keys + i);
+		TQS_RETURN_IF_ERROR(t, (pos < 0), "Delete key failed #%d",
+					keys[i]);
+		/* Start the quiescent state query process */
+		ret = rte_tqs_start(t, num_1qs, &token);
+		TQS_RETURN_IF_ERROR(t, (ret != 0), "TQS Start Failed");
+
+		/* Check the quiescent state status */
+		rte_tqs_check(t, token, true);
+		TQS_RETURN_IF_ERROR(t, (*hash_data[i] != COUNTER_VALUE),
+				    "Reader did not complete %d",
+				    *hash_data[i]);
+
+		rte_hash_free_key_with_position(handle, pos);
+		TQS_RETURN_IF_ERROR(t, (ret < 0),
+				    "Failed to free the key #%d", keys[i]);
+		rte_free(hash_data[i]);
+	}
+	writer_done = 1;
+	/* Wait until all readers have exited */
+	rte_eal_mp_wait_lcore();
+
+	/* Free TQS variable */
+	rte_tqs_free(t);
+	rte_hash_free(handle);
+	rte_free(keys);
+
+	return 0;
+}
+
+static int
+test_tqs_main(void)
+{
+	if (get_enabled_cores_mask() != 0)
+		return -1;
+
+	/* Error-checking test cases */
+	if (test_tqs_alloc() < 0)
+		goto test_fail;
+
+	if (test_tqs_register_lcore() < 0)
+		goto test_fail;
+
+	if (test_tqs_unregister_lcore() < 0)
+		goto test_fail;
+
+	if (test_tqs_start() < 0)
+		goto test_fail;
+
+	if (test_tqs_check() < 0)
+		goto test_fail;
+
+	if (test_tqs_lookup() < 0)
+		goto test_fail;
+
+	if (test_tqs_list_dump() < 0)
+		goto test_fail;
+
+	if (test_tqs_dump() < 0)
+		goto test_fail;
+
+
+	/* Functional test cases */
+	if (test_tqs_sw_sv_1qs() < 0)
+		goto test_fail;
+
+	printf("\n");
+	return 0;
+
+ test_fail:
+	return -1;
+}
+
+REGISTER_TEST_COMMAND(tqs_autotest, test_tqs_main);
-- 
2.17.1