[dpdk-dev] [RFC v2 2/2] test/rcu_qsbr: add API and functional tests

[Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>]

From: Dharmik Thakkar <dharmik.thakkar@arm.com>

Add API positive/negative test cases and functional tests.

Signed-off-by: Malvika Gupta <malvika.gupta@arm.com>
Signed-off-by: Dharmik Thakkar <dharmik.thakkar@arm.com>
Signed-off-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
Reviewed-by: Gavin Hu <gavin.hu@arm.com>
---
 test/test/Makefile         |   2 +
 test/test/autotest_data.py |   6 +
 test/test/meson.build      |   5 +-
 test/test/test_rcu_qsbr.c  | 801 +++++++++++++++++++++++++++++++++++++
 4 files changed, 813 insertions(+), 1 deletion(-)
 create mode 100644 test/test/test_rcu_qsbr.c

diff --git a/test/test/Makefile b/test/test/Makefile
index ab4fec34a..dfc0325e4 100644
--- a/test/test/Makefile
+++ b/test/test/Makefile
@@ -207,6 +207,8 @@ SRCS-$(CONFIG_RTE_LIBRTE_KVARGS) += test_kvargs.c
 
 SRCS-$(CONFIG_RTE_LIBRTE_BPF) += test_bpf.c
 
+SRCS-$(CONFIG_RTE_LIBRTE_RCU) += test_rcu_qsbr.c
+
 CFLAGS += -DALLOW_EXPERIMENTAL_API
 
 CFLAGS += -O3
diff --git a/test/test/autotest_data.py b/test/test/autotest_data.py
index 0fb7866db..cbd1f94ad 100644
--- a/test/test/autotest_data.py
+++ b/test/test/autotest_data.py
@@ -676,6 +676,12 @@
         "Func":    default_autotest,
         "Report":  None,
     },
+    {
+        "Name":    "RCU QSBR autotest",
+        "Command": "rcu_qsbr_autotest",
+        "Func":    default_autotest,
+        "Report":  None,
+    },
     #
     # Please always make sure that ring_perf is the last test!
     #
diff --git a/test/test/meson.build b/test/test/meson.build
index 554e9945f..3be21be27 100644
--- a/test/test/meson.build
+++ b/test/test/meson.build
@@ -100,6 +100,7 @@ test_sources = files('commands.c',
 	'test_timer.c',
 	'test_timer_perf.c',
 	'test_timer_racecond.c',
+	'test_rcu_qsbr.c',
 	'test_version.c',
 	'virtual_pmd.c'
 )
@@ -122,7 +123,8 @@ test_deps = ['acl',
 	'port',
 	'reorder',
 	'ring',
-	'timer'
+	'timer',
+	'rcu'
 ]
 
 test_names = [
@@ -228,6 +230,7 @@ test_names = [
 	'timer_autotest',
 	'timer_perf__autotest',
 	'timer_racecond_autotest',
+	'rcu_qsbr_autotest',
 	'user_delay_us',
 	'version_autotest',
 ]
diff --git a/test/test/test_rcu_qsbr.c b/test/test/test_rcu_qsbr.c
new file mode 100644
index 000000000..c9efd3e21
--- /dev/null
+++ b/test/test/test_rcu_qsbr.c
@@ -0,0 +1,801 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2018 Arm Limited
+ */
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <rte_pause.h>
+#include <rte_rcu_qsbr.h>
+#include <rte_hash.h>
+#include <rte_hash_crc.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+
+#include "test.h"
+
+/* Check condition and return an error if true. */
+#define RCU_QSBR_RETURN_IF_ERROR(cond, str, ...) do { \
+	if (cond) { \
+		printf("ERROR file %s, line %d: " str "\n", __FILE__, \
+			__LINE__, ##__VA_ARGS__); \
+		return -1; \
+	} \
+} while (0)
+
+#define RTE_RCU_MAX_LCORE 64
+uint16_t enabled_core_ids[RTE_RCU_MAX_LCORE];
+uint8_t num_cores;
+uint16_t num_1qs = 1; /* Number of quiescent states = 1 */
+uint16_t num_2qs = 2; /* Number of quiescent states = 2 */
+uint16_t num_3qs = 3; /* Number of quiescent states = 3 */
+
+static uint32_t *keys;
+#define TOTAL_ENTRY (1024 * 8)
+#define COUNTER_VALUE 4096
+uint32_t *hash_data[RTE_RCU_MAX_LCORE][TOTAL_ENTRY];
+uint8_t writer_done;
+
+struct rte_rcu_qsbr t[RTE_RCU_MAX_LCORE];
+struct rte_hash *h[RTE_RCU_MAX_LCORE];
+char hash_name[RTE_RCU_MAX_LCORE][8];
+
+static inline int
+get_enabled_cores_mask(void)
+{
+	uint16_t core_id;
+	uint32_t max_cores = rte_lcore_count();
+	if (max_cores > RTE_RCU_MAX_LCORE) {
+		printf("Number of cores exceed %d\n", RTE_RCU_MAX_LCORE);
+		return -1;
+	}
+
+	core_id = 0;
+	num_cores = 0;
+	RTE_LCORE_FOREACH_SLAVE(core_id) {
+		enabled_core_ids[num_cores] = core_id;
+		num_cores++;
+	}
+
+	return 0;
+}
+
+/*
+ * rte_rcu_qsbr_register_thread: Add a reader thread, to the list of threads
+ * reporting their quiescent state on a QS variable.
+ */
+static int
+test_rcu_qsbr_register_thread(void)
+{
+	printf("\nTest rte_rcu_qsbr_register_thread()\n");
+
+	rte_rcu_qsbr_init(&t[0]);
+
+	rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[0]);
+	return 0;
+}
+
+/*
+ * rte_rcu_qsbr_unregister_thread: Remove a reader thread, from the list of
+ * threads reporting their quiescent state on a QS variable.
+ */
+static int
+test_rcu_qsbr_unregister_thread(void)
+{
+	int i, j, ret;
+	uint64_t token;
+	uint8_t num_threads[3] = {1, RTE_RCU_MAX_THREADS, 1};
+
+	printf("\nTest rte_rcu_qsbr_unregister_thread()\n");
+
+	rte_rcu_qsbr_init(&t[0]);
+
+	rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[0]);
+
+	/* Find first disabled core */
+	for (i = 0; i < RTE_RCU_MAX_LCORE; i++) {
+		if (enabled_core_ids[i] == 0)
+			break;
+	}
+	/* Test with disabled lcore */
+	rte_rcu_qsbr_unregister_thread(&t[0], i);
+
+	/* Test with enabled lcore */
+	rte_rcu_qsbr_unregister_thread(&t[0], enabled_core_ids[0]);
+
+	/*
+	 * Test with different thread_ids:
+	 * 1 - thread_id = 0
+	 * 2 - All possible thread_ids, from 0 to RTE_RCU_MAX_THREADS
+	 * 3 - thread_id = RTE_RCU_MAX_THREADS - 1
+	 */
+	for (j = 0; j < 3; j++) {
+		rte_rcu_qsbr_init(&t[0]);
+
+		for (i = 0; i < num_threads[j]; i++)
+			rte_rcu_qsbr_register_thread(&t[0],
+				(j == 2) ? (RTE_RCU_MAX_THREADS - 1) : i);
+
+		rte_rcu_qsbr_start(&t[0], 1, &token);
+		RCU_QSBR_RETURN_IF_ERROR((token != 1), "QSBR Start");
+		/* Update quiescent state counter */
+		for (i = 0; i < num_threads[j]; i++) {
+			/* Skip one update */
+			if (i == 72)
+				continue;
+			rte_rcu_qsbr_update(&t[0],
+				(j == 2) ? (RTE_RCU_MAX_THREADS - 1) : i);
+		}
+
+		if (j == 1) {
+			/* Validate the updates */
+			ret = rte_rcu_qsbr_check(&t[0], token, false);
+			RCU_QSBR_RETURN_IF_ERROR((ret == 1), "Non-blocking QSBR check");
+			/* Update the previously skipped thread */
+			rte_rcu_qsbr_update(&t[0], 72);
+		}
+
+		/* Validate the updates */
+		ret = rte_rcu_qsbr_check(&t[0], token, false);
+		RCU_QSBR_RETURN_IF_ERROR((ret == 0), "Non-blocking QSBR check");
+
+		for (i = 0; i < num_threads[j]; i++)
+			rte_rcu_qsbr_unregister_thread(&t[0],
+				(j == 2) ? (RTE_RCU_MAX_THREADS - 1) : i);
+
+		/* Check with no thread registered */
+		ret = rte_rcu_qsbr_check(&t[0], token, true);
+		RCU_QSBR_RETURN_IF_ERROR((ret == 0), "Blocking QSBR check");
+	}
+	return 0;
+}
+
+/*
+ * rte_rcu_qsbr_start: Trigger the worker threads to report the quiescent state
+ * status.
+ */
+static int
+test_rcu_qsbr_start(void)
+{
+	uint64_t token;
+	int i;
+
+	printf("\nTest rte_rcu_qsbr_start()\n");
+
+	rte_rcu_qsbr_init(&t[0]);
+
+	for (i = 0; i < 3; i++)
+		rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[i]);
+
+	rte_rcu_qsbr_start(&t[0], 1, &token);
+	RCU_QSBR_RETURN_IF_ERROR((token != 1), "QSBR Start");
+	return 0;
+}
+
+/*
+ * rte_rcu_qsbr_check: Checks if all the worker threads have entered the queis-
+ * cent state 'n' number of times. 'n' is provided in rte_rcu_qsbr_start API.
+ */
+static int
+test_rcu_qsbr_check(void)
+{
+	int i, ret;
+	uint64_t token;
+
+	printf("\nTest rte_rcu_qsbr_check()\n");
+
+	rte_rcu_qsbr_init(&t[0]);
+
+	rte_rcu_qsbr_start(&t[0], 1, &token);
+	RCU_QSBR_RETURN_IF_ERROR((token != 1), "QSBR Start");
+
+	ret = rte_rcu_qsbr_check(&t[0], 0, true);
+	RCU_QSBR_RETURN_IF_ERROR((ret == 0), "Token = 0");
+
+	ret = rte_rcu_qsbr_check(&t[0], token, true);
+	RCU_QSBR_RETURN_IF_ERROR((ret == 0), "Blocking QSBR check");
+
+	for (i = 0; i < 3; i++)
+		rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[i]);
+
+	ret = rte_rcu_qsbr_check(&t[0], token, false);
+	RCU_QSBR_RETURN_IF_ERROR((ret == 0), "Non-blocking QSBR check");
+
+	rte_rcu_qsbr_start(&t[0], 1, &token);
+	RCU_QSBR_RETURN_IF_ERROR((token != 2), "QSBR Start");
+
+	ret = rte_rcu_qsbr_check(&t[0], token, false);
+	RCU_QSBR_RETURN_IF_ERROR((ret == 1), "Non-blocking QSBR check");
+
+	for (i = 0; i < 3; i++)
+		rte_rcu_qsbr_unregister_thread(&t[0], enabled_core_ids[i]);
+
+	ret = rte_rcu_qsbr_check(&t[0], token, true);
+	RCU_QSBR_RETURN_IF_ERROR((ret == 0), "Blocking QSBR check");
+
+	return 0;
+}
+
+/*
+ * rte_rcu_qsbr_dump: Dump status of a single QS variable to a file
+ */
+static int
+test_rcu_qsbr_dump(void)
+{
+	int i;
+
+	printf("\nTest rte_rcu_qsbr_dump()\n");
+
+	rte_rcu_qsbr_init(&t[0]);
+	rte_rcu_qsbr_init(&t[1]);
+
+	/* QS variable with 0 core mask */
+	rte_rcu_qsbr_dump(stdout, &t[0]);
+
+	rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[0]);
+
+	for (i = 1; i < 3; i++)
+		rte_rcu_qsbr_register_thread(&t[1], enabled_core_ids[i]);
+
+	rte_rcu_qsbr_dump(stdout, &t[0]);
+	rte_rcu_qsbr_dump(stdout, &t[1]);
+	printf("\n");
+	return 0;
+}
+
+static int
+test_rcu_qsbr_reader(void *arg)
+{
+	struct rte_rcu_qsbr *temp;
+	struct rte_hash *hash = NULL;
+	int i;
+	uint32_t lcore_id = rte_lcore_id();
+	uint8_t read_type = (uint8_t)((uintptr_t)arg);
+	uint32_t *pdata;
+
+	temp = &t[read_type];
+	hash = h[read_type];
+
+	do {
+		rte_rcu_qsbr_register_thread(temp, lcore_id);
+		for (i = 0; i < TOTAL_ENTRY; i++) {
+			if (rte_hash_lookup_data(hash, keys+i,
+					(void **)&pdata) != -ENOENT) {
+				*pdata = 0;
+				while (*pdata < COUNTER_VALUE)
+					++*pdata;
+			}
+		}
+		/* Update quiescent state counter */
+		rte_rcu_qsbr_update(temp, lcore_id);
+		rte_rcu_qsbr_unregister_thread(temp, lcore_id);
+	} while (!writer_done);
+
+	return 0;
+}
+
+static int
+test_rcu_qsbr_writer(void *arg)
+{
+	uint64_t token;
+	int32_t pos;
+	struct rte_rcu_qsbr *temp;
+	struct rte_hash *hash = NULL;
+	uint8_t writer_type = (uint8_t)((uintptr_t)arg);
+
+	temp = &t[(writer_type/2) % RTE_RCU_MAX_LCORE];
+	hash = h[(writer_type/2) % RTE_RCU_MAX_LCORE];
+
+	/* Delete element from the shared data structure */
+	pos = rte_hash_del_key(hash, keys + (writer_type % TOTAL_ENTRY));
+	if (pos < 0) {
+		printf("Delete key failed #%d\n",
+		       keys[writer_type % TOTAL_ENTRY]);
+		return -1;
+	}
+	/*
+	 * Start the quiescent state query process
+	 * Note: Expected Quiescent states kept greater than 1 for test only
+	 */
+	rte_rcu_qsbr_start(temp, writer_type + 1, &token);
+	/* Check the quiescent state status */
+	rte_rcu_qsbr_check(temp, token, true);
+	if (*hash_data[(writer_type/2) % RTE_RCU_MAX_LCORE]
+	    [writer_type % TOTAL_ENTRY] != COUNTER_VALUE &&
+	    *hash_data[(writer_type/2) % RTE_RCU_MAX_LCORE]
+	    [writer_type % TOTAL_ENTRY] != 0) {
+		printf("Reader did not complete #%d = %d\t", writer_type,
+			*hash_data[(writer_type/2) % RTE_RCU_MAX_LCORE]
+				[writer_type % TOTAL_ENTRY]);
+		return -1;
+	}
+
+	if (rte_hash_free_key_with_position(hash, pos) < 0) {
+		printf("Failed to free the key #%d\n",
+		       keys[writer_type % TOTAL_ENTRY]);
+		return -1;
+	}
+	rte_free(hash_data[(writer_type/2) % RTE_RCU_MAX_LCORE]
+				[writer_type % TOTAL_ENTRY]);
+	hash_data[(writer_type/2) % RTE_RCU_MAX_LCORE]
+			[writer_type % TOTAL_ENTRY] = NULL;
+
+	return 0;
+}
+
+static struct rte_hash *
+init_hash(int hash_id)
+{
+	int i;
+	struct rte_hash *h = NULL;
+	sprintf(hash_name[hash_id], "hash%d", hash_id);
+	struct rte_hash_parameters hash_params = {
+		.entries = TOTAL_ENTRY,
+		.key_len = sizeof(uint32_t),
+		.hash_func_init_val = 0,
+		.socket_id = rte_socket_id(),
+		.hash_func = rte_hash_crc,
+		.extra_flag =
+			RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF,
+		.name = hash_name[hash_id],
+	};
+
+	h = rte_hash_create(&hash_params);
+	if (h == NULL) {
+		printf("Hash create Failed\n");
+		return NULL;
+	}
+
+	for (i = 0; i < TOTAL_ENTRY; i++) {
+		hash_data[hash_id][i] = rte_zmalloc(NULL, sizeof(uint32_t), 0);
+		if (hash_data[hash_id][i] == NULL) {
+			printf("No memory\n");
+			return NULL;
+		}
+	}
+	keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
+	if (keys == NULL) {
+		printf("No memory\n");
+		return NULL;
+	}
+
+	for (i = 0; i < TOTAL_ENTRY; i++)
+		keys[i] = i;
+
+	for (i = 0; i < TOTAL_ENTRY; i++) {
+		if (rte_hash_add_key_data(h, keys + i,
+				(void *)((uintptr_t)hash_data[hash_id][i]))
+				< 0) {
+			printf("Hash key add Failed #%d\n", i);
+			return NULL;
+		}
+	}
+	return h;
+}
+
+/*
+ * Functional test:
+ * Single writer, Single QS variable Single QSBR query, Blocking rcu_qsbr_check
+ */
+static int
+test_rcu_qsbr_sw_sv_1qs(void)
+{
+	uint64_t token;
+	int i;
+	int32_t pos;
+	writer_done = 0;
+
+	printf("\nTest: 1 writer, 1 QSBR variable, 1 QSBR Query, "
+	       "Blocking QSBR Check\n");
+
+	/* QS variable is initialized */
+	rte_rcu_qsbr_init(&t[0]);
+
+	/* Register worker threads on 4 cores */
+	for (i = 0; i < 4; i++)
+		rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[i]);
+
+	/* Shared data structure created */
+	h[0] = init_hash(0);
+	if (h[0] == NULL) {
+		printf("Hash init failed\n");
+		goto error;
+	}
+
+	/* Reader threads are launched */
+	for (i = 0; i < 4; i++)
+		rte_eal_remote_launch(test_rcu_qsbr_reader, NULL,
+					enabled_core_ids[i]);
+
+	for (i = 0; i < TOTAL_ENTRY; i++) {
+		/* Delete elements from the shared data structure */
+		pos = rte_hash_del_key(h[0], keys + i);
+		if (pos < 0) {
+			printf("Delete key failed #%d\n", keys[i]);
+			goto error;
+		}
+		/* Start the quiescent state query process */
+		rte_rcu_qsbr_start(&t[0], num_1qs, &token);
+
+		/* Check the quiescent state status */
+		rte_rcu_qsbr_check(&t[0], token, true);
+		if (*hash_data[0][i] != COUNTER_VALUE &&
+			*hash_data[0][i] != 0) {
+			printf("Reader did not complete #%d =  %d\n", i,
+							*hash_data[0][i]);
+			goto error;
+		}
+
+		if (rte_hash_free_key_with_position(h[0], pos) < 0) {
+			printf("Failed to free the key #%d\n", keys[i]);
+			goto error;
+		}
+		rte_free(hash_data[0][i]);
+		hash_data[0][i] = NULL;
+	}
+	writer_done = 1;
+	/* Wait until all readers have exited */
+	rte_eal_mp_wait_lcore();
+	/* Check return value from threads */
+	for (i = 0; i < 4; i++)
+		if (lcore_config[enabled_core_ids[i]].ret < 0)
+			goto error;
+	rte_hash_free(h[0]);
+	rte_free(keys);
+
+	return 0;
+
+error:
+	writer_done = 1;
+	/* Wait until all readers have exited */
+	rte_eal_mp_wait_lcore();
+
+	rte_hash_free(h[0]);
+	rte_free(keys);
+	for (i = 0; i < TOTAL_ENTRY; i++)
+		rte_free(hash_data[0][i]);
+
+	return -1;
+}
+
+/*
+ * Functional test:
+ * Single writer, Single QS variable, Single QSBR query,
+ * Non-blocking rcu_qsbr_check
+ */
+static int
+test_rcu_qsbr_sw_sv_1qs_non_blocking(void)
+{
+	uint64_t token;
+	int i, ret;
+	int32_t pos;
+	writer_done = 0;
+
+	printf("Test: 1 writer, 1 QSBR variable, 1 QSBR Query, "
+	       "Non-Blocking QSBR check\n");
+
+	rte_rcu_qsbr_init(&t[0]);
+	/* Register worker threads on 4 cores */
+	for (i = 0; i < 4; i++)
+		rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[i]);
+
+	/* Shared data structure created */
+	h[0] = init_hash(0);
+	if (h[0] == NULL) {
+		printf("Hash init failed\n");
+		goto error;
+	}
+
+	/* Reader threads are launched */
+	for (i = 0; i < 4; i++)
+		rte_eal_remote_launch(test_rcu_qsbr_reader, NULL,
+					enabled_core_ids[i]);
+
+	for (i = 0; i < TOTAL_ENTRY; i++) {
+		/* Delete elements from the shared data structure */
+		pos = rte_hash_del_key(h[0], keys + i);
+		if (pos < 0) {
+			printf("Delete key failed #%d\n", keys[i]);
+			goto error;
+		}
+		/* Start the quiescent state query process */
+		rte_rcu_qsbr_start(&t[0], num_1qs, &token);
+
+		/* Check the quiescent state status */
+		do {
+			ret = rte_rcu_qsbr_check(&t[0], token, false);
+		} while (ret == 0);
+		if (*hash_data[0][i] != COUNTER_VALUE &&
+			*hash_data[0][i] != 0) {
+			printf("Reader did not complete  #%d = %d\n", i,
+							*hash_data[0][i]);
+			goto error;
+		}
+
+		if (rte_hash_free_key_with_position(h[0], pos) < 0) {
+			printf("Failed to free the key #%d\n", keys[i]);
+			goto error;
+		}
+		rte_free(hash_data[0][i]);
+		hash_data[0][i] = NULL;
+	}
+	writer_done = 1;
+	/* Wait until all readers have exited */
+	rte_eal_mp_wait_lcore();
+	/* Check return value from threads */
+	for (i = 0; i < 4; i++)
+		if (lcore_config[enabled_core_ids[i]].ret < 0)
+			goto error;
+	rte_hash_free(h[0]);
+	rte_free(keys);
+
+	return 0;
+
+error:
+	writer_done = 1;
+	/* Wait until all readers have exited */
+	rte_eal_mp_wait_lcore();
+
+	rte_hash_free(h[0]);
+	rte_free(keys);
+	for (i = 0; i < TOTAL_ENTRY; i++)
+		rte_free(hash_data[0][i]);
+
+	return -1;
+}
+
+/*
+ * Functional test:
+ * Single writer, Single QS variable, simultaneous QSBR Queries
+ */
+static int
+test_rcu_qsbr_sw_sv_3qs(void)
+{
+	uint64_t token[3];
+	int i;
+	int32_t pos[3];
+	writer_done = 0;
+
+	printf("Test: 1 writer, 1 QSBR variable, simultaneous QSBR queries\n");
+
+	rte_rcu_qsbr_init(&t[0]);
+
+	/* Register worker threads on 4 cores */
+	for (i = 0; i < 4; i++)
+		rte_rcu_qsbr_register_thread(&t[0], enabled_core_ids[i]);
+
+	/* Shared data structure created */
+	h[0] = init_hash(0);
+	if (h[0] == NULL) {
+		printf("Hash init failed\n");
+		goto error;
+	}
+
+	/* Reader threads are launched */
+	for (i = 0; i < 4; i++)
+		rte_eal_remote_launch(test_rcu_qsbr_reader, NULL,
+					enabled_core_ids[i]);
+
+	/* Delete element from the shared data structure */
+	pos[0] = rte_hash_del_key(h[0], keys + 0);
+	if (pos[0] < 0) {
+		printf("Delete key failed #%d\n", keys[0]);
+		goto error;
+	}
+	/* Start the quiescent state query process */
+	rte_rcu_qsbr_start(&t[0], num_1qs, &token[0]);
+
+	/* Delete element from the shared data structure */
+	pos[1] = rte_hash_del_key(h[0], keys + 3);
+	if (pos[1] < 0) {
+		printf("Delete key failed #%d\n", keys[3]);
+		goto error;
+	}
+	/*
+	 * Start the quiescent state query process
+	 * Note: num_2qs kept greater than 1 for test only
+	 */
+	rte_rcu_qsbr_start(&t[0], num_2qs, &token[1]);
+
+	/* Delete element from the shared data structure */
+	pos[2] = rte_hash_del_key(h[0], keys + 6);
+	if (pos[2] < 0) {
+		printf("Delete key failed #%d\n", keys[6]);
+		goto error;
+	}
+	/*
+	 * Start the quiescent state query process
+	 * Note: num_3qs kept greater than 1 for test only
+	 */
+	rte_rcu_qsbr_start(&t[0], num_3qs, &token[2]);
+
+	/* Check the quiescent state status */
+	rte_rcu_qsbr_check(&t[0], token[0], true);
+	if (*hash_data[0][0] != COUNTER_VALUE && *hash_data[0][0] != 0) {
+		printf("Reader did not complete #0 = %d\n", *hash_data[0][0]);
+		goto error;
+	}
+
+	if (rte_hash_free_key_with_position(h[0], pos[0]) < 0) {
+		printf("Failed to free the key #%d\n", keys[0]);
+		goto error;
+	}
+	rte_free(hash_data[0][0]);
+	hash_data[0][0] = NULL;
+
+	/* Check the quiescent state status */
+	rte_rcu_qsbr_check(&t[0], token[1], true);
+	if (*hash_data[0][3] != COUNTER_VALUE && *hash_data[0][3] != 0) {
+		printf("Reader did not complete #3 = %d\n", *hash_data[0][3]);
+		goto error;
+	}
+
+	if (rte_hash_free_key_with_position(h[0], pos[1]) < 0) {
+		printf("Failed to free the key #%d\n", keys[3]);
+		goto error;
+	}
+	rte_free(hash_data[0][3]);
+	hash_data[0][3] = NULL;
+
+	/* Check the quiescent state status */
+	rte_rcu_qsbr_check(&t[0], token[2], true);
+	if (*hash_data[0][6] != COUNTER_VALUE && *hash_data[0][6] != 0) {
+		printf("Reader did not complete #6 = %d\n", *hash_data[0][6]);
+		goto error;
+	}
+
+	if (rte_hash_free_key_with_position(h[0], pos[2]) < 0) {
+		printf("Failed to free the key #%d\n", keys[6]);
+		goto error;
+	}
+	rte_free(hash_data[0][6]);
+	hash_data[0][6] = NULL;
+
+	writer_done = 1;
+	/* Wait until all readers have exited */
+	rte_eal_mp_wait_lcore();
+	/* Check return value from threads */
+	for (i = 0; i < 4; i++)
+		if (lcore_config[enabled_core_ids[i]].ret < 0)
+			goto error;
+	rte_hash_free(h[0]);
+	rte_free(keys);
+
+	return 0;
+
+error:
+	writer_done = 1;
+	/* Wait until all readers have exited */
+	rte_eal_mp_wait_lcore();
+
+	rte_hash_free(h[0]);
+	rte_free(keys);
+	for (i = 0; i < TOTAL_ENTRY; i++)
+		rte_free(hash_data[0][i]);
+
+	return -1;
+}
+
+/*
+ * Multi writer, Multiple QS variable, simultaneous QSBR queries
+ */
+static int
+test_rcu_qsbr_mw_mv_mqs(void)
+{
+	int i, j;
+	writer_done = 0;
+	uint8_t test_cores;
+	test_cores = num_cores / 4;
+	test_cores = test_cores * 4;
+
+	printf("Test: %d writers, %d QSBR variable, Simultaneous QSBR queries\n"
+	       , test_cores / 2, test_cores / 4);
+
+	for (i = 0; i < num_cores / 4; i++) {
+		rte_rcu_qsbr_init(&t[i]);
+		h[i] = init_hash(i);
+		if (h[i] == NULL) {
+			printf("Hash init failed\n");
+			goto error;
+		}
+	}
+
+	/* Register worker threads on 2 cores */
+	for (i = 0; i < test_cores / 2; i += 2) {
+		rte_rcu_qsbr_register_thread(&t[i / 2], enabled_core_ids[i]);
+		rte_rcu_qsbr_register_thread(&t[i / 2],
+					     enabled_core_ids[i + 1]);
+	}
+
+	/* Reader threads are launched */
+	for (i = 0; i < test_cores / 2; i++)
+		rte_eal_remote_launch(test_rcu_qsbr_reader,
+				      (void *)(uintptr_t)(i / 2),
+					enabled_core_ids[i]);
+
+	/* Writer threads are launched */
+	for (; i < test_cores; i++)
+		rte_eal_remote_launch(test_rcu_qsbr_writer,
+				      (void *)(uintptr_t)(i - (test_cores / 2)),
+					enabled_core_ids[i]);
+	/* Wait for writers to complete */
+	for (i = test_cores / 2; i < test_cores;  i++)
+		rte_eal_wait_lcore(enabled_core_ids[i]);
+
+	writer_done = 1;
+	/* Wait for readers to complete */
+	rte_eal_mp_wait_lcore();
+
+	/* Check return value from threads */
+	for (i = 0; i < test_cores; i++)
+		if (lcore_config[enabled_core_ids[i]].ret < 0)
+			goto error;
+
+	for (i = 0; i < num_cores / 4; i++)
+		rte_hash_free(h[i]);
+
+	rte_free(keys);
+
+	return 0;
+
+error:
+	writer_done = 1;
+	/* Wait until all readers have exited */
+	rte_eal_mp_wait_lcore();
+
+	for (i = 0; i < num_cores / 4; i++)
+		rte_hash_free(h[i]);
+	rte_free(keys);
+	for (j = 0; j < RTE_RCU_MAX_LCORE; j++)
+		for (i = 0; i < TOTAL_ENTRY; i++)
+			rte_free(hash_data[j][i]);
+
+	return -1;
+}
+
+static int
+test_rcu_qsbr_main(void)
+{
+	if (get_enabled_cores_mask() != 0)
+		return -1;
+
+	/* Error-checking test cases */
+	if (test_rcu_qsbr_register_thread() < 0)
+		goto test_fail;
+
+	if (test_rcu_qsbr_unregister_thread() < 0)
+		goto test_fail;
+
+	if (test_rcu_qsbr_start() < 0)
+		goto test_fail;
+
+	if (test_rcu_qsbr_check() < 0)
+		goto test_fail;
+
+	if (test_rcu_qsbr_dump() < 0)
+		goto test_fail;
+
+	/* Functional test cases */
+	if (num_cores < 4) {
+		printf("Test failed! Need 4 or more cores\n");
+		goto test_fail;
+	}
+	if (test_rcu_qsbr_sw_sv_1qs() < 0)
+		goto test_fail;
+
+	if (test_rcu_qsbr_sw_sv_1qs_non_blocking() < 0)
+		goto test_fail;
+
+	if (test_rcu_qsbr_sw_sv_3qs() < 0)
+		goto test_fail;
+
+	if (test_rcu_qsbr_mw_mv_mqs() < 0)
+		goto test_fail;
+
+	printf("\n");
+	return 0;
+
+test_fail:
+	return -1;
+}
+
+REGISTER_TEST_COMMAND(rcu_qsbr_autotest, test_rcu_qsbr_main);
-- 
2.17.1