[dpdk-dev] [PATCH v5 4/4] test/hash: add tests for integrated RCU QSBR

[Dharmik Thakkar <dharmik.thakkar@arm.com>]

Add functional and performance tests for the integrated RCU QSBR.

Suggested-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
Signed-off-by: Dharmik Thakkar <dharmik.thakkar@arm.com>
Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
---
 app/test/test_hash.c                   | 390 ++++++++++++++++++++++++-
 app/test/test_hash_readwrite_lf_perf.c | 170 ++++++++++-
 2 files changed, 556 insertions(+), 4 deletions(-)

diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index 990a1815f893..22b47b3e7728 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -52,7 +52,7 @@ static uint32_t hashtest_key_lens[] = {0, 2, 4, 5, 6, 7, 8, 10, 11, 15, 16, 21,
 	}								\
 } while(0)
 
-#define RETURN_IF_ERROR_FBK(cond, str, ...) do {				\
+#define RETURN_IF_ERROR_FBK(cond, str, ...) do {			\
 	if (cond) {							\
 		printf("ERROR line %d: " str "\n", __LINE__, ##__VA_ARGS__); \
 		if (handle) rte_fbk_hash_free(handle);			\
@@ -60,6 +60,20 @@ static uint32_t hashtest_key_lens[] = {0, 2, 4, 5, 6, 7, 8, 10, 11, 15, 16, 21,
 	}								\
 } while(0)
 
+#define RETURN_IF_ERROR_RCU_QSBR(cond, str, ...) do {			\
+	if (cond) {							\
+		printf("ERROR line %d: " str "\n", __LINE__, ##__VA_ARGS__); \
+		if (rcu_cfg.mode == RTE_HASH_QSBR_MODE_SYNC) {		\
+			writer_done = 1;				\
+			/* Wait until reader exited. */			\
+			rte_eal_mp_wait_lcore();			\
+		}							\
+		if (g_handle) rte_hash_free(g_handle);			\
+		if (g_qsv) rte_free(g_qsv);				\
+		return -1;						\
+	}								\
+} while(0)
+
 /* 5-tuple key type */
 struct flow_key {
 	uint32_t ip_src;
@@ -1801,6 +1815,365 @@ test_hash_add_delete_jhash_3word(void)
 	return ret;
 }
 
+static struct rte_hash *g_handle;
+static struct rte_rcu_qsbr *g_qsv;
+static volatile uint8_t writer_done;
+struct flow_key g_rand_keys[9];
+/*
+ * rte_hash_rcu_qsbr_add positive and negative tests.
+ *  - Add RCU QSBR variable to Hash
+ *  - Add another RCU QSBR variable to Hash
+ *  - Check returns
+ */
+static int
+test_hash_rcu_qsbr_add(void)
+{
+	size_t sz;
+	struct rte_rcu_qsbr *qsv2 = NULL;
+	int32_t status;
+	struct rte_hash_rcu_config rcu_cfg = {0};
+
+	struct rte_hash_parameters params;
+
+	printf("\n# Running RCU QSBR add tests\n");
+	memcpy(&params, &ut_params, sizeof(params));
+	params.name = "test_hash_rcu_qsbr_add";
+	params.extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF |
+				RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
+	g_handle = rte_hash_create(&params);
+	RETURN_IF_ERROR_RCU_QSBR(g_handle == NULL, "Hash creation failed");
+
+	/* Create RCU QSBR variable */
+	sz = rte_rcu_qsbr_get_memsize(RTE_MAX_LCORE);
+	g_qsv = (struct rte_rcu_qsbr *)rte_zmalloc_socket(NULL, sz,
+					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+	RETURN_IF_ERROR_RCU_QSBR(g_qsv == NULL,
+				 "RCU QSBR variable creation failed");
+
+	status = rte_rcu_qsbr_init(g_qsv, RTE_MAX_LCORE);
+	RETURN_IF_ERROR_RCU_QSBR(status != 0,
+				 "RCU QSBR variable initialization failed");
+
+	rcu_cfg.v = g_qsv;
+	/* Invalid QSBR mode */
+	rcu_cfg.mode = 2;
+	status = rte_hash_rcu_qsbr_add(g_handle, &rcu_cfg);
+	RETURN_IF_ERROR_RCU_QSBR(status == 0, "Invalid QSBR mode test failed");
+
+	rcu_cfg.mode = RTE_HASH_QSBR_MODE_DQ;
+	/* Attach RCU QSBR to hash table */
+	status = rte_hash_rcu_qsbr_add(g_handle, &rcu_cfg);
+	RETURN_IF_ERROR_RCU_QSBR(status != 0,
+				 "Attach RCU QSBR to hash table failed");
+
+	/* Create and attach another RCU QSBR to hash table */
+	qsv2 = (struct rte_rcu_qsbr *)rte_zmalloc_socket(NULL, sz,
+					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+	RETURN_IF_ERROR_RCU_QSBR(qsv2 == NULL,
+				 "RCU QSBR variable creation failed");
+
+	rcu_cfg.v = qsv2;
+	rcu_cfg.mode = RTE_HASH_QSBR_MODE_SYNC;
+	status = rte_hash_rcu_qsbr_add(g_handle, &rcu_cfg);
+	rte_free(qsv2);
+	RETURN_IF_ERROR_RCU_QSBR(status == 0,
+			"Attach RCU QSBR to hash table succeeded where failure"
+			" is expected");
+
+	rte_hash_free(g_handle);
+	rte_free(g_qsv);
+
+	return 0;
+}
+
+/*
+ * rte_hash_rcu_qsbr_add DQ mode functional test.
+ * Reader and writer are in the same thread in this test.
+ *  - Create hash which supports maximum 8 (9 if ext bkt is enabled) entries
+ *  - Add RCU QSBR variable to hash
+ *  - Add 8 hash entries and fill the bucket
+ *  - If ext bkt is enabled, add 1 extra entry that is available in the ext bkt
+ *  - Register a reader thread (not a real thread)
+ *  - Reader lookup existing entry
+ *  - Writer deletes the entry
+ *  - Reader lookup the entry
+ *  - Writer re-add the entry (no available free index)
+ *  - Reader report quiescent state and unregister
+ *  - Writer re-add the entry
+ *  - Reader lookup the entry
+ */
+static int
+test_hash_rcu_qsbr_dq_mode(uint8_t ext_bkt)
+{
+	uint32_t total_entries = (ext_bkt == 0) ? 8 : 9;
+
+	uint8_t hash_extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF;
+
+	if (ext_bkt)
+		hash_extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE;
+
+	struct rte_hash_parameters params_pseudo_hash = {
+		.name = "test_hash_rcu_qsbr_dq_mode",
+		.entries = total_entries,
+		.key_len = sizeof(struct flow_key), /* 13 */
+		.hash_func = pseudo_hash,
+		.hash_func_init_val = 0,
+		.socket_id = 0,
+		.extra_flag = hash_extra_flag,
+	};
+	int pos[total_entries];
+	int expected_pos[total_entries];
+	unsigned i;
+	size_t sz;
+	int32_t status;
+	struct rte_hash_rcu_config rcu_cfg = {0};
+
+	g_qsv = NULL;
+	g_handle = NULL;
+
+	for (i = 0; i < total_entries; i++) {
+		g_rand_keys[i].port_dst = i;
+		g_rand_keys[i].port_src = i+1;
+	}
+
+	if (ext_bkt)
+		printf("\n# Running RCU QSBR DQ mode functional test with"
+		       " ext bkt\n");
+	else
+		printf("\n# Running RCU QSBR DQ mode functional test\n");
+
+	g_handle = rte_hash_create(&params_pseudo_hash);
+	RETURN_IF_ERROR_RCU_QSBR(g_handle == NULL, "Hash creation failed");
+
+	/* Create RCU QSBR variable */
+	sz = rte_rcu_qsbr_get_memsize(RTE_MAX_LCORE);
+	g_qsv = (struct rte_rcu_qsbr *)rte_zmalloc_socket(NULL, sz,
+					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+	RETURN_IF_ERROR_RCU_QSBR(g_qsv == NULL,
+				 "RCU QSBR variable creation failed");
+
+	status = rte_rcu_qsbr_init(g_qsv, RTE_MAX_LCORE);
+	RETURN_IF_ERROR_RCU_QSBR(status != 0,
+				 "RCU QSBR variable initialization failed");
+
+	rcu_cfg.v = g_qsv;
+	rcu_cfg.mode = RTE_HASH_QSBR_MODE_DQ;
+	/* Attach RCU QSBR to hash table */
+	status = rte_hash_rcu_qsbr_add(g_handle, &rcu_cfg);
+	RETURN_IF_ERROR_RCU_QSBR(status != 0,
+				 "Attach RCU QSBR to hash table failed");
+
+	/* Fill bucket */
+	for (i = 0; i < total_entries; i++) {
+		pos[i] = rte_hash_add_key(g_handle, &g_rand_keys[i]);
+		print_key_info("Add", &g_rand_keys[i], pos[i]);
+		RETURN_IF_ERROR_RCU_QSBR(pos[i] < 0,
+					 "failed to add key (pos[%u]=%d)", i,
+					 pos[i]);
+		expected_pos[i] = pos[i];
+	}
+
+	/* Register pseudo reader */
+	status = rte_rcu_qsbr_thread_register(g_qsv, 0);
+	RETURN_IF_ERROR_RCU_QSBR(status != 0,
+				 "RCU QSBR thread registration failed");
+	rte_rcu_qsbr_thread_online(g_qsv, 0);
+
+	/* Lookup */
+	pos[0] = rte_hash_lookup(g_handle, &g_rand_keys[0]);
+	print_key_info("Lkp", &g_rand_keys[0], pos[0]);
+	RETURN_IF_ERROR_RCU_QSBR(pos[0] != expected_pos[0],
+				 "failed to find correct key (pos[%u]=%d)", 0,
+				 pos[0]);
+
+	/* Writer update */
+	pos[0] = rte_hash_del_key(g_handle, &g_rand_keys[0]);
+	print_key_info("Del", &g_rand_keys[0], pos[0]);
+	RETURN_IF_ERROR_RCU_QSBR(pos[0] != expected_pos[0],
+				 "failed to del correct key (pos[%u]=%d)", 0,
+				 pos[0]);
+
+	/* Lookup */
+	pos[0] = rte_hash_lookup(g_handle, &g_rand_keys[0]);
+	print_key_info("Lkp", &g_rand_keys[0], pos[0]);
+	RETURN_IF_ERROR_RCU_QSBR(pos[0] != -ENOENT,
+				 "found deleted key (pos[%u]=%d)", 0, pos[0]);
+
+	/* Fill bucket */
+	pos[0] = rte_hash_add_key(g_handle, &g_rand_keys[0]);
+	print_key_info("Add", &g_rand_keys[0], pos[0]);
+	RETURN_IF_ERROR_RCU_QSBR(pos[0] != -ENOSPC,
+				 "Added key successfully (pos[%u]=%d)", 0, pos[0]);
+
+	/* Reader quiescent */
+	rte_rcu_qsbr_quiescent(g_qsv, 0);
+
+	/* Fill bucket */
+	pos[0] = rte_hash_add_key(g_handle, &g_rand_keys[0]);
+	print_key_info("Add", &g_rand_keys[0], pos[0]);
+	RETURN_IF_ERROR_RCU_QSBR(pos[0] < 0,
+				 "failed to add key (pos[%u]=%d)", 0, pos[0]);
+	expected_pos[0] = pos[0];
+
+	rte_rcu_qsbr_thread_offline(g_qsv, 0);
+	(void)rte_rcu_qsbr_thread_unregister(g_qsv, 0);
+
+	/* Lookup */
+	pos[0] = rte_hash_lookup(g_handle, &g_rand_keys[0]);
+	print_key_info("Lkp", &g_rand_keys[0], pos[0]);
+	RETURN_IF_ERROR_RCU_QSBR(pos[0] != expected_pos[0],
+				 "failed to find correct key (pos[%u]=%d)", 0,
+				 pos[0]);
+
+	rte_hash_free(g_handle);
+	rte_free(g_qsv);
+	return 0;
+
+}
+
+/* Report quiescent state interval every 1024 lookups. Larger critical
+ * sections in reader will result in writer polling multiple times.
+ */
+#define QSBR_REPORTING_INTERVAL 1024
+#define WRITER_ITERATIONS	512
+
+/*
+ * Reader thread using rte_hash data structure with RCU.
+ */
+static int
+test_hash_rcu_qsbr_reader(void *arg)
+{
+	int i;
+
+	RTE_SET_USED(arg);
+	/* Register this thread to report quiescent state */
+	(void)rte_rcu_qsbr_thread_register(g_qsv, 0);
+	rte_rcu_qsbr_thread_online(g_qsv, 0);
+
+	do {
+		for (i = 0; i < QSBR_REPORTING_INTERVAL; i++)
+			rte_hash_lookup(g_handle, &g_rand_keys[0]);
+
+		/* Update quiescent state */
+		rte_rcu_qsbr_quiescent(g_qsv, 0);
+	} while (!writer_done);
+
+	rte_rcu_qsbr_thread_offline(g_qsv, 0);
+	(void)rte_rcu_qsbr_thread_unregister(g_qsv, 0);
+
+	return 0;
+}
+
+/*
+ * rte_hash_rcu_qsbr_add sync mode functional test.
+ * 1 Reader and 1 writer. They cannot be in the same thread in this test.
+ *  - Create hash which supports maximum 8 (9 if ext bkt is enabled) entries
+ *  - Add RCU QSBR variable to hash
+ *  - Register a reader thread. Reader keeps looking up a specific key.
+ *  - Writer keeps adding and deleting a specific key.
+ */
+static int
+test_hash_rcu_qsbr_sync_mode(uint8_t ext_bkt)
+{
+	uint32_t total_entries = (ext_bkt == 0) ? 8 : 9;
+
+	uint8_t hash_extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF;
+
+	if (ext_bkt)
+		hash_extra_flag |= RTE_HASH_EXTRA_FLAGS_EXT_TABLE;
+
+	struct rte_hash_parameters params_pseudo_hash = {
+		.name = "test_hash_rcu_qsbr_sync_mode",
+		.entries = total_entries,
+		.key_len = sizeof(struct flow_key), /* 13 */
+		.hash_func = pseudo_hash,
+		.hash_func_init_val = 0,
+		.socket_id = 0,
+		.extra_flag = hash_extra_flag,
+	};
+	int pos[total_entries];
+	int expected_pos[total_entries];
+	unsigned i;
+	size_t sz;
+	int32_t status;
+	struct rte_hash_rcu_config rcu_cfg = {0};
+
+	g_qsv = NULL;
+	g_handle = NULL;
+
+	for (i = 0; i < total_entries; i++) {
+		g_rand_keys[i].port_dst = i;
+		g_rand_keys[i].port_src = i+1;
+	}
+
+	if (ext_bkt)
+		printf("\n# Running RCU QSBR sync mode functional test with"
+		       " ext bkt\n");
+	else
+		printf("\n# Running RCU QSBR sync mode functional test\n");
+
+	g_handle = rte_hash_create(&params_pseudo_hash);
+	RETURN_IF_ERROR_RCU_QSBR(g_handle == NULL, "Hash creation failed");
+
+	/* Create RCU QSBR variable */
+	sz = rte_rcu_qsbr_get_memsize(RTE_MAX_LCORE);
+	g_qsv = (struct rte_rcu_qsbr *)rte_zmalloc_socket(NULL, sz,
+					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+	RETURN_IF_ERROR_RCU_QSBR(g_qsv == NULL,
+				 "RCU QSBR variable creation failed");
+
+	status = rte_rcu_qsbr_init(g_qsv, RTE_MAX_LCORE);
+	RETURN_IF_ERROR_RCU_QSBR(status != 0,
+				 "RCU QSBR variable initialization failed");
+
+	rcu_cfg.v = g_qsv;
+	rcu_cfg.mode = RTE_HASH_QSBR_MODE_SYNC;
+	/* Attach RCU QSBR to hash table */
+	status = rte_hash_rcu_qsbr_add(g_handle, &rcu_cfg);
+	RETURN_IF_ERROR_RCU_QSBR(status != 0,
+				 "Attach RCU QSBR to hash table failed");
+
+	/* Launch reader thread */
+	rte_eal_remote_launch(test_hash_rcu_qsbr_reader, NULL,
+				rte_get_next_lcore(-1, 1, 0));
+
+	/* Fill bucket */
+	for (i = 0; i < total_entries; i++) {
+		pos[i] = rte_hash_add_key(g_handle, &g_rand_keys[i]);
+		print_key_info("Add", &g_rand_keys[i], pos[i]);
+		RETURN_IF_ERROR_RCU_QSBR(pos[i] < 0,
+				"failed to add key (pos[%u]=%d)", i, pos[i]);
+		expected_pos[i] = pos[i];
+	}
+	writer_done = 0;
+
+	/* Writer Update */
+	for (i = 0; i < WRITER_ITERATIONS; i++) {
+		expected_pos[0] = pos[0];
+		pos[0] = rte_hash_del_key(g_handle, &g_rand_keys[0]);
+		print_key_info("Del", &g_rand_keys[0], status);
+		RETURN_IF_ERROR_RCU_QSBR(pos[0] != expected_pos[0],
+					 "failed to del correct key (pos[%u]=%d)"
+					 , 0, pos[0]);
+
+		pos[0] = rte_hash_add_key(g_handle, &g_rand_keys[0]);
+		print_key_info("Add", &g_rand_keys[0], pos[0]);
+		RETURN_IF_ERROR_RCU_QSBR(pos[0] < 0,
+					 "failed to add key (pos[%u]=%d)", 0,
+					 pos[0]);
+	}
+
+	writer_done = 1;
+	/* Wait until reader exited. */
+	rte_eal_mp_wait_lcore();
+
+	rte_hash_free(g_handle);
+	rte_free(g_qsv);
+
+	return  0;
+
+}
+
 /*
  * Do all unit and performance tests.
  */
@@ -1862,6 +2235,21 @@ test_hash(void)
 	if (test_crc32_hash_alg_equiv() < 0)
 		return -1;
 
+	if (test_hash_rcu_qsbr_add() < 0)
+		return -1;
+
+	if (test_hash_rcu_qsbr_dq_mode(0) < 0)
+		return -1;
+
+	if (test_hash_rcu_qsbr_dq_mode(1) < 0)
+		return -1;
+
+	if (test_hash_rcu_qsbr_sync_mode(0) < 0)
+		return -1;
+
+	if (test_hash_rcu_qsbr_sync_mode(1) < 0)
+		return -1;
+
 	return 0;
 }
 
diff --git a/app/test/test_hash_readwrite_lf_perf.c b/app/test/test_hash_readwrite_lf_perf.c
index 328fa5116f65..caf18b69c1d4 100644
--- a/app/test/test_hash_readwrite_lf_perf.c
+++ b/app/test/test_hash_readwrite_lf_perf.c
@@ -48,6 +48,9 @@
 #define WRITE_EXT_BKT 2
 
 #define NUM_TEST 3
+
+#define QSBR_REPORTING_INTERVAL 1024
+
 static unsigned int rwc_core_cnt[NUM_TEST] = {1, 2, 4};
 
 struct rwc_perf {
@@ -58,6 +61,7 @@ struct rwc_perf {
 	uint32_t w_ks_r_miss[2][NUM_TEST];
 	uint32_t multi_rw[NUM_TEST - 1][2][NUM_TEST];
 	uint32_t w_ks_r_hit_extbkt[2][NUM_TEST];
+	uint32_t writer_add_del[NUM_TEST];
 };
 
 static struct rwc_perf rwc_lf_results, rwc_non_lf_results;
@@ -84,6 +88,8 @@ static struct {
 
 static uint64_t gread_cycles;
 static uint64_t greads;
+static uint64_t gwrite_cycles;
+static uint64_t gwrites;
 
 static volatile uint8_t writer_done;
 
@@ -1044,7 +1050,7 @@ test_hash_add_ks_lookup_miss(struct rwc_perf *rwc_perf_results, int rwc_lf, int
 /*
  * Test lookup perf for multi-writer:
  * Reader(s) lookup keys present in the table and likely on the shift-path while
- * Writers add keys causing key-shiftsi.
+ * Writers add keys causing key-shifts.
  * Writers are running in parallel, on different data plane cores.
  */
 static int
@@ -1223,6 +1229,163 @@ test_hash_add_ks_lookup_hit_extbkt(struct rwc_perf *rwc_perf_results,
 	return -1;
 }
 
+static struct rte_rcu_qsbr *rv;
+
+/*
+ * Reader thread using rte_hash data structure with RCU
+ */
+static int
+test_hash_rcu_qsbr_reader(void *arg)
+{
+	unsigned int i, j;
+	uint32_t num_keys = tbl_rwc_test_param.count_keys_no_ks
+				- QSBR_REPORTING_INTERVAL;
+	uint32_t *keys = tbl_rwc_test_param.keys_no_ks;
+	uint32_t lcore_id = rte_lcore_id();
+	RTE_SET_USED(arg);
+
+	(void)rte_rcu_qsbr_thread_register(rv, lcore_id);
+	rte_rcu_qsbr_thread_online(rv, lcore_id);
+	do {
+		for (i = 0; i < num_keys; i += j) {
+			for (j = 0; j < QSBR_REPORTING_INTERVAL; j++)
+				rte_hash_lookup(tbl_rwc_test_param.h,
+						keys + i + j);
+			/* Update quiescent state counter */
+			rte_rcu_qsbr_quiescent(rv, lcore_id);
+		}
+	} while (!writer_done);
+	rte_rcu_qsbr_thread_offline(rv, lcore_id);
+	(void)rte_rcu_qsbr_thread_unregister(rv, lcore_id);
+
+	return 0;
+}
+
+/*
+ * Writer thread using rte_hash data structure with RCU
+ */
+static int
+test_hash_rcu_qsbr_writer(void *arg)
+{
+	uint32_t i, offset;
+	uint64_t begin, cycles;
+	uint8_t pos_core = (uint32_t)((uintptr_t)arg);
+	offset = pos_core * tbl_rwc_test_param.single_insert;
+
+	begin = rte_rdtsc_precise();
+	for (i = offset; i < offset + tbl_rwc_test_param.single_insert; i++) {
+		/* Delete element from the shared data structure */
+		rte_hash_del_key(tbl_rwc_test_param.h,
+					tbl_rwc_test_param.keys_no_ks + i);
+		rte_hash_add_key(tbl_rwc_test_param.h,
+				tbl_rwc_test_param.keys_no_ks + i);
+	}
+	cycles = rte_rdtsc_precise() - begin;
+	__atomic_fetch_add(&gwrite_cycles, cycles, __ATOMIC_RELAXED);
+	__atomic_fetch_add(&gwrites, tbl_rwc_test_param.single_insert,
+			   __ATOMIC_RELAXED);
+	return 0;
+}
+
+/*
+ * Writer perf test with RCU QSBR in DQ mode:
+ * Writer(s) delete and add keys in the table.
+ * Readers lookup keys in the hash table
+ */
+static int
+test_hash_rcu_qsbr_writer_perf(struct rwc_perf *rwc_perf_results, int rwc_lf,
+				int htm, int ext_bkt)
+{
+	unsigned int n;
+	uint64_t i;
+	uint8_t write_type;
+	int use_jhash = 0;
+	struct rte_hash_rcu_config rcu_config = {0};
+	uint32_t sz;
+	uint8_t pos_core;
+
+	printf("\nTest: Writer perf with integrated RCU\n");
+
+	if (init_params(rwc_lf, use_jhash, htm, ext_bkt) != 0)
+		goto err;
+
+	sz = rte_rcu_qsbr_get_memsize(RTE_MAX_LCORE);
+	rv = (struct rte_rcu_qsbr *)rte_zmalloc(NULL, sz, RTE_CACHE_LINE_SIZE);
+	rcu_config.v = rv;
+
+	if (rte_hash_rcu_qsbr_add(tbl_rwc_test_param.h, &rcu_config) < 0) {
+		printf("RCU init in hash failed\n");
+		goto err;
+	}
+
+	for (n = 0; n < NUM_TEST; n++) {
+		unsigned int tot_lcore = rte_lcore_count();
+		if (tot_lcore < rwc_core_cnt[n] + 3)
+			goto finish;
+
+		/* Calculate keys added by each writer */
+		tbl_rwc_test_param.single_insert =
+			tbl_rwc_test_param.count_keys_no_ks /
+				rwc_core_cnt[n];
+		printf("\nNumber of writers: %u\n", rwc_core_cnt[n]);
+
+		__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
+		__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
+
+		rte_hash_reset(tbl_rwc_test_param.h);
+		rte_rcu_qsbr_init(rv, RTE_MAX_LCORE);
+
+		write_type = WRITE_NO_KEY_SHIFT;
+		if (write_keys(write_type) < 0)
+			goto err;
+		write_type = WRITE_KEY_SHIFT;
+		if (write_keys(write_type) < 0)
+			goto err;
+
+		/* Launch 2 readers */
+		for (i = 1; i <= 2; i++)
+			rte_eal_remote_launch(test_hash_rcu_qsbr_reader, NULL,
+					      enabled_core_ids[i]);
+		pos_core = 0;
+		/* Launch writer(s) */
+		for (; i <= rwc_core_cnt[n] + 2; i++) {
+			rte_eal_remote_launch(test_hash_rcu_qsbr_writer,
+				(void *)(uintptr_t)pos_core,
+				enabled_core_ids[i]);
+			pos_core++;
+		}
+
+		/* Wait for writers to complete */
+		for (i = 3; i <= rwc_core_cnt[n] + 2; i++)
+			rte_eal_wait_lcore(enabled_core_ids[i]);
+
+		writer_done = 1;
+
+		/* Wait for readers to complete */
+		rte_eal_mp_wait_lcore();
+
+		unsigned long long cycles_per_write_operation =
+			__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
+			__atomic_load_n(&gwrites, __ATOMIC_RELAXED);
+		rwc_perf_results->writer_add_del[n]
+					= cycles_per_write_operation;
+		printf("Cycles per write operation: %llu\n",
+				cycles_per_write_operation);
+	}
+
+finish:
+	rte_hash_free(tbl_rwc_test_param.h);
+	rte_free(rv);
+	return 0;
+
+err:
+	writer_done = 1;
+	rte_eal_mp_wait_lcore();
+	rte_hash_free(tbl_rwc_test_param.h);
+	rte_free(rv);
+	return -1;
+}
+
 static int
 test_hash_readwrite_lf_perf_main(void)
 {
@@ -1235,7 +1398,6 @@ test_hash_readwrite_lf_perf_main(void)
 	int rwc_lf = 0;
 	int htm;
 	int ext_bkt = 0;
-
 	if (rte_lcore_count() < 2) {
 		printf("Not enough cores for hash_readwrite_lf_perf_autotest, expecting at least 2\n");
 		return TEST_SKIPPED;
@@ -1255,7 +1417,6 @@ test_hash_readwrite_lf_perf_main(void)
 		return -1;
 	if (get_enabled_cores_list() != 0)
 		return -1;
-
 	if (RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY_LF) {
 		rwc_lf = 1;
 		ext_bkt = 1;
@@ -1282,6 +1443,9 @@ test_hash_readwrite_lf_perf_main(void)
 		if (test_hash_add_ks_lookup_hit_extbkt(&rwc_lf_results, rwc_lf,
 							htm, ext_bkt) < 0)
 			return -1;
+		if (test_hash_rcu_qsbr_writer_perf(&rwc_lf_results, rwc_lf,
+						   htm, ext_bkt) < 0)
+			return -1;
 	}
 	printf("\nTest lookup with read-write concurrency lock free support"
 	       " disabled\n");
-- 
2.17.1