From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga03.intel.com (mga03.intel.com [134.134.136.65]) by dpdk.org (Postfix) with ESMTP id D9BEE1BB4B for ; Fri, 8 Jun 2018 19:56:41 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by orsmga103.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 08 Jun 2018 10:56:37 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.49,490,1520924400"; d="scan'208";a="55270345" Received: from skx-yipeng.jf.intel.com ([10.54.81.175]) by fmsmga002.fm.intel.com with ESMTP; 08 Jun 2018 10:56:37 -0700 From: Yipeng Wang To: pablo.de.lara.guarch@intel.com Cc: dev@dpdk.org, yipeng1.wang@intel.com, john.mcnamara@intel.com, bruce.richardson@intel.com, honnappa.nagarahalli@arm.com, vguvva@caviumnetworks.com, brijesh.s.singh@gmail.com Date: Fri, 8 Jun 2018 03:51:17 -0700 Message-Id: <1528455078-328182-3-git-send-email-yipeng1.wang@intel.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1528455078-328182-1-git-send-email-yipeng1.wang@intel.com> References: <1528455078-328182-1-git-send-email-yipeng1.wang@intel.com> Subject: [dpdk-dev] [PATCH v1 2/3] test: add test case for read write concurrency X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 08 Jun 2018 17:56:42 -0000 This commit adds a new test case for testing read/write concurrency. Signed-off-by: Yipeng Wang --- test/test/Makefile | 1 + test/test/test_hash_perf.c | 36 ++- test/test/test_hash_readwrite.c | 649 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 675 insertions(+), 11 deletions(-) create mode 100644 test/test/test_hash_readwrite.c diff --git a/test/test/Makefile b/test/test/Makefile index eccc8ef..6ce66c9 100644 --- a/test/test/Makefile +++ b/test/test/Makefile @@ -113,6 +113,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_perf.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_functions.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_scaling.c SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_multiwriter.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_readwrite.c SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm.c SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm_perf.c diff --git a/test/test/test_hash_perf.c b/test/test/test_hash_perf.c index a81d0c7..33dcb9f 100644 --- a/test/test/test_hash_perf.c +++ b/test/test/test_hash_perf.c @@ -76,7 +76,8 @@ static struct rte_hash_parameters ut_params = { }; static int -create_table(unsigned with_data, unsigned table_index) +create_table(unsigned int with_data, unsigned int table_index, + unsigned int with_locks) { char name[RTE_HASH_NAMESIZE]; @@ -86,6 +87,14 @@ create_table(unsigned with_data, unsigned table_index) else sprintf(name, "test_hash%d", hashtest_key_lens[table_index]); + + if (with_locks) + ut_params.extra_flag = + RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT + | RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY; + else + ut_params.extra_flag = 0; + ut_params.name = name; ut_params.key_len = hashtest_key_lens[table_index]; ut_params.socket_id = rte_socket_id(); @@ -459,7 +468,7 @@ reset_table(unsigned table_index) } static int -run_all_tbl_perf_tests(unsigned with_pushes) +run_all_tbl_perf_tests(unsigned int with_pushes, unsigned int with_locks) { unsigned i, j, with_data, with_hash; @@ -468,7 +477,7 @@ run_all_tbl_perf_tests(unsigned with_pushes) for (with_data = 0; with_data <= 1; with_data++) { for (i = 0; i < NUM_KEYSIZES; i++) { - if (create_table(with_data, i) < 0) + if (create_table(with_data, i, with_locks) < 0) return -1; if (get_input_keys(with_pushes, i) < 0) @@ -611,15 +620,20 @@ fbk_hash_perf_test(void) static int test_hash_perf(void) { - unsigned with_pushes; - - for (with_pushes = 0; with_pushes <= 1; with_pushes++) { - if (with_pushes == 0) - printf("\nALL ELEMENTS IN PRIMARY LOCATION\n"); + unsigned int with_pushes, with_locks; + for (with_locks = 0; with_locks <= 1; with_locks++) { + if (with_locks) + printf("\nWith locks in the code\n"); else - printf("\nELEMENTS IN PRIMARY OR SECONDARY LOCATION\n"); - if (run_all_tbl_perf_tests(with_pushes) < 0) - return -1; + printf("\nWithout locks in the code\n"); + for (with_pushes = 0; with_pushes <= 1; with_pushes++) { + if (with_pushes == 0) + printf("\nALL ELEMENTS IN PRIMARY LOCATION\n"); + else + printf("\nELEMENTS IN PRIMARY OR SECONDARY LOCATION\n"); + if (run_all_tbl_perf_tests(with_pushes, with_locks) < 0) + return -1; + } } if (fbk_hash_perf_test() < 0) return -1; diff --git a/test/test/test_hash_readwrite.c b/test/test/test_hash_readwrite.c new file mode 100644 index 0000000..ef3bbe5 --- /dev/null +++ b/test/test/test_hash_readwrite.c @@ -0,0 +1,649 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "test.h" + + +#define RTE_APP_TEST_HASH_MULTIWRITER_FAILED 0 + +#define TOTAL_ENTRY (16*1024*1024) +#define TOTAL_INSERT (15*1024*1024) + +#define NUM_TEST 3 + + +unsigned int core_cnt[NUM_TEST] = {2, 4, 8}; + +struct perf { + uint32_t single_read; + uint32_t single_write; + uint32_t read_only[NUM_TEST]; + uint32_t write_only[NUM_TEST]; + uint32_t read_write_r[NUM_TEST]; + uint32_t read_write_w[NUM_TEST]; +}; + +static struct perf htm_results, non_htm_results; + +struct { + uint32_t *keys; + uint32_t *found; + uint32_t nb_tsx_insertion; + uint32_t rounded_nb_total_tsx_insertion; + struct rte_hash *h; +} tbl_multiwriter_test_params; + +static rte_atomic64_t gcycles; +static rte_atomic64_t ginsertions; + +static rte_atomic64_t gread_cycles; +static rte_atomic64_t gwrite_cycles; + +static rte_atomic64_t greads; +static rte_atomic64_t gwrites; + +static int use_htm; + +static int reader_faster; + +static int +test_hash_readwrite_worker(__attribute__((unused)) void *arg) +{ + uint64_t i, offset; + uint32_t lcore_id = rte_lcore_id(); + uint64_t begin, cycles; + int ret; + + offset = (lcore_id - rte_get_master_lcore()) + * tbl_multiwriter_test_params.nb_tsx_insertion; + + printf("Core #%d inserting and reading %d: %'"PRId64" - %'"PRId64"\n", + lcore_id, tbl_multiwriter_test_params.nb_tsx_insertion, + offset, offset + tbl_multiwriter_test_params.nb_tsx_insertion); + + begin = rte_rdtsc_precise(); + + + for (i = offset; + i < offset + tbl_multiwriter_test_params.nb_tsx_insertion; + i++) { + + if (rte_hash_lookup(tbl_multiwriter_test_params.h, + tbl_multiwriter_test_params.keys + i) > 0) + break; + + ret = rte_hash_add_key(tbl_multiwriter_test_params.h, + tbl_multiwriter_test_params.keys + i); + if (ret < 0) + break; + + if (rte_hash_lookup(tbl_multiwriter_test_params.h, + tbl_multiwriter_test_params.keys + i) != ret) + break; + } + + cycles = rte_rdtsc_precise() - begin; + rte_atomic64_add(&gcycles, cycles); + rte_atomic64_add(&ginsertions, i - offset); + + for (; i < offset + tbl_multiwriter_test_params.nb_tsx_insertion; i++) + tbl_multiwriter_test_params.keys[i] + = RTE_APP_TEST_HASH_MULTIWRITER_FAILED; + + + return 0; +} + + +static int +init_params(void) +{ + unsigned int i; + + uint32_t *keys = NULL; + uint32_t *found = NULL; + struct rte_hash *handle; + char name[RTE_HASH_NAMESIZE]; + + + struct rte_hash_parameters hash_params = { + .entries = TOTAL_ENTRY, + .key_len = sizeof(uint32_t), + .hash_func = rte_hash_crc, + .hash_func_init_val = 0, + .socket_id = rte_socket_id(), + }; + if (use_htm) + hash_params.extra_flag = + RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT + | RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY; + else + hash_params.extra_flag = + RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY; + + snprintf(name, 32, "tests"); + hash_params.name = name; + + handle = rte_hash_create(&hash_params); + if (handle == NULL) { + printf("hash creation failed"); + return -1; + } + + tbl_multiwriter_test_params.h = handle; + keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0); + + if (keys == NULL) { + printf("RTE_MALLOC failed\n"); + goto err1; + } + + found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0); + if (found == NULL) { + printf("RTE_ZMALLOC failed\n"); + goto err2; + } + + + tbl_multiwriter_test_params.keys = keys; + tbl_multiwriter_test_params.found = found; + + for (i = 0; i < TOTAL_ENTRY; i++) + keys[i] = i; + + return 0; + +err2: + rte_free(keys); +err1: + rte_hash_free(handle); + + return -1; +} + +static int +test_hash_readwrite_functional(void) +{ + unsigned int i; + const void *next_key; + void *next_data; + uint32_t iter = 0; + + uint32_t duplicated_keys = 0; + uint32_t lost_keys = 0; + + rte_atomic64_init(&gcycles); + rte_atomic64_clear(&gcycles); + + rte_atomic64_init(&ginsertions); + rte_atomic64_clear(&ginsertions); + + if (init_params() != 0) + goto err; + + tbl_multiwriter_test_params.nb_tsx_insertion = + TOTAL_INSERT / rte_lcore_count(); + + tbl_multiwriter_test_params.rounded_nb_total_tsx_insertion = + tbl_multiwriter_test_params.nb_tsx_insertion + * rte_lcore_count(); + + printf("++++++++Start function tests:+++++++++\n"); + + /* Fire all threads. */ + rte_eal_mp_remote_launch(test_hash_readwrite_worker, + NULL, CALL_MASTER); + rte_eal_mp_wait_lcore(); + + while (rte_hash_iterate(tbl_multiwriter_test_params.h, &next_key, + &next_data, &iter) >= 0) { + /* Search for the key in the list of keys added .*/ + i = *(const uint32_t *)next_key; + tbl_multiwriter_test_params.found[i]++; + } + + for (i = 0; + i < tbl_multiwriter_test_params.rounded_nb_total_tsx_insertion; + i++) { + if (tbl_multiwriter_test_params.keys[i] + != RTE_APP_TEST_HASH_MULTIWRITER_FAILED) { + if (tbl_multiwriter_test_params.found[i] > 1) { + duplicated_keys++; + break; + } + if (tbl_multiwriter_test_params.found[i] == 0) { + lost_keys++; + printf("key %d is lost\n", i); + break; + } + } + } + + if (duplicated_keys > 0) { + printf("%d key duplicated\n", duplicated_keys); + goto err_free; + } + + if (lost_keys > 0) { + printf("%d key lost\n", lost_keys); + goto err_free; + } + + printf("No key corrupted during multiwriter insertion.\n"); + + unsigned long long int cycles_per_insertion = + rte_atomic64_read(&gcycles)/ + rte_atomic64_read(&ginsertions); + + printf(" cycles per insertion and lookup: %llu\n", + cycles_per_insertion); + + rte_free(tbl_multiwriter_test_params.found); + rte_free(tbl_multiwriter_test_params.keys); + rte_hash_free(tbl_multiwriter_test_params.h); + printf("+++++++++Complete function tests+++++++++\n"); + return 0; + +err_free: + rte_free(tbl_multiwriter_test_params.found); + rte_free(tbl_multiwriter_test_params.keys); + rte_hash_free(tbl_multiwriter_test_params.h); +err: + return -1; +} + + +static int +test_rw_reader(__attribute__((unused)) void *arg) +{ + unsigned int i; + uint64_t begin, cycles; + uint64_t read_cnt = (uint64_t)((uintptr_t)arg); + + begin = rte_rdtsc_precise(); + for (i = 0; i < read_cnt; i++) { + void *data; + rte_hash_lookup_data(tbl_multiwriter_test_params.h, + tbl_multiwriter_test_params.keys + i, + &data); + if (i != (uint64_t)data) { + printf("lookup find wrong value %d, %ld\n", i, + (uint64_t)data); + break; + } + } + + cycles = rte_rdtsc_precise() - begin; + rte_atomic64_add(&gread_cycles, cycles); + rte_atomic64_add(&greads, i); + return 0; + +} + +static int +test_rw_writer(__attribute__((unused)) void *arg) +{ + unsigned int i; + uint32_t lcore_id = rte_lcore_id(); + uint64_t begin, cycles; + int ret; + uint64_t start_coreid = (uint64_t)arg; + uint64_t offset; + + offset = TOTAL_INSERT / 2 + (lcore_id - start_coreid) + * tbl_multiwriter_test_params.nb_tsx_insertion; + begin = rte_rdtsc_precise(); + for (i = offset; i < offset + + tbl_multiwriter_test_params.nb_tsx_insertion; + i++) { + ret = rte_hash_add_key_data(tbl_multiwriter_test_params.h, + tbl_multiwriter_test_params.keys + i, + (void *)((uintptr_t)i)); + if (ret < 0) { + printf("writer failed %d\n", i); + break; + } + } + + cycles = rte_rdtsc_precise() - begin; + rte_atomic64_add(&gwrite_cycles, cycles); + rte_atomic64_add(&gwrites, + tbl_multiwriter_test_params.nb_tsx_insertion); + return 0; +} + + +static int +test_hash_readwrite_perf(struct perf *perf_results) +{ + unsigned int i, n; + int ret; + int start_coreid; + uint64_t read_cnt; + + const void *next_key; + void *next_data; + uint32_t iter = 0; + + uint32_t duplicated_keys = 0; + uint32_t lost_keys = 0; + + uint64_t start = 0, end = 0; + + rte_atomic64_init(&greads); + rte_atomic64_init(&gwrites); + rte_atomic64_clear(&gwrites); + rte_atomic64_clear(&greads); + + rte_atomic64_init(&gread_cycles); + rte_atomic64_clear(&gread_cycles); + rte_atomic64_init(&gwrite_cycles); + rte_atomic64_clear(&gwrite_cycles); + + if (init_params() != 0) + goto err; + + if (reader_faster) { + printf("++++++Start perf test: reader++++++++\n"); + read_cnt = TOTAL_INSERT / 10; + } else { + printf("++++++Start perf test: writer++++++++\n"); + read_cnt = TOTAL_INSERT / 2; + } + + + /* We first test single thread performance */ + start = rte_rdtsc_precise(); + /* Insert half of the keys */ + for (i = 0; i < TOTAL_INSERT / 2; i++) { + ret = rte_hash_add_key_data(tbl_multiwriter_test_params.h, + tbl_multiwriter_test_params.keys + i, + (void *)((uintptr_t)i)); + if (ret < 0) { + printf("Failed to insert half of keys\n"); + goto err_free; + } + } + end = rte_rdtsc_precise() - start; + perf_results->single_write = end / i; + + start = rte_rdtsc_precise(); + + for (i = 0; i < read_cnt; i++) { + void *data; + rte_hash_lookup_data(tbl_multiwriter_test_params.h, + tbl_multiwriter_test_params.keys + i, + &data); + if (i != (uint64_t)data) { + printf("lookup find wrong value %d, %ld\n", i, + (uint64_t)data); + break; + } + } + end = rte_rdtsc_precise() - start; + perf_results->single_read = end / i; + + + + for (n = 0; n < NUM_TEST; n++) { + + rte_atomic64_clear(&greads); + rte_atomic64_clear(&gread_cycles); + rte_atomic64_clear(&gwrites); + rte_atomic64_clear(&gwrite_cycles); + + rte_hash_reset(tbl_multiwriter_test_params.h); + + tbl_multiwriter_test_params.nb_tsx_insertion = TOTAL_INSERT / + 2 / core_cnt[n]; + tbl_multiwriter_test_params.rounded_nb_total_tsx_insertion = + TOTAL_INSERT / 2 + + tbl_multiwriter_test_params.nb_tsx_insertion * + core_cnt[n]; + + + for (i = 0; i < TOTAL_INSERT / 2; i++) { + ret = rte_hash_add_key_data( + tbl_multiwriter_test_params.h, + tbl_multiwriter_test_params.keys + i, + (void *)((uintptr_t)i)); + if (ret < 0) { + printf("Failed to insert half of keys\n"); + goto err_free; + } + } + + /* Then test multiple thread case but only all reads or + * all writes + */ + + /* Test only reader cases */ + for (i = 1; i <= core_cnt[n]; i++) + rte_eal_remote_launch(test_rw_reader, + (void *)read_cnt, i); + + rte_eal_mp_wait_lcore(); + + start_coreid = i; + /* Test only writer cases */ + for (; i <= core_cnt[n] * 2; i++) + rte_eal_remote_launch(test_rw_writer, + (void *)((uintptr_t)start_coreid), i); + + + rte_eal_mp_wait_lcore(); + + if (reader_faster) { + unsigned long long int cycles_per_insertion = + rte_atomic64_read(&gread_cycles)/ + rte_atomic64_read(&greads); + perf_results->read_only[n] = cycles_per_insertion; + printf("Reader only: cycles per lookup: %llu\n", + cycles_per_insertion); + } + + else { + unsigned long long int cycles_per_insertion = + rte_atomic64_read(&gwrite_cycles)/ + rte_atomic64_read(&gwrites); + perf_results->write_only[n] = cycles_per_insertion; + printf("Writer only: cycles per writes: %llu\n", + cycles_per_insertion); + } + + rte_atomic64_clear(&greads); + rte_atomic64_clear(&gread_cycles); + rte_atomic64_clear(&gwrites); + rte_atomic64_clear(&gwrite_cycles); + + rte_hash_reset(tbl_multiwriter_test_params.h); + + for (i = 0; i < TOTAL_INSERT / 2; i++) { + ret = rte_hash_add_key_data( + tbl_multiwriter_test_params.h, + tbl_multiwriter_test_params.keys + i, + (void *)((uintptr_t)i)); + if (ret < 0) { + printf("Failed to insert half of keys\n"); + goto err_free; + } + } + + + start_coreid = core_cnt[n] + 1; + + if (reader_faster) { + for (i = core_cnt[n] + 1; i <= core_cnt[n] * 2; i++) + rte_eal_remote_launch(test_rw_writer, + (void *)((uintptr_t)start_coreid), i); + for (i = 1; i <= core_cnt[n]; i++) + rte_eal_remote_launch(test_rw_reader, + (void *)read_cnt, i); + } else { + for (i = 1; i <= core_cnt[n]; i++) + rte_eal_remote_launch(test_rw_reader, + (void *)read_cnt, i); + for (; i <= core_cnt[n] * 2; i++) + rte_eal_remote_launch(test_rw_writer, + (void *)((uintptr_t)start_coreid), i); + } + + rte_eal_mp_wait_lcore(); + + while (rte_hash_iterate(tbl_multiwriter_test_params.h, + &next_key, &next_data, &iter) >= 0) { + /* Search for the key in the list of keys added .*/ + i = *(const uint32_t *)next_key; + tbl_multiwriter_test_params.found[i]++; + } + + + for (i = 0; i < + tbl_multiwriter_test_params.rounded_nb_total_tsx_insertion; + i++) { + if (tbl_multiwriter_test_params.keys[i] + != RTE_APP_TEST_HASH_MULTIWRITER_FAILED) { + if (tbl_multiwriter_test_params.found[i] > 1) { + duplicated_keys++; + break; + } + if (tbl_multiwriter_test_params.found[i] == 0) { + lost_keys++; + printf("key %d is lost\n", i); + break; + } + } + } + + if (duplicated_keys > 0) { + printf("%d key duplicated\n", duplicated_keys); + goto err_free; + } + + if (lost_keys > 0) { + printf("%d key lost\n", lost_keys); + goto err_free; + } + + printf("No key corrupted during multiwriter insertion.\n"); + + if (reader_faster) { + unsigned long long int cycles_per_insertion = + rte_atomic64_read(&gread_cycles) / + rte_atomic64_read(&greads); + perf_results->read_write_r[n] = cycles_per_insertion; + printf("Read-write cycles per lookup: %llu\n", + cycles_per_insertion); + } + + else { + unsigned long long int cycles_per_insertion = + rte_atomic64_read(&gwrite_cycles) / + rte_atomic64_read(&gwrites); + perf_results->read_write_w[n] = cycles_per_insertion; + printf("Read-write cycles per writes: %llu\n", + cycles_per_insertion); + } + } + + rte_free(tbl_multiwriter_test_params.found); + rte_free(tbl_multiwriter_test_params.keys); + rte_hash_free(tbl_multiwriter_test_params.h); + return 0; + +err_free: + rte_free(tbl_multiwriter_test_params.found); + rte_free(tbl_multiwriter_test_params.keys); + rte_hash_free(tbl_multiwriter_test_params.h); + +err: + return -1; +} + + +static int +test_hash_readwrite_main(void) +{ + if (rte_lcore_count() == 1) { + printf("More than one lcore is required " + "to do read write test\n"); + return 0; + } + + + setlocale(LC_NUMERIC, ""); + + if (rte_tm_supported()) { + printf("Hardware transactional memory (lock elision) " + "is supported\n"); + + printf("Test multi-writer with Hardware " + "transactional memory\n"); + + use_htm = 1; + if (test_hash_readwrite_functional() < 0) + return -1; + + reader_faster = 1; + if (test_hash_readwrite_perf(&htm_results) < 0) + return -1; + + reader_faster = 0; + if (test_hash_readwrite_perf(&htm_results) < 0) + return -1; + } else { + printf("Hardware transactional memory (lock elision) " + "is NOT supported\n"); + } + + printf("Test multi-writer without Hardware transactional memory\n"); + use_htm = 0; + if (test_hash_readwrite_functional() < 0) + return -1; + reader_faster = 1; + if (test_hash_readwrite_perf(&non_htm_results) < 0) + return -1; + reader_faster = 0; + if (test_hash_readwrite_perf(&non_htm_results) < 0) + return -1; + + + printf("Results summary:\n"); + + int i; + + printf("single read: %u\n", htm_results.single_read); + printf("single write: %u\n", htm_results.single_write); + for (i = 0; i < NUM_TEST; i++) { + printf("core_cnt: %u\n", core_cnt[i]); + printf("HTM:\n"); + printf("read only: %u\n", htm_results.read_only[i]); + printf("write only: %u\n", htm_results.write_only[i]); + printf("read-write read: %u\n", htm_results.read_write_r[i]); + printf("read-write write: %u\n", htm_results.read_write_w[i]); + + printf("non HTM:\n"); + printf("read only: %u\n", non_htm_results.read_only[i]); + printf("write only: %u\n", non_htm_results.write_only[i]); + printf("read-write read: %u\n", + non_htm_results.read_write_r[i]); + printf("read-write write: %u\n", + non_htm_results.read_write_w[i]); + } + + return 0; +} + +REGISTER_TEST_COMMAND(hash_readwrite_autotest, test_hash_readwrite_main); -- 2.7.4