* [dpdk-dev] [PATCH v3 01/11] eal: add const in prefetch functions
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Pablo de Lara
@ 2015-06-28 22:25 ` Pablo de Lara
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 02/11] hash: move rte_hash structure to C file and make it internal Pablo de Lara
` (11 subsequent siblings)
12 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-06-28 22:25 UTC (permalink / raw)
To: dev
rte_prefetchX functions included volatile void *p as parameter,
but the function does not modify it, so it should include the const keyword.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
lib/librte_eal/common/include/arch/ppc_64/rte_prefetch.h | 6 +++---
lib/librte_eal/common/include/arch/x86/rte_prefetch.h | 14 +++++++-------
lib/librte_eal/common/include/generic/rte_prefetch.h | 8 ++++----
3 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_prefetch.h b/lib/librte_eal/common/include/arch/ppc_64/rte_prefetch.h
index 9df0d13..fea3be1 100644
--- a/lib/librte_eal/common/include/arch/ppc_64/rte_prefetch.h
+++ b/lib/librte_eal/common/include/arch/ppc_64/rte_prefetch.h
@@ -39,17 +39,17 @@ extern "C" {
#include "generic/rte_prefetch.h"
-static inline void rte_prefetch0(volatile void *p)
+static inline void rte_prefetch0(const volatile void *p)
{
asm volatile ("dcbt 0,%[p],1" : : [p] "r" (p));
}
-static inline void rte_prefetch1(volatile void *p)
+static inline void rte_prefetch1(const volatile void *p)
{
asm volatile ("dcbt 0,%[p],1" : : [p] "r" (p));
}
-static inline void rte_prefetch2(volatile void *p)
+static inline void rte_prefetch2(const volatile void *p)
{
asm volatile ("dcbt 0,%[p],1" : : [p] "r" (p));
}
diff --git a/lib/librte_eal/common/include/arch/x86/rte_prefetch.h b/lib/librte_eal/common/include/arch/x86/rte_prefetch.h
index ec2454d..8e6e02c 100644
--- a/lib/librte_eal/common/include/arch/x86/rte_prefetch.h
+++ b/lib/librte_eal/common/include/arch/x86/rte_prefetch.h
@@ -1,7 +1,7 @@
/*-
* BSD LICENSE
*
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -40,19 +40,19 @@ extern "C" {
#include "generic/rte_prefetch.h"
-static inline void rte_prefetch0(volatile void *p)
+static inline void rte_prefetch0(const volatile void *p)
{
- asm volatile ("prefetcht0 %[p]" : [p] "+m" (*(volatile char *)p));
+ asm volatile ("prefetcht0 %[p]" : : [p] "m" (*(const volatile char *)p));
}
-static inline void rte_prefetch1(volatile void *p)
+static inline void rte_prefetch1(const volatile void *p)
{
- asm volatile ("prefetcht1 %[p]" : [p] "+m" (*(volatile char *)p));
+ asm volatile ("prefetcht1 %[p]" : : [p] "m" (*(const volatile char *)p));
}
-static inline void rte_prefetch2(volatile void *p)
+static inline void rte_prefetch2(const volatile void *p)
{
- asm volatile ("prefetcht2 %[p]" : [p] "+m" (*(volatile char *)p));
+ asm volatile ("prefetcht2 %[p]" : : [p] "m" (*(const volatile char *)p));
}
#ifdef __cplusplus
diff --git a/lib/librte_eal/common/include/generic/rte_prefetch.h b/lib/librte_eal/common/include/generic/rte_prefetch.h
index 217f319..725715f 100644
--- a/lib/librte_eal/common/include/generic/rte_prefetch.h
+++ b/lib/librte_eal/common/include/generic/rte_prefetch.h
@@ -1,7 +1,7 @@
/*-
* BSD LICENSE
*
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -51,14 +51,14 @@
* @param p
* Address to prefetch
*/
-static inline void rte_prefetch0(volatile void *p);
+static inline void rte_prefetch0(const volatile void *p);
/**
* Prefetch a cache line into all cache levels except the 0th cache level.
* @param p
* Address to prefetch
*/
-static inline void rte_prefetch1(volatile void *p);
+static inline void rte_prefetch1(const volatile void *p);
/**
* Prefetch a cache line into all cache levels except the 0th and 1th cache
@@ -66,6 +66,6 @@ static inline void rte_prefetch1(volatile void *p);
* @param p
* Address to prefetch
*/
-static inline void rte_prefetch2(volatile void *p);
+static inline void rte_prefetch2(const volatile void *p);
#endif /* _RTE_PREFETCH_H_ */
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v3 02/11] hash: move rte_hash structure to C file and make it internal
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Pablo de Lara
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 01/11] eal: add const in prefetch functions Pablo de Lara
@ 2015-06-28 22:25 ` Pablo de Lara
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 03/11] test/hash: enhance hash unit tests Pablo de Lara
` (10 subsequent siblings)
12 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-06-28 22:25 UTC (permalink / raw)
To: dev
rte_hash structure should not be a public structure,
and therefore it should be moved to the C file and be declared
as internal.
This patch also removes part of a unit test that was checking
a field of the structure.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 6 +-----
lib/librte_hash/rte_hash.c | 30 +++++++++++++++++++++++++++++-
lib/librte_hash/rte_hash.h | 37 +++++--------------------------------
3 files changed, 35 insertions(+), 38 deletions(-)
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index 4ecb11b..4300de9 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -1,7 +1,7 @@
/*-
* BSD LICENSE
*
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -1110,10 +1110,6 @@ test_hash_creation_with_good_parameters(void)
printf("Creating hash with null hash_func failed\n");
return -1;
}
- if (handle->hash_func == NULL) {
- printf("Hash function should have been DEFAULT_HASH_FUNC\n");
- return -1;
- }
/* this test is trying to create a hash with the same name as previous one.
* this should return a pointer to the hash we previously created.
diff --git a/lib/librte_hash/rte_hash.c b/lib/librte_hash/rte_hash.c
index 67dff5b..5100a75 100644
--- a/lib/librte_hash/rte_hash.c
+++ b/lib/librte_hash/rte_hash.c
@@ -1,7 +1,7 @@
/*-
* BSD LICENSE
*
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -92,6 +92,27 @@ EAL_REGISTER_TAILQ(rte_hash_tailq)
/* The high bit is always set in real signatures */
#define NULL_SIGNATURE 0
+struct rte_hash {
+ char name[RTE_HASH_NAMESIZE]; /**< Name of the hash. */
+ uint32_t entries; /**< Total table entries. */
+ uint32_t bucket_entries; /**< Bucket entries. */
+ uint32_t key_len; /**< Length of hash key. */
+ rte_hash_function hash_func; /**< Function used to calculate hash. */
+ uint32_t hash_func_init_val; /**< Init value used by hash_func. */
+ uint32_t num_buckets; /**< Number of buckets in table. */
+ uint32_t bucket_bitmask; /**< Bitmask for getting bucket index
+ from hash signature. */
+ hash_sig_t sig_msb; /**< MSB is always set in valid signatures. */
+ uint8_t *sig_tbl; /**< Flat array of hash signature buckets. */
+ uint32_t sig_tbl_bucket_size; /**< Signature buckets may be padded for
+ alignment reasons, and this is the
+ bucket size used by sig_tbl. */
+ uint8_t *key_tbl; /**< Flat array of key value buckets. */
+ uint32_t key_tbl_key_size; /**< Keys may be padded for alignment
+ reasons, and this is the key size
+ used by key_tbl. */
+};
+
/* Returns a pointer to the first signature in specified bucket. */
static inline hash_sig_t *
get_sig_tbl_bucket(const struct rte_hash *h, uint32_t bucket_index)
@@ -291,6 +312,13 @@ rte_hash_free(struct rte_hash *h)
rte_free(te);
}
+hash_sig_t
+rte_hash_hash(const struct rte_hash *h, const void *key)
+{
+ /* calc hash result by key */
+ return h->hash_func(key, h->key_len, h->hash_func_init_val);
+}
+
static inline int32_t
__rte_hash_add_key_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig)
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index 821a9d4..79827a6 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -1,7 +1,7 @@
/*-
* BSD LICENSE
*
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -40,9 +40,6 @@
* RTE Hash Table
*/
-#include <stdint.h>
-#include <sys/queue.h>
-
#ifdef __cplusplus
extern "C" {
#endif
@@ -84,27 +81,8 @@ struct rte_hash_parameters {
int socket_id; /**< NUMA Socket ID for memory. */
};
-/** A hash table structure. */
-struct rte_hash {
- char name[RTE_HASH_NAMESIZE]; /**< Name of the hash. */
- uint32_t entries; /**< Total table entries. */
- uint32_t bucket_entries; /**< Bucket entries. */
- uint32_t key_len; /**< Length of hash key. */
- rte_hash_function hash_func; /**< Function used to calculate hash. */
- uint32_t hash_func_init_val; /**< Init value used by hash_func. */
- uint32_t num_buckets; /**< Number of buckets in table. */
- uint32_t bucket_bitmask; /**< Bitmask for getting bucket index
- from hash signature. */
- hash_sig_t sig_msb; /**< MSB is always set in valid signatures. */
- uint8_t *sig_tbl; /**< Flat array of hash signature buckets. */
- uint32_t sig_tbl_bucket_size; /**< Signature buckets may be padded for
- alignment reasons, and this is the
- bucket size used by sig_tbl. */
- uint8_t *key_tbl; /**< Flat array of key value buckets. */
- uint32_t key_tbl_key_size; /**< Keys may be padded for alignment
- reasons, and this is the key size
- used by key_tbl. */
-};
+/** @internal A hash table structure. */
+struct rte_hash;
/**
* Create a new hash table.
@@ -262,7 +240,6 @@ int32_t
rte_hash_lookup_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig);
-
/**
* Calc a hash value by key. This operation is not multi-process safe.
*
@@ -273,12 +250,8 @@ rte_hash_lookup_with_hash(const struct rte_hash *h,
* @return
* - hash value
*/
-static inline hash_sig_t
-rte_hash_hash(const struct rte_hash *h, const void *key)
-{
- /* calc hash result by key */
- return h->hash_func(key, h->key_len, h->hash_func_init_val);
-}
+hash_sig_t
+rte_hash_hash(const struct rte_hash *h, const void *key);
#define rte_hash_lookup_multi rte_hash_lookup_bulk
/**
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v3 03/11] test/hash: enhance hash unit tests
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Pablo de Lara
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 01/11] eal: add const in prefetch functions Pablo de Lara
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 02/11] hash: move rte_hash structure to C file and make it internal Pablo de Lara
@ 2015-06-28 22:25 ` Pablo de Lara
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 04/11] test/hash: rename new hash perf unit test back to original name Pablo de Lara
` (9 subsequent siblings)
12 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-06-28 22:25 UTC (permalink / raw)
To: dev
Add new unit test for calculating the average table utilization,
using random keys, based on number of entries that can be added
until we encounter one that cannot be added (bucket if full)
Also, replace current hash_perf unit test to see performance more clear.
The current hash_perf unit test takes too long and add keys that
may or may not fit in the table and look up/delete that may not be
in the table. This new unit test gets a set of keys that we know
that fits in the table, and then measure the time to add/look up/delete
them.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/Makefile | 2 +-
app/test/test_hash.c | 59 ++++
app/test/test_hash_perf.c | 702 ------------------------------------------
app/test/test_hash_perf_new.c | 560 +++++++++++++++++++++++++++++++++
4 files changed, 620 insertions(+), 703 deletions(-)
delete mode 100644 app/test/test_hash_perf.c
create mode 100644 app/test/test_hash_perf_new.c
diff --git a/app/test/Makefile b/app/test/Makefile
index 2e2758c..8624e95 100644
--- a/app/test/Makefile
+++ b/app/test/Makefile
@@ -82,7 +82,7 @@ SRCS-y += test_memcpy.c
SRCS-y += test_memcpy_perf.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash.c
-SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_perf.c
+SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_perf_new.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_functions.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_scaling.c
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index 4300de9..46174db 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -1147,6 +1147,63 @@ test_hash_creation_with_good_parameters(void)
return 0;
}
+#define ITERATIONS 50
+/*
+ * Test to see the average table utilization (entries added/max entries)
+ * before hitting a random entry that cannot be added
+ */
+static int test_average_table_utilization(void)
+{
+ struct rte_hash *handle;
+ void *simple_key;
+ unsigned i, j, no_space = 0;
+ double added_keys_until_no_space = 0;
+ int ret;
+
+ ut_params.entries = 1 << 20;
+ ut_params.name = "test_average_utilization";
+ ut_params.hash_func = rte_hash_crc;
+ handle = rte_hash_create(&ut_params);
+ RETURN_IF_ERROR(handle == NULL, "hash creation failed");
+
+ simple_key = rte_zmalloc(NULL, ut_params.key_len, 0);
+
+ for (j = 0; j < ITERATIONS; j++) {
+ while (!no_space) {
+ for (i = 0; i < ut_params.key_len; i++)
+ ((uint8_t *) simple_key)[i] = rte_rand() % 255;
+
+ ret = rte_hash_add_key(handle, simple_key);
+ print_key_info("Add", simple_key, ret);
+
+ if (ret == -ENOSPC) {
+ if (-ENOENT != rte_hash_lookup(handle, simple_key))
+ printf("Found key that should not be present\n");
+ no_space = 1;
+ } else {
+ if (ret < 0)
+ rte_free(simple_key);
+ RETURN_IF_ERROR(ret < 0, "failed to add key (ret=%d)", ret);
+ added_keys_until_no_space++;
+ }
+ }
+ no_space = 0;
+
+ /* Reset the table */
+ rte_hash_free(handle);
+ rte_hash_create(&ut_params);
+ }
+
+ const unsigned average_keys_added = added_keys_until_no_space / ITERATIONS;
+
+ printf("Average table utilization = %.2f%% (%u/%u)\n",
+ ((double) average_keys_added / ut_params.entries * 100),
+ average_keys_added, ut_params.entries);
+ rte_hash_free(handle);
+
+ return 0;
+}
+
static uint8_t key[16] = {0x00, 0x01, 0x02, 0x03,
0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b,
@@ -1405,6 +1462,8 @@ test_hash(void)
return -1;
if (test_hash_creation_with_good_parameters() < 0)
return -1;
+ if (test_average_table_utilization() < 0)
+ return -1;
run_hash_func_tests();
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
deleted file mode 100644
index d0e5ce0..0000000
--- a/app/test/test_hash_perf.c
+++ /dev/null
@@ -1,702 +0,0 @@
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <stdio.h>
-#include <stdint.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <errno.h>
-#include <sys/queue.h>
-
-#include <rte_common.h>
-#include <rte_lcore.h>
-#include <rte_malloc.h>
-#include <rte_cycles.h>
-#include <rte_random.h>
-#include <rte_memory.h>
-#include <rte_memzone.h>
-#include <rte_eal.h>
-#include <rte_ip.h>
-#include <rte_string_fns.h>
-
-#include "test.h"
-
-#include <rte_hash.h>
-#include <rte_fbk_hash.h>
-#include <rte_jhash.h>
-#include <rte_hash_crc.h>
-
-/* Types of hash table performance test that can be performed */
-enum hash_test_t {
- ADD_ON_EMPTY, /*< Add keys to empty table */
- DELETE_ON_EMPTY, /*< Attempt to delete keys from empty table */
- LOOKUP_ON_EMPTY, /*< Attempt to find keys in an empty table */
- ADD_UPDATE, /*< Add/update keys in a full table */
- DELETE, /*< Delete keys from a full table */
- LOOKUP /*< Find keys in a full table */
-};
-
-/* Function type for hash table operations. */
-typedef int32_t (*hash_operation)(const struct rte_hash *h, const void *key);
-
-/* Structure to hold parameters used to run a hash table performance test */
-struct tbl_perf_test_params {
- enum hash_test_t test_type;
- uint32_t num_iterations;
- uint32_t entries;
- uint32_t bucket_entries;
- uint32_t key_len;
- rte_hash_function hash_func;
- uint32_t hash_func_init_val;
-};
-
-#define ITERATIONS 10000
-#define LOCAL_FBK_HASH_ENTRIES_MAX (1 << 15)
-
-/*******************************************************************************
- * Hash table performance test configuration section.
- */
-struct tbl_perf_test_params tbl_perf_params[] =
-{
-/* Small table, add */
-/* Test type | Iterations | Entries | BucketSize | KeyLen | HashFunc | InitVal */
-{ ADD_ON_EMPTY, 1024, 1024, 1, 16, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 2, 16, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 4, 16, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 8, 16, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 16, 16, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 1, 32, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 2, 32, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 4, 32, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 8, 32, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 16, 32, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 1, 48, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 2, 48, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 4, 48, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 8, 48, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 16, 48, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 1, 64, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 2, 64, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 4, 64, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 8, 64, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 16, 64, rte_jhash, 0},
-/* Small table, update */
-/* Test type | Iterations | Entries | BucketSize | KeyLen | HashFunc | InitVal */
-{ ADD_UPDATE, ITERATIONS, 1024, 1, 16, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 2, 16, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 4, 16, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 8, 16, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 16, 16, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 1, 32, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 2, 32, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 4, 32, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 8, 32, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 16, 32, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 1, 48, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 2, 48, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 4, 48, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 8, 48, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 16, 48, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 1, 64, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 2, 64, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 4, 64, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 8, 64, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 16, 64, rte_jhash, 0},
-/* Small table, lookup */
-/* Test type | Iterations | Entries | BucketSize | KeyLen | HashFunc | InitVal */
-{ LOOKUP, ITERATIONS, 1024, 1, 16, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 2, 16, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 4, 16, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 8, 16, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 16, 16, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 1, 32, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 2, 32, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 4, 32, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 8, 32, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 16, 32, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 1, 48, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 2, 48, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 4, 48, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 8, 48, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 16, 48, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 1, 64, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 2, 64, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 4, 64, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 8, 64, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1024, 16, 64, rte_jhash, 0},
-/* Big table, add */
-/* Test type | Iterations | Entries | BucketSize | KeyLen | HashFunc | InitVal */
-{ ADD_ON_EMPTY, 1048576, 1048576, 1, 16, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 2, 16, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 4, 16, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 8, 16, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 16, 16, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 1, 32, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 2, 32, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 4, 32, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 8, 32, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 16, 32, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 1, 48, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 2, 48, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 4, 48, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 8, 48, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 16, 48, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 1, 64, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 2, 64, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 4, 64, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 8, 64, rte_jhash, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 16, 64, rte_jhash, 0},
-/* Big table, update */
-/* Test type | Iterations | Entries | BucketSize | KeyLen | HashFunc | InitVal */
-{ ADD_UPDATE, ITERATIONS, 1048576, 1, 16, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 2, 16, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 4, 16, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 8, 16, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 16, 16, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 1, 32, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 2, 32, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 4, 32, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 8, 32, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 16, 32, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 1, 48, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 2, 48, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 4, 48, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 8, 48, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 16, 48, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 1, 64, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 2, 64, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 4, 64, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 8, 64, rte_jhash, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 16, 64, rte_jhash, 0},
-/* Big table, lookup */
-/* Test type | Iterations | Entries | BucketSize | KeyLen | HashFunc | InitVal */
-{ LOOKUP, ITERATIONS, 1048576, 1, 16, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 2, 16, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 4, 16, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 8, 16, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 16, 16, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 1, 32, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 2, 32, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 4, 32, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 8, 32, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 16, 32, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 1, 48, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 2, 48, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 4, 48, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 8, 48, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 16, 48, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 1, 64, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 2, 64, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 4, 64, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 8, 64, rte_jhash, 0},
-{ LOOKUP, ITERATIONS, 1048576, 16, 64, rte_jhash, 0},
-/* Small table, add */
-/* Test type | Iterations | Entries | BucketSize | KeyLen | HashFunc | InitVal */
-{ ADD_ON_EMPTY, 1024, 1024, 1, 16, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 2, 16, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 4, 16, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 8, 16, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 16, 16, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 1, 32, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 2, 32, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 4, 32, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 8, 32, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 16, 32, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 1, 48, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 2, 48, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 4, 48, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 8, 48, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 16, 48, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 1, 64, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 2, 64, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 4, 64, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 8, 64, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1024, 1024, 16, 64, rte_hash_crc, 0},
-/* Small table, update */
-/* Test type | Iterations | Entries | BucketSize | KeyLen | HashFunc | InitVal */
-{ ADD_UPDATE, ITERATIONS, 1024, 1, 16, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 2, 16, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 4, 16, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 8, 16, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 16, 16, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 1, 32, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 2, 32, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 4, 32, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 8, 32, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 16, 32, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 1, 48, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 2, 48, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 4, 48, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 8, 48, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 16, 48, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 1, 64, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 2, 64, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 4, 64, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 8, 64, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1024, 16, 64, rte_hash_crc, 0},
-/* Small table, lookup */
-/* Test type | Iterations | Entries | BucketSize | KeyLen | HashFunc | InitVal */
-{ LOOKUP, ITERATIONS, 1024, 1, 16, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 2, 16, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 4, 16, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 8, 16, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 16, 16, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 1, 32, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 2, 32, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 4, 32, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 8, 32, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 16, 32, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 1, 48, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 2, 48, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 4, 48, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 8, 48, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 16, 48, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 1, 64, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 2, 64, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 4, 64, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 8, 64, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1024, 16, 64, rte_hash_crc, 0},
-/* Big table, add */
-/* Test type | Iterations | Entries | BucketSize | KeyLen | HashFunc | InitVal */
-{ ADD_ON_EMPTY, 1048576, 1048576, 1, 16, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 2, 16, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 4, 16, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 8, 16, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 16, 16, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 1, 32, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 2, 32, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 4, 32, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 8, 32, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 16, 32, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 1, 48, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 2, 48, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 4, 48, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 8, 48, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 16, 48, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 1, 64, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 2, 64, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 4, 64, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 8, 64, rte_hash_crc, 0},
-{ ADD_ON_EMPTY, 1048576, 1048576, 16, 64, rte_hash_crc, 0},
-/* Big table, update */
-/* Test type | Iterations | Entries | BucketSize | KeyLen | HashFunc | InitVal */
-{ ADD_UPDATE, ITERATIONS, 1048576, 1, 16, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 2, 16, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 4, 16, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 8, 16, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 16, 16, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 1, 32, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 2, 32, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 4, 32, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 8, 32, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 16, 32, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 1, 48, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 2, 48, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 4, 48, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 8, 48, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 16, 48, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 1, 64, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 2, 64, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 4, 64, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 8, 64, rte_hash_crc, 0},
-{ ADD_UPDATE, ITERATIONS, 1048576, 16, 64, rte_hash_crc, 0},
-/* Big table, lookup */
-/* Test type | Iterations | Entries | BucketSize | KeyLen | HashFunc | InitVal */
-{ LOOKUP, ITERATIONS, 1048576, 1, 16, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 2, 16, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 4, 16, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 8, 16, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 16, 16, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 1, 32, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 2, 32, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 4, 32, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 8, 32, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 16, 32, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 1, 48, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 2, 48, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 4, 48, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 8, 48, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 16, 48, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 1, 64, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 2, 64, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 4, 64, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 8, 64, rte_hash_crc, 0},
-{ LOOKUP, ITERATIONS, 1048576, 16, 64, rte_hash_crc, 0},
-};
-
-/******************************************************************************/
-
-/*
- * Check condition and return an error if true. Assumes that "handle" is the
- * name of the hash structure pointer to be freed.
- */
-#define RETURN_IF_ERROR(cond, str, ...) do { \
- if (cond) { \
- printf("ERROR line %d: " str "\n", __LINE__, ##__VA_ARGS__); \
- if (handle) rte_hash_free(handle); \
- return -1; \
- } \
-} while(0)
-
-#define RETURN_IF_ERROR_FBK(cond, str, ...) do { \
- if (cond) { \
- printf("ERROR line %d: " str "\n", __LINE__, ##__VA_ARGS__); \
- if (handle) rte_fbk_hash_free(handle); \
- rte_free(keys); \
- return -1; \
- } \
-} while(0)
-
-/*
- * Find average of array of numbers.
- */
-static double
-get_avg(const uint32_t *array, uint32_t size)
-{
- double sum = 0;
- unsigned i;
- for (i = 0; i < size; i++)
- sum += array[i];
- return sum / (double)size;
-}
-
-/*
- * To help print out name of hash functions.
- */
-static const char *get_hash_name(rte_hash_function f)
-{
- if (f == rte_jhash)
- return "jhash";
-
- if (f == rte_hash_crc)
- return "rte_hash_crc";
-
- return "UnknownHash";
-}
-
-/*
- * Do a single performance test, of one type of operation.
- *
- * @param h
- * hash table to run test on
- * @param func
- * function to call (add, delete or lookup function)
- * @param avg_occupancy
- * The average number of entries in each bucket of the hash table
- * @param invalid_pos_count
- * The amount of errors (e.g. due to a full bucket).
- * @return
- * The average number of ticks per hash function call. A negative number
- * signifies failure.
- */
-static double
-run_single_tbl_perf_test(const struct rte_hash *h, hash_operation func,
- const struct tbl_perf_test_params *params, double *avg_occupancy,
- uint32_t *invalid_pos_count)
-{
- uint64_t begin, end, ticks = 0;
- uint8_t *key = NULL;
- uint32_t *bucket_occupancies = NULL;
- uint32_t num_buckets, i, j;
- int32_t pos;
-
- /* Initialise */
- num_buckets = params->entries / params->bucket_entries;
- key = rte_zmalloc("hash key",
- params->key_len * sizeof(uint8_t), 16);
- if (key == NULL)
- return -1;
-
- bucket_occupancies = rte_calloc("bucket occupancies",
- num_buckets, sizeof(uint32_t), 16);
- if (bucket_occupancies == NULL) {
- rte_free(key);
- return -1;
- }
-
- ticks = 0;
- *invalid_pos_count = 0;
-
- for (i = 0; i < params->num_iterations; i++) {
- /* Prepare inputs for the current iteration */
- for (j = 0; j < params->key_len; j++)
- key[j] = (uint8_t) rte_rand();
-
- /* Perform operation, and measure time it takes */
- begin = rte_rdtsc();
- pos = func(h, key);
- end = rte_rdtsc();
- ticks += end - begin;
-
- /* Other work per iteration */
- if (pos < 0)
- *invalid_pos_count += 1;
- else
- bucket_occupancies[pos / params->bucket_entries]++;
- }
- *avg_occupancy = get_avg(bucket_occupancies, num_buckets);
-
- rte_free(bucket_occupancies);
- rte_free(key);
-
- return (double)ticks / params->num_iterations;
-}
-
-/*
- * To help print out what tests are being done.
- */
-static const char *
-get_tbl_perf_test_desc(enum hash_test_t type)
-{
- switch (type){
- case ADD_ON_EMPTY: return "Add on Empty";
- case DELETE_ON_EMPTY: return "Delete on Empty";
- case LOOKUP_ON_EMPTY: return "Lookup on Empty";
- case ADD_UPDATE: return "Add Update";
- case DELETE: return "Delete";
- case LOOKUP: return "Lookup";
- default: return "UNKNOWN";
- }
-}
-
-/*
- * Run a hash table performance test based on params.
- */
-static int
-run_tbl_perf_test(struct tbl_perf_test_params *params)
-{
- static unsigned calledCount = 5;
- struct rte_hash_parameters hash_params = {
- .entries = params->entries,
- .bucket_entries = params->bucket_entries,
- .key_len = params->key_len,
- .hash_func = params->hash_func,
- .hash_func_init_val = params->hash_func_init_val,
- .socket_id = rte_socket_id(),
- };
- struct rte_hash *handle;
- double avg_occupancy = 0, ticks = 0;
- uint32_t num_iterations, invalid_pos;
- char name[RTE_HASH_NAMESIZE];
- char hashname[RTE_HASH_NAMESIZE];
-
- snprintf(name, 32, "test%u", calledCount++);
- hash_params.name = name;
-
- handle = rte_hash_create(&hash_params);
- RETURN_IF_ERROR(handle == NULL, "hash creation failed");
-
- switch (params->test_type){
- case ADD_ON_EMPTY:
- ticks = run_single_tbl_perf_test(handle, rte_hash_add_key,
- params, &avg_occupancy, &invalid_pos);
- break;
- case DELETE_ON_EMPTY:
- ticks = run_single_tbl_perf_test(handle, rte_hash_del_key,
- params, &avg_occupancy, &invalid_pos);
- break;
- case LOOKUP_ON_EMPTY:
- ticks = run_single_tbl_perf_test(handle, rte_hash_lookup,
- params, &avg_occupancy, &invalid_pos);
- break;
- case ADD_UPDATE:
- num_iterations = params->num_iterations;
- params->num_iterations = params->entries;
- run_single_tbl_perf_test(handle, rte_hash_add_key, params,
- &avg_occupancy, &invalid_pos);
- params->num_iterations = num_iterations;
- ticks = run_single_tbl_perf_test(handle, rte_hash_add_key,
- params, &avg_occupancy, &invalid_pos);
- break;
- case DELETE:
- num_iterations = params->num_iterations;
- params->num_iterations = params->entries;
- run_single_tbl_perf_test(handle, rte_hash_add_key, params,
- &avg_occupancy, &invalid_pos);
-
- params->num_iterations = num_iterations;
- ticks = run_single_tbl_perf_test(handle, rte_hash_del_key,
- params, &avg_occupancy, &invalid_pos);
- break;
- case LOOKUP:
- num_iterations = params->num_iterations;
- params->num_iterations = params->entries;
- run_single_tbl_perf_test(handle, rte_hash_add_key, params,
- &avg_occupancy, &invalid_pos);
-
- params->num_iterations = num_iterations;
- ticks = run_single_tbl_perf_test(handle, rte_hash_lookup,
- params, &avg_occupancy, &invalid_pos);
- break;
- default: return -1;
- }
-
- snprintf(hashname, RTE_HASH_NAMESIZE, "%s", get_hash_name(params->hash_func));
-
- printf("%-12s, %-15s, %-16u, %-7u, %-18u, %-8u, %-19.2f, %.2f\n",
- hashname,
- get_tbl_perf_test_desc(params->test_type),
- (unsigned) params->key_len,
- (unsigned) params->entries,
- (unsigned) params->bucket_entries,
- (unsigned) invalid_pos,
- avg_occupancy,
- ticks
- );
-
- /* Free */
- rte_hash_free(handle);
- return 0;
-}
-
-/*
- * Run all hash table performance tests.
- */
-static int run_all_tbl_perf_tests(void)
-{
- unsigned i;
-
- printf(" *** Hash table performance test results ***\n");
- printf("Hash Func. , Operation , Key size (bytes), Entries, "
- "Entries per bucket, Errors , Avg. bucket entries, Ticks/Op.\n");
-
- /* Loop through every combination of test parameters */
- for (i = 0;
- i < sizeof(tbl_perf_params) / sizeof(struct tbl_perf_test_params);
- i++) {
-
- /* Perform test */
- if (run_tbl_perf_test(&tbl_perf_params[i]) < 0)
- return -1;
- }
- return 0;
-}
-
-/* Control operation of performance testing of fbk hash. */
-#define LOAD_FACTOR 0.667 /* How full to make the hash table. */
-#define TEST_SIZE 1000000 /* How many operations to time. */
-#define TEST_ITERATIONS 30 /* How many measurements to take. */
-#define ENTRIES (1 << 15) /* How many entries. */
-
-static int
-fbk_hash_perf_test(void)
-{
- struct rte_fbk_hash_params params = {
- .name = "fbk_hash_test",
- .entries = ENTRIES,
- .entries_per_bucket = 4,
- .socket_id = rte_socket_id(),
- };
- struct rte_fbk_hash_table *handle = NULL;
- uint32_t *keys = NULL;
- unsigned indexes[TEST_SIZE];
- uint64_t lookup_time = 0;
- unsigned added = 0;
- unsigned value = 0;
- unsigned i, j;
-
- handle = rte_fbk_hash_create(¶ms);
- RETURN_IF_ERROR_FBK(handle == NULL, "fbk hash creation failed");
-
- keys = rte_zmalloc(NULL, ENTRIES * sizeof(*keys), 0);
- RETURN_IF_ERROR_FBK(keys == NULL,
- "fbk hash: memory allocation for key store failed");
-
- /* Generate random keys and values. */
- for (i = 0; i < ENTRIES; i++) {
- uint32_t key = (uint32_t)rte_rand();
- key = ((uint64_t)key << 32) | (uint64_t)rte_rand();
- uint16_t val = (uint16_t)rte_rand();
-
- if (rte_fbk_hash_add_key(handle, key, val) == 0) {
- keys[added] = key;
- added++;
- }
- if (added > (LOAD_FACTOR * ENTRIES)) {
- break;
- }
- }
-
- for (i = 0; i < TEST_ITERATIONS; i++) {
- uint64_t begin;
- uint64_t end;
-
- /* Generate random indexes into keys[] array. */
- for (j = 0; j < TEST_SIZE; j++) {
- indexes[j] = rte_rand() % added;
- }
-
- begin = rte_rdtsc();
- /* Do lookups */
- for (j = 0; j < TEST_SIZE; j++) {
- value += rte_fbk_hash_lookup(handle, keys[indexes[j]]);
- }
- end = rte_rdtsc();
- lookup_time += (double)(end - begin);
- }
-
- printf("\n\n *** FBK Hash function performance test results ***\n");
- /*
- * The use of the 'value' variable ensures that the hash lookup is not
- * being optimised out by the compiler.
- */
- if (value != 0)
- printf("Number of ticks per lookup = %g\n",
- (double)lookup_time /
- ((double)TEST_ITERATIONS * (double)TEST_SIZE));
-
- rte_fbk_hash_free(handle);
-
- return 0;
-}
-
-/*
- * Do all unit and performance tests.
- */
-static int
-test_hash_perf(void)
-{
- if (run_all_tbl_perf_tests() < 0)
- return -1;
-
- if (fbk_hash_perf_test() < 0)
- return -1;
- return 0;
-}
-
-static struct test_command hash_perf_cmd = {
- .command = "hash_perf_autotest",
- .callback = test_hash_perf,
-};
-REGISTER_TEST_COMMAND(hash_perf_cmd);
diff --git a/app/test/test_hash_perf_new.c b/app/test/test_hash_perf_new.c
new file mode 100644
index 0000000..1da1839
--- /dev/null
+++ b/app/test/test_hash_perf_new.c
@@ -0,0 +1,560 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <inttypes.h>
+
+#include <rte_lcore.h>
+#include <rte_cycles.h>
+#include <rte_malloc.h>
+#include <rte_hash.h>
+#include <rte_hash_crc.h>
+#include <rte_jhash.h>
+#include <rte_fbk_hash.h>
+#include <rte_random.h>
+#include <rte_string_fns.h>
+
+#include "test.h"
+
+#define KEYS_TO_ADD (1 << 18)
+#define MAX_ENTRIES (KEYS_TO_ADD * 4) /* 25% table utilization */
+#define NUM_LOOKUPS (KEYS_TO_ADD * 10) /* Loop among keys added, several times */
+#define BUCKET_SIZE 4
+#define NUM_BUCKETS (MAX_ENTRIES / BUCKET_SIZE)
+#define MAX_KEYSIZE 64
+#define NUM_KEYSIZES 10
+#define NUM_SHUFFLES 10
+#define BURST_SIZE 16
+
+enum operations {
+ ADD = 0,
+ LOOKUP,
+ LOOKUP_MULTI,
+ DELETE,
+ NUM_OPERATIONS
+};
+
+static uint32_t hashtest_key_lens[] = {
+ 4, 8, 16, 32, 48, 64, /* standard key sizes */
+ 9, /* IPv4 SRC + DST + protocol, unpadded */
+ 13, /* IPv4 5-tuple, unpadded */
+ 37, /* IPv6 5-tuple, unpadded */
+ 40 /* IPv6 5-tuple, padded to 8-byte boundary */
+};
+struct rte_hash *h[NUM_KEYSIZES];
+/* Array that stores if a slot is full */
+uint8_t slot_taken[MAX_ENTRIES];
+/* Array to store number of cycles per operation */
+uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS][2];
+/* Array to store all input keys */
+uint8_t keys[KEYS_TO_ADD][MAX_KEYSIZE];
+/* Array to store the precomputed hash for 'keys' */
+hash_sig_t signatures[KEYS_TO_ADD];
+/* Array to store how many busy entries have each bucket */
+uint8_t buckets[NUM_BUCKETS];
+
+/* Parameters used for hash table in unit test functions. */
+static struct rte_hash_parameters ut_params = {
+ .entries = MAX_ENTRIES,
+ .bucket_entries = BUCKET_SIZE,
+ .hash_func = rte_jhash,
+ .hash_func_init_val = 0,
+};
+
+static int
+create_table(unsigned table_index)
+{
+ char name[RTE_HASH_NAMESIZE];
+
+ sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
+ ut_params.name = name;
+ ut_params.key_len = hashtest_key_lens[table_index];
+ ut_params.socket_id = rte_socket_id();
+ h[table_index] = rte_hash_find_existing(name);
+ if (h[table_index] != NULL)
+ /*
+ * If table was already created, free it to create it again,
+ * so we force it is empty
+ */
+ rte_hash_free(h[table_index]);
+ h[table_index] = rte_hash_create(&ut_params);
+ if (h[table_index] == NULL) {
+ printf("Error creating table\n");
+ return -1;
+ }
+ return 0;
+
+}
+
+/* Shuffle the keys that have been added, so lookups will be totally random */
+static void
+shuffle_input_keys(unsigned table_index)
+{
+ unsigned i;
+ uint32_t swap_idx;
+ uint8_t temp_key[RTE_HASH_KEY_LENGTH_MAX];
+ hash_sig_t temp_signature;
+
+ for (i = 0; i < KEYS_TO_ADD; i++) {
+ do
+ swap_idx = rte_rand() % KEYS_TO_ADD;
+ while (swap_idx == i);
+
+ memcpy(temp_key, keys[i], hashtest_key_lens[table_index]);
+ temp_signature = signatures[i];
+
+ memcpy(keys[i], keys[swap_idx], hashtest_key_lens[table_index]);
+ signatures[i] = signatures[swap_idx];
+
+ memcpy(keys[swap_idx], temp_key, hashtest_key_lens[table_index]);
+ signatures[swap_idx] = temp_signature;
+ }
+}
+
+/*
+ * Creates the table and looks for random keys which
+ * ALL can fit in hash table (no errors)
+ */
+static int
+get_input_keys(unsigned table_index)
+{
+ unsigned i, j;
+ unsigned bucket_idx, incr, success = 1;
+ uint8_t k = 0;
+ int32_t ret;
+ const uint32_t bucket_bitmask = NUM_BUCKETS - 1;
+
+ /* Reset all arrays */
+ for (i = 0; i < MAX_ENTRIES; i++)
+ slot_taken[i] = 0;
+
+ for (i = 0; i < NUM_BUCKETS; i++)
+ buckets[i] = 0;
+
+ for (j = 0; j < hashtest_key_lens[table_index]; j++)
+ keys[0][j] = 0;
+
+ /*
+ * Add only entries that are not duplicated and that fits in the table
+ * (cannot store more than BUCKET_SIZE entries in a bucket).
+ * Regardless a key has been added correctly or not (success),
+ * the next one to try will be increased by 1.
+ */
+ for (i = 0; i < KEYS_TO_ADD;) {
+ incr = 0;
+ if (i != 0) {
+ keys[i][0] = ++k;
+ /* Overflow, need to increment the next byte */
+ if (keys[i][0] == 0)
+ incr = 1;
+ for (j = 1; j < hashtest_key_lens[table_index]; j++) {
+ /* Do not increase next byte */
+ if (incr == 0)
+ if (success == 1)
+ keys[i][j] = keys[i - 1][j];
+ else
+ keys[i][j] = keys[i][j];
+ /* Increase next byte by one */
+ else {
+ if (success == 1)
+ keys[i][j] = keys[i-1][j] + 1;
+ else
+ keys[i][j] = keys[i][j] + 1;
+ if (keys[i][j] == 0)
+ incr = 1;
+ else
+ incr = 0;
+ }
+ }
+ }
+ success = 0;
+ signatures[i] = rte_hash_hash(h[table_index], keys[i]);
+ bucket_idx = signatures[i] & bucket_bitmask;
+ /* If bucket is full, do not try to insert the key */
+ if (buckets[bucket_idx] == BUCKET_SIZE)
+ continue;
+ /* If key can be added, leave in successful key arrays "keys" */
+ ret = rte_hash_add_key_with_hash(h[table_index], keys[i],
+ signatures[i]);
+ if (ret >= 0) {
+ /* If key is already added, ignore the entry and do not store */
+ if (slot_taken[ret])
+ continue;
+ else {
+ /* Store the returned position and mark slot as taken */
+ slot_taken[ret] = 1;
+ buckets[bucket_idx]++;
+ success = 1;
+ i++;
+ }
+ }
+ }
+
+ /* Reset the table, so we can measure the time to add all the entries */
+ rte_hash_free(h[table_index]);
+ h[table_index] = rte_hash_create(&ut_params);
+
+ return 0;
+}
+
+static int
+timed_adds(unsigned with_hash, unsigned table_index) {
+ unsigned i;
+ const uint64_t start_tsc = rte_rdtsc();
+
+ for (i = 0; i < KEYS_TO_ADD; i++) {
+ if (with_hash)
+ rte_hash_add_key_with_hash(h[table_index],
+ (const void *) keys[i],
+ signatures[i]);
+ else
+ rte_hash_add_key(h[table_index], keys[i]);
+ }
+
+ const uint64_t end_tsc = rte_rdtsc();
+ const uint64_t time_taken = end_tsc - start_tsc;
+ const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
+
+ cycles[table_index][ADD][with_hash] = time_taken/KEYS_TO_ADD;
+
+ printf("\n%"PRIu64" adds in %f seconds\n", (uint64_t)KEYS_TO_ADD,
+ seconds_taken);
+ printf("Average %"PRIu64" tsc ticks per add\n",
+ cycles[table_index][ADD][with_hash]);
+ printf("Average %"PRIu64" adds per second\n",
+ (KEYS_TO_ADD * rte_get_tsc_hz())/time_taken);
+ return 0;
+}
+
+static int
+timed_lookups(unsigned with_hash, unsigned table_index)
+{
+ unsigned i, j;
+ const uint64_t start_tsc = rte_rdtsc();
+
+ for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
+ for (j = 0; j < KEYS_TO_ADD; j++) {
+ if (with_hash)
+ rte_hash_lookup_with_hash(h[table_index],
+ (const void *) keys[j],
+ signatures[j]);
+ else
+ rte_hash_lookup(h[table_index], keys[j]);
+ }
+ }
+
+ const uint64_t end_tsc = rte_rdtsc();
+ const uint64_t time_taken = end_tsc - start_tsc;
+ const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
+
+ cycles[table_index][LOOKUP][with_hash] = time_taken/NUM_LOOKUPS;
+
+ printf("%"PRIu64" lookups in %f seconds\n", (uint64_t) NUM_LOOKUPS,
+ seconds_taken);
+ printf("Average %"PRIu64" tsc ticks per lookup\n",
+ cycles[table_index][LOOKUP][with_hash]);
+ printf("Average %"PRIu64" lookups per second\n",
+ (NUM_LOOKUPS * rte_get_tsc_hz())/time_taken);
+ return 0;
+}
+
+static int
+timed_lookups_multi(unsigned table_index)
+{
+ unsigned i, j, k;
+ int32_t positions_burst[BURST_SIZE];
+ const void *keys_burst[BURST_SIZE];
+ const uint64_t start_tsc = rte_rdtsc();
+
+ for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
+ for (j = 0; j < KEYS_TO_ADD/BURST_SIZE; j++) {
+ for (k = 0; k < BURST_SIZE; k++)
+ keys_burst[k] = keys[j * BURST_SIZE + k];
+ rte_hash_lookup_bulk(h[table_index],
+ (const void **) keys_burst,
+ BURST_SIZE,
+ positions_burst);
+ }
+ }
+
+ const uint64_t end_tsc = rte_rdtsc();
+ const uint64_t time_taken = end_tsc - start_tsc;
+ const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
+
+ cycles[table_index][LOOKUP_MULTI][0] = time_taken/NUM_LOOKUPS;
+
+ printf("%"PRIu64" lookups in %f seconds\n", (uint64_t)NUM_LOOKUPS,
+ seconds_taken);
+ printf("Average %"PRIu64" tsc ticks per lookup\n",
+ cycles[table_index][LOOKUP_MULTI][0]);
+ printf("Average %"PRIu64" lookups per second\n",
+ (NUM_LOOKUPS * rte_get_tsc_hz())/time_taken);
+ return 0;
+}
+
+static int
+timed_deletes(unsigned with_hash, unsigned table_index)
+{
+ unsigned i;
+ const uint64_t start_tsc = rte_rdtsc();
+
+ for (i = 0; i < KEYS_TO_ADD; i++) {
+ if (with_hash)
+ rte_hash_del_key_with_hash(h[table_index],
+ (const void *) keys[i],
+ signatures[i]);
+ else
+ rte_hash_del_key(h[table_index],
+ (const void *) keys[i]);
+ }
+
+ const uint64_t end_tsc = rte_rdtsc();
+ const uint64_t time_taken = end_tsc - start_tsc;
+ const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
+
+ cycles[table_index][DELETE][with_hash] = time_taken/KEYS_TO_ADD;
+
+ printf("\n%"PRIu64" deletions in %f seconds\n", (uint64_t) KEYS_TO_ADD,
+ seconds_taken);
+ printf("Average %"PRIu64" tsc ticks per deletion\n",
+ cycles[table_index][DELETE][with_hash]);
+ printf("Average %"PRIu64" deletions per second\n",
+ (KEYS_TO_ADD * rte_get_tsc_hz())/time_taken);
+ return 0;
+}
+
+static void
+free_table(unsigned table_index)
+{
+ rte_hash_free(h[table_index]);
+}
+
+static int
+reset_table(unsigned table_index)
+{
+ free_table(table_index);
+ if (create_table(table_index) != 0)
+ return -1;
+
+ return 0;
+}
+
+static int
+run_all_tbl_perf_tests(void)
+{
+ unsigned i, j;
+
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ if (create_table(i) < 0)
+ return -1;
+
+ if (get_input_keys(i) < 0)
+ return -1;
+
+ printf("\n------ KEY SIZE = %u ----------\n\n",
+ hashtest_key_lens[i]);
+ printf("\n ----- WITH PRECOMPUTED HASH VALUES -----\n\n");
+
+ printf("\nTimed additions\n");
+ printf("------------------\n");
+ if (timed_adds(1, i) < 0)
+ return -1;
+
+ for (j = 0; j < NUM_SHUFFLES; j++)
+ shuffle_input_keys(i);
+
+ printf("\nTimed lookups\n");
+ printf("------------------\n");
+ if (timed_lookups(1, i) < 0)
+ return -1;
+
+ printf("\nTimed deletions\n");
+ printf("------------------\n");
+ if (timed_deletes(1, i) < 0)
+ return -1;
+
+ if (reset_table(i) < 0)
+ return -1;
+
+ printf("\n ----- WITH JUST KEYS -----\n\n");
+
+ printf("\nTimed additions\n");
+ printf("------------------\n");
+ if (timed_adds(0, i) < 0)
+ return -1;
+
+ for (j = 0; j < NUM_SHUFFLES; j++)
+ shuffle_input_keys(i);
+
+ printf("\nTimed lookups\n");
+ printf("------------------\n");
+ if (timed_lookups(0, i) < 0)
+ return -1;
+
+ printf("\nTimed lookups multi\n");
+ printf("------------------\n");
+ if (timed_lookups_multi(i) < 0)
+ return -1;
+
+ printf("\nTimed deletions\n");
+ printf("------------------\n");
+ if (timed_deletes(0, i) < 0)
+ return -1;
+
+ free_table(i);
+
+ }
+ printf("\nResults (in CPU cycles/operation)\n");
+ printf("-----------------------------------\n");
+ printf("\nWith precomputed hash\n");
+ printf("\n%-18s%-18s%-18s%-18s%-18s\n",
+ "Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ printf("%-18d", hashtest_key_lens[i]);
+ for (j = 0; j < NUM_OPERATIONS; j++)
+ printf("%-18"PRIu64, cycles[i][j][1]);
+ printf("\n");
+ }
+ printf("\nWith just keys\n");
+ printf("\n%-18s%-18s%-18s%-18s%-18s\n",
+ "Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ printf("%-18d", hashtest_key_lens[i]);
+ for (j = 0; j < NUM_OPERATIONS; j++)
+ printf("%-18"PRIu64, cycles[i][j][0]);
+ printf("\n");
+ }
+
+ return 0;
+}
+
+/* Control operation of performance testing of fbk hash. */
+#define LOAD_FACTOR 0.667 /* How full to make the hash table. */
+#define TEST_SIZE 1000000 /* How many operations to time. */
+#define TEST_ITERATIONS 30 /* How many measurements to take. */
+#define ENTRIES (1 << 15) /* How many entries. */
+
+static int
+fbk_hash_perf_test(void)
+{
+ struct rte_fbk_hash_params params = {
+ .name = "fbk_hash_test",
+ .entries = ENTRIES,
+ .entries_per_bucket = 4,
+ .socket_id = rte_socket_id(),
+ };
+ struct rte_fbk_hash_table *handle = NULL;
+ uint32_t *keys = NULL;
+ unsigned indexes[TEST_SIZE];
+ uint64_t lookup_time = 0;
+ unsigned added = 0;
+ unsigned value = 0;
+ uint32_t key;
+ uint16_t val;
+ unsigned i, j;
+
+ handle = rte_fbk_hash_create(¶ms);
+ if (handle == NULL) {
+ printf("Error creating table\n");
+ return -1;
+ }
+
+ keys = rte_zmalloc(NULL, ENTRIES * sizeof(*keys), 0);
+ if (keys == NULL) {
+ printf("fbk hash: memory allocation for key store failed\n");
+ return -1;
+ }
+
+ /* Generate random keys and values. */
+ for (i = 0; i < ENTRIES; i++) {
+ key = (uint32_t)rte_rand();
+ key = ((uint64_t)key << 32) | (uint64_t)rte_rand();
+ val = (uint16_t)rte_rand();
+
+ if (rte_fbk_hash_add_key(handle, key, val) == 0) {
+ keys[added] = key;
+ added++;
+ }
+ if (added > (LOAD_FACTOR * ENTRIES))
+ break;
+ }
+
+ for (i = 0; i < TEST_ITERATIONS; i++) {
+ uint64_t begin;
+ uint64_t end;
+
+ /* Generate random indexes into keys[] array. */
+ for (j = 0; j < TEST_SIZE; j++)
+ indexes[j] = rte_rand() % added;
+
+ begin = rte_rdtsc();
+ /* Do lookups */
+ for (j = 0; j < TEST_SIZE; j++)
+ value += rte_fbk_hash_lookup(handle, keys[indexes[j]]);
+
+ end = rte_rdtsc();
+ lookup_time += (double)(end - begin);
+ }
+
+ printf("\n\n *** FBK Hash function performance test results ***\n");
+ /*
+ * The use of the 'value' variable ensures that the hash lookup is not
+ * being optimised out by the compiler.
+ */
+ if (value != 0)
+ printf("Number of ticks per lookup = %g\n",
+ (double)lookup_time /
+ ((double)TEST_ITERATIONS * (double)TEST_SIZE));
+
+ rte_fbk_hash_free(handle);
+
+ return 0;
+}
+
+static int
+test_hash_perf_new(void)
+{
+ if (run_all_tbl_perf_tests() < 0)
+ return -1;
+
+ if (fbk_hash_perf_test() < 0)
+ return -1;
+
+ return 0;
+}
+
+static struct test_command hash_perf_new_cmd = {
+ .command = "hash_perf_new_autotest",
+ .callback = test_hash_perf_new,
+};
+REGISTER_TEST_COMMAND(hash_perf_new_cmd);
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v3 04/11] test/hash: rename new hash perf unit test back to original name
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Pablo de Lara
` (2 preceding siblings ...)
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 03/11] test/hash: enhance hash unit tests Pablo de Lara
@ 2015-06-28 22:25 ` Pablo de Lara
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 05/11] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
` (8 subsequent siblings)
12 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-06-28 22:25 UTC (permalink / raw)
To: dev
To be able to see the diff more clear, new performance unit test was
named differently from the old unit test. This patch renames the
new unit test as the old one.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/Makefile | 2 +-
app/test/test_hash_perf.c | 560 ++++++++++++++++++++++++++++++++++++++++++
app/test/test_hash_perf_new.c | 560 ------------------------------------------
3 files changed, 561 insertions(+), 561 deletions(-)
create mode 100644 app/test/test_hash_perf.c
delete mode 100644 app/test/test_hash_perf_new.c
diff --git a/app/test/Makefile b/app/test/Makefile
index 8624e95..2e2758c 100644
--- a/app/test/Makefile
+++ b/app/test/Makefile
@@ -82,7 +82,7 @@ SRCS-y += test_memcpy.c
SRCS-y += test_memcpy_perf.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash.c
-SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_perf_new.c
+SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_perf.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_functions.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_scaling.c
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
new file mode 100644
index 0000000..978731c
--- /dev/null
+++ b/app/test/test_hash_perf.c
@@ -0,0 +1,560 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <inttypes.h>
+
+#include <rte_lcore.h>
+#include <rte_cycles.h>
+#include <rte_malloc.h>
+#include <rte_hash.h>
+#include <rte_hash_crc.h>
+#include <rte_jhash.h>
+#include <rte_fbk_hash.h>
+#include <rte_random.h>
+#include <rte_string_fns.h>
+
+#include "test.h"
+
+#define KEYS_TO_ADD (1 << 18)
+#define MAX_ENTRIES (KEYS_TO_ADD * 4) /* 25% table utilization */
+#define NUM_LOOKUPS (KEYS_TO_ADD * 10) /* Loop among keys added, several times */
+#define BUCKET_SIZE 4
+#define NUM_BUCKETS (MAX_ENTRIES / BUCKET_SIZE)
+#define MAX_KEYSIZE 64
+#define NUM_KEYSIZES 10
+#define NUM_SHUFFLES 10
+#define BURST_SIZE 16
+
+enum operations {
+ ADD = 0,
+ LOOKUP,
+ LOOKUP_MULTI,
+ DELETE,
+ NUM_OPERATIONS
+};
+
+static uint32_t hashtest_key_lens[] = {
+ 4, 8, 16, 32, 48, 64, /* standard key sizes */
+ 9, /* IPv4 SRC + DST + protocol, unpadded */
+ 13, /* IPv4 5-tuple, unpadded */
+ 37, /* IPv6 5-tuple, unpadded */
+ 40 /* IPv6 5-tuple, padded to 8-byte boundary */
+};
+struct rte_hash *h[NUM_KEYSIZES];
+/* Array that stores if a slot is full */
+uint8_t slot_taken[MAX_ENTRIES];
+/* Array to store number of cycles per operation */
+uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS][2];
+/* Array to store all input keys */
+uint8_t keys[KEYS_TO_ADD][MAX_KEYSIZE];
+/* Array to store the precomputed hash for 'keys' */
+hash_sig_t signatures[KEYS_TO_ADD];
+/* Array to store how many busy entries have each bucket */
+uint8_t buckets[NUM_BUCKETS];
+
+/* Parameters used for hash table in unit test functions. */
+static struct rte_hash_parameters ut_params = {
+ .entries = MAX_ENTRIES,
+ .bucket_entries = BUCKET_SIZE,
+ .hash_func = rte_jhash,
+ .hash_func_init_val = 0,
+};
+
+static int
+create_table(unsigned table_index)
+{
+ char name[RTE_HASH_NAMESIZE];
+
+ sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
+ ut_params.name = name;
+ ut_params.key_len = hashtest_key_lens[table_index];
+ ut_params.socket_id = rte_socket_id();
+ h[table_index] = rte_hash_find_existing(name);
+ if (h[table_index] != NULL)
+ /*
+ * If table was already created, free it to create it again,
+ * so we force it is empty
+ */
+ rte_hash_free(h[table_index]);
+ h[table_index] = rte_hash_create(&ut_params);
+ if (h[table_index] == NULL) {
+ printf("Error creating table\n");
+ return -1;
+ }
+ return 0;
+
+}
+
+/* Shuffle the keys that have been added, so lookups will be totally random */
+static void
+shuffle_input_keys(unsigned table_index)
+{
+ unsigned i;
+ uint32_t swap_idx;
+ uint8_t temp_key[RTE_HASH_KEY_LENGTH_MAX];
+ hash_sig_t temp_signature;
+
+ for (i = 0; i < KEYS_TO_ADD; i++) {
+ do
+ swap_idx = rte_rand() % KEYS_TO_ADD;
+ while (swap_idx == i);
+
+ memcpy(temp_key, keys[i], hashtest_key_lens[table_index]);
+ temp_signature = signatures[i];
+
+ memcpy(keys[i], keys[swap_idx], hashtest_key_lens[table_index]);
+ signatures[i] = signatures[swap_idx];
+
+ memcpy(keys[swap_idx], temp_key, hashtest_key_lens[table_index]);
+ signatures[swap_idx] = temp_signature;
+ }
+}
+
+/*
+ * Creates the table and looks for random keys which
+ * ALL can fit in hash table (no errors)
+ */
+static int
+get_input_keys(unsigned table_index)
+{
+ unsigned i, j;
+ unsigned bucket_idx, incr, success = 1;
+ uint8_t k = 0;
+ int32_t ret;
+ const uint32_t bucket_bitmask = NUM_BUCKETS - 1;
+
+ /* Reset all arrays */
+ for (i = 0; i < MAX_ENTRIES; i++)
+ slot_taken[i] = 0;
+
+ for (i = 0; i < NUM_BUCKETS; i++)
+ buckets[i] = 0;
+
+ for (j = 0; j < hashtest_key_lens[table_index]; j++)
+ keys[0][j] = 0;
+
+ /*
+ * Add only entries that are not duplicated and that fits in the table
+ * (cannot store more than BUCKET_SIZE entries in a bucket).
+ * Regardless a key has been added correctly or not (success),
+ * the next one to try will be increased by 1.
+ */
+ for (i = 0; i < KEYS_TO_ADD;) {
+ incr = 0;
+ if (i != 0) {
+ keys[i][0] = ++k;
+ /* Overflow, need to increment the next byte */
+ if (keys[i][0] == 0)
+ incr = 1;
+ for (j = 1; j < hashtest_key_lens[table_index]; j++) {
+ /* Do not increase next byte */
+ if (incr == 0)
+ if (success == 1)
+ keys[i][j] = keys[i - 1][j];
+ else
+ keys[i][j] = keys[i][j];
+ /* Increase next byte by one */
+ else {
+ if (success == 1)
+ keys[i][j] = keys[i-1][j] + 1;
+ else
+ keys[i][j] = keys[i][j] + 1;
+ if (keys[i][j] == 0)
+ incr = 1;
+ else
+ incr = 0;
+ }
+ }
+ }
+ success = 0;
+ signatures[i] = rte_hash_hash(h[table_index], keys[i]);
+ bucket_idx = signatures[i] & bucket_bitmask;
+ /* If bucket is full, do not try to insert the key */
+ if (buckets[bucket_idx] == BUCKET_SIZE)
+ continue;
+ /* If key can be added, leave in successful key arrays "keys" */
+ ret = rte_hash_add_key_with_hash(h[table_index], keys[i],
+ signatures[i]);
+ if (ret >= 0) {
+ /* If key is already added, ignore the entry and do not store */
+ if (slot_taken[ret])
+ continue;
+ else {
+ /* Store the returned position and mark slot as taken */
+ slot_taken[ret] = 1;
+ buckets[bucket_idx]++;
+ success = 1;
+ i++;
+ }
+ }
+ }
+
+ /* Reset the table, so we can measure the time to add all the entries */
+ rte_hash_free(h[table_index]);
+ h[table_index] = rte_hash_create(&ut_params);
+
+ return 0;
+}
+
+static int
+timed_adds(unsigned with_hash, unsigned table_index) {
+ unsigned i;
+ const uint64_t start_tsc = rte_rdtsc();
+
+ for (i = 0; i < KEYS_TO_ADD; i++) {
+ if (with_hash)
+ rte_hash_add_key_with_hash(h[table_index],
+ (const void *) keys[i],
+ signatures[i]);
+ else
+ rte_hash_add_key(h[table_index], keys[i]);
+ }
+
+ const uint64_t end_tsc = rte_rdtsc();
+ const uint64_t time_taken = end_tsc - start_tsc;
+ const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
+
+ cycles[table_index][ADD][with_hash] = time_taken/KEYS_TO_ADD;
+
+ printf("\n%"PRIu64" adds in %f seconds\n", (uint64_t)KEYS_TO_ADD,
+ seconds_taken);
+ printf("Average %"PRIu64" tsc ticks per add\n",
+ cycles[table_index][ADD][with_hash]);
+ printf("Average %"PRIu64" adds per second\n",
+ (KEYS_TO_ADD * rte_get_tsc_hz())/time_taken);
+ return 0;
+}
+
+static int
+timed_lookups(unsigned with_hash, unsigned table_index)
+{
+ unsigned i, j;
+ const uint64_t start_tsc = rte_rdtsc();
+
+ for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
+ for (j = 0; j < KEYS_TO_ADD; j++) {
+ if (with_hash)
+ rte_hash_lookup_with_hash(h[table_index],
+ (const void *) keys[j],
+ signatures[j]);
+ else
+ rte_hash_lookup(h[table_index], keys[j]);
+ }
+ }
+
+ const uint64_t end_tsc = rte_rdtsc();
+ const uint64_t time_taken = end_tsc - start_tsc;
+ const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
+
+ cycles[table_index][LOOKUP][with_hash] = time_taken/NUM_LOOKUPS;
+
+ printf("%"PRIu64" lookups in %f seconds\n", (uint64_t) NUM_LOOKUPS,
+ seconds_taken);
+ printf("Average %"PRIu64" tsc ticks per lookup\n",
+ cycles[table_index][LOOKUP][with_hash]);
+ printf("Average %"PRIu64" lookups per second\n",
+ (NUM_LOOKUPS * rte_get_tsc_hz())/time_taken);
+ return 0;
+}
+
+static int
+timed_lookups_multi(unsigned table_index)
+{
+ unsigned i, j, k;
+ int32_t positions_burst[BURST_SIZE];
+ const void *keys_burst[BURST_SIZE];
+ const uint64_t start_tsc = rte_rdtsc();
+
+ for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
+ for (j = 0; j < KEYS_TO_ADD/BURST_SIZE; j++) {
+ for (k = 0; k < BURST_SIZE; k++)
+ keys_burst[k] = keys[j * BURST_SIZE + k];
+ rte_hash_lookup_bulk(h[table_index],
+ (const void **) keys_burst,
+ BURST_SIZE,
+ positions_burst);
+ }
+ }
+
+ const uint64_t end_tsc = rte_rdtsc();
+ const uint64_t time_taken = end_tsc - start_tsc;
+ const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
+
+ cycles[table_index][LOOKUP_MULTI][0] = time_taken/NUM_LOOKUPS;
+
+ printf("%"PRIu64" lookups in %f seconds\n", (uint64_t)NUM_LOOKUPS,
+ seconds_taken);
+ printf("Average %"PRIu64" tsc ticks per lookup\n",
+ cycles[table_index][LOOKUP_MULTI][0]);
+ printf("Average %"PRIu64" lookups per second\n",
+ (NUM_LOOKUPS * rte_get_tsc_hz())/time_taken);
+ return 0;
+}
+
+static int
+timed_deletes(unsigned with_hash, unsigned table_index)
+{
+ unsigned i;
+ const uint64_t start_tsc = rte_rdtsc();
+
+ for (i = 0; i < KEYS_TO_ADD; i++) {
+ if (with_hash)
+ rte_hash_del_key_with_hash(h[table_index],
+ (const void *) keys[i],
+ signatures[i]);
+ else
+ rte_hash_del_key(h[table_index],
+ (const void *) keys[i]);
+ }
+
+ const uint64_t end_tsc = rte_rdtsc();
+ const uint64_t time_taken = end_tsc - start_tsc;
+ const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
+
+ cycles[table_index][DELETE][with_hash] = time_taken/KEYS_TO_ADD;
+
+ printf("\n%"PRIu64" deletions in %f seconds\n", (uint64_t) KEYS_TO_ADD,
+ seconds_taken);
+ printf("Average %"PRIu64" tsc ticks per deletion\n",
+ cycles[table_index][DELETE][with_hash]);
+ printf("Average %"PRIu64" deletions per second\n",
+ (KEYS_TO_ADD * rte_get_tsc_hz())/time_taken);
+ return 0;
+}
+
+static void
+free_table(unsigned table_index)
+{
+ rte_hash_free(h[table_index]);
+}
+
+static int
+reset_table(unsigned table_index)
+{
+ free_table(table_index);
+ if (create_table(table_index) != 0)
+ return -1;
+
+ return 0;
+}
+
+static int
+run_all_tbl_perf_tests(void)
+{
+ unsigned i, j;
+
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ if (create_table(i) < 0)
+ return -1;
+
+ if (get_input_keys(i) < 0)
+ return -1;
+
+ printf("\n------ KEY SIZE = %u ----------\n\n",
+ hashtest_key_lens[i]);
+ printf("\n ----- WITH PRECOMPUTED HASH VALUES -----\n\n");
+
+ printf("\nTimed additions\n");
+ printf("------------------\n");
+ if (timed_adds(1, i) < 0)
+ return -1;
+
+ for (j = 0; j < NUM_SHUFFLES; j++)
+ shuffle_input_keys(i);
+
+ printf("\nTimed lookups\n");
+ printf("------------------\n");
+ if (timed_lookups(1, i) < 0)
+ return -1;
+
+ printf("\nTimed deletions\n");
+ printf("------------------\n");
+ if (timed_deletes(1, i) < 0)
+ return -1;
+
+ if (reset_table(i) < 0)
+ return -1;
+
+ printf("\n ----- WITH JUST KEYS -----\n\n");
+
+ printf("\nTimed additions\n");
+ printf("------------------\n");
+ if (timed_adds(0, i) < 0)
+ return -1;
+
+ for (j = 0; j < NUM_SHUFFLES; j++)
+ shuffle_input_keys(i);
+
+ printf("\nTimed lookups\n");
+ printf("------------------\n");
+ if (timed_lookups(0, i) < 0)
+ return -1;
+
+ printf("\nTimed lookups multi\n");
+ printf("------------------\n");
+ if (timed_lookups_multi(i) < 0)
+ return -1;
+
+ printf("\nTimed deletions\n");
+ printf("------------------\n");
+ if (timed_deletes(0, i) < 0)
+ return -1;
+
+ free_table(i);
+
+ }
+ printf("\nResults (in CPU cycles/operation)\n");
+ printf("-----------------------------------\n");
+ printf("\nWith precomputed hash\n");
+ printf("\n%-18s%-18s%-18s%-18s%-18s\n",
+ "Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ printf("%-18d", hashtest_key_lens[i]);
+ for (j = 0; j < NUM_OPERATIONS; j++)
+ printf("%-18"PRIu64, cycles[i][j][1]);
+ printf("\n");
+ }
+ printf("\nWith just keys\n");
+ printf("\n%-18s%-18s%-18s%-18s%-18s\n",
+ "Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ printf("%-18d", hashtest_key_lens[i]);
+ for (j = 0; j < NUM_OPERATIONS; j++)
+ printf("%-18"PRIu64, cycles[i][j][0]);
+ printf("\n");
+ }
+
+ return 0;
+}
+
+/* Control operation of performance testing of fbk hash. */
+#define LOAD_FACTOR 0.667 /* How full to make the hash table. */
+#define TEST_SIZE 1000000 /* How many operations to time. */
+#define TEST_ITERATIONS 30 /* How many measurements to take. */
+#define ENTRIES (1 << 15) /* How many entries. */
+
+static int
+fbk_hash_perf_test(void)
+{
+ struct rte_fbk_hash_params params = {
+ .name = "fbk_hash_test",
+ .entries = ENTRIES,
+ .entries_per_bucket = 4,
+ .socket_id = rte_socket_id(),
+ };
+ struct rte_fbk_hash_table *handle = NULL;
+ uint32_t *keys = NULL;
+ unsigned indexes[TEST_SIZE];
+ uint64_t lookup_time = 0;
+ unsigned added = 0;
+ unsigned value = 0;
+ uint32_t key;
+ uint16_t val;
+ unsigned i, j;
+
+ handle = rte_fbk_hash_create(¶ms);
+ if (handle == NULL) {
+ printf("Error creating table\n");
+ return -1;
+ }
+
+ keys = rte_zmalloc(NULL, ENTRIES * sizeof(*keys), 0);
+ if (keys == NULL) {
+ printf("fbk hash: memory allocation for key store failed\n");
+ return -1;
+ }
+
+ /* Generate random keys and values. */
+ for (i = 0; i < ENTRIES; i++) {
+ key = (uint32_t)rte_rand();
+ key = ((uint64_t)key << 32) | (uint64_t)rte_rand();
+ val = (uint16_t)rte_rand();
+
+ if (rte_fbk_hash_add_key(handle, key, val) == 0) {
+ keys[added] = key;
+ added++;
+ }
+ if (added > (LOAD_FACTOR * ENTRIES))
+ break;
+ }
+
+ for (i = 0; i < TEST_ITERATIONS; i++) {
+ uint64_t begin;
+ uint64_t end;
+
+ /* Generate random indexes into keys[] array. */
+ for (j = 0; j < TEST_SIZE; j++)
+ indexes[j] = rte_rand() % added;
+
+ begin = rte_rdtsc();
+ /* Do lookups */
+ for (j = 0; j < TEST_SIZE; j++)
+ value += rte_fbk_hash_lookup(handle, keys[indexes[j]]);
+
+ end = rte_rdtsc();
+ lookup_time += (double)(end - begin);
+ }
+
+ printf("\n\n *** FBK Hash function performance test results ***\n");
+ /*
+ * The use of the 'value' variable ensures that the hash lookup is not
+ * being optimised out by the compiler.
+ */
+ if (value != 0)
+ printf("Number of ticks per lookup = %g\n",
+ (double)lookup_time /
+ ((double)TEST_ITERATIONS * (double)TEST_SIZE));
+
+ rte_fbk_hash_free(handle);
+
+ return 0;
+}
+
+static int
+test_hash_perf(void)
+{
+ if (run_all_tbl_perf_tests() < 0)
+ return -1;
+
+ if (fbk_hash_perf_test() < 0)
+ return -1;
+
+ return 0;
+}
+
+static struct test_command hash_perf_cmd = {
+ .command = "hash_perf_autotest",
+ .callback = test_hash_perf,
+};
+REGISTER_TEST_COMMAND(hash_perf_cmd);
diff --git a/app/test/test_hash_perf_new.c b/app/test/test_hash_perf_new.c
deleted file mode 100644
index 1da1839..0000000
--- a/app/test/test_hash_perf_new.c
+++ /dev/null
@@ -1,560 +0,0 @@
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <stdio.h>
-#include <inttypes.h>
-
-#include <rte_lcore.h>
-#include <rte_cycles.h>
-#include <rte_malloc.h>
-#include <rte_hash.h>
-#include <rte_hash_crc.h>
-#include <rte_jhash.h>
-#include <rte_fbk_hash.h>
-#include <rte_random.h>
-#include <rte_string_fns.h>
-
-#include "test.h"
-
-#define KEYS_TO_ADD (1 << 18)
-#define MAX_ENTRIES (KEYS_TO_ADD * 4) /* 25% table utilization */
-#define NUM_LOOKUPS (KEYS_TO_ADD * 10) /* Loop among keys added, several times */
-#define BUCKET_SIZE 4
-#define NUM_BUCKETS (MAX_ENTRIES / BUCKET_SIZE)
-#define MAX_KEYSIZE 64
-#define NUM_KEYSIZES 10
-#define NUM_SHUFFLES 10
-#define BURST_SIZE 16
-
-enum operations {
- ADD = 0,
- LOOKUP,
- LOOKUP_MULTI,
- DELETE,
- NUM_OPERATIONS
-};
-
-static uint32_t hashtest_key_lens[] = {
- 4, 8, 16, 32, 48, 64, /* standard key sizes */
- 9, /* IPv4 SRC + DST + protocol, unpadded */
- 13, /* IPv4 5-tuple, unpadded */
- 37, /* IPv6 5-tuple, unpadded */
- 40 /* IPv6 5-tuple, padded to 8-byte boundary */
-};
-struct rte_hash *h[NUM_KEYSIZES];
-/* Array that stores if a slot is full */
-uint8_t slot_taken[MAX_ENTRIES];
-/* Array to store number of cycles per operation */
-uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS][2];
-/* Array to store all input keys */
-uint8_t keys[KEYS_TO_ADD][MAX_KEYSIZE];
-/* Array to store the precomputed hash for 'keys' */
-hash_sig_t signatures[KEYS_TO_ADD];
-/* Array to store how many busy entries have each bucket */
-uint8_t buckets[NUM_BUCKETS];
-
-/* Parameters used for hash table in unit test functions. */
-static struct rte_hash_parameters ut_params = {
- .entries = MAX_ENTRIES,
- .bucket_entries = BUCKET_SIZE,
- .hash_func = rte_jhash,
- .hash_func_init_val = 0,
-};
-
-static int
-create_table(unsigned table_index)
-{
- char name[RTE_HASH_NAMESIZE];
-
- sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
- ut_params.name = name;
- ut_params.key_len = hashtest_key_lens[table_index];
- ut_params.socket_id = rte_socket_id();
- h[table_index] = rte_hash_find_existing(name);
- if (h[table_index] != NULL)
- /*
- * If table was already created, free it to create it again,
- * so we force it is empty
- */
- rte_hash_free(h[table_index]);
- h[table_index] = rte_hash_create(&ut_params);
- if (h[table_index] == NULL) {
- printf("Error creating table\n");
- return -1;
- }
- return 0;
-
-}
-
-/* Shuffle the keys that have been added, so lookups will be totally random */
-static void
-shuffle_input_keys(unsigned table_index)
-{
- unsigned i;
- uint32_t swap_idx;
- uint8_t temp_key[RTE_HASH_KEY_LENGTH_MAX];
- hash_sig_t temp_signature;
-
- for (i = 0; i < KEYS_TO_ADD; i++) {
- do
- swap_idx = rte_rand() % KEYS_TO_ADD;
- while (swap_idx == i);
-
- memcpy(temp_key, keys[i], hashtest_key_lens[table_index]);
- temp_signature = signatures[i];
-
- memcpy(keys[i], keys[swap_idx], hashtest_key_lens[table_index]);
- signatures[i] = signatures[swap_idx];
-
- memcpy(keys[swap_idx], temp_key, hashtest_key_lens[table_index]);
- signatures[swap_idx] = temp_signature;
- }
-}
-
-/*
- * Creates the table and looks for random keys which
- * ALL can fit in hash table (no errors)
- */
-static int
-get_input_keys(unsigned table_index)
-{
- unsigned i, j;
- unsigned bucket_idx, incr, success = 1;
- uint8_t k = 0;
- int32_t ret;
- const uint32_t bucket_bitmask = NUM_BUCKETS - 1;
-
- /* Reset all arrays */
- for (i = 0; i < MAX_ENTRIES; i++)
- slot_taken[i] = 0;
-
- for (i = 0; i < NUM_BUCKETS; i++)
- buckets[i] = 0;
-
- for (j = 0; j < hashtest_key_lens[table_index]; j++)
- keys[0][j] = 0;
-
- /*
- * Add only entries that are not duplicated and that fits in the table
- * (cannot store more than BUCKET_SIZE entries in a bucket).
- * Regardless a key has been added correctly or not (success),
- * the next one to try will be increased by 1.
- */
- for (i = 0; i < KEYS_TO_ADD;) {
- incr = 0;
- if (i != 0) {
- keys[i][0] = ++k;
- /* Overflow, need to increment the next byte */
- if (keys[i][0] == 0)
- incr = 1;
- for (j = 1; j < hashtest_key_lens[table_index]; j++) {
- /* Do not increase next byte */
- if (incr == 0)
- if (success == 1)
- keys[i][j] = keys[i - 1][j];
- else
- keys[i][j] = keys[i][j];
- /* Increase next byte by one */
- else {
- if (success == 1)
- keys[i][j] = keys[i-1][j] + 1;
- else
- keys[i][j] = keys[i][j] + 1;
- if (keys[i][j] == 0)
- incr = 1;
- else
- incr = 0;
- }
- }
- }
- success = 0;
- signatures[i] = rte_hash_hash(h[table_index], keys[i]);
- bucket_idx = signatures[i] & bucket_bitmask;
- /* If bucket is full, do not try to insert the key */
- if (buckets[bucket_idx] == BUCKET_SIZE)
- continue;
- /* If key can be added, leave in successful key arrays "keys" */
- ret = rte_hash_add_key_with_hash(h[table_index], keys[i],
- signatures[i]);
- if (ret >= 0) {
- /* If key is already added, ignore the entry and do not store */
- if (slot_taken[ret])
- continue;
- else {
- /* Store the returned position and mark slot as taken */
- slot_taken[ret] = 1;
- buckets[bucket_idx]++;
- success = 1;
- i++;
- }
- }
- }
-
- /* Reset the table, so we can measure the time to add all the entries */
- rte_hash_free(h[table_index]);
- h[table_index] = rte_hash_create(&ut_params);
-
- return 0;
-}
-
-static int
-timed_adds(unsigned with_hash, unsigned table_index) {
- unsigned i;
- const uint64_t start_tsc = rte_rdtsc();
-
- for (i = 0; i < KEYS_TO_ADD; i++) {
- if (with_hash)
- rte_hash_add_key_with_hash(h[table_index],
- (const void *) keys[i],
- signatures[i]);
- else
- rte_hash_add_key(h[table_index], keys[i]);
- }
-
- const uint64_t end_tsc = rte_rdtsc();
- const uint64_t time_taken = end_tsc - start_tsc;
- const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
-
- cycles[table_index][ADD][with_hash] = time_taken/KEYS_TO_ADD;
-
- printf("\n%"PRIu64" adds in %f seconds\n", (uint64_t)KEYS_TO_ADD,
- seconds_taken);
- printf("Average %"PRIu64" tsc ticks per add\n",
- cycles[table_index][ADD][with_hash]);
- printf("Average %"PRIu64" adds per second\n",
- (KEYS_TO_ADD * rte_get_tsc_hz())/time_taken);
- return 0;
-}
-
-static int
-timed_lookups(unsigned with_hash, unsigned table_index)
-{
- unsigned i, j;
- const uint64_t start_tsc = rte_rdtsc();
-
- for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
- for (j = 0; j < KEYS_TO_ADD; j++) {
- if (with_hash)
- rte_hash_lookup_with_hash(h[table_index],
- (const void *) keys[j],
- signatures[j]);
- else
- rte_hash_lookup(h[table_index], keys[j]);
- }
- }
-
- const uint64_t end_tsc = rte_rdtsc();
- const uint64_t time_taken = end_tsc - start_tsc;
- const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
-
- cycles[table_index][LOOKUP][with_hash] = time_taken/NUM_LOOKUPS;
-
- printf("%"PRIu64" lookups in %f seconds\n", (uint64_t) NUM_LOOKUPS,
- seconds_taken);
- printf("Average %"PRIu64" tsc ticks per lookup\n",
- cycles[table_index][LOOKUP][with_hash]);
- printf("Average %"PRIu64" lookups per second\n",
- (NUM_LOOKUPS * rte_get_tsc_hz())/time_taken);
- return 0;
-}
-
-static int
-timed_lookups_multi(unsigned table_index)
-{
- unsigned i, j, k;
- int32_t positions_burst[BURST_SIZE];
- const void *keys_burst[BURST_SIZE];
- const uint64_t start_tsc = rte_rdtsc();
-
- for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
- for (j = 0; j < KEYS_TO_ADD/BURST_SIZE; j++) {
- for (k = 0; k < BURST_SIZE; k++)
- keys_burst[k] = keys[j * BURST_SIZE + k];
- rte_hash_lookup_bulk(h[table_index],
- (const void **) keys_burst,
- BURST_SIZE,
- positions_burst);
- }
- }
-
- const uint64_t end_tsc = rte_rdtsc();
- const uint64_t time_taken = end_tsc - start_tsc;
- const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
-
- cycles[table_index][LOOKUP_MULTI][0] = time_taken/NUM_LOOKUPS;
-
- printf("%"PRIu64" lookups in %f seconds\n", (uint64_t)NUM_LOOKUPS,
- seconds_taken);
- printf("Average %"PRIu64" tsc ticks per lookup\n",
- cycles[table_index][LOOKUP_MULTI][0]);
- printf("Average %"PRIu64" lookups per second\n",
- (NUM_LOOKUPS * rte_get_tsc_hz())/time_taken);
- return 0;
-}
-
-static int
-timed_deletes(unsigned with_hash, unsigned table_index)
-{
- unsigned i;
- const uint64_t start_tsc = rte_rdtsc();
-
- for (i = 0; i < KEYS_TO_ADD; i++) {
- if (with_hash)
- rte_hash_del_key_with_hash(h[table_index],
- (const void *) keys[i],
- signatures[i]);
- else
- rte_hash_del_key(h[table_index],
- (const void *) keys[i]);
- }
-
- const uint64_t end_tsc = rte_rdtsc();
- const uint64_t time_taken = end_tsc - start_tsc;
- const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
-
- cycles[table_index][DELETE][with_hash] = time_taken/KEYS_TO_ADD;
-
- printf("\n%"PRIu64" deletions in %f seconds\n", (uint64_t) KEYS_TO_ADD,
- seconds_taken);
- printf("Average %"PRIu64" tsc ticks per deletion\n",
- cycles[table_index][DELETE][with_hash]);
- printf("Average %"PRIu64" deletions per second\n",
- (KEYS_TO_ADD * rte_get_tsc_hz())/time_taken);
- return 0;
-}
-
-static void
-free_table(unsigned table_index)
-{
- rte_hash_free(h[table_index]);
-}
-
-static int
-reset_table(unsigned table_index)
-{
- free_table(table_index);
- if (create_table(table_index) != 0)
- return -1;
-
- return 0;
-}
-
-static int
-run_all_tbl_perf_tests(void)
-{
- unsigned i, j;
-
- for (i = 0; i < NUM_KEYSIZES; i++) {
- if (create_table(i) < 0)
- return -1;
-
- if (get_input_keys(i) < 0)
- return -1;
-
- printf("\n------ KEY SIZE = %u ----------\n\n",
- hashtest_key_lens[i]);
- printf("\n ----- WITH PRECOMPUTED HASH VALUES -----\n\n");
-
- printf("\nTimed additions\n");
- printf("------------------\n");
- if (timed_adds(1, i) < 0)
- return -1;
-
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
-
- printf("\nTimed lookups\n");
- printf("------------------\n");
- if (timed_lookups(1, i) < 0)
- return -1;
-
- printf("\nTimed deletions\n");
- printf("------------------\n");
- if (timed_deletes(1, i) < 0)
- return -1;
-
- if (reset_table(i) < 0)
- return -1;
-
- printf("\n ----- WITH JUST KEYS -----\n\n");
-
- printf("\nTimed additions\n");
- printf("------------------\n");
- if (timed_adds(0, i) < 0)
- return -1;
-
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
-
- printf("\nTimed lookups\n");
- printf("------------------\n");
- if (timed_lookups(0, i) < 0)
- return -1;
-
- printf("\nTimed lookups multi\n");
- printf("------------------\n");
- if (timed_lookups_multi(i) < 0)
- return -1;
-
- printf("\nTimed deletions\n");
- printf("------------------\n");
- if (timed_deletes(0, i) < 0)
- return -1;
-
- free_table(i);
-
- }
- printf("\nResults (in CPU cycles/operation)\n");
- printf("-----------------------------------\n");
- printf("\nWith precomputed hash\n");
- printf("\n%-18s%-18s%-18s%-18s%-18s\n",
- "Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
- for (i = 0; i < NUM_KEYSIZES; i++) {
- printf("%-18d", hashtest_key_lens[i]);
- for (j = 0; j < NUM_OPERATIONS; j++)
- printf("%-18"PRIu64, cycles[i][j][1]);
- printf("\n");
- }
- printf("\nWith just keys\n");
- printf("\n%-18s%-18s%-18s%-18s%-18s\n",
- "Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
- for (i = 0; i < NUM_KEYSIZES; i++) {
- printf("%-18d", hashtest_key_lens[i]);
- for (j = 0; j < NUM_OPERATIONS; j++)
- printf("%-18"PRIu64, cycles[i][j][0]);
- printf("\n");
- }
-
- return 0;
-}
-
-/* Control operation of performance testing of fbk hash. */
-#define LOAD_FACTOR 0.667 /* How full to make the hash table. */
-#define TEST_SIZE 1000000 /* How many operations to time. */
-#define TEST_ITERATIONS 30 /* How many measurements to take. */
-#define ENTRIES (1 << 15) /* How many entries. */
-
-static int
-fbk_hash_perf_test(void)
-{
- struct rte_fbk_hash_params params = {
- .name = "fbk_hash_test",
- .entries = ENTRIES,
- .entries_per_bucket = 4,
- .socket_id = rte_socket_id(),
- };
- struct rte_fbk_hash_table *handle = NULL;
- uint32_t *keys = NULL;
- unsigned indexes[TEST_SIZE];
- uint64_t lookup_time = 0;
- unsigned added = 0;
- unsigned value = 0;
- uint32_t key;
- uint16_t val;
- unsigned i, j;
-
- handle = rte_fbk_hash_create(¶ms);
- if (handle == NULL) {
- printf("Error creating table\n");
- return -1;
- }
-
- keys = rte_zmalloc(NULL, ENTRIES * sizeof(*keys), 0);
- if (keys == NULL) {
- printf("fbk hash: memory allocation for key store failed\n");
- return -1;
- }
-
- /* Generate random keys and values. */
- for (i = 0; i < ENTRIES; i++) {
- key = (uint32_t)rte_rand();
- key = ((uint64_t)key << 32) | (uint64_t)rte_rand();
- val = (uint16_t)rte_rand();
-
- if (rte_fbk_hash_add_key(handle, key, val) == 0) {
- keys[added] = key;
- added++;
- }
- if (added > (LOAD_FACTOR * ENTRIES))
- break;
- }
-
- for (i = 0; i < TEST_ITERATIONS; i++) {
- uint64_t begin;
- uint64_t end;
-
- /* Generate random indexes into keys[] array. */
- for (j = 0; j < TEST_SIZE; j++)
- indexes[j] = rte_rand() % added;
-
- begin = rte_rdtsc();
- /* Do lookups */
- for (j = 0; j < TEST_SIZE; j++)
- value += rte_fbk_hash_lookup(handle, keys[indexes[j]]);
-
- end = rte_rdtsc();
- lookup_time += (double)(end - begin);
- }
-
- printf("\n\n *** FBK Hash function performance test results ***\n");
- /*
- * The use of the 'value' variable ensures that the hash lookup is not
- * being optimised out by the compiler.
- */
- if (value != 0)
- printf("Number of ticks per lookup = %g\n",
- (double)lookup_time /
- ((double)TEST_ITERATIONS * (double)TEST_SIZE));
-
- rte_fbk_hash_free(handle);
-
- return 0;
-}
-
-static int
-test_hash_perf_new(void)
-{
- if (run_all_tbl_perf_tests() < 0)
- return -1;
-
- if (fbk_hash_perf_test() < 0)
- return -1;
-
- return 0;
-}
-
-static struct test_command hash_perf_new_cmd = {
- .command = "hash_perf_new_autotest",
- .callback = test_hash_perf_new,
-};
-REGISTER_TEST_COMMAND(hash_perf_new_cmd);
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v3 05/11] hash: replace existing hash library with cuckoo hash implementation
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Pablo de Lara
` (3 preceding siblings ...)
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 04/11] test/hash: rename new hash perf unit test back to original name Pablo de Lara
@ 2015-06-28 22:25 ` Pablo de Lara
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 06/11] hash: add new lookup_bulk_with_hash function Pablo de Lara
` (7 subsequent siblings)
12 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-06-28 22:25 UTC (permalink / raw)
To: dev
This patch replaces the existing hash library with another approach,
using the Cuckoo Hash method to resolve collisions (open addressing),
which pushes items from a full bucket when a new entry tries
to be added in it, storing the evicted entry in an alternative location,
using a secondary hash function.
This gives the user the ability to store more entries when a bucket
is full, in comparison with the previous implementation.
Therefore, the unit test has been updated, as some scenarios have changed
(such as the previous removed restriction).
Also note that the API has not been changed, although new fields
have been added in the rte_hash structure (structure is internal now).
The main change when creating a new table is that the number of entries
per bucket is fixed now, so its parameter is ignored now
(still there to maintain the same parameters structure).
The hash unit test has been updated to reflect these changes.
As a last note, the maximum burst size in lookup_burst function
hash been increased to 64, to improve performance.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 106 +---
lib/librte_hash/Makefile | 8 +-
lib/librte_hash/rte_cuckoo_hash.c | 1056 +++++++++++++++++++++++++++++++++++++
lib/librte_hash/rte_hash.c | 499 ------------------
lib/librte_hash/rte_hash.h | 67 ++-
5 files changed, 1128 insertions(+), 608 deletions(-)
create mode 100644 lib/librte_hash/rte_cuckoo_hash.c
delete mode 100644 lib/librte_hash/rte_hash.c
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index 46174db..0176219 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -169,7 +169,6 @@ static struct flow_key keys[5] = { {
/* Parameters used for hash table in unit test functions. Name set later. */
static struct rte_hash_parameters ut_params = {
.entries = 64,
- .bucket_entries = 4,
.key_len = sizeof(struct flow_key), /* 13 */
.hash_func = rte_jhash,
.hash_func_init_val = 0,
@@ -516,9 +515,18 @@ static int test_five_keys(void)
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != -ENOENT,
- "failed to find key (pos[%u]=%d)", i, pos[i]);
+ "found non-existent key (pos[%u]=%d)", i, pos[i]);
}
+ /* Lookup multi */
+ ret = rte_hash_lookup_multi(handle, &key_array[0], 5, (int32_t *)pos);
+ if (ret == 0)
+ for (i = 0; i < 5; i++) {
+ print_key_info("Lkp", key_array[i], pos[i]);
+ RETURN_IF_ERROR(pos[i] != -ENOENT,
+ "found not-existent key (pos[%u]=%d)", i, pos[i]);
+ }
+
rte_hash_free(handle);
return 0;
@@ -527,21 +535,18 @@ static int test_five_keys(void)
/*
* Add keys to the same bucket until bucket full.
* - add 5 keys to the same bucket (hash created with 4 keys per bucket):
- * first 4 successful, 5th unsuccessful
- * - lookup the 5 keys: 4 hits, 1 miss
- * - add the 5 keys again: 4 OK, one error as bucket is full
- * - lookup the 5 keys: 4 hits (updated data), 1 miss
- * - delete the 5 keys: 5 OK (even if the 5th is not in the table)
+ * first 4 successful, 5th successful, pushing existing item in bucket
+ * - lookup the 5 keys: 5 hits
+ * - add the 5 keys again: 5 OK
+ * - lookup the 5 keys: 5 hits (updated data)
+ * - delete the 5 keys: 5 OK
* - lookup the 5 keys: 5 misses
- * - add the 5th key: OK
- * - lookup the 5th key: hit
*/
static int test_full_bucket(void)
{
struct rte_hash_parameters params_pseudo_hash = {
.name = "test4",
.entries = 64,
- .bucket_entries = 4,
.key_len = sizeof(struct flow_key), /* 13 */
.hash_func = pseudo_hash,
.hash_func_init_val = 0,
@@ -555,7 +560,7 @@ static int test_full_bucket(void)
handle = rte_hash_create(¶ms_pseudo_hash);
RETURN_IF_ERROR(handle == NULL, "hash creation failed");
- /* Fill bucket*/
+ /* Fill bucket */
for (i = 0; i < 4; i++) {
pos[i] = rte_hash_add_key(handle, &keys[i]);
print_key_info("Add", &keys[i], pos[i]);
@@ -563,47 +568,39 @@ static int test_full_bucket(void)
"failed to add key (pos[%u]=%d)", i, pos[i]);
expected_pos[i] = pos[i];
}
- /* This shouldn't work because the bucket is full */
+ /*
+ * This should work and will push one of the items
+ * in the bucket because it is full
+ */
pos[4] = rte_hash_add_key(handle, &keys[4]);
print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOSPC,
- "fail: added key to full bucket (pos[4]=%d)", pos[4]);
+ RETURN_IF_ERROR(pos[4] < 0,
+ "failed to add key (pos[4]=%d)", pos[4]);
+ expected_pos[4] = pos[4];
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to find key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: found non-existent key (pos[4]=%d)", pos[4]);
/* Add - update */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_add_key(handle, &keys[i]);
print_key_info("Add", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to add key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_add_key(handle, &keys[4]);
- print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOSPC,
- "fail: added key to full bucket (pos[4]=%d)", pos[4]);
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to find key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: found non-existent key (pos[4]=%d)", pos[4]);
/* Delete 1 key, check other keys are still found */
pos[1] = rte_hash_del_key(handle, &keys[1]);
@@ -623,35 +620,21 @@ static int test_full_bucket(void)
RETURN_IF_ERROR(pos[1] < 0, "failed to add key (pos[1]=%d)", pos[1]);
/* Delete */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_del_key(handle, &keys[i]);
print_key_info("Del", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to delete key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_del_key(handle, &keys[4]);
- print_key_info("Del", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: deleted non-existent key (pos[4]=%d)", pos[4]);
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != -ENOENT,
"fail: found non-existent key (pos[%u]=%d)", i, pos[i]);
}
- /* Add and lookup the 5th key */
- pos[4] = rte_hash_add_key(handle, &keys[4]);
- print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] < 0, "failed to add key (pos[4]=%d)", pos[4]);
- expected_pos[4] = pos[4];
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != expected_pos[4],
- "failed to find key (pos[4]=%d)", pos[4]);
-
rte_hash_free(handle);
/* Cover the NULL case. */
@@ -1017,18 +1000,8 @@ static int test_hash_creation_with_bad_parameters(void)
}
memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_1";
- params.bucket_entries = RTE_HASH_BUCKET_ENTRIES_MAX + 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully with bucket_entries in parameter exceeded\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
params.name = "creation_with_bad_parameters_2";
- params.entries = params.bucket_entries - 1;
+ params.entries = RTE_HASH_BUCKET_ENTRIES - 1;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
rte_hash_free(handle);
@@ -1048,16 +1021,6 @@ static int test_hash_creation_with_bad_parameters(void)
memcpy(¶ms, &ut_params, sizeof(params));
params.name = "creation_with_bad_parameters_4";
- params.bucket_entries = params.bucket_entries - 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully if bucket_entries in parameter is not power of 2\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_5";
params.key_len = 0;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
@@ -1068,16 +1031,6 @@ static int test_hash_creation_with_bad_parameters(void)
memcpy(¶ms, &ut_params, sizeof(params));
params.name = "creation_with_bad_parameters_6";
- params.key_len = RTE_HASH_KEY_LENGTH_MAX + 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully if key_len is greater than the maximum\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_7";
params.socket_id = RTE_MAX_NUMA_NODES + 1;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
@@ -1211,7 +1164,6 @@ static uint8_t key[16] = {0x00, 0x01, 0x02, 0x03,
static struct rte_hash_parameters hash_params_ex = {
.name = NULL,
.entries = 64,
- .bucket_entries = 4,
.key_len = 0,
.hash_func = NULL,
.hash_func_init_val = 0,
diff --git a/lib/librte_hash/Makefile b/lib/librte_hash/Makefile
index 3696cb1..1dc8a9c 100644
--- a/lib/librte_hash/Makefile
+++ b/lib/librte_hash/Makefile
@@ -1,6 +1,6 @@
# BSD LICENSE
#
-# Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+# Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@@ -42,7 +42,7 @@ EXPORT_MAP := rte_hash_version.map
LIBABIVER := 1
# all source are stored in SRCS-y
-SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_hash.c
+SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c
# install this header file
@@ -51,7 +51,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_hash_crc.h
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h
-# this lib needs eal
-DEPDIRS-$(CONFIG_RTE_LIBRTE_HASH) += lib/librte_eal lib/librte_malloc
+# this lib needs eal, malloc and ring
+DEPDIRS-$(CONFIG_RTE_LIBRTE_HASH) += lib/librte_eal lib/librte_malloc lib/librte_ring
include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
new file mode 100644
index 0000000..10febdc
--- /dev/null
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -0,0 +1,1056 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <sys/queue.h>
+
+#include <rte_common.h>
+#include <rte_memory.h> /* for definition of RTE_CACHE_LINE_SIZE */
+#include <rte_log.h>
+#include <rte_memcpy.h>
+#include <rte_prefetch.h>
+#include <rte_branch_prediction.h>
+#include <rte_memzone.h>
+#include <rte_malloc.h>
+#include <rte_eal.h>
+#include <rte_eal_memconfig.h>
+#include <rte_per_lcore.h>
+#include <rte_errno.h>
+#include <rte_string_fns.h>
+#include <rte_cpuflags.h>
+#include <rte_log.h>
+#include <rte_rwlock.h>
+#include <rte_spinlock.h>
+
+#include "rte_hash.h"
+
+TAILQ_HEAD(rte_hash_list, rte_tailq_entry);
+
+static struct rte_tailq_elem rte_hash_tailq = {
+ .name = "RTE_HASH",
+};
+EAL_REGISTER_TAILQ(rte_hash_tailq)
+
+/* Macro to enable/disable run-time checking of function parameters */
+#if defined(RTE_LIBRTE_HASH_DEBUG)
+#define RETURN_IF_TRUE(cond, retval) do { \
+ if (cond) \
+ return retval; \
+} while (0)
+#else
+#define RETURN_IF_TRUE(cond, retval)
+#endif
+
+/* Hash function used if none is specified */
+#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
+#include <rte_hash_crc.h>
+#define DEFAULT_HASH_FUNC rte_hash_crc
+#else
+#include <rte_jhash.h>
+#define DEFAULT_HASH_FUNC rte_jhash
+#endif
+
+#define NULL_SIGNATURE 0
+
+typedef int (*rte_hash_cmp_eq_t)(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k48_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k64_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k80_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k96_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k112_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k128_cmp_eq(const void *key1, const void *key2, size_t key_len);
+
+/** A hash table structure. */
+struct rte_hash {
+ char name[RTE_HASH_NAMESIZE]; /**< Name of the hash. */
+ uint32_t entries; /**< Total table entries. */
+ uint16_t key_len; /**< Length of hash key. */
+ rte_hash_function hash_func; /**< Function used to calculate hash. */
+ rte_hash_cmp_eq_t rte_hash_cmp_eq; /**< Function used to compare keys. */
+ uint32_t hash_func_init_val; /**< Init value used by hash_func. */
+ uint32_t num_buckets; /**< Number of buckets in table. */
+ uint32_t bucket_bitmask; /**< Bitmask for getting bucket index
+ from hash signature. */
+ uint32_t key_entry_size; /**< Size of each key entry. */
+
+ struct rte_ring *free_slots; /**< Ring that stores all indexes
+ of the free slots in the key table */
+ void *key_store; /**< Table storing all keys and data */
+ struct rte_hash_bucket *buckets; /**< Table with buckets storing all the
+ hash values and key indexes
+ to the key table*/
+} __rte_cache_aligned;
+
+/* Structure storing both primary and secondary hashes */
+struct rte_hash_signatures {
+ union {
+ struct {
+ hash_sig_t current;
+ hash_sig_t alt;
+ };
+ uint64_t sig;
+ };
+};
+
+/** Bucket structure */
+struct rte_hash_bucket {
+ struct rte_hash_signatures signatures[RTE_HASH_BUCKET_ENTRIES];
+ /* Includes dummy key index that always contains index 0 */
+ uint32_t key_idx[RTE_HASH_BUCKET_ENTRIES + 1];
+} __rte_cache_aligned;
+
+struct rte_hash *
+rte_hash_find_existing(const char *name)
+{
+ struct rte_hash *h = NULL;
+ struct rte_tailq_entry *te;
+ struct rte_hash_list *hash_list;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK);
+ TAILQ_FOREACH(te, hash_list, next) {
+ h = (struct rte_hash *) te->data;
+ if (strncmp(name, h->name, RTE_HASH_NAMESIZE) == 0)
+ break;
+ }
+ rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ if (te == NULL) {
+ rte_errno = ENOENT;
+ return NULL;
+ }
+ return h;
+}
+
+struct rte_hash *
+rte_hash_create(const struct rte_hash_parameters *params)
+{
+ struct rte_hash *h = NULL;
+ struct rte_tailq_entry *te = NULL;
+ struct rte_hash_list *hash_list;
+ struct rte_ring *r = NULL;
+ char hash_name[RTE_HASH_NAMESIZE];
+ void *ptr, *k = NULL;
+ char ring_name[RTE_RING_NAMESIZE];
+ unsigned i;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ if (params == NULL) {
+ RTE_LOG(ERR, HASH, "rte_hash_create has no parameters\n");
+ return NULL;
+ }
+
+ /* Check for valid parameters */
+ if ((params->entries > RTE_HASH_ENTRIES_MAX) ||
+ (params->entries < RTE_HASH_BUCKET_ENTRIES) ||
+ !rte_is_power_of_2(params->entries) ||
+ !rte_is_power_of_2(RTE_HASH_BUCKET_ENTRIES) ||
+ (params->key_len == 0)) {
+ rte_errno = EINVAL;
+ RTE_LOG(ERR, HASH, "rte_hash_create has invalid parameters\n");
+ return NULL;
+ }
+
+ snprintf(hash_name, sizeof(hash_name), "HT_%s", params->name);
+
+ /* Guarantee there's no existing */
+ h = rte_hash_find_existing(params->name);
+ if (h != NULL)
+ return h;
+
+ /* Calculate hash dimensions */
+ const uint32_t num_buckets = params->entries / RTE_HASH_BUCKET_ENTRIES;
+
+ /* Total memory required for hash context */
+ const uint32_t mem_size = sizeof(struct rte_hash) +
+ num_buckets * sizeof(struct rte_hash_bucket);
+ const uint32_t key_entry_size = params->key_len;
+ /* Store all keys and leave the first entry as a dummy entry for lookup_bulk */
+ const uint64_t key_tbl_size = key_entry_size * (params->entries + 1);
+
+ te = rte_zmalloc("HASH_TAILQ_ENTRY", sizeof(*te), 0);
+ if (te == NULL) {
+ RTE_LOG(ERR, HASH, "tailq entry allocation failed\n");
+ goto err;
+ }
+
+ h = (struct rte_hash *)rte_zmalloc_socket(hash_name, mem_size,
+ RTE_CACHE_LINE_SIZE, params->socket_id);
+
+ if (h == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ k = rte_zmalloc_socket(NULL, key_tbl_size,
+ RTE_CACHE_LINE_SIZE, params->socket_id);
+
+ if (k == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ /* Select function to compare keys */
+ switch (params->key_len) {
+ case 16:
+ h->rte_hash_cmp_eq = rte_hash_k16_cmp_eq;
+ break;
+ case 32:
+ h->rte_hash_cmp_eq = rte_hash_k32_cmp_eq;
+ break;
+ case 48:
+ h->rte_hash_cmp_eq = rte_hash_k48_cmp_eq;
+ break;
+ case 64:
+ h->rte_hash_cmp_eq = rte_hash_k64_cmp_eq;
+ break;
+ case 80:
+ h->rte_hash_cmp_eq = rte_hash_k80_cmp_eq;
+ break;
+ case 96:
+ h->rte_hash_cmp_eq = rte_hash_k96_cmp_eq;
+ break;
+ case 112:
+ h->rte_hash_cmp_eq = rte_hash_k112_cmp_eq;
+ break;
+ case 128:
+ h->rte_hash_cmp_eq = rte_hash_k128_cmp_eq;
+ break;
+ default:
+ /* If key is not multiple of 16, use generic memcmp */
+ h->rte_hash_cmp_eq = memcmp;
+ }
+
+ snprintf(ring_name, sizeof(ring_name), "HT_%s", params->name);
+ r = rte_ring_lookup(ring_name);
+ if (r != NULL) {
+ /* clear the free ring */
+ while (rte_ring_dequeue(r, &ptr) == 0)
+ rte_pause();
+ } else
+ r = rte_ring_create(ring_name, rte_align32pow2(params->entries + 1),
+ params->socket_id, 0);
+ if (r == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ /* Setup hash context */
+ snprintf(h->name, sizeof(h->name), "%s", params->name);
+ h->entries = params->entries;
+ h->key_len = params->key_len;
+ h->key_entry_size = key_entry_size;
+ h->hash_func_init_val = params->hash_func_init_val;
+
+ h->num_buckets = num_buckets;
+ h->bucket_bitmask = h->num_buckets - 1;
+ h->buckets = (void *) &h[1];
+ h->hash_func = (params->hash_func == NULL) ?
+ DEFAULT_HASH_FUNC : params->hash_func;
+
+ h->key_store = k;
+ h->free_slots = r;
+ te->data = (void *) h;
+
+ /* populate the free slots ring. Entry zero is reserved for key misses */
+ for (i = 1; i < params->entries + 1; i++)
+ rte_ring_sp_enqueue(r, (void *)((uintptr_t) i));
+
+ rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
+ TAILQ_INSERT_TAIL(hash_list, te, next);
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ return h;
+err:
+ rte_free(te);
+ rte_free(h);
+ rte_free(k);
+ return NULL;
+}
+
+void
+rte_hash_free(struct rte_hash *h)
+{
+ struct rte_tailq_entry *te;
+ struct rte_hash_list *hash_list;
+
+ if (h == NULL)
+ return;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
+
+ /* find out tailq entry */
+ TAILQ_FOREACH(te, hash_list, next) {
+ if (te->data == (void *) h)
+ break;
+ }
+
+ if (te == NULL) {
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+ return;
+ }
+
+ TAILQ_REMOVE(hash_list, te, next);
+
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ rte_free(h->key_store);
+ rte_free(h);
+ rte_free(te);
+}
+
+hash_sig_t
+rte_hash_hash(const struct rte_hash *h, const void *key)
+{
+ /* calc hash result by key */
+ return h->hash_func(key, h->key_len, h->hash_func_init_val);
+}
+
+/* Calc the secondary hash value from the primary hash value of a given key */
+static inline hash_sig_t
+rte_hash_secondary_hash(const hash_sig_t primary_hash)
+{
+ static const unsigned all_bits_shift = 12;
+ static const unsigned alt_bits_xor = 0x5bd1e995;
+
+ uint32_t tag = primary_hash >> all_bits_shift;
+
+ return (primary_hash ^ ((tag + 1) * alt_bits_xor));
+}
+
+/*
+ * Try to insert a new entry. If bucket has space, hash value and key index
+ * to the key table are copied.
+ * If bucket is full, one of the entries in the bucket is evicted to
+ * its alternative location, based on the entry that has space
+ * in its alternative location.
+ * If none of the entries has space in its alternative location,
+ * we pick the last one to be pushed.
+ * Algorithm finishes when the entry has been successfully added.
+ * If we are trying to add AGAIN the same entry,
+ * entry cannot be added.
+ */
+static inline int32_t
+run_cuckoo(const struct rte_hash *h, struct rte_hash_bucket *bkt,
+ uint32_t key_idx, hash_sig_t current_hash,
+ hash_sig_t alt_hash, hash_sig_t original_hash,
+ const void *original_key)
+{
+ static unsigned number_pushes;
+ void *k, *keys = h->key_store;
+ unsigned i, j;
+
+ hash_sig_t current_hash_stored, alt_hash_stored;
+ uint32_t key_idx_stored;
+ uint32_t bucket_stored_idx;
+ struct rte_hash_bucket *bkt_stored;
+
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ /* Check if slot is available */
+ if (likely(bkt->signatures[i].sig == NULL_SIGNATURE)) {
+ bkt->signatures[i].current = current_hash;
+ bkt->signatures[i].alt = alt_hash;
+ bkt->key_idx[i] = key_idx;
+ number_pushes = 0;
+ return 0;
+ }
+ }
+
+ /*
+ * If the entry that has been just evicted (hash) is the entry
+ * that is trying to be added in the hash table,
+ * we just entered in a loop and key cannot be added
+ */
+ if (++number_pushes > 1 && current_hash == original_hash) {
+ k = (char *)keys + key_idx * h->key_entry_size;
+ if (!h->rte_hash_cmp_eq(k, original_key, h->key_len)) {
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t)key_idx));
+ number_pushes = 0;
+ return -ENOSPC;
+ }
+ }
+
+ /*
+ * Push existing item (search for bucket with space in
+ * alternative locations) to its alternative location
+ */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ key_idx_stored = bkt->key_idx[i];
+ current_hash_stored = bkt->signatures[i].current;
+ alt_hash_stored = bkt->signatures[i].alt;
+
+ /* Search for space in alternative locations */
+ bucket_stored_idx = alt_hash_stored & h->bucket_bitmask;
+ bkt_stored = &h->buckets[bucket_stored_idx];
+
+ for (j = 0; j < RTE_HASH_BUCKET_ENTRIES; j++) {
+ if (bkt_stored->signatures[j].sig == NULL_SIGNATURE)
+ break;
+ }
+
+ if (j != RTE_HASH_BUCKET_ENTRIES)
+ break;
+ }
+
+ /* Push existing item (if all alternative buckets are full, pick the last one) */
+ if (i == RTE_HASH_BUCKET_ENTRIES)
+ i -= 1;
+
+ /* Replace evicted entry with new entry */
+ bkt->signatures[i].current = current_hash;
+ bkt->signatures[i].alt = alt_hash;
+ bkt->key_idx[i] = key_idx;
+
+ /* There is an empty slot in the alternative bucket */
+ if (j != RTE_HASH_BUCKET_ENTRIES) {
+ /*
+ * Swap current and alt hashes as we have pushed the item
+ * to its alternative location
+ */
+ bkt_stored->signatures[j].current = alt_hash_stored;
+ bkt_stored->signatures[j].alt = current_hash_stored;
+ bkt_stored->key_idx[j] = key_idx_stored;
+ number_pushes = 0;
+ return 0;
+ } else
+ return run_cuckoo(h, bkt_stored, key_idx_stored, alt_hash_stored,
+ current_hash_stored, original_hash, original_key);
+}
+
+static inline int32_t
+__rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ hash_sig_t hash0, hash1;
+ uint32_t bucket_idx0, bucket_idx1;
+ unsigned i;
+ struct rte_hash_bucket *bkt0, *bkt1;
+ void *new_k, *k, *keys = h->key_store;
+ void *slot_id;
+ int ret;
+
+ /* Get a new slot for storing the new key */
+ if (rte_ring_sc_dequeue(h->free_slots, &slot_id) != 0)
+ return -ENOSPC;
+ new_k = (char *)keys + (uintptr_t)slot_id * h->key_entry_size;
+ rte_prefetch0(new_k);
+
+ hash0 = sig;
+ bucket_idx0 = hash0 & h->bucket_bitmask;
+ bkt0 = &h->buckets[bucket_idx0];
+
+ hash1 = rte_hash_secondary_hash(hash0);
+ bucket_idx1 = hash1 & h->bucket_bitmask;
+ bkt1 = &h->buckets[bucket_idx1];
+ rte_prefetch0(bkt1);
+
+ /* Check if key is already inserted in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt0->signatures[i].current == hash0 &&
+ bkt0->signatures[i].alt == hash1) {
+ k = (char *)keys + bkt0->key_idx[i] * h->key_entry_size;
+ if (!h->rte_hash_cmp_eq(key, k, h->key_len)) {
+ rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt0->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Check if key is already inserted in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt1->signatures[i].alt == hash0 &&
+ bkt1->signatures[i].current == hash1) {
+ k = (char *)keys + bkt1->key_idx[i] * h->key_entry_size;
+ if (!h->rte_hash_cmp_eq(key, k, h->key_len)) {
+ rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt1->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Copy key */
+ rte_memcpy(new_k, key, h->key_len);
+
+ /*
+ * Run cuckoo algorithm
+ * Notice that for the first call of run_cuckoo,
+ * 4th and 5th parameter are the same (hash0),
+ * but this will change in next calls (recursive function),
+ * since the 4th parameter is the hash value we are currently trying
+ * to insert in the current bucket (NOT NEW in next calls)
+ * and the 5th parameter is the NEW hash value
+ * we are trying to add in the hash table
+ */
+ ret = run_cuckoo(h, bkt0, (uint32_t)((uintptr_t) slot_id), hash0,
+ hash1, hash0, key);
+
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ * or error
+ */
+ if (ret == 0)
+ return (int32_t)((uintptr_t) slot_id - 1);
+ else
+ return ret;
+}
+
+int32_t
+rte_hash_add_key_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_add_key_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_add_key(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+static inline int32_t
+__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ uint32_t bucket_idx;
+ hash_sig_t alt_hash;
+ unsigned i;
+ struct rte_hash_bucket *bkt;
+ void *k, *keys = h->key_store;
+
+ bucket_idx = sig & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == sig &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (!h->rte_hash_cmp_eq(key, k, h->key_len))
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+
+ /* Calculate secondary hash */
+ alt_hash = rte_hash_secondary_hash(sig);
+ bucket_idx = alt_hash & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == alt_hash &&
+ bkt->signatures[i].alt == sig) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (!h->rte_hash_cmp_eq(key, k, h->key_len))
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+
+ return -ENOENT;
+}
+
+int32_t
+rte_hash_lookup_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_lookup(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+static inline int32_t
+__rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ uint32_t bucket_idx;
+ hash_sig_t alt_hash;
+ unsigned i;
+ struct rte_hash_bucket *bkt;
+ void *k, *keys = h->key_store;
+
+ bucket_idx = sig & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == sig &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (!h->rte_hash_cmp_eq(key, k, h->key_len)) {
+ bkt->signatures[i].sig = NULL_SIGNATURE;
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t)bkt->key_idx[i]));
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Calculate secondary hash */
+ alt_hash = rte_hash_secondary_hash(sig);
+ bucket_idx = alt_hash & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == alt_hash &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (!h->rte_hash_cmp_eq(key, k, h->key_len)) {
+ bkt->signatures[i].sig = NULL_SIGNATURE;
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t)bkt->key_idx[i]));
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ return -ENOENT;
+}
+
+int32_t
+rte_hash_del_key_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_del_key_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_del_key(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_del_key_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+/* Lookup bulk stage 0: Calculate next primary/secondary hash value (from new key) */
+static inline void
+lookup_stage0(unsigned *idx, uint64_t *lookup_mask,
+ hash_sig_t *primary_hash, hash_sig_t *secondary_hash,
+ const void * const *keys, const struct rte_hash *h)
+{
+ *idx = __builtin_ctzl(*lookup_mask);
+ if (*lookup_mask == 0)
+ *idx = 0;
+
+ *primary_hash = rte_hash_hash(h, keys[*idx]);
+ *secondary_hash = rte_hash_secondary_hash(*primary_hash);
+
+ *lookup_mask &= ~(1llu << *idx);
+}
+
+
+/* Lookup bulk stage 1: Prefetch primary/secondary buckets */
+static inline void
+lookup_stage1(hash_sig_t primary_hash, hash_sig_t secondary_hash,
+ const struct rte_hash_bucket **primary_bkt,
+ const struct rte_hash_bucket **secondary_bkt,
+ const struct rte_hash *h)
+{
+ *primary_bkt = &h->buckets[primary_hash & h->bucket_bitmask];
+ *secondary_bkt = &h->buckets[secondary_hash & h->bucket_bitmask];
+
+ rte_prefetch0(*primary_bkt);
+ rte_prefetch0(*secondary_bkt);
+}
+
+/*
+ * Lookup bulk stage 2: Search for match hashes in primary/secondary locations
+ * and prefetch first key slot
+ */
+static inline void
+lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
+ const struct rte_hash_bucket *prim_bkt,
+ const struct rte_hash_bucket *sec_bkt,
+ const void **key_slot, int32_t *positions,
+ uint64_t *extra_hits_mask, const void *keys,
+ const struct rte_hash *h)
+{
+ unsigned prim_hash_matches, sec_hash_matches, key_idx, i;
+ unsigned total_hash_matches;
+
+ prim_hash_matches = 1 << RTE_HASH_BUCKET_ENTRIES;
+ sec_hash_matches = 1 << RTE_HASH_BUCKET_ENTRIES;
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ prim_hash_matches |= ((prim_hash == prim_bkt->signatures[i].current) << i);
+ sec_hash_matches |= ((sec_hash == sec_bkt->signatures[i].current) << i);
+ }
+
+ key_idx = prim_bkt->key_idx[__builtin_ctzl(prim_hash_matches)];
+ if (key_idx == 0)
+ key_idx = sec_bkt->key_idx[__builtin_ctzl(sec_hash_matches)];
+
+ total_hash_matches = (prim_hash_matches |
+ (sec_hash_matches << (RTE_HASH_BUCKET_ENTRIES + 1)));
+ *key_slot = (const char *)keys + key_idx * h->key_entry_size;
+
+ rte_prefetch0(*key_slot);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ positions[idx] = (key_idx - 1);
+
+ *extra_hits_mask |= (uint64_t)(__builtin_popcount(prim_hash_matches) > 2) << idx;
+ *extra_hits_mask |= (uint64_t)(__builtin_popcount(sec_hash_matches) > 2) << idx;
+ *extra_hits_mask |= (uint64_t)(__builtin_popcount(total_hash_matches) > 3) << idx;
+
+}
+
+
+/* Lookup bulk stage 3: Check if key matches, update hit mask */
+static inline void
+lookup_stage3(unsigned idx, const void *key_slot,
+ const void * const *keys, int32_t *positions,
+ uint64_t *hits, const struct rte_hash *h)
+{
+ unsigned hit;
+
+ hit = !h->rte_hash_cmp_eq(key_slot, keys[idx], h->key_len);
+ if (unlikely(hit == 0))
+ positions[idx] = -ENOENT;
+ *hits = (uint64_t)(hit) << idx;
+}
+
+static inline int
+__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, int32_t *positions) {
+
+ uint64_t hits = 0;
+ uint64_t next_mask = 0;
+ uint64_t extra_hits_mask = 0;
+ uint64_t lookup_mask;
+ unsigned idx;
+ const void *key_store = h->key_store;
+
+ unsigned idx00, idx01, idx10, idx11, idx20, idx21, idx30, idx31;
+ const struct rte_hash_bucket *primary_bkt10, *primary_bkt11;
+ const struct rte_hash_bucket *secondary_bkt10, *secondary_bkt11;
+ const struct rte_hash_bucket *primary_bkt20, *primary_bkt21;
+ const struct rte_hash_bucket *secondary_bkt20, *secondary_bkt21;
+ const void *k_slot20, *k_slot21, *k_slot30, *k_slot31;
+ hash_sig_t primary_hash00, primary_hash01;
+ hash_sig_t secondary_hash00, secondary_hash01;
+ hash_sig_t primary_hash10, primary_hash11;
+ hash_sig_t secondary_hash10, secondary_hash11;
+ hash_sig_t primary_hash20, primary_hash21;
+ hash_sig_t secondary_hash20, secondary_hash21;
+
+ if (num_keys == RTE_HASH_LOOKUP_BULK_MAX)
+ lookup_mask = 0xffffffffffffffff;
+ else
+ lookup_mask = (1ULL << num_keys) - 1;
+
+ lookup_stage0(&idx00, &lookup_mask, &primary_hash00,
+ &secondary_hash00, keys, h);
+ lookup_stage0(&idx01, &lookup_mask, &primary_hash01,
+ &secondary_hash01, keys, h);
+
+ primary_hash10 = primary_hash00;
+ primary_hash11 = primary_hash01;
+ secondary_hash10 = secondary_hash00;
+ secondary_hash11 = secondary_hash01;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, &primary_hash00,
+ &secondary_hash00, keys, h);
+ lookup_stage0(&idx01, &lookup_mask, &primary_hash01,
+ &secondary_hash01, keys, h);
+ lookup_stage1(primary_hash10, secondary_hash10, &primary_bkt10,
+ &secondary_bkt10, h);
+ lookup_stage1(primary_hash11, secondary_hash11, &primary_bkt11,
+ &secondary_bkt11, h);
+
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ primary_hash10 = primary_hash00;
+ primary_hash11 = primary_hash01;
+ secondary_hash10 = secondary_hash00;
+ secondary_hash11 = secondary_hash01;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, &primary_hash00,
+ &secondary_hash00, keys, h);
+ lookup_stage0(&idx01, &lookup_mask, &primary_hash01,
+ &secondary_hash01, keys, h);
+ lookup_stage1(primary_hash10, secondary_hash10, &primary_bkt10,
+ &secondary_bkt10, h);
+ lookup_stage1(primary_hash11, secondary_hash11, &primary_bkt11,
+ &secondary_bkt11, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+
+ while (lookup_mask) {
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ primary_hash10 = primary_hash00;
+ primary_hash11 = primary_hash01;
+ secondary_hash10 = secondary_hash00;
+ secondary_hash11 = secondary_hash01;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, &primary_hash00,
+ &secondary_hash00, keys, h);
+ lookup_stage0(&idx01, &lookup_mask, &primary_hash01,
+ &secondary_hash01, keys, h);
+ lookup_stage1(primary_hash10, secondary_hash10, &primary_bkt10,
+ &secondary_bkt10, h);
+ lookup_stage1(primary_hash11, secondary_hash11, &primary_bkt11,
+ &secondary_bkt11, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20,
+ primary_bkt20, secondary_bkt20, &k_slot20, positions,
+ &extra_hits_mask, key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21,
+ primary_bkt21, secondary_bkt21, &k_slot21, positions,
+ &extra_hits_mask, key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+ }
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ primary_hash10 = primary_hash00;
+ primary_hash11 = primary_hash01;
+ secondary_hash10 = secondary_hash00;
+ secondary_hash11 = secondary_hash01;
+ idx10 = idx00, idx11 = idx01;
+ lookup_stage1(primary_hash10, secondary_hash10, &primary_bkt10,
+ &secondary_bkt10, h);
+ lookup_stage1(primary_hash11, secondary_hash11, &primary_bkt11,
+ &secondary_bkt11, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+
+ lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+
+ /* handle extra_hits_mask */
+ next_mask |= extra_hits_mask;
+
+ /* ignore any items we have already found */
+ next_mask &= ~hits;
+
+ if (unlikely(next_mask)) {
+ /* run a single search for each remaining item */
+ do {
+ idx = __builtin_ctzl(next_mask);
+ positions[idx] = rte_hash_lookup(h, keys[idx]);
+ next_mask &= ~(1llu << idx);
+ } while (next_mask);
+ }
+
+ return 0;
+}
+
+int
+rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, int32_t *positions)
+{
+ RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
+ (num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
+ (positions == NULL)), -EINVAL);
+
+ return __rte_hash_lookup_bulk(h, keys, num_keys, positions);
+}
+
+/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
+static int
+rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
+{
+ const __m128i k1 = _mm_load_si128((const __m128i *) key1);
+ const __m128i k2 = _mm_load_si128((const __m128i *) key2);
+ const __m128i x = _mm_xor_si128(k1, k2);
+
+ return !_mm_test_all_zeros(x, x);
+}
+
+static int
+rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k16_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 16,
+ (const char *) key2 + 16, key_len);
+}
+
+static int
+rte_hash_k48_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k16_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 16,
+ (const char *) key2 + 16, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 32,
+ (const char *) key2 + 32, key_len);
+}
+
+static int
+rte_hash_k64_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k32_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 32,
+ (const char *) key2 + 32, key_len);
+}
+
+static int
+rte_hash_k80_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
+
+static int
+rte_hash_k96_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
+
+static int
+rte_hash_k112_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 96,
+ (const char *) key2 + 96, key_len);
+}
+
+static int
+rte_hash_k128_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k64_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
diff --git a/lib/librte_hash/rte_hash.c b/lib/librte_hash/rte_hash.c
deleted file mode 100644
index 5100a75..0000000
--- a/lib/librte_hash/rte_hash.c
+++ /dev/null
@@ -1,499 +0,0 @@
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <string.h>
-#include <stdint.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <sys/queue.h>
-
-#include <rte_common.h>
-#include <rte_memory.h> /* for definition of RTE_CACHE_LINE_SIZE */
-#include <rte_log.h>
-#include <rte_memcpy.h>
-#include <rte_prefetch.h>
-#include <rte_branch_prediction.h>
-#include <rte_memzone.h>
-#include <rte_malloc.h>
-#include <rte_eal.h>
-#include <rte_eal_memconfig.h>
-#include <rte_per_lcore.h>
-#include <rte_errno.h>
-#include <rte_string_fns.h>
-#include <rte_cpuflags.h>
-#include <rte_log.h>
-#include <rte_rwlock.h>
-#include <rte_spinlock.h>
-
-#include "rte_hash.h"
-
-TAILQ_HEAD(rte_hash_list, rte_tailq_entry);
-
-static struct rte_tailq_elem rte_hash_tailq = {
- .name = "RTE_HASH",
-};
-EAL_REGISTER_TAILQ(rte_hash_tailq)
-
-/* Macro to enable/disable run-time checking of function parameters */
-#if defined(RTE_LIBRTE_HASH_DEBUG)
-#define RETURN_IF_TRUE(cond, retval) do { \
- if (cond) return (retval); \
-} while (0)
-#else
-#define RETURN_IF_TRUE(cond, retval)
-#endif
-
-/* Hash function used if none is specified */
-#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
-#include <rte_hash_crc.h>
-#define DEFAULT_HASH_FUNC rte_hash_crc
-#else
-#include <rte_jhash.h>
-#define DEFAULT_HASH_FUNC rte_jhash
-#endif
-
-/* Signature bucket size is a multiple of this value */
-#define SIG_BUCKET_ALIGNMENT 16
-
-/* Stoered key size is a multiple of this value */
-#define KEY_ALIGNMENT 16
-
-/* The high bit is always set in real signatures */
-#define NULL_SIGNATURE 0
-
-struct rte_hash {
- char name[RTE_HASH_NAMESIZE]; /**< Name of the hash. */
- uint32_t entries; /**< Total table entries. */
- uint32_t bucket_entries; /**< Bucket entries. */
- uint32_t key_len; /**< Length of hash key. */
- rte_hash_function hash_func; /**< Function used to calculate hash. */
- uint32_t hash_func_init_val; /**< Init value used by hash_func. */
- uint32_t num_buckets; /**< Number of buckets in table. */
- uint32_t bucket_bitmask; /**< Bitmask for getting bucket index
- from hash signature. */
- hash_sig_t sig_msb; /**< MSB is always set in valid signatures. */
- uint8_t *sig_tbl; /**< Flat array of hash signature buckets. */
- uint32_t sig_tbl_bucket_size; /**< Signature buckets may be padded for
- alignment reasons, and this is the
- bucket size used by sig_tbl. */
- uint8_t *key_tbl; /**< Flat array of key value buckets. */
- uint32_t key_tbl_key_size; /**< Keys may be padded for alignment
- reasons, and this is the key size
- used by key_tbl. */
-};
-
-/* Returns a pointer to the first signature in specified bucket. */
-static inline hash_sig_t *
-get_sig_tbl_bucket(const struct rte_hash *h, uint32_t bucket_index)
-{
- return RTE_PTR_ADD(h->sig_tbl, (bucket_index *
- h->sig_tbl_bucket_size));
-}
-
-/* Returns a pointer to the first key in specified bucket. */
-static inline uint8_t *
-get_key_tbl_bucket(const struct rte_hash *h, uint32_t bucket_index)
-{
- return RTE_PTR_ADD(h->key_tbl, (bucket_index * h->bucket_entries *
- h->key_tbl_key_size));
-}
-
-/* Returns a pointer to a key at a specific position in a specified bucket. */
-static inline void *
-get_key_from_bucket(const struct rte_hash *h, uint8_t *bkt, uint32_t pos)
-{
- return RTE_PTR_ADD(bkt, pos * h->key_tbl_key_size);
-}
-
-/* Does integer division with rounding-up of result. */
-static inline uint32_t
-div_roundup(uint32_t numerator, uint32_t denominator)
-{
- return (numerator + denominator - 1) / denominator;
-}
-
-/* Increases a size (if needed) to a multiple of alignment. */
-static inline uint32_t
-align_size(uint32_t val, uint32_t alignment)
-{
- return alignment * div_roundup(val, alignment);
-}
-
-/* Returns the index into the bucket of the first occurrence of a signature. */
-static inline int
-find_first(uint32_t sig, const uint32_t *sig_bucket, uint32_t num_sigs)
-{
- uint32_t i;
- for (i = 0; i < num_sigs; i++) {
- if (sig == sig_bucket[i])
- return i;
- }
- return -1;
-}
-
-struct rte_hash *
-rte_hash_find_existing(const char *name)
-{
- struct rte_hash *h = NULL;
- struct rte_tailq_entry *te;
- struct rte_hash_list *hash_list;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK);
- TAILQ_FOREACH(te, hash_list, next) {
- h = (struct rte_hash *) te->data;
- if (strncmp(name, h->name, RTE_HASH_NAMESIZE) == 0)
- break;
- }
- rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- if (te == NULL) {
- rte_errno = ENOENT;
- return NULL;
- }
- return h;
-}
-
-struct rte_hash *
-rte_hash_create(const struct rte_hash_parameters *params)
-{
- struct rte_hash *h = NULL;
- struct rte_tailq_entry *te;
- uint32_t num_buckets, sig_bucket_size, key_size,
- hash_tbl_size, sig_tbl_size, key_tbl_size, mem_size;
- char hash_name[RTE_HASH_NAMESIZE];
- struct rte_hash_list *hash_list;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- /* Check for valid parameters */
- if ((params == NULL) ||
- (params->entries > RTE_HASH_ENTRIES_MAX) ||
- (params->bucket_entries > RTE_HASH_BUCKET_ENTRIES_MAX) ||
- (params->entries < params->bucket_entries) ||
- !rte_is_power_of_2(params->entries) ||
- !rte_is_power_of_2(params->bucket_entries) ||
- (params->key_len == 0) ||
- (params->key_len > RTE_HASH_KEY_LENGTH_MAX)) {
- rte_errno = EINVAL;
- RTE_LOG(ERR, HASH, "rte_hash_create has invalid parameters\n");
- return NULL;
- }
-
- snprintf(hash_name, sizeof(hash_name), "HT_%s", params->name);
-
- /* Calculate hash dimensions */
- num_buckets = params->entries / params->bucket_entries;
- sig_bucket_size = align_size(params->bucket_entries *
- sizeof(hash_sig_t), SIG_BUCKET_ALIGNMENT);
- key_size = align_size(params->key_len, KEY_ALIGNMENT);
-
- hash_tbl_size = align_size(sizeof(struct rte_hash), RTE_CACHE_LINE_SIZE);
- sig_tbl_size = align_size(num_buckets * sig_bucket_size,
- RTE_CACHE_LINE_SIZE);
- key_tbl_size = align_size(num_buckets * key_size *
- params->bucket_entries, RTE_CACHE_LINE_SIZE);
-
- /* Total memory required for hash context */
- mem_size = hash_tbl_size + sig_tbl_size + key_tbl_size;
-
- rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
-
- /* guarantee there's no existing */
- TAILQ_FOREACH(te, hash_list, next) {
- h = (struct rte_hash *) te->data;
- if (strncmp(params->name, h->name, RTE_HASH_NAMESIZE) == 0)
- break;
- }
- if (te != NULL)
- goto exit;
-
- te = rte_zmalloc("HASH_TAILQ_ENTRY", sizeof(*te), 0);
- if (te == NULL) {
- RTE_LOG(ERR, HASH, "tailq entry allocation failed\n");
- goto exit;
- }
-
- h = (struct rte_hash *)rte_zmalloc_socket(hash_name, mem_size,
- RTE_CACHE_LINE_SIZE, params->socket_id);
- if (h == NULL) {
- RTE_LOG(ERR, HASH, "memory allocation failed\n");
- rte_free(te);
- goto exit;
- }
-
- /* Setup hash context */
- snprintf(h->name, sizeof(h->name), "%s", params->name);
- h->entries = params->entries;
- h->bucket_entries = params->bucket_entries;
- h->key_len = params->key_len;
- h->hash_func_init_val = params->hash_func_init_val;
- h->num_buckets = num_buckets;
- h->bucket_bitmask = h->num_buckets - 1;
- h->sig_msb = 1 << (sizeof(hash_sig_t) * 8 - 1);
- h->sig_tbl = (uint8_t *)h + hash_tbl_size;
- h->sig_tbl_bucket_size = sig_bucket_size;
- h->key_tbl = h->sig_tbl + sig_tbl_size;
- h->key_tbl_key_size = key_size;
- h->hash_func = (params->hash_func == NULL) ?
- DEFAULT_HASH_FUNC : params->hash_func;
-
- te->data = (void *) h;
-
- TAILQ_INSERT_TAIL(hash_list, te, next);
-
-exit:
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- return h;
-}
-
-void
-rte_hash_free(struct rte_hash *h)
-{
- struct rte_tailq_entry *te;
- struct rte_hash_list *hash_list;
-
- if (h == NULL)
- return;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
-
- /* find out tailq entry */
- TAILQ_FOREACH(te, hash_list, next) {
- if (te->data == (void *) h)
- break;
- }
-
- if (te == NULL) {
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
- return;
- }
-
- TAILQ_REMOVE(hash_list, te, next);
-
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- rte_free(h);
- rte_free(te);
-}
-
-hash_sig_t
-rte_hash_hash(const struct rte_hash *h, const void *key)
-{
- /* calc hash result by key */
- return h->hash_func(key, h->key_len, h->hash_func_init_val);
-}
-
-static inline int32_t
-__rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
- int32_t pos;
-
- /* Get the hash signature and bucket index */
- sig |= h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- /* Check if any free slot within the bucket to add the new key */
- pos = find_first(NULL_SIGNATURE, sig_bucket, h->bucket_entries);
-
- if (unlikely(pos < 0))
- return -ENOSPC;
-
- /* Add the new key to the bucket */
- sig_bucket[pos] = sig;
- rte_memcpy(get_key_from_bucket(h, key_bucket, pos), key, h->key_len);
- return bucket_index * h->bucket_entries + pos;
-}
-
-int32_t
-rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_add_key(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-static inline int32_t
-__rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
-
- /* Get the hash signature and bucket index */
- sig = sig | h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- sig_bucket[i] = NULL_SIGNATURE;
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- return -ENOENT;
-}
-
-int32_t
-rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_del_key_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_del_key(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_del_key_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-static inline int32_t
-__rte_hash_lookup_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
-
- /* Get the hash signature and bucket index */
- sig |= h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- return -ENOENT;
-}
-
-int32_t
-rte_hash_lookup_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_lookup(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-int
-rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
- uint32_t num_keys, int32_t *positions)
-{
- uint32_t i, j, bucket_index;
- hash_sig_t sigs[RTE_HASH_LOOKUP_BULK_MAX];
-
- RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
- (num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
- (positions == NULL)), -EINVAL);
-
- /* Get the hash signature and bucket index */
- for (i = 0; i < num_keys; i++) {
- sigs[i] = h->hash_func(keys[i], h->key_len,
- h->hash_func_init_val) | h->sig_msb;
- bucket_index = sigs[i] & h->bucket_bitmask;
-
- /* Pre-fetch relevant buckets */
- rte_prefetch1((void *) get_sig_tbl_bucket(h, bucket_index));
- rte_prefetch1((void *) get_key_tbl_bucket(h, bucket_index));
- }
-
- /* Check if key is already present in the hash */
- for (i = 0; i < num_keys; i++) {
- bucket_index = sigs[i] & h->bucket_bitmask;
- hash_sig_t *sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- uint8_t *key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- positions[i] = -ENOENT;
-
- for (j = 0; j < h->bucket_entries; j++) {
- if ((sigs[i] == sig_bucket[j]) &&
- likely(memcmp(keys[i],
- get_key_from_bucket(h, key_bucket, j),
- h->key_len) == 0)) {
- positions[i] = bucket_index *
- h->bucket_entries + j;
- break;
- }
- }
- }
-
- return 0;
-}
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index 79827a6..13fad73 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -40,26 +40,31 @@
* RTE Hash Table
*/
+#include <rte_ring.h>
+
#ifdef __cplusplus
extern "C" {
#endif
/** Maximum size of hash table that can be created. */
-#define RTE_HASH_ENTRIES_MAX (1 << 26)
+#define RTE_HASH_ENTRIES_MAX (1 << 30)
-/** Maximum bucket size that can be created. */
-#define RTE_HASH_BUCKET_ENTRIES_MAX 16
+/** @deprecated Maximum bucket size that can be created. */
+#define RTE_HASH_BUCKET_ENTRIES_MAX 4
-/** Maximum length of key that can be used. */
-#define RTE_HASH_KEY_LENGTH_MAX 64
+/** Number of items per bucket. */
+#define RTE_HASH_BUCKET_ENTRIES 4
-/** Max number of keys that can be searched for using rte_hash_lookup_multi. */
-#define RTE_HASH_LOOKUP_BULK_MAX 16
-#define RTE_HASH_LOOKUP_MULTI_MAX RTE_HASH_LOOKUP_BULK_MAX
+/** @deprecated Maximum length of key that can be used. */
+#define RTE_HASH_KEY_LENGTH_MAX 64
-/** Max number of characters in hash name.*/
+/** Maximum number of characters in hash name.*/
#define RTE_HASH_NAMESIZE 32
+/** Maximum number of keys that can be searched for using rte_hash_lookup_bulk. */
+#define RTE_HASH_LOOKUP_BULK_MAX 64
+#define RTE_HASH_LOOKUP_MULTI_MAX RTE_HASH_LOOKUP_BULK_MAX
+
/** Signature of key that is stored internally. */
typedef uint32_t hash_sig_t;
@@ -74,9 +79,9 @@ typedef uint32_t (*rte_hash_function)(const void *key, uint32_t key_len,
struct rte_hash_parameters {
const char *name; /**< Name of the hash. */
uint32_t entries; /**< Total hash table entries. */
- uint32_t bucket_entries; /**< Bucket entries. */
+ uint32_t bucket_entries; /**< Bucket entries. */
uint32_t key_len; /**< Length of hash key. */
- rte_hash_function hash_func; /**< Function used to calculate hash. */
+ rte_hash_function hash_func; /**< Primary Hash function used to calculate hash. */
uint32_t hash_func_init_val; /**< Init value used by hash_func. */
int socket_id; /**< NUMA Socket ID for memory. */
};
@@ -84,6 +89,7 @@ struct rte_hash_parameters {
/** @internal A hash table structure. */
struct rte_hash;
+
/**
* Create a new hash table.
*
@@ -104,7 +110,6 @@ struct rte_hash;
struct rte_hash *
rte_hash_create(const struct rte_hash_parameters *params);
-
/**
* Find an existing hash table object and return a pointer to it.
*
@@ -127,7 +132,8 @@ void
rte_hash_free(struct rte_hash *h);
/**
- * Add a key to an existing hash table. This operation is not multi-thread safe
+ * Add a key to an existing hash table.
+ * This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
@@ -144,7 +150,8 @@ int32_t
rte_hash_add_key(const struct rte_hash *h, const void *key);
/**
- * Add a key to an existing hash table. This operation is not multi-thread safe
+ * Add a key to an existing hash table.
+ * This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
@@ -152,7 +159,7 @@ rte_hash_add_key(const struct rte_hash *h, const void *key);
* @param key
* Key to add to the hash table.
* @param sig
- * Hash value to add to the hash table.
+ * Precomputed hash value for 'key'.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOSPC if there is no space in the hash for this key.
@@ -160,11 +167,11 @@ rte_hash_add_key(const struct rte_hash *h, const void *key);
* array of user data. This value is unique for this key.
*/
int32_t
-rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig);
+rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
/**
- * Remove a key from an existing hash table. This operation is not multi-thread
+ * Remove a key from an existing hash table.
+ * This operation is not multi-thread
* safe and should only be called from one thread.
*
* @param h
@@ -182,7 +189,8 @@ int32_t
rte_hash_del_key(const struct rte_hash *h, const void *key);
/**
- * Remove a key from an existing hash table. This operation is not multi-thread
+ * Remove a key from an existing hash table.
+ * This operation is not multi-thread
* safe and should only be called from one thread.
*
* @param h
@@ -190,7 +198,7 @@ rte_hash_del_key(const struct rte_hash *h, const void *key);
* @param key
* Key to remove from the hash table.
* @param sig
- * Hash value to remove from the hash table.
+ * Precomputed hash value for 'key'.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOENT if the key is not found.
@@ -199,12 +207,11 @@ rte_hash_del_key(const struct rte_hash *h, const void *key);
* value that was returned when the key was added.
*/
int32_t
-rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig);
-
+rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
/**
- * Find a key in the hash table. This operation is multi-thread safe.
+ * Find a key in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
@@ -221,14 +228,15 @@ int32_t
rte_hash_lookup(const struct rte_hash *h, const void *key);
/**
- * Find a key in the hash table. This operation is multi-thread safe.
+ * Find a key in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
* @param key
* Key to find.
* @param sig
- * Hash value to find.
+ * Hash value to remove from the hash table.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOENT if the key is not found.
@@ -241,7 +249,8 @@ rte_hash_lookup_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig);
/**
- * Calc a hash value by key. This operation is not multi-process safe.
+ * Calc a hash value by key.
+ * This operation is not multi-process safe.
*
* @param h
* Hash table to look in.
@@ -255,7 +264,8 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
#define rte_hash_lookup_multi rte_hash_lookup_bulk
/**
- * Find multiple keys in the hash table. This operation is multi-thread safe.
+ * Find multiple keys in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
@@ -275,6 +285,7 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions);
+
#ifdef __cplusplus
}
#endif
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v3 06/11] hash: add new lookup_bulk_with_hash function
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Pablo de Lara
` (4 preceding siblings ...)
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 05/11] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
@ 2015-06-28 22:25 ` Pablo de Lara
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 07/11] hash: add new function rte_hash_reset Pablo de Lara
` (6 subsequent siblings)
12 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-06-28 22:25 UTC (permalink / raw)
To: dev
Previous implementation was lacking a function
to look up a burst of entries, given precalculated hash values.
This patch implements such function, quite useful for
looking up keys from packets that have precalculated hash values
from a 5-tuple key.
Added the function in the hash unit tests as well.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 19 ++-
app/test/test_hash_perf.c | 24 +++-
lib/librte_hash/rte_cuckoo_hash.c | 222 ++++++++++++++++++++++++++++++++++-
lib/librte_hash/rte_hash.h | 27 +++++
lib/librte_hash/rte_hash_version.map | 8 ++
5 files changed, 291 insertions(+), 9 deletions(-)
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index 0176219..52de1bd 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -456,6 +456,7 @@ static int test_five_keys(void)
{
struct rte_hash *handle;
const void *key_array[5] = {0};
+ hash_sig_t hashes[5];
int pos[5];
int expected_pos[5];
unsigned i;
@@ -475,12 +476,24 @@ static int test_five_keys(void)
}
/* Lookup */
- for(i = 0; i < 5; i++)
+ for (i = 0; i < 5; i++) {
key_array[i] = &keys[i];
+ hashes[i] = rte_hash_hash(handle, &keys[i]);
+ }
ret = rte_hash_lookup_multi(handle, &key_array[0], 5, (int32_t *)pos);
- if(ret == 0)
- for(i = 0; i < 5; i++) {
+ if (ret == 0)
+ for (i = 0; i < 5; i++) {
+ print_key_info("Lkp", key_array[i], pos[i]);
+ RETURN_IF_ERROR(pos[i] != expected_pos[i],
+ "failed to find key (pos[%u]=%d)", i, pos[i]);
+ }
+
+ /* Lookup with precalculated hashes */
+ ret = rte_hash_lookup_multi_with_hash(handle, &key_array[0], hashes,
+ 5, (int32_t *)pos);
+ if (ret == 0)
+ for (i = 0; i < 5; i++) {
print_key_info("Lkp", key_array[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to find key (pos[%u]=%d)", i, pos[i]);
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index 978731c..86295b8 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -289,18 +289,27 @@ timed_lookups(unsigned with_hash, unsigned table_index)
}
static int
-timed_lookups_multi(unsigned table_index)
+timed_lookups_multi(unsigned with_hash, unsigned table_index)
{
unsigned i, j, k;
int32_t positions_burst[BURST_SIZE];
const void *keys_burst[BURST_SIZE];
const uint64_t start_tsc = rte_rdtsc();
+ hash_sig_t *hash_array;
for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
for (j = 0; j < KEYS_TO_ADD/BURST_SIZE; j++) {
for (k = 0; k < BURST_SIZE; k++)
keys_burst[k] = keys[j * BURST_SIZE + k];
- rte_hash_lookup_bulk(h[table_index],
+ if (with_hash) {
+ hash_array = &signatures[j * BURST_SIZE];
+ rte_hash_lookup_bulk_with_hash(h[table_index],
+ (const void **) keys_burst,
+ (const hash_sig_t *) hash_array,
+ BURST_SIZE,
+ positions_burst);
+ } else
+ rte_hash_lookup_bulk(h[table_index],
(const void **) keys_burst,
BURST_SIZE,
positions_burst);
@@ -311,12 +320,12 @@ timed_lookups_multi(unsigned table_index)
const uint64_t time_taken = end_tsc - start_tsc;
const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
- cycles[table_index][LOOKUP_MULTI][0] = time_taken/NUM_LOOKUPS;
+ cycles[table_index][LOOKUP_MULTI][with_hash] = time_taken/NUM_LOOKUPS;
printf("%"PRIu64" lookups in %f seconds\n", (uint64_t)NUM_LOOKUPS,
seconds_taken);
printf("Average %"PRIu64" tsc ticks per lookup\n",
- cycles[table_index][LOOKUP_MULTI][0]);
+ cycles[table_index][LOOKUP_MULTI][with_hash]);
printf("Average %"PRIu64" lookups per second\n",
(NUM_LOOKUPS * rte_get_tsc_hz())/time_taken);
return 0;
@@ -398,6 +407,11 @@ run_all_tbl_perf_tests(void)
if (timed_lookups(1, i) < 0)
return -1;
+ printf("\nTimed lookups multi\n");
+ printf("------------------\n");
+ if (timed_lookups_multi(1, i) < 0)
+ return -1;
+
printf("\nTimed deletions\n");
printf("------------------\n");
if (timed_deletes(1, i) < 0)
@@ -423,7 +437,7 @@ run_all_tbl_perf_tests(void)
printf("\nTimed lookups multi\n");
printf("------------------\n");
- if (timed_lookups_multi(i) < 0)
+ if (timed_lookups_multi(0, i) < 0)
return -1;
printf("\nTimed deletions\n");
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index 10febdc..1e70069 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -711,6 +711,21 @@ lookup_stage0(unsigned *idx, uint64_t *lookup_mask,
*lookup_mask &= ~(1llu << *idx);
}
+/* Lookup bulk stage 0: Get primary hash value and calculate secondary hash value */
+static inline void
+lookup_stage0_with_hash(unsigned *idx, uint64_t *lookup_mask,
+ hash_sig_t *primary_hash, hash_sig_t *secondary_hash,
+ const hash_sig_t *hash_vals)
+{
+ *idx = __builtin_ctzl(*lookup_mask);
+ if (*lookup_mask == 0)
+ *idx = 0;
+
+ *primary_hash = hash_vals[*idx];
+ *secondary_hash = rte_hash_secondary_hash(*primary_hash);
+
+ *lookup_mask &= ~(1llu << *idx);
+}
/* Lookup bulk stage 1: Prefetch primary/secondary buckets */
static inline void
@@ -727,7 +742,7 @@ lookup_stage1(hash_sig_t primary_hash, hash_sig_t secondary_hash,
}
/*
- * Lookup bulk stage 2: Search for match hashes in primary/secondary locations
+ * Lookup bulk stage 2: Search for match hashes in primary/secondary locations
* and prefetch first key slot
*/
static inline void
@@ -973,6 +988,198 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
return 0;
}
+static inline int
+__rte_hash_lookup_bulk_with_hash(const struct rte_hash *h, const void **keys,
+ const hash_sig_t *hash_vals, uint32_t num_keys,
+ int32_t *positions)
+{
+ uint64_t hits = 0;
+ uint64_t next_mask = 0;
+ uint64_t extra_hits_mask = 0;
+ uint64_t lookup_mask;
+ unsigned idx;
+ const void *key_store = h->key_store;
+
+ unsigned idx00, idx01, idx10, idx11, idx20, idx21, idx30, idx31;
+ const struct rte_hash_bucket *primary_bkt10, *primary_bkt11;
+ const struct rte_hash_bucket *secondary_bkt10, *secondary_bkt11;
+ const struct rte_hash_bucket *primary_bkt20, *primary_bkt21;
+ const struct rte_hash_bucket *secondary_bkt20, *secondary_bkt21;
+ const void *k_slot20, *k_slot21, *k_slot30, *k_slot31;
+ hash_sig_t primary_hash00, primary_hash01;
+ hash_sig_t secondary_hash00, secondary_hash01;
+ hash_sig_t primary_hash10, primary_hash11;
+ hash_sig_t secondary_hash10, secondary_hash11;
+ hash_sig_t primary_hash20, primary_hash21;
+ hash_sig_t secondary_hash20, secondary_hash21;
+
+ if (num_keys == RTE_HASH_LOOKUP_BULK_MAX)
+ lookup_mask = 0xffffffffffffffff;
+ else
+ lookup_mask = (1ULL << num_keys) - 1;
+
+ lookup_stage0_with_hash(&idx00, &lookup_mask, &primary_hash00,
+ &secondary_hash00, hash_vals);
+ lookup_stage0_with_hash(&idx01, &lookup_mask, &primary_hash01,
+ &secondary_hash01, hash_vals);
+
+ primary_hash10 = primary_hash00;
+ primary_hash11 = primary_hash01;
+ secondary_hash10 = secondary_hash00;
+ secondary_hash11 = secondary_hash01;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0_with_hash(&idx00, &lookup_mask, &primary_hash00,
+ &secondary_hash00, hash_vals);
+ lookup_stage0_with_hash(&idx01, &lookup_mask, &primary_hash01,
+ &secondary_hash01, hash_vals);
+ lookup_stage1(primary_hash10, secondary_hash10, &primary_bkt10,
+ &secondary_bkt10, h);
+ lookup_stage1(primary_hash11, secondary_hash11, &primary_bkt11,
+ &secondary_bkt11, h);
+
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ primary_hash10 = primary_hash00;
+ primary_hash11 = primary_hash01;
+ secondary_hash10 = secondary_hash00;
+ secondary_hash11 = secondary_hash01;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0_with_hash(&idx00, &lookup_mask, &primary_hash00,
+ &secondary_hash00, hash_vals);
+ lookup_stage0_with_hash(&idx01, &lookup_mask, &primary_hash01,
+ &secondary_hash01, hash_vals);
+ lookup_stage1(primary_hash10, secondary_hash10, &primary_bkt10,
+ &secondary_bkt10, h);
+ lookup_stage1(primary_hash11, secondary_hash11, &primary_bkt11,
+ &secondary_bkt11, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+
+ while (lookup_mask) {
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ primary_hash10 = primary_hash00;
+ primary_hash11 = primary_hash01;
+ secondary_hash10 = secondary_hash00;
+ secondary_hash11 = secondary_hash01;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0_with_hash(&idx00, &lookup_mask, &primary_hash00,
+ &secondary_hash00, hash_vals);
+ lookup_stage0_with_hash(&idx01, &lookup_mask, &primary_hash01,
+ &secondary_hash01, hash_vals);
+ lookup_stage1(primary_hash10, secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, h);
+ lookup_stage1(primary_hash11, secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20,
+ primary_bkt20, secondary_bkt20, &k_slot20, positions,
+ &extra_hits_mask, key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21,
+ primary_bkt21, secondary_bkt21, &k_slot21, positions,
+ &extra_hits_mask, key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+ }
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ primary_hash10 = primary_hash00;
+ primary_hash11 = primary_hash01;
+ secondary_hash10 = secondary_hash00;
+ secondary_hash11 = secondary_hash01;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage1(primary_hash10, secondary_hash10, &primary_bkt10,
+ &secondary_bkt10, h);
+ lookup_stage1(primary_hash11, secondary_hash11, &primary_bkt11,
+ &secondary_bkt11, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+
+ lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+
+ /* handle extra_hits_mask */
+ next_mask |= extra_hits_mask;
+
+ /* ignore any items we have already found */
+ next_mask &= ~hits;
+
+ if (unlikely(next_mask)) {
+ /* run a single search for each remaining item */
+ do {
+ idx = __builtin_ctzl(next_mask);
+ positions[idx] = rte_hash_lookup_with_hash(h, keys[idx],
+ hash_vals[idx]);
+ next_mask &= ~(1llu << idx);
+ } while (next_mask);
+ }
+
+ return 0;
+}
+
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions)
@@ -984,6 +1191,19 @@ rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
return __rte_hash_lookup_bulk(h, keys, num_keys, positions);
}
+int
+rte_hash_lookup_bulk_with_hash(const struct rte_hash *h, const void **keys,
+ const hash_sig_t *hash_vals, uint32_t num_keys,
+ int32_t *positions)
+{
+ RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
+ (num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
+ (positions == NULL)), -EINVAL);
+
+ return __rte_hash_lookup_bulk_with_hash(h, keys, hash_vals, num_keys,
+ positions);
+}
+
/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
static int
rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index 13fad73..7f7e75f 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -263,6 +263,7 @@ hash_sig_t
rte_hash_hash(const struct rte_hash *h, const void *key);
#define rte_hash_lookup_multi rte_hash_lookup_bulk
+#define rte_hash_lookup_multi_with_hash rte_hash_lookup_bulk_with_hash
/**
* Find multiple keys in the hash table.
* This operation is multi-thread safe.
@@ -286,6 +287,32 @@ int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions);
+/**
+ * Find multiple keys in the hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param keys
+ * A pointer to a list of keys to look for.
+ * @param hash_vals
+ * A pointer to a list of pre-calculated hash values.
+ * @param num_keys
+ * How many keys are in the keys list (less than RTE_HASH_LOOKUP_BULK_MAX).
+ * @param positions
+ * Output containing a list of values, corresponding to the list of keys that
+ * can be used by the caller as an offset into an array of user data. These
+ * values are unique for each key, and are the same values that were returned
+ * when each key was added. If a key in the list was not found, then -ENOENT
+ * will be the value.
+ * @return
+ * -EINVAL if there's an error, otherwise 0.
+ */
+int
+rte_hash_lookup_bulk_with_hash(const struct rte_hash *h, const void **keys,
+ const hash_sig_t *hash_vals, uint32_t num_keys,
+ int32_t *positions);
+
#ifdef __cplusplus
}
#endif
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index 0b749e8..fd92def 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -17,3 +17,11 @@ DPDK_2.0 {
local: *;
};
+
+DPDK_2.1 {
+ global:
+
+ rte_hash_lookup_bulk_with_hash;
+
+ local: *;
+} DPDK_2.0;
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v3 07/11] hash: add new function rte_hash_reset
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Pablo de Lara
` (5 preceding siblings ...)
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 06/11] hash: add new lookup_bulk_with_hash function Pablo de Lara
@ 2015-06-28 22:25 ` Pablo de Lara
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 08/11] hash: add new functionality to store data in hash table Pablo de Lara
` (5 subsequent siblings)
12 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-06-28 22:25 UTC (permalink / raw)
To: dev
Added reset function to be able to empty the table,
without having to destroy and create it again.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 3 +--
app/test/test_hash_perf.c | 11 +++--------
lib/librte_hash/rte_cuckoo_hash.c | 21 +++++++++++++++++++++
lib/librte_hash/rte_hash.h | 11 +++++++++--
lib/librte_hash/rte_hash_version.map | 1 +
5 files changed, 35 insertions(+), 12 deletions(-)
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index 52de1bd..b1ca939 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -1156,8 +1156,7 @@ static int test_average_table_utilization(void)
no_space = 0;
/* Reset the table */
- rte_hash_free(handle);
- rte_hash_create(&ut_params);
+ rte_hash_reset(handle);
}
const unsigned average_keys_added = added_keys_until_no_space / ITERATIONS;
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index 86295b8..6e8b255 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -368,14 +368,10 @@ free_table(unsigned table_index)
rte_hash_free(h[table_index]);
}
-static int
+static void
reset_table(unsigned table_index)
{
- free_table(table_index);
- if (create_table(table_index) != 0)
- return -1;
-
- return 0;
+ rte_hash_reset(h[table_index]);
}
static int
@@ -417,8 +413,7 @@ run_all_tbl_perf_tests(void)
if (timed_deletes(1, i) < 0)
return -1;
- if (reset_table(i) < 0)
- return -1;
+ reset_table(i);
printf("\n ----- WITH JUST KEYS -----\n\n");
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index 1e70069..44f87fb 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -358,6 +358,27 @@ rte_hash_secondary_hash(const hash_sig_t primary_hash)
return (primary_hash ^ ((tag + 1) * alt_bits_xor));
}
+void
+rte_hash_reset(struct rte_hash *h)
+{
+ void *ptr;
+ unsigned i;
+
+ if (h == NULL)
+ return;
+
+ memset(h->buckets, 0, h->num_buckets * sizeof(struct rte_hash_bucket));
+ memset(h->key_store, 0, h->key_entry_size * (h->entries + 1));
+
+ /* clear the free ring */
+ while (rte_ring_dequeue(h->free_slots, &ptr) == 0)
+ rte_pause();
+
+ /* Repopulate the free slots ring. Entry zero is reserved for key misses */
+ for (i = 1; i < h->entries + 1; i++)
+ rte_ring_sp_enqueue(h->free_slots, (void *)((uintptr_t) i));
+}
+
/*
* Try to insert a new entry. If bucket has space, hash value and key index
* to the key table are copied.
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index 7f7e75f..fa327c2 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -132,8 +132,15 @@ void
rte_hash_free(struct rte_hash *h);
/**
- * Add a key to an existing hash table.
- * This operation is not multi-thread safe
+ * Reset all hash structure, by zeroing all entries
+ * @param h
+ * Hash table to reset
+ */
+void
+rte_hash_reset(struct rte_hash *h);
+
+/**
+ * Add a key to an existing hash table. This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index fd92def..f011054 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -22,6 +22,7 @@ DPDK_2.1 {
global:
rte_hash_lookup_bulk_with_hash;
+ rte_hash_reset;
local: *;
} DPDK_2.0;
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v3 08/11] hash: add new functionality to store data in hash table
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Pablo de Lara
` (6 preceding siblings ...)
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 07/11] hash: add new function rte_hash_reset Pablo de Lara
@ 2015-06-28 22:25 ` Pablo de Lara
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 09/11] MAINTAINERS: claim responsability for hash library Pablo de Lara
` (4 subsequent siblings)
12 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-06-28 22:25 UTC (permalink / raw)
To: dev
Usually hash tables not only store keys, but also data associated
to them. In order to maintain the existing API,
the key index will still be returned when
adding/looking up/deleting an entry, but user will be able
to store/look up data associated to a key.
Unit tests have been updated to use these new functions.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/Makefile | 1 +
app/test/test_hash_perf.c | 246 ++++++++++++++++++++---------------
lib/librte_hash/rte_cuckoo_hash.c | 167 +++++++++++++++++++-----
lib/librte_hash/rte_hash.h | 145 ++++++++++++++++++++-
lib/librte_hash/rte_hash_version.map | 6 +
5 files changed, 424 insertions(+), 141 deletions(-)
diff --git a/app/test/Makefile b/app/test/Makefile
index 2e2758c..5481ca1 100644
--- a/app/test/Makefile
+++ b/app/test/Makefile
@@ -83,6 +83,7 @@ SRCS-y += test_memcpy_perf.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_perf.c
+SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_cuckoo_hash_perf.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_functions.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_scaling.c
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index 6e8b255..2fcb9a6 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -55,6 +55,7 @@
#define NUM_KEYSIZES 10
#define NUM_SHUFFLES 10
#define BURST_SIZE 16
+#define DATA_MULTIPLIER 13 /* Used for storing data */
enum operations {
ADD = 0,
@@ -75,7 +76,7 @@ struct rte_hash *h[NUM_KEYSIZES];
/* Array that stores if a slot is full */
uint8_t slot_taken[MAX_ENTRIES];
/* Array to store number of cycles per operation */
-uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS][2];
+uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS][2][2];
/* Array to store all input keys */
uint8_t keys[KEYS_TO_ADD][MAX_KEYSIZE];
/* Array to store the precomputed hash for 'keys' */
@@ -92,11 +93,19 @@ static struct rte_hash_parameters ut_params = {
};
static int
-create_table(unsigned table_index)
+create_table(unsigned with_data, unsigned table_index)
{
char name[RTE_HASH_NAMESIZE];
- sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
+ if (with_data) {
+ sprintf(name, "test_hash%d_data", hashtest_key_lens[table_index]);
+ /* Table will store 8-byte data */
+ ut_params.data_len = sizeof(uint64_t);
+ } else {
+ sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
+ ut_params.data_len = 0;
+ }
+
ut_params.name = name;
ut_params.key_len = hashtest_key_lens[table_index];
ut_params.socket_id = rte_socket_id();
@@ -228,15 +237,25 @@ get_input_keys(unsigned table_index)
}
static int
-timed_adds(unsigned with_hash, unsigned table_index) {
+timed_adds(unsigned with_hash, unsigned with_data, unsigned table_index) {
unsigned i;
const uint64_t start_tsc = rte_rdtsc();
+ uint64_t data;
for (i = 0; i < KEYS_TO_ADD; i++) {
- if (with_hash)
+ data = signatures[i] * DATA_MULTIPLIER;
+ if (with_hash && with_data)
+ rte_hash_add_key_with_hash_data(h[table_index],
+ (const void *) keys[i],
+ signatures[i], &data);
+ else if (with_hash && !with_data)
rte_hash_add_key_with_hash(h[table_index],
(const void *) keys[i],
signatures[i]);
+ else if (!with_hash && with_data)
+ rte_hash_add_key_data(h[table_index],
+ (const void *) keys[i],
+ &data);
else
rte_hash_add_key(h[table_index], keys[i]);
}
@@ -245,29 +264,38 @@ timed_adds(unsigned with_hash, unsigned table_index) {
const uint64_t time_taken = end_tsc - start_tsc;
const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
- cycles[table_index][ADD][with_hash] = time_taken/KEYS_TO_ADD;
+ cycles[table_index][ADD][with_hash][with_data] = time_taken/KEYS_TO_ADD;
printf("\n%"PRIu64" adds in %f seconds\n", (uint64_t)KEYS_TO_ADD,
seconds_taken);
printf("Average %"PRIu64" tsc ticks per add\n",
- cycles[table_index][ADD][with_hash]);
+ cycles[table_index][ADD][with_hash][with_data]);
printf("Average %"PRIu64" adds per second\n",
(KEYS_TO_ADD * rte_get_tsc_hz())/time_taken);
return 0;
}
static int
-timed_lookups(unsigned with_hash, unsigned table_index)
+timed_lookups(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i, j;
const uint64_t start_tsc = rte_rdtsc();
+ uint64_t ret_data;
for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
for (j = 0; j < KEYS_TO_ADD; j++) {
- if (with_hash)
+ if (with_hash && with_data)
+ rte_hash_lookup_with_hash_data(h[table_index],
+ (const void *) keys[j],
+ signatures[j], &ret_data);
+ else if (with_hash && !with_data)
rte_hash_lookup_with_hash(h[table_index],
(const void *) keys[j],
signatures[j]);
+ else if (!with_hash && with_data)
+ rte_hash_lookup_data(h[table_index],
+ (const void *) keys[j],
+ &ret_data);
else
rte_hash_lookup(h[table_index], keys[j]);
}
@@ -277,23 +305,29 @@ timed_lookups(unsigned with_hash, unsigned table_index)
const uint64_t time_taken = end_tsc - start_tsc;
const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
- cycles[table_index][LOOKUP][with_hash] = time_taken/NUM_LOOKUPS;
+ cycles[table_index][LOOKUP][with_hash][with_data] = time_taken/NUM_LOOKUPS;
printf("%"PRIu64" lookups in %f seconds\n", (uint64_t) NUM_LOOKUPS,
seconds_taken);
printf("Average %"PRIu64" tsc ticks per lookup\n",
- cycles[table_index][LOOKUP][with_hash]);
+ cycles[table_index][LOOKUP][with_hash][with_data]);
printf("Average %"PRIu64" lookups per second\n",
(NUM_LOOKUPS * rte_get_tsc_hz())/time_taken);
return 0;
}
static int
-timed_lookups_multi(unsigned with_hash, unsigned table_index)
+timed_lookups_multi(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i, j, k;
int32_t positions_burst[BURST_SIZE];
const void *keys_burst[BURST_SIZE];
+ uint64_t ret_data[BURST_SIZE];
+ void *data_ptrs[BURST_SIZE];
+
+ for (i = 0; i < BURST_SIZE; i++)
+ data_ptrs[i] = &ret_data[i];
+
const uint64_t start_tsc = rte_rdtsc();
hash_sig_t *hash_array;
@@ -301,18 +335,32 @@ timed_lookups_multi(unsigned with_hash, unsigned table_index)
for (j = 0; j < KEYS_TO_ADD/BURST_SIZE; j++) {
for (k = 0; k < BURST_SIZE; k++)
keys_burst[k] = keys[j * BURST_SIZE + k];
- if (with_hash) {
+ if (with_hash && with_data) {
+ hash_array = &signatures[j * BURST_SIZE];
+ rte_hash_lookup_bulk_with_hash_data(h[table_index],
+ (const void **) keys_burst,
+ (const hash_sig_t *) hash_array,
+ BURST_SIZE,
+ positions_burst,
+ data_ptrs);
+ } else if (with_hash && !with_data) {
hash_array = &signatures[j * BURST_SIZE];
rte_hash_lookup_bulk_with_hash(h[table_index],
- (const void **) keys_burst,
- (const hash_sig_t *) hash_array,
- BURST_SIZE,
- positions_burst);
- } else
+ (const void **) keys_burst,
+ (const hash_sig_t *) hash_array,
+ BURST_SIZE,
+ positions_burst);
+ } else if (!with_hash && with_data)
+ rte_hash_lookup_bulk_data(h[table_index],
+ (const void **) keys_burst,
+ BURST_SIZE,
+ positions_burst,
+ data_ptrs);
+ else
rte_hash_lookup_bulk(h[table_index],
- (const void **) keys_burst,
- BURST_SIZE,
- positions_burst);
+ (const void **) keys_burst,
+ BURST_SIZE,
+ positions_burst);
}
}
@@ -320,24 +368,25 @@ timed_lookups_multi(unsigned with_hash, unsigned table_index)
const uint64_t time_taken = end_tsc - start_tsc;
const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
- cycles[table_index][LOOKUP_MULTI][with_hash] = time_taken/NUM_LOOKUPS;
+ cycles[table_index][LOOKUP_MULTI][with_hash][with_data] = time_taken/NUM_LOOKUPS;
printf("%"PRIu64" lookups in %f seconds\n", (uint64_t)NUM_LOOKUPS,
seconds_taken);
printf("Average %"PRIu64" tsc ticks per lookup\n",
- cycles[table_index][LOOKUP_MULTI][with_hash]);
+ cycles[table_index][LOOKUP_MULTI][with_hash][with_data]);
printf("Average %"PRIu64" lookups per second\n",
(NUM_LOOKUPS * rte_get_tsc_hz())/time_taken);
return 0;
}
static int
-timed_deletes(unsigned with_hash, unsigned table_index)
+timed_deletes(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i;
const uint64_t start_tsc = rte_rdtsc();
for (i = 0; i < KEYS_TO_ADD; i++) {
+ /* There are no delete functions with data, so just call two functions */
if (with_hash)
rte_hash_del_key_with_hash(h[table_index],
(const void *) keys[i],
@@ -351,12 +400,12 @@ timed_deletes(unsigned with_hash, unsigned table_index)
const uint64_t time_taken = end_tsc - start_tsc;
const float seconds_taken = (float)time_taken/rte_get_tsc_hz();
- cycles[table_index][DELETE][with_hash] = time_taken/KEYS_TO_ADD;
+ cycles[table_index][DELETE][with_hash][with_data] = time_taken/KEYS_TO_ADD;
printf("\n%"PRIu64" deletions in %f seconds\n", (uint64_t) KEYS_TO_ADD,
seconds_taken);
printf("Average %"PRIu64" tsc ticks per deletion\n",
- cycles[table_index][DELETE][with_hash]);
+ cycles[table_index][DELETE][with_hash][with_data]);
printf("Average %"PRIu64" deletions per second\n",
(KEYS_TO_ADD * rte_get_tsc_hz())/time_taken);
return 0;
@@ -377,93 +426,80 @@ reset_table(unsigned table_index)
static int
run_all_tbl_perf_tests(void)
{
- unsigned i, j;
-
- for (i = 0; i < NUM_KEYSIZES; i++) {
- if (create_table(i) < 0)
- return -1;
-
- if (get_input_keys(i) < 0)
- return -1;
-
- printf("\n------ KEY SIZE = %u ----------\n\n",
- hashtest_key_lens[i]);
- printf("\n ----- WITH PRECOMPUTED HASH VALUES -----\n\n");
-
- printf("\nTimed additions\n");
- printf("------------------\n");
- if (timed_adds(1, i) < 0)
- return -1;
-
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
-
- printf("\nTimed lookups\n");
- printf("------------------\n");
- if (timed_lookups(1, i) < 0)
- return -1;
-
- printf("\nTimed lookups multi\n");
- printf("------------------\n");
- if (timed_lookups_multi(1, i) < 0)
- return -1;
+ unsigned i, j, with_data, with_hash;
- printf("\nTimed deletions\n");
- printf("------------------\n");
- if (timed_deletes(1, i) < 0)
- return -1;
-
- reset_table(i);
-
- printf("\n ----- WITH JUST KEYS -----\n\n");
-
- printf("\nTimed additions\n");
- printf("------------------\n");
- if (timed_adds(0, i) < 0)
- return -1;
-
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
-
- printf("\nTimed lookups\n");
- printf("------------------\n");
- if (timed_lookups(0, i) < 0)
- return -1;
-
- printf("\nTimed lookups multi\n");
- printf("------------------\n");
- if (timed_lookups_multi(0, i) < 0)
- return -1;
-
- printf("\nTimed deletions\n");
- printf("------------------\n");
- if (timed_deletes(0, i) < 0)
- return -1;
+ for (with_data = 0; with_data <= 1; with_data++) {
+ if (with_data)
+ printf("\n--- OPERATIONS WITH 8-BYTE DATA ---\n");
+ else
+ printf("\n--- OPERATIONS WITHOUT DATA ---\n");
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ printf("\n--------- KEY SIZE = %u ---------\n\n",
+ hashtest_key_lens[i]);
+ if (create_table(with_data, i) < 0)
+ return -1;
+
+ if (get_input_keys(i) < 0)
+ return -1;
+ for (with_hash = 0; with_hash <= 1; with_hash++) {
+ if (with_hash)
+ printf("\n--- WITH PRECOMPUTED HASH VALUES ---\n\n");
+ else
+ printf("\n--- WITH JUST KEYS ---\n\n");
+
+ printf("\nTimed additions\n");
+ printf("------------------\n");
+ if (timed_adds(with_hash, with_data, i) < 0)
+ return -1;
+
+ for (j = 0; j < NUM_SHUFFLES; j++)
+ shuffle_input_keys(i);
+
+ printf("\nTimed lookups\n");
+ printf("------------------\n");
+ if (timed_lookups(with_hash, with_data, i) < 0)
+ return -1;
+
+ printf("\nTimed lookups multi\n");
+ printf("------------------\n");
+ if (timed_lookups_multi(with_hash, with_data, i) < 0)
+ return -1;
+
+ printf("\nTimed deletions\n");
+ printf("------------------\n");
+ if (timed_deletes(with_hash, with_data, i) < 0)
+ return -1;
+
+ reset_table(i);
+ }
- free_table(i);
+ free_table(i);
+ }
}
printf("\nResults (in CPU cycles/operation)\n");
printf("-----------------------------------\n");
- printf("\nWith precomputed hash\n");
- printf("\n%-18s%-18s%-18s%-18s%-18s\n",
- "Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
- for (i = 0; i < NUM_KEYSIZES; i++) {
- printf("%-18d", hashtest_key_lens[i]);
- for (j = 0; j < NUM_OPERATIONS; j++)
- printf("%-18"PRIu64, cycles[i][j][1]);
- printf("\n");
- }
- printf("\nWith just keys\n");
- printf("\n%-18s%-18s%-18s%-18s%-18s\n",
+ for (with_data = 0; with_data <= 1; with_data++) {
+ if (with_data)
+ printf("\n Operations with 8-byte data\n");
+ else
+ printf("\n Operations without data\n");
+ for (with_hash = 0; with_hash <= 1; with_hash++) {
+ if (with_hash)
+ printf("\nWith precomputed hash\n");
+ else
+ printf("\nWith just keys\n");
+
+ printf("\n%-18s%-18s%-18s%-18s%-18s\n",
"Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
- for (i = 0; i < NUM_KEYSIZES; i++) {
- printf("%-18d", hashtest_key_lens[i]);
- for (j = 0; j < NUM_OPERATIONS; j++)
- printf("%-18"PRIu64, cycles[i][j][0]);
- printf("\n");
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ printf("%-18d", hashtest_key_lens[i]);
+ for (j = 0; j < NUM_OPERATIONS; j++)
+ printf("%-18"PRIu64, cycles[i][j][with_hash][with_data]);
+ printf("\n");
+ }
+ }
}
-
return 0;
}
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index 44f87fb..ad8ac9b 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -84,7 +84,9 @@ EAL_REGISTER_TAILQ(rte_hash_tailq)
#define DEFAULT_HASH_FUNC rte_jhash
#endif
-#define NULL_SIGNATURE 0
+#define NULL_SIGNATURE 0
+/* Stored key size is a multiple of this value */
+#define KEY_ALIGNMENT 16
typedef int (*rte_hash_cmp_eq_t)(const void *key1, const void *key2, size_t key_len);
static int rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len);
@@ -101,6 +103,7 @@ struct rte_hash {
char name[RTE_HASH_NAMESIZE]; /**< Name of the hash. */
uint32_t entries; /**< Total table entries. */
uint16_t key_len; /**< Length of hash key. */
+ uint16_t data_len; /**< Length of data. */
rte_hash_function hash_func; /**< Function used to calculate hash. */
rte_hash_cmp_eq_t rte_hash_cmp_eq; /**< Function used to compare keys. */
uint32_t hash_func_init_val; /**< Init value used by hash_func. */
@@ -170,6 +173,7 @@ rte_hash_create(const struct rte_hash_parameters *params)
void *ptr, *k = NULL;
char ring_name[RTE_RING_NAMESIZE];
unsigned i;
+ uint32_t key_entry_size;
hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
@@ -202,7 +206,12 @@ rte_hash_create(const struct rte_hash_parameters *params)
/* Total memory required for hash context */
const uint32_t mem_size = sizeof(struct rte_hash) +
num_buckets * sizeof(struct rte_hash_bucket);
- const uint32_t key_entry_size = params->key_len;
+ /* If key size is multiple of KEY_ALIGNMENT, need to align for fast comparison */
+ if ((params->key_len % KEY_ALIGNMENT) == 0)
+ key_entry_size = RTE_ALIGN(params->key_len + params->data_len, KEY_ALIGNMENT);
+ else
+ key_entry_size = params->key_len + params->data_len;
+
/* Store all keys and leave the first entry as a dummy entry for lookup_bulk */
const uint64_t key_tbl_size = key_entry_size * (params->entries + 1);
@@ -277,6 +286,7 @@ rte_hash_create(const struct rte_hash_parameters *params)
snprintf(h->name, sizeof(h->name), "%s", params->name);
h->entries = params->entries;
h->key_len = params->key_len;
+ h->data_len = params->data_len;
h->key_entry_size = key_entry_size;
h->hash_func_init_val = params->hash_func_init_val;
@@ -481,14 +491,14 @@ run_cuckoo(const struct rte_hash *h, struct rte_hash_bucket *bkt,
static inline int32_t
__rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
- hash_sig_t sig)
+ hash_sig_t sig, const void *data)
{
hash_sig_t hash0, hash1;
uint32_t bucket_idx0, bucket_idx1;
unsigned i;
struct rte_hash_bucket *bkt0, *bkt1;
void *new_k, *k, *keys = h->key_store;
- void *slot_id;
+ void *slot_id, *data_entry;
int ret;
/* Get a new slot for storing the new key */
@@ -512,6 +522,11 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
bkt0->signatures[i].alt == hash1) {
k = (char *)keys + bkt0->key_idx[i] * h->key_entry_size;
if (!h->rte_hash_cmp_eq(key, k, h->key_len)) {
+ if (data != NULL) {
+ data_entry = (char *) k + h->key_len;
+ /* Update data */
+ rte_memcpy(data_entry, data, h->data_len);
+ }
rte_ring_sp_enqueue(h->free_slots, &slot_id);
/*
* Return index where key is stored,
@@ -528,6 +543,11 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
bkt1->signatures[i].current == hash1) {
k = (char *)keys + bkt1->key_idx[i] * h->key_entry_size;
if (!h->rte_hash_cmp_eq(key, k, h->key_len)) {
+ if (data != NULL) {
+ data_entry = (char *) k + h->key_len;
+ /* Update data */
+ rte_memcpy(data_entry, data, h->data_len);
+ }
rte_ring_sp_enqueue(h->free_slots, &slot_id);
/*
* Return index where key is stored,
@@ -540,6 +560,11 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
/* Copy key */
rte_memcpy(new_k, key, h->key_len);
+ if (data != NULL) {
+ data_entry = (char *) new_k + h->key_len;
+ /* Copy data */
+ rte_memcpy(data_entry, data, h->data_len);
+ }
/*
* Run cuckoo algorithm
@@ -570,19 +595,33 @@ rte_hash_add_key_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, sig);
+ return __rte_hash_add_key_with_hash(h, key, sig, NULL);
}
int32_t
rte_hash_add_key(const struct rte_hash *h, const void *key)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
+ return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key), NULL);
}
+int32_t
+rte_hash_add_key_with_hash_data(const struct rte_hash *h,
+ const void *key, hash_sig_t sig, const void *data)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_add_key_with_hash(h, key, sig, data);
+}
+
+int32_t
+rte_hash_add_key_data(const struct rte_hash *h, const void *key, const void *data)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key), data);
+}
static inline int32_t
__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
- hash_sig_t sig)
+ hash_sig_t sig, void *data)
{
uint32_t bucket_idx;
hash_sig_t alt_hash;
@@ -598,12 +637,19 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
if (bkt->signatures[i].current == sig &&
bkt->signatures[i].sig != NULL_SIGNATURE) {
k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (!h->rte_hash_cmp_eq(key, k, h->key_len))
+ if (!h->rte_hash_cmp_eq(key, k, h->key_len)) {
+ if (data != NULL) {
+ const void *data_entry = (char *) k + h->key_len;
+ /* Return data */
+ rte_memcpy(data, data_entry, h->data_len);
+ }
+
/*
* Return index where key is stored,
* substracting the first dummy index
*/
return (bkt->key_idx[i] - 1);
+ }
}
}
@@ -617,12 +663,18 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
if (bkt->signatures[i].current == alt_hash &&
bkt->signatures[i].alt == sig) {
k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (!h->rte_hash_cmp_eq(key, k, h->key_len))
+ if (!h->rte_hash_cmp_eq(key, k, h->key_len)) {
+ if (data != NULL) {
+ const void *data_entry = (char *) k + h->key_len;
+ /* Return data */
+ rte_memcpy(data, data_entry, h->data_len);
+ }
/*
* Return index where key is stored,
* substracting the first dummy index
*/
return (bkt->key_idx[i] - 1);
+ }
}
}
@@ -634,16 +686,30 @@ rte_hash_lookup_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, sig);
+ return __rte_hash_lookup_with_hash(h, key, sig, NULL);
}
int32_t
rte_hash_lookup(const struct rte_hash *h, const void *key)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key), NULL);
+}
+
+int32_t
+rte_hash_lookup_with_hash_data(const struct rte_hash *h,
+ const void *key, hash_sig_t sig, void *data)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, sig, data);
}
+int32_t
+rte_hash_lookup_data(const struct rte_hash *h, const void *key, void *data)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key), data);
+}
static inline int32_t
__rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
hash_sig_t sig)
@@ -806,23 +872,30 @@ lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
}
-/* Lookup bulk stage 3: Check if key matches, update hit mask */
+/* Lookup bulk stage 3: Check if key matches, update hit mask and return data */
static inline void
-lookup_stage3(unsigned idx, const void *key_slot,
- const void * const *keys, int32_t *positions,
+lookup_stage3(unsigned idx, const void *key_slot, const void * const *keys,
+ void * const *data, int32_t *positions,
uint64_t *hits, const struct rte_hash *h)
{
unsigned hit;
hit = !h->rte_hash_cmp_eq(key_slot, keys[idx], h->key_len);
- if (unlikely(hit == 0))
+ if (unlikely(hit == 0)) {
positions[idx] = -ENOENT;
+ return;
+ }
+ if (data != NULL) {
+ const void *data_entry = (const char *) key_slot + h->key_len;
+
+ rte_memcpy(data[idx], data_entry, h->data_len);
+ }
*hits = (uint64_t)(hit) << idx;
}
static inline int
__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
- uint32_t num_keys, int32_t *positions) {
+ uint32_t num_keys, int32_t *positions, void **data) {
uint64_t hits = 0;
uint64_t next_mask = 0;
@@ -931,8 +1004,8 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21,
primary_bkt21, secondary_bkt21, &k_slot21, positions,
&extra_hits_mask, key_store, h);
- lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, positions, &hits, h);
}
k_slot30 = k_slot20, k_slot31 = k_slot21;
@@ -961,8 +1034,8 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
key_store, h);
- lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, positions, &hits, h);
k_slot30 = k_slot20, k_slot31 = k_slot21;
idx30 = idx20, idx31 = idx21;
@@ -982,14 +1055,14 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
key_store, h);
- lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, positions, &hits, h);
k_slot30 = k_slot20, k_slot31 = k_slot21;
idx30 = idx20, idx31 = idx21;
- lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, positions, &hits, h);
/* handle extra_hits_mask */
next_mask |= extra_hits_mask;
@@ -1012,7 +1085,7 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
static inline int
__rte_hash_lookup_bulk_with_hash(const struct rte_hash *h, const void **keys,
const hash_sig_t *hash_vals, uint32_t num_keys,
- int32_t *positions)
+ int32_t *positions, void **data)
{
uint64_t hits = 0;
uint64_t next_mask = 0;
@@ -1121,8 +1194,8 @@ __rte_hash_lookup_bulk_with_hash(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21,
primary_bkt21, secondary_bkt21, &k_slot21, positions,
&extra_hits_mask, key_store, h);
- lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, positions, &hits, h);
}
k_slot30 = k_slot20, k_slot31 = k_slot21;
@@ -1152,8 +1225,8 @@ __rte_hash_lookup_bulk_with_hash(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
key_store, h);
- lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, positions, &hits, h);
k_slot30 = k_slot20, k_slot31 = k_slot21;
idx30 = idx20, idx31 = idx21;
@@ -1173,14 +1246,14 @@ __rte_hash_lookup_bulk_with_hash(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
key_store, h);
- lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, positions, &hits, h);
k_slot30 = k_slot20, k_slot31 = k_slot21;
idx30 = idx20, idx31 = idx21;
- lookup_stage3(idx30, k_slot30, keys, positions, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, positions, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, positions, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, positions, &hits, h);
/* handle extra_hits_mask */
next_mask |= extra_hits_mask;
@@ -1209,7 +1282,7 @@ rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
(num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
(positions == NULL)), -EINVAL);
- return __rte_hash_lookup_bulk(h, keys, num_keys, positions);
+ return __rte_hash_lookup_bulk(h, keys, num_keys, positions, NULL);
}
int
@@ -1222,7 +1295,31 @@ rte_hash_lookup_bulk_with_hash(const struct rte_hash *h, const void **keys,
(positions == NULL)), -EINVAL);
return __rte_hash_lookup_bulk_with_hash(h, keys, hash_vals, num_keys,
- positions);
+ positions, NULL);
+}
+
+int
+rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, int32_t *positions, void **data)
+{
+ RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
+ (num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
+ (positions == NULL)), -EINVAL);
+
+ return __rte_hash_lookup_bulk(h, keys, num_keys, positions, data);
+}
+
+int
+rte_hash_lookup_bulk_with_hash_data(const struct rte_hash *h, const void **keys,
+ const hash_sig_t *hash_vals, uint32_t num_keys,
+ int32_t *positions, void **data)
+{
+ RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
+ (num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
+ (positions == NULL)), -EINVAL);
+
+ return __rte_hash_lookup_bulk_with_hash(h, keys, hash_vals, num_keys,
+ positions, data);
}
/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index fa327c2..e917fea 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -84,12 +84,12 @@ struct rte_hash_parameters {
rte_hash_function hash_func; /**< Primary Hash function used to calculate hash. */
uint32_t hash_func_init_val; /**< Init value used by hash_func. */
int socket_id; /**< NUMA Socket ID for memory. */
+ uint32_t data_len; /**< Length of data. */
};
/** @internal A hash table structure. */
struct rte_hash;
-
/**
* Create a new hash table.
*
@@ -140,6 +140,50 @@ void
rte_hash_reset(struct rte_hash *h);
/**
+ * Add a key-value pair to an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
+ *
+ * @param h
+ * Hash table to add the key to.
+ * @param key
+ * Key to add to the hash table.
+ * @param data
+ * Data to add to the hash table.
+ * @return
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOSPC if there is no space in the hash for this key.
+ * - A positive value that can be used by the caller as an offset into an
+ * array of user data. This value is unique for this key.
+ */
+int32_t
+rte_hash_add_key_data(const struct rte_hash *h, const void *key, const void *data);
+
+/**
+ * Add a key-value pair with a pre-computed hash value
+ * to an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
+ *
+ * @param h
+ * Hash table to add the key to.
+ * @param key
+ * Key to add to the hash table.
+ * @param sig
+ * Precomputed hash value for 'key'
+ * @param data
+ * Data to add to the hash table.
+ * @return
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOSPC if there is no space in the hash for this key.
+ * - A positive value that can be used by the caller as an offset into an
+ * array of user data. This value is unique for this key.
+ */
+int32_t
+rte_hash_add_key_with_hash_data(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, const void *data);
+
+/**
* Add a key to an existing hash table. This operation is not multi-thread safe
* and should only be called from one thread.
*
@@ -216,6 +260,51 @@ rte_hash_del_key(const struct rte_hash *h, const void *key);
int32_t
rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
+
+/**
+ * Find a key-value pair in the hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param key
+ * Key to find.
+ * @param data
+ * Data to return.
+ * @return
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOENT if the key is not found.
+ * - A positive value that can be used by the caller as an offset into an
+ * array of user data. This value is unique for this key, and is the same
+ * value that was returned when the key was added.
+ */
+int32_t
+rte_hash_lookup_data(const struct rte_hash *h, const void *key, void *data);
+
+/**
+ * Find a key-value pair with a pre-computed hash value
+ * to an existing hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param key
+ * Key to find.
+ * @param sig
+ * Precomputed hash value for 'key'
+ * @param data
+ * Pointer to data returned from the hash table.
+ * @return
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOENT if the key is not found.
+ * - A positive value that can be used by the caller as an offset into an
+ * array of user data. This value is unique for this key, and is the same
+ * value that was returned when the key was added.
+ */
+int32_t
+rte_hash_lookup_with_hash_data(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, void *data);
+
/**
* Find a key in the hash table.
* This operation is multi-thread safe.
@@ -270,7 +359,34 @@ hash_sig_t
rte_hash_hash(const struct rte_hash *h, const void *key);
#define rte_hash_lookup_multi rte_hash_lookup_bulk
+#define rte_hash_lookup_multi_data rte_hash_lookup_bulk_data
#define rte_hash_lookup_multi_with_hash rte_hash_lookup_bulk_with_hash
+#define rte_hash_lookup_multi_with_hash_data rte_hash_lookup_bulk_with_hash_data
+/**
+ * Find multiple keys in the hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param keys
+ * A pointer to a list of keys to look for.
+ * @param num_keys
+ * How many keys are in the keys list (less than RTE_HASH_LOOKUP_BULK_MAX).
+ * @param positions
+ * Output containing a list of values, corresponding to the list of keys that
+ * can be used by the caller as an offset into an array of user data. These
+ * values are unique for each key, and are the same values that were returned
+ * when each key was added. If a key in the list was not found, then -ENOENT
+ * will be the value.
+ * @param data
+ * Output containing array of data returned from all the successful lookups.
+ * @return
+ * -EINVAL if there's an error, otherwise 0.
+ */
+int
+rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, int32_t *positions, void **data);
+
/**
* Find multiple keys in the hash table.
* This operation is multi-thread safe.
@@ -303,6 +419,33 @@ rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
* @param keys
* A pointer to a list of keys to look for.
* @param hash_vals
+ * A pointer to a list of pre-calculated hash values for 'keys'.
+ * @param num_keys
+ * How many keys are in the keys list (less than RTE_HASH_LOOKUP_BULK_MAX).
+ * @param positions
+ * Output containing a list of values, corresponding to the list of keys that
+ * can be used by the caller as an offset into an array of user data. These
+ * values are unique for each key, and are the same values that were returned
+ * when each key was added. If a key in the list was not found, then -ENOENT
+ * will be the value.
+ * @param data
+ * Output containing array of data returned from all the successful lookups.
+ * @return
+ * -EINVAL if there's an error, otherwise 0.
+ */
+int
+rte_hash_lookup_bulk_with_hash_data(const struct rte_hash *h, const void **keys,
+ const hash_sig_t *hash_vals, uint32_t num_keys,
+ int32_t *positions, void **data);
+
+/**
+ * Find multiple keys in the hash table. This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param keys
+ * A pointer to a list of keys to look for.
+ * @param hash_vals
* A pointer to a list of pre-calculated hash values.
* @param num_keys
* How many keys are in the keys list (less than RTE_HASH_LOOKUP_BULK_MAX).
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index f011054..3a4e1a3 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -21,7 +21,13 @@ DPDK_2.0 {
DPDK_2.1 {
global:
+ rte_hash_add_key_data;
+ rte_hash_add_key_with_hash_data;
+ rte_hash_lookup_bulk_data;
rte_hash_lookup_bulk_with_hash;
+ rte_hash_lookup_bulk_with_hash_data;
+ rte_hash_lookup_data;
+ rte_hash_lookup_with_hash_data;
rte_hash_reset;
local: *;
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v3 09/11] MAINTAINERS: claim responsability for hash library
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Pablo de Lara
` (7 preceding siblings ...)
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 08/11] hash: add new functionality to store data in hash table Pablo de Lara
@ 2015-06-28 22:25 ` Pablo de Lara
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 10/11] doc: announce ABI change of librte_hash Pablo de Lara
` (3 subsequent siblings)
12 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-06-28 22:25 UTC (permalink / raw)
To: dev
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
MAINTAINERS | 1 +
1 file changed, 1 insertion(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 54f0973..a536992 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -339,6 +339,7 @@ F: doc/guides/sample_app_ug/l3_forward_access_ctrl.rst
Hashes
M: Bruce Richardson <bruce.richardson@intel.com>
+M: Pablo de Lara <pablo.de.lara.guarch@intel.com>
F: lib/librte_hash/
F: doc/guides/prog_guide/hash_lib.rst
F: app/test/test_hash*
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v3 10/11] doc: announce ABI change of librte_hash
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Pablo de Lara
` (8 preceding siblings ...)
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 09/11] MAINTAINERS: claim responsability for hash library Pablo de Lara
@ 2015-06-28 22:25 ` Pablo de Lara
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 11/11] doc: update hash documentation Pablo de Lara
` (2 subsequent siblings)
12 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-06-28 22:25 UTC (permalink / raw)
To: dev
rte_hash structure is now private for version 2.1, and two
of the macros in rte_hash.h are now deprecated, so this patch
adds notice of these changes.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
doc/guides/rel_notes/abi.rst | 2 ++
1 file changed, 2 insertions(+)
diff --git a/doc/guides/rel_notes/abi.rst b/doc/guides/rel_notes/abi.rst
index f00a6ee..fae09fd 100644
--- a/doc/guides/rel_notes/abi.rst
+++ b/doc/guides/rel_notes/abi.rst
@@ -38,3 +38,5 @@ Examples of Deprecation Notices
Deprecation Notices
-------------------
+* Structure rte_hash in librte_hash library has been changed and has been made private in relese 2.1, as applications should have never accessed to its internal data (library should have been marked as internal).
+* The Macros #RTE_HASH_BUCKET_ENTRIES_MAX and #RTE_HASH_KEY_LENGTH_MAX are deprecated and will be removed with version 2.2.
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v3 11/11] doc: update hash documentation
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Pablo de Lara
` (9 preceding siblings ...)
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 10/11] doc: announce ABI change of librte_hash Pablo de Lara
@ 2015-06-28 22:25 ` Pablo de Lara
2015-07-08 23:23 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Thomas Monjalon
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of " Pablo de Lara
12 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-06-28 22:25 UTC (permalink / raw)
To: dev
Updates hash library documentation, reflecting
the new implementation changes.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
doc/guides/prog_guide/hash_lib.rst | 77 +++++++++++++++++++++++++++++++-------
1 file changed, 64 insertions(+), 13 deletions(-)
diff --git a/doc/guides/prog_guide/hash_lib.rst b/doc/guides/prog_guide/hash_lib.rst
index 9b83835..40e24bf 100644
--- a/doc/guides/prog_guide/hash_lib.rst
+++ b/doc/guides/prog_guide/hash_lib.rst
@@ -1,5 +1,5 @@
.. BSD LICENSE
- Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
All rights reserved.
Redistribution and use in source and binary forms, with or without
@@ -50,8 +50,6 @@ The hash also allows the configuration of some low-level implementation related
* Hash function to translate the key into a bucket index
-* Number of entries per bucket
-
The main methods exported by the hash are:
* Add entry with key: The key is provided as input. If a new entry is successfully added to the hash for the specified key,
@@ -65,10 +63,27 @@ The main methods exported by the hash are:
* Lookup for entry with key: The key is provided as input. If an entry with the specified key is found in the hash (lookup hit),
then the position of the entry is returned, otherwise (lookup miss) a negative value is returned.
-The current hash implementation handles the key management only.
-The actual data associated with each key has to be managed by the user using a separate table that
+Apart from these method explained above, the API allows the user three more options:
+
+* Add / lookup / delete with key and precomputed hash: Both the key and its precomputed hash are provided as input. This allows
+ the user to perform these operations faster, as hash is already computed.
+
+* Add / lookup / delete with key and data: A pair of key-value is provided as input. This allows the user to store
+ not only the key, but also a user-defined number of bytes of data, which should perform better
+ than storing the data in an external table
+
+* Combination of the two options above: User can provide key, precomputed hash and data.
+
+Also, the API contains a method to allow the user to look up entries in bursts, achieving higher performance
+than looking up individual entries, as the function prefetches next entries at the time it is operating
+with the first ones, which reduces significantly the impact of the necessary memory accesses.
+Notice that this method uses a pipeline of 8 entries (4 stages of 2 entries), so it is highly recommended
+to use at least 8 entries per burst.
+
+The actual data associated with each key can be either managed by the user using a separate table that
mirrors the hash in terms of number of entries and position of each entry,
-as shown in the Flow Classification use case describes in the following sections.
+as shown in the Flow Classification use case describes in the following sections,
+or stored in the hash table itself.
The example hash tables in the L2/L3 Forwarding sample applications defines which port to forward a packet to based on a packet flow identified by the five-tuple lookup.
However, this table could also be used for more sophisticated features and provide many other functions and actions that could be performed on the packets and flows.
@@ -76,17 +91,26 @@ However, this table could also be used for more sophisticated features and provi
Implementation Details
----------------------
-The hash table is implemented as an array of entries which is further divided into buckets,
-with the same number of consecutive array entries in each bucket.
-For any input key, there is always a single bucket where that key can be stored in the hash,
-therefore only the entries within that bucket need to be examined when the key is looked up.
+The hash table has two main tables:
+
+* First table is an array of entries which is further divided into buckets,
+ with the same number of consecutive array entries in each bucket. Each entry contains the computed primary
+ and secondary hashes of a given key (explained below), and an index to the second table.
+
+* The second table is an array of all the keys stored in the hash table and its data associated to each key.
+
+The hash library uses the cuckoo hash method to resolve collisions.
+For any input key, there are two possible buckets (primary and secondary/alternative location)
+where that key can be stored in the hash, therefore only the entries within those bucket need to be examined
+when the key is looked up.
The lookup speed is achieved by reducing the number of entries to be scanned from the total
-number of hash entries down to the number of entries in a hash bucket,
+number of hash entries down to the number of entries in the two hash buckets,
as opposed to the basic method of linearly scanning all the entries in the array.
The hash uses a hash function (configurable) to translate the input key into a 4-byte key signature.
The bucket index is the key signature modulo the number of hash buckets.
-Once the bucket is identified, the scope of the hash add,
-delete and lookup operations is reduced to the entries in that bucket.
+
+Once the buckets are identified, the scope of the hash add,
+delete and lookup operations is reduced to the entries in those buckets (it is very likely that entries are in the primary bucket).
To speed up the search logic within the bucket, each hash entry stores the 4-byte key signature together with the full key for each hash entry.
For large key sizes, comparing the input key against a key from the bucket can take significantly more time than
@@ -95,6 +119,33 @@ Therefore, the signature comparison is done first and the full key comparison do
The full key comparison is still necessary, as two input keys from the same bucket can still potentially have the same 4-byte hash signature,
although this event is relatively rare for hash functions providing good uniform distributions for the set of input keys.
+Example of lookup:
+
+First of all, the primary bucket is identified and entry is likely to be stored there.
+If signature was stored there, we compare its key against the one provided and return the position
+where it was stored and/or the data associated to that key if there is a match.
+If signature is not in the primary bucket, the secondary bucket is looked up, where same procedure
+is carried out. If there is no match there either, key is considered not to be in the table.
+
+Example of addition:
+
+Like lookup, the primary and secondary buckets are indentified. If there is an empty slot in
+the primary bucket, primary and secondary signatures are stored in that slot, key and data (if any) are added to
+the second table and an index to the position in the second table is stored in the slot of the first table.
+If there is no space in the primary bucket, one of the entries on that bucket is pushed to its alternative location,
+and the key to be added is inserted in its position.
+To know where the alternative bucket of the evicted entry is, the secondary signature is looked up and alternative bucket index
+is calculated from doing the modulo, as seen above. If there is room in the alternative bucket, the evicted entry
+is stored in it. If not, same process is repeated (one of the entries gets pushed) until a non full bucket is found.
+Notice that despite all the entry movement in the first table, the second table is not touched, which would impact
+greatly in performance.
+
+In the very unlikely event that table enters in a loop where same entries are being evicted indefinitely,
+key is considered not able to be stored.
+With random keys, this method allows the user to get around 90% of the table utilization, without
+having to drop any stored entry (LRU) or allocate more memory (extended buckets).
+
+
Use Case: Flow Classification
-----------------------------
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* Re: [dpdk-dev] [PATCH v3 00/11] Cuckoo hash
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Pablo de Lara
` (10 preceding siblings ...)
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 11/11] doc: update hash documentation Pablo de Lara
@ 2015-07-08 23:23 ` Thomas Monjalon
2015-07-09 8:02 ` Bruce Richardson
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of " Pablo de Lara
12 siblings, 1 reply; 92+ messages in thread
From: Thomas Monjalon @ 2015-07-08 23:23 UTC (permalink / raw)
To: bruce.richardson; +Cc: dev
Bruce, what is the status of this series?
2015-06-28 23:25, Pablo de Lara:
> This patchset is to replace the existing hash library with
> a more efficient and functional approach, using the Cuckoo hash
> method to deal with collisions. This method is based on using
> two different hash functions to have two possible locations
> in the hash table where an entry can be.
> So, if a bucket is full, a new entry can push one of the items
> in that bucket to its alternative location, making space for itself.
>
> Advantages
> ~~~~~~
> - Offers the option to store more entries when the target bucket is full
> (unlike the previous implementation)
> - Memory efficient: for storing those entries, it is not necessary to
> request new memory, as the entries will be stored in the same table
> - Constant worst lookup time: in worst case scenario, it always takes
> the same time to look up an entry, as there are only two possible locations
> where an entry can be.
> - Storing data: user can store data in the hash table, unlike the
> previous implementation, but he can still use the old API
>
> This implementation tipically offers over 90% utilization.
> Notice that API has been extended, but old API remains. The main
> change in ABI is that rte_hash structure is now private and the
> deprecation of two macros.
>
> Changes in v3:
>
> - Now user can store variable size data, instead of 32 or 64-bit size data,
> using the new parameter "data_len" in rte_hash_parameters
> - Add lookup_bulk_with_hash function in performance unit tests
> - Add new functions that handle data in performance unit tests
> - Remove duplicates in performance unit tests
> - Fix rte_hash_reset, which was not reseting the last entry
[...]
> Pablo de Lara (11):
> eal: add const in prefetch functions
> hash: move rte_hash structure to C file and make it internal
> test/hash: enhance hash unit tests
> test/hash: rename new hash perf unit test back to original name
> hash: replace existing hash library with cuckoo hash implementation
> hash: add new lookup_bulk_with_hash function
> hash: add new function rte_hash_reset
> hash: add new functionality to store data in hash table
> MAINTAINERS: claim responsability for hash library
> doc: announce ABI change of librte_hash
> doc: update hash documentation
^ permalink raw reply [flat|nested] 92+ messages in thread* Re: [dpdk-dev] [PATCH v3 00/11] Cuckoo hash
2015-07-08 23:23 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Thomas Monjalon
@ 2015-07-09 8:02 ` Bruce Richardson
0 siblings, 0 replies; 92+ messages in thread
From: Bruce Richardson @ 2015-07-09 8:02 UTC (permalink / raw)
To: Thomas Monjalon; +Cc: dev
On Thu, Jul 09, 2015 at 01:23:54AM +0200, Thomas Monjalon wrote:
> Bruce, what is the status of this series?
>
The parts of the series can that are independent of the cuckoo hash update
have already been sent out as updates so changes to those can quicker be resolved
and merged. Pablo should be sending out a V4 of the cuckoo hash implementation
very shortly.
/Bruce
> 2015-06-28 23:25, Pablo de Lara:
> > This patchset is to replace the existing hash library with
> > a more efficient and functional approach, using the Cuckoo hash
> > method to deal with collisions. This method is based on using
> > two different hash functions to have two possible locations
> > in the hash table where an entry can be.
> > So, if a bucket is full, a new entry can push one of the items
> > in that bucket to its alternative location, making space for itself.
> >
> > Advantages
> > ~~~~~~
> > - Offers the option to store more entries when the target bucket is full
> > (unlike the previous implementation)
> > - Memory efficient: for storing those entries, it is not necessary to
> > request new memory, as the entries will be stored in the same table
> > - Constant worst lookup time: in worst case scenario, it always takes
> > the same time to look up an entry, as there are only two possible locations
> > where an entry can be.
> > - Storing data: user can store data in the hash table, unlike the
> > previous implementation, but he can still use the old API
> >
> > This implementation tipically offers over 90% utilization.
> > Notice that API has been extended, but old API remains. The main
> > change in ABI is that rte_hash structure is now private and the
> > deprecation of two macros.
> >
> > Changes in v3:
> >
> > - Now user can store variable size data, instead of 32 or 64-bit size data,
> > using the new parameter "data_len" in rte_hash_parameters
> > - Add lookup_bulk_with_hash function in performance unit tests
> > - Add new functions that handle data in performance unit tests
> > - Remove duplicates in performance unit tests
> > - Fix rte_hash_reset, which was not reseting the last entry
> [...]
> > Pablo de Lara (11):
> > eal: add const in prefetch functions
> > hash: move rte_hash structure to C file and make it internal
> > test/hash: enhance hash unit tests
> > test/hash: rename new hash perf unit test back to original name
> > hash: replace existing hash library with cuckoo hash implementation
> > hash: add new lookup_bulk_with_hash function
> > hash: add new function rte_hash_reset
> > hash: add new functionality to store data in hash table
> > MAINTAINERS: claim responsability for hash library
> > doc: announce ABI change of librte_hash
> > doc: update hash documentation
>
^ permalink raw reply [flat|nested] 92+ messages in thread
* [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of Cuckoo hash
2015-06-28 22:25 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Pablo de Lara
` (11 preceding siblings ...)
2015-07-08 23:23 ` [dpdk-dev] [PATCH v3 00/11] Cuckoo hash Thomas Monjalon
@ 2015-07-10 17:24 ` Pablo de Lara
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 1/7] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
` (8 more replies)
12 siblings, 9 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 17:24 UTC (permalink / raw)
To: dev
This patchset is to replace the existing hash library with
a more efficient and functional approach, using the Cuckoo hash
method to deal with collisions. This method is based on using
two different hash functions to have two possible locations
in the hash table where an entry can be.
So, if a bucket is full, a new entry can push one of the items
in that bucket to its alternative location, making space for itself.
Advantages
~~~~~
- Offers the option to store more entries when the target bucket is full
(unlike the previous implementation)
- Memory efficient: for storing those entries, it is not necessary to
request new memory, as the entries will be stored in the same table
- Constant worst lookup time: in worst case scenario, it always takes
the same time to look up an entry, as there are only two possible locations
where an entry can be.
- Storing data: user can store data in the hash table, unlike the
previous implementation, but he can still use the old API
This implementation tipically offers over 90% utilization.
Notice that API has been extended, but old API remains.
Check documentation included to know more about this new implementation
(including how entries are distributed as table utilization increases).
Changes in v4:
- Unit tests enhancements are not part of this patchset anymore.
- rte_hash structure has been made internal in another patch,
so it is not part of this patchset anymore.
- Add function to iterate through the hash table, as rte_hash
structure has been made private.
- Added extra_flag parameter in rte_hash_parameter to be able
to add new parameters in the future without breaking the ABI
- Remove proposed lookup_bulk_with_hash function, as it is
not of much use with the existing hash functions
(there are no vector hash functions).
- User can store 8-byte integer or pointer as data, instead
of variable size data, as discussed in the mailing list.
Changes in v3:
- Now user can store variable size data, instead of 32 or 64-bit size data,
using the new parameter "data_len" in rte_hash_parameters
- Add lookup_bulk_with_hash function in performance unit tests
- Add new functions that handle data in performance unit tests
- Remove duplicates in performance unit tests
- Fix rte_hash_reset, which was not reseting the last entry
Changes in v2:
- Fixed issue where table could not store maximum number of entries
- Fixed issue where lookup burst could not be more than 32 (instead of 64)
- Remove unnecessary macros and add other useful ones
- Added missing library dependencies
- Used directly rte_hash_secondary instead of rte_hash_alt
- Renamed rte_hash.c to rte_cuckoo_hash.c to ease the view of the new library
- Renamed test_hash_perf.c temporarily to ease the view of the improved unit test
- Moved rte_hash, rte_bucket and rte_hash_key structures to rte_cuckoo_hash.c to
make them private
- Corrected copyright dates
- Added an optimized function to compare keys that are multiple of 16 bytes
- Improved the way to use primary/secondary signatures. Now both are stored in
the bucket, so there is no need to calculate them if required.
Also, there is no need to use the MSB of a signature to differenciate between
an empty entry and signature 0, since we are storing both signatures,
which cannot be both 0.
- Removed rte_hash_rehash, as it was a very expensive operation.
Therefore, the add function returns now -ENOSPC if key cannot be added
because of a loop.
- Prefetched new slot for new key in add function to improve performance.
- Made doxygen comments more clear.
- Removed unnecessary rte_hash_del_key_data and rte_hash_del_key_with_data,
as we can use the lookup functions if we want to get the data before deleting.
- Removed some unnecessary includes in rte_hash.h
- Removed some unnecessary variables in rte_cuckoo_hash.c
- Removed some unnecessary checks before creating a new hash table
- Added documentation (in release notes and programmers guide)
- Added new unit tests and replaced the performance one for hash tables
Pablo de Lara (7):
hash: replace existing hash library with cuckoo hash implementation
hash: add new function rte_hash_reset
hash: add new functionality to store data in hash table
hash: add iterate function
MAINTAINERS: claim responsability for hash library
doc: announce ABI change of librte_hash
doc: update hash documentation
MAINTAINERS | 1 +
app/test/test_hash.c | 189 +++---
app/test/test_hash_perf.c | 305 ++++++---
doc/guides/prog_guide/hash_lib.rst | 138 +++-
doc/guides/prog_guide/index.rst | 4 +
doc/guides/rel_notes/abi.rst | 1 +
lib/librte_hash/Makefile | 8 +-
lib/librte_hash/rte_cuckoo_hash.c | 1194 ++++++++++++++++++++++++++++++++++
lib/librte_hash/rte_hash.c | 499 --------------
lib/librte_hash/rte_hash.h | 198 +++++-
lib/librte_hash/rte_hash_version.map | 15 +
11 files changed, 1812 insertions(+), 740 deletions(-)
create mode 100644 lib/librte_hash/rte_cuckoo_hash.c
delete mode 100644 lib/librte_hash/rte_hash.c
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v4 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of " Pablo de Lara
@ 2015-07-10 17:24 ` Pablo de Lara
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 2/7] hash: add new function rte_hash_reset Pablo de Lara
` (7 subsequent siblings)
8 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 17:24 UTC (permalink / raw)
To: dev
This patch replaces the existing hash library with another approach,
using the Cuckoo Hash method to resolve collisions (open addressing),
which pushes items from a full bucket when a new entry tries
to be added in it, storing the evicted entry in an alternative location,
using a secondary hash function.
This gives the user the ability to store more entries when a bucket
is full, in comparison with the previous implementation.
Therefore, the unit test has been updated, as some scenarios have changed
(such as the previous removed restriction).
Also note that the API has not been changed, although new fields
have been added in the rte_hash structure (structure is internal now).
The main change when creating a new table is that the number of entries
per bucket is fixed now, so its parameter is ignored now
(still there to maintain the same parameters structure).
The hash unit test has been updated to reflect these changes.
As a last note, the maximum burst size in lookup_burst function
hash been increased to 64, to improve performance.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 119 ++---
app/test/test_hash_perf.c | 79 ++-
lib/librte_hash/Makefile | 8 +-
lib/librte_hash/rte_cuckoo_hash.c | 1027 +++++++++++++++++++++++++++++++++++++
lib/librte_hash/rte_hash.c | 499 ------------------
lib/librte_hash/rte_hash.h | 69 +--
6 files changed, 1138 insertions(+), 663 deletions(-)
create mode 100644 lib/librte_hash/rte_cuckoo_hash.c
delete mode 100644 lib/librte_hash/rte_hash.c
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index 7c71ed6..e70d859 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -169,7 +169,6 @@ static struct flow_key keys[5] = { {
/* Parameters used for hash table in unit test functions. Name set later. */
static struct rte_hash_parameters ut_params = {
.entries = 64,
- .bucket_entries = 4,
.key_len = sizeof(struct flow_key), /* 13 */
.hash_func = rte_jhash,
.hash_func_init_val = 0,
@@ -516,9 +515,18 @@ static int test_five_keys(void)
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != -ENOENT,
- "failed to find key (pos[%u]=%d)", i, pos[i]);
+ "found non-existent key (pos[%u]=%d)", i, pos[i]);
}
+ /* Lookup multi */
+ ret = rte_hash_lookup_multi(handle, &key_array[0], 5, (int32_t *)pos);
+ if (ret == 0)
+ for (i = 0; i < 5; i++) {
+ print_key_info("Lkp", key_array[i], pos[i]);
+ RETURN_IF_ERROR(pos[i] != -ENOENT,
+ "found not-existent key (pos[%u]=%d)", i, pos[i]);
+ }
+
rte_hash_free(handle);
return 0;
@@ -527,21 +535,18 @@ static int test_five_keys(void)
/*
* Add keys to the same bucket until bucket full.
* - add 5 keys to the same bucket (hash created with 4 keys per bucket):
- * first 4 successful, 5th unsuccessful
- * - lookup the 5 keys: 4 hits, 1 miss
- * - add the 5 keys again: 4 OK, one error as bucket is full
- * - lookup the 5 keys: 4 hits (updated data), 1 miss
- * - delete the 5 keys: 5 OK (even if the 5th is not in the table)
+ * first 4 successful, 5th successful, pushing existing item in bucket
+ * - lookup the 5 keys: 5 hits
+ * - add the 5 keys again: 5 OK
+ * - lookup the 5 keys: 5 hits (updated data)
+ * - delete the 5 keys: 5 OK
* - lookup the 5 keys: 5 misses
- * - add the 5th key: OK
- * - lookup the 5th key: hit
*/
static int test_full_bucket(void)
{
struct rte_hash_parameters params_pseudo_hash = {
.name = "test4",
.entries = 64,
- .bucket_entries = 4,
.key_len = sizeof(struct flow_key), /* 13 */
.hash_func = pseudo_hash,
.hash_func_init_val = 0,
@@ -555,7 +560,7 @@ static int test_full_bucket(void)
handle = rte_hash_create(¶ms_pseudo_hash);
RETURN_IF_ERROR(handle == NULL, "hash creation failed");
- /* Fill bucket*/
+ /* Fill bucket */
for (i = 0; i < 4; i++) {
pos[i] = rte_hash_add_key(handle, &keys[i]);
print_key_info("Add", &keys[i], pos[i]);
@@ -563,47 +568,39 @@ static int test_full_bucket(void)
"failed to add key (pos[%u]=%d)", i, pos[i]);
expected_pos[i] = pos[i];
}
- /* This shouldn't work because the bucket is full */
+ /*
+ * This should work and will push one of the items
+ * in the bucket because it is full
+ */
pos[4] = rte_hash_add_key(handle, &keys[4]);
print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOSPC,
- "fail: added key to full bucket (pos[4]=%d)", pos[4]);
+ RETURN_IF_ERROR(pos[4] < 0,
+ "failed to add key (pos[4]=%d)", pos[4]);
+ expected_pos[4] = pos[4];
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to find key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: found non-existent key (pos[4]=%d)", pos[4]);
/* Add - update */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_add_key(handle, &keys[i]);
print_key_info("Add", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to add key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_add_key(handle, &keys[4]);
- print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOSPC,
- "fail: added key to full bucket (pos[4]=%d)", pos[4]);
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to find key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: found non-existent key (pos[4]=%d)", pos[4]);
/* Delete 1 key, check other keys are still found */
pos[1] = rte_hash_del_key(handle, &keys[1]);
@@ -623,35 +620,21 @@ static int test_full_bucket(void)
RETURN_IF_ERROR(pos[1] < 0, "failed to add key (pos[1]=%d)", pos[1]);
/* Delete */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_del_key(handle, &keys[i]);
print_key_info("Del", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to delete key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_del_key(handle, &keys[4]);
- print_key_info("Del", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: deleted non-existent key (pos[4]=%d)", pos[4]);
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != -ENOENT,
"fail: found non-existent key (pos[%u]=%d)", i, pos[i]);
}
- /* Add and lookup the 5th key */
- pos[4] = rte_hash_add_key(handle, &keys[4]);
- print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] < 0, "failed to add key (pos[4]=%d)", pos[4]);
- expected_pos[4] = pos[4];
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != expected_pos[4],
- "failed to find key (pos[4]=%d)", pos[4]);
-
rte_hash_free(handle);
/* Cover the NULL case. */
@@ -991,6 +974,7 @@ static int test_fbk_hash_find_existing(void)
return 0;
}
+#define BUCKET_ENTRIES 4
/*
* Do tests for hash creation with bad parameters.
*/
@@ -1017,18 +1001,8 @@ static int test_hash_creation_with_bad_parameters(void)
}
memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_1";
- params.bucket_entries = RTE_HASH_BUCKET_ENTRIES_MAX + 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully with bucket_entries in parameter exceeded\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
params.name = "creation_with_bad_parameters_2";
- params.entries = params.bucket_entries - 1;
+ params.entries = BUCKET_ENTRIES - 1;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
rte_hash_free(handle);
@@ -1038,26 +1012,6 @@ static int test_hash_creation_with_bad_parameters(void)
memcpy(¶ms, &ut_params, sizeof(params));
params.name = "creation_with_bad_parameters_3";
- params.entries = params.entries - 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully if entries in parameter is not power of 2\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_4";
- params.bucket_entries = params.bucket_entries - 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully if bucket_entries in parameter is not power of 2\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_5";
params.key_len = 0;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
@@ -1067,17 +1021,7 @@ static int test_hash_creation_with_bad_parameters(void)
}
memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_6";
- params.key_len = RTE_HASH_KEY_LENGTH_MAX + 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully if key_len is greater than the maximum\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_7";
+ params.name = "creation_with_bad_parameters_4";
params.socket_id = RTE_MAX_NUMA_NODES + 1;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
@@ -1214,7 +1158,6 @@ static uint8_t key[16] = {0x00, 0x01, 0x02, 0x03,
static struct rte_hash_parameters hash_params_ex = {
.name = NULL,
.entries = 64,
- .bucket_entries = 4,
.key_len = 0,
.hash_func = NULL,
.hash_func_init_val = 0,
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index a3876c1..a54f3c1 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -162,7 +162,7 @@ shuffle_input_keys(unsigned table_index)
* ALL can fit in hash table (no errors)
*/
static int
-get_input_keys(unsigned table_index)
+get_input_keys(unsigned with_pushes, unsigned table_index)
{
unsigned i, j;
unsigned bucket_idx, incr, success = 1;
@@ -216,9 +216,14 @@ get_input_keys(unsigned table_index)
success = 0;
signatures[i] = rte_hash_hash(h[table_index], keys[i]);
bucket_idx = signatures[i] & bucket_bitmask;
- /* If bucket is full, do not try to insert the key */
- if (buckets[bucket_idx] == BUCKET_SIZE)
- continue;
+ /*
+ * If we are not inserting keys in secondary location,
+ * when bucket is full, do not try to insert the key
+ */
+ if (with_pushes == 0)
+ if (buckets[bucket_idx] == BUCKET_SIZE)
+ continue;
+
/* If key can be added, leave in successful key arrays "keys" */
ret = rte_hash_add_key_with_hash(h[table_index], keys[i],
signatures[i]);
@@ -388,9 +393,9 @@ reset_table(unsigned table_index)
}
static int
-run_all_tbl_perf_tests(void)
+run_all_tbl_perf_tests(unsigned with_pushes)
{
- unsigned i, j;
+ unsigned i, j, with_hash;
printf("Measuring performance, please wait");
fflush(stdout);
@@ -398,46 +403,32 @@ run_all_tbl_perf_tests(void)
if (create_table(i) < 0)
return -1;
- if (get_input_keys(i) < 0)
- return -1;
-
- if (timed_adds(0, i) < 0)
- return -1;
-
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
-
- if (timed_lookups(0, i) < 0)
- return -1;
-
- if (timed_lookups_multi(i) < 0)
- return -1;
-
- if (timed_deletes(0, i) < 0)
+ if (get_input_keys(with_pushes, i) < 0)
return -1;
+ for (with_hash = 0; with_hash <= 1; with_hash++) {
+ if (timed_adds(with_hash, i) < 0)
+ return -1;
- /* Print a dot to show progress on operations */
- printf(".");
- fflush(stdout);
+ for (j = 0; j < NUM_SHUFFLES; j++)
+ shuffle_input_keys(i);
- if (reset_table(i) < 0)
- return -1;
+ if (timed_lookups(with_hash, i) < 0)
+ return -1;
- if (timed_adds(1, i) < 0)
- return -1;
+ if (timed_lookups_multi(i) < 0)
+ return -1;
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
+ if (timed_deletes(with_hash, i) < 0)
+ return -1;
- if (timed_lookups(1, i) < 0)
- return -1;
+ if (reset_table(i) < 0)
+ return -1;
- if (timed_deletes(1, i) < 0)
- return -1;
+ /* Print a dot to show progress on operations */
+ printf(".");
+ fflush(stdout);
- /* Print a dot to show progress on operations */
- printf(".");
- fflush(stdout);
+ }
free_table(i);
}
@@ -551,8 +542,16 @@ fbk_hash_perf_test(void)
static int
test_hash_perf(void)
{
- if (run_all_tbl_perf_tests() < 0)
- return -1;
+ unsigned with_pushes;
+
+ for (with_pushes = 0; with_pushes <= 1; with_pushes++) {
+ if (with_pushes == 0)
+ printf("\nALL ELEMENTS IN PRIMARY LOCATION\n");
+ else
+ printf("\nELEMENTS IN PRIMARY OR SECONDARY LOCATION\n");
+ if (run_all_tbl_perf_tests(with_pushes) < 0)
+ return -1;
+ }
if (fbk_hash_perf_test() < 0)
return -1;
diff --git a/lib/librte_hash/Makefile b/lib/librte_hash/Makefile
index 981230b..039da46 100644
--- a/lib/librte_hash/Makefile
+++ b/lib/librte_hash/Makefile
@@ -1,6 +1,6 @@
# BSD LICENSE
#
-# Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+# Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@@ -42,7 +42,7 @@ EXPORT_MAP := rte_hash_version.map
LIBABIVER := 1
# all source are stored in SRCS-y
-SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_hash.c
+SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c
# install this header file
@@ -52,7 +52,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_thash.h
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h
-# this lib needs eal
-DEPDIRS-$(CONFIG_RTE_LIBRTE_HASH) += lib/librte_eal lib/librte_malloc
+# this lib needs eal and ring
+DEPDIRS-$(CONFIG_RTE_LIBRTE_HASH) += lib/librte_eal lib/librte_malloc lib/librte_ring
include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
new file mode 100644
index 0000000..50e3acd
--- /dev/null
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -0,0 +1,1027 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <sys/queue.h>
+
+#include <rte_common.h>
+#include <rte_memory.h> /* for definition of RTE_CACHE_LINE_SIZE */
+#include <rte_log.h>
+#include <rte_memcpy.h>
+#include <rte_prefetch.h>
+#include <rte_branch_prediction.h>
+#include <rte_memzone.h>
+#include <rte_malloc.h>
+#include <rte_eal.h>
+#include <rte_eal_memconfig.h>
+#include <rte_per_lcore.h>
+#include <rte_errno.h>
+#include <rte_string_fns.h>
+#include <rte_cpuflags.h>
+#include <rte_log.h>
+#include <rte_rwlock.h>
+#include <rte_spinlock.h>
+#include <rte_ring.h>
+
+#include "rte_hash.h"
+
+TAILQ_HEAD(rte_hash_list, rte_tailq_entry);
+
+static struct rte_tailq_elem rte_hash_tailq = {
+ .name = "RTE_HASH",
+};
+EAL_REGISTER_TAILQ(rte_hash_tailq)
+
+/* Macro to enable/disable run-time checking of function parameters */
+#if defined(RTE_LIBRTE_HASH_DEBUG)
+#define RETURN_IF_TRUE(cond, retval) do { \
+ if (cond) \
+ return retval; \
+} while (0)
+#else
+#define RETURN_IF_TRUE(cond, retval)
+#endif
+
+/* Hash function used if none is specified */
+#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
+#include <rte_hash_crc.h>
+#define DEFAULT_HASH_FUNC rte_hash_crc
+#else
+#include <rte_jhash.h>
+#define DEFAULT_HASH_FUNC rte_jhash
+#endif
+
+/** Number of items per bucket. */
+#define RTE_HASH_BUCKET_ENTRIES 4
+
+#define NULL_SIGNATURE 0
+
+typedef int (*rte_hash_cmp_eq_t)(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k48_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k64_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k80_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k96_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k112_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k128_cmp_eq(const void *key1, const void *key2, size_t key_len);
+
+/** A hash table structure. */
+struct rte_hash {
+ char name[RTE_HASH_NAMESIZE]; /**< Name of the hash. */
+ uint32_t entries; /**< Total table entries. */
+ uint32_t num_buckets; /**< Number of buckets in table. */
+ uint32_t key_len; /**< Length of hash key. */
+ rte_hash_function hash_func; /**< Function used to calculate hash. */
+ rte_hash_cmp_eq_t rte_hash_cmp_eq; /**< Function used to compare keys. */
+ uint32_t hash_func_init_val; /**< Init value used by hash_func. */
+ uint32_t bucket_bitmask; /**< Bitmask for getting bucket index
+ from hash signature. */
+ uint32_t key_entry_size; /**< Size of each key entry. */
+
+ struct rte_ring *free_slots; /**< Ring that stores all indexes
+ of the free slots in the key table */
+ void *key_store; /**< Table storing all keys and data */
+ struct rte_hash_bucket *buckets; /**< Table with buckets storing all the
+ hash values and key indexes
+ to the key table*/
+} __rte_cache_aligned;
+
+/* Structure storing both primary and secondary hashes */
+struct rte_hash_signatures {
+ union {
+ struct {
+ hash_sig_t current;
+ hash_sig_t alt;
+ };
+ uint64_t sig;
+ };
+};
+
+/** Bucket structure */
+struct rte_hash_bucket {
+ struct rte_hash_signatures signatures[RTE_HASH_BUCKET_ENTRIES];
+ /* Includes dummy key index that always contains index 0 */
+ uint32_t key_idx[RTE_HASH_BUCKET_ENTRIES + 1];
+ uint8_t flag[RTE_HASH_BUCKET_ENTRIES];
+} __rte_cache_aligned;
+
+struct rte_hash *
+rte_hash_find_existing(const char *name)
+{
+ struct rte_hash *h = NULL;
+ struct rte_tailq_entry *te;
+ struct rte_hash_list *hash_list;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK);
+ TAILQ_FOREACH(te, hash_list, next) {
+ h = (struct rte_hash *) te->data;
+ if (strncmp(name, h->name, RTE_HASH_NAMESIZE) == 0)
+ break;
+ }
+ rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ if (te == NULL) {
+ rte_errno = ENOENT;
+ return NULL;
+ }
+ return h;
+}
+
+struct rte_hash *
+rte_hash_create(const struct rte_hash_parameters *params)
+{
+ struct rte_hash *h = NULL;
+ struct rte_tailq_entry *te = NULL;
+ struct rte_hash_list *hash_list;
+ struct rte_ring *r = NULL;
+ char hash_name[RTE_HASH_NAMESIZE];
+ void *ptr, *k = NULL;
+ void *buckets = NULL;
+ char ring_name[RTE_RING_NAMESIZE];
+ unsigned i;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ if (params == NULL) {
+ RTE_LOG(ERR, HASH, "rte_hash_create has no parameters\n");
+ return NULL;
+ }
+
+ /* Check for valid parameters */
+ if ((params->entries > RTE_HASH_ENTRIES_MAX) ||
+ (params->entries < RTE_HASH_BUCKET_ENTRIES) ||
+ !rte_is_power_of_2(RTE_HASH_BUCKET_ENTRIES) ||
+ (params->key_len == 0)) {
+ rte_errno = EINVAL;
+ RTE_LOG(ERR, HASH, "rte_hash_create has invalid parameters\n");
+ return NULL;
+ }
+
+ snprintf(hash_name, sizeof(hash_name), "HT_%s", params->name);
+
+ /* Guarantee there's no existing */
+ h = rte_hash_find_existing(params->name);
+ if (h != NULL)
+ return h;
+
+ te = rte_zmalloc("HASH_TAILQ_ENTRY", sizeof(*te), 0);
+ if (te == NULL) {
+ RTE_LOG(ERR, HASH, "tailq entry allocation failed\n");
+ goto err;
+ }
+
+ h = (struct rte_hash *)rte_zmalloc_socket(hash_name, sizeof(struct rte_hash),
+ RTE_CACHE_LINE_SIZE, params->socket_id);
+
+ if (h == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ const uint32_t num_buckets = rte_align32pow2(params->entries)
+ / RTE_HASH_BUCKET_ENTRIES;
+
+ buckets = rte_zmalloc_socket(NULL,
+ num_buckets * sizeof(struct rte_hash_bucket),
+ RTE_CACHE_LINE_SIZE, params->socket_id);
+
+ if (buckets == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ const uint32_t key_entry_size = params->key_len;
+ /* Store all keys and leave the first entry as a dummy entry for lookup_bulk */
+ const uint64_t key_tbl_size = key_entry_size * (params->entries + 1);
+
+ k = rte_zmalloc_socket(NULL, key_tbl_size,
+ RTE_CACHE_LINE_SIZE, params->socket_id);
+
+ if (k == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ /* Select function to compare keys */
+ switch (params->key_len) {
+ case 16:
+ h->rte_hash_cmp_eq = rte_hash_k16_cmp_eq;
+ break;
+ case 32:
+ h->rte_hash_cmp_eq = rte_hash_k32_cmp_eq;
+ break;
+ case 48:
+ h->rte_hash_cmp_eq = rte_hash_k48_cmp_eq;
+ break;
+ case 64:
+ h->rte_hash_cmp_eq = rte_hash_k64_cmp_eq;
+ break;
+ case 80:
+ h->rte_hash_cmp_eq = rte_hash_k80_cmp_eq;
+ break;
+ case 96:
+ h->rte_hash_cmp_eq = rte_hash_k96_cmp_eq;
+ break;
+ case 112:
+ h->rte_hash_cmp_eq = rte_hash_k112_cmp_eq;
+ break;
+ case 128:
+ h->rte_hash_cmp_eq = rte_hash_k128_cmp_eq;
+ break;
+ default:
+ /* If key is not multiple of 16, use generic memcmp */
+ h->rte_hash_cmp_eq = memcmp;
+ }
+
+ snprintf(ring_name, sizeof(ring_name), "HT_%s", params->name);
+ r = rte_ring_lookup(ring_name);
+ if (r != NULL) {
+ /* clear the free ring */
+ while (rte_ring_dequeue(r, &ptr) == 0)
+ rte_pause();
+ } else
+ r = rte_ring_create(ring_name, rte_align32pow2(params->entries + 1),
+ params->socket_id, 0);
+ if (r == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ /* Setup hash context */
+ snprintf(h->name, sizeof(h->name), "%s", params->name);
+ h->entries = params->entries;
+ h->key_len = params->key_len;
+ h->key_entry_size = key_entry_size;
+ h->hash_func_init_val = params->hash_func_init_val;
+
+ h->num_buckets = num_buckets;
+ h->bucket_bitmask = h->num_buckets - 1;
+ h->buckets = buckets;
+ h->hash_func = (params->hash_func == NULL) ?
+ DEFAULT_HASH_FUNC : params->hash_func;
+
+ h->key_store = k;
+ h->free_slots = r;
+
+ /* populate the free slots ring. Entry zero is reserved for key misses */
+ for (i = 1; i < params->entries + 1; i++)
+ rte_ring_sp_enqueue(r, (void *)((uintptr_t) i));
+
+ rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
+ te->data = (void *) h;
+ TAILQ_INSERT_TAIL(hash_list, te, next);
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ return h;
+err:
+ rte_free(te);
+ rte_free(h);
+ rte_free(buckets);
+ rte_free(k);
+ return NULL;
+}
+
+void
+rte_hash_free(struct rte_hash *h)
+{
+ struct rte_tailq_entry *te;
+ struct rte_hash_list *hash_list;
+
+ if (h == NULL)
+ return;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
+
+ /* find out tailq entry */
+ TAILQ_FOREACH(te, hash_list, next) {
+ if (te->data == (void *) h)
+ break;
+ }
+
+ if (te == NULL) {
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+ return;
+ }
+
+ TAILQ_REMOVE(hash_list, te, next);
+
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ rte_free(h->key_store);
+ rte_free(h->buckets);
+ rte_free(h);
+ rte_free(te);
+}
+
+hash_sig_t
+rte_hash_hash(const struct rte_hash *h, const void *key)
+{
+ /* calc hash result by key */
+ return h->hash_func(key, h->key_len, h->hash_func_init_val);
+}
+
+/* Calc the secondary hash value from the primary hash value of a given key */
+static inline hash_sig_t
+rte_hash_secondary_hash(const hash_sig_t primary_hash)
+{
+ static const unsigned all_bits_shift = 12;
+ static const unsigned alt_bits_xor = 0x5bd1e995;
+
+ uint32_t tag = primary_hash >> all_bits_shift;
+
+ return (primary_hash ^ ((tag + 1) * alt_bits_xor));
+}
+
+/* Search for an entry that can be pushed to its alternative location */
+static inline int
+make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt)
+{
+ unsigned i, j;
+ int ret;
+ uint32_t next_bucket_idx;
+ struct rte_hash_bucket *next_bkt[RTE_HASH_BUCKET_ENTRIES];
+
+ /*
+ * Push existing item (search for bucket with space in
+ * alternative locations) to its alternative location
+ */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ /* Search for space in alternative locations */
+ next_bucket_idx = bkt->signatures[i].alt & h->bucket_bitmask;
+ next_bkt[i] = &h->buckets[next_bucket_idx];
+ for (j = 0; j < RTE_HASH_BUCKET_ENTRIES; j++) {
+ if (next_bkt[i]->signatures[j].sig == NULL_SIGNATURE)
+ break;
+ }
+
+ if (j != RTE_HASH_BUCKET_ENTRIES)
+ break;
+ }
+
+ /* Alternative location has spare room (end of recursive function) */
+ if (i != RTE_HASH_BUCKET_ENTRIES) {
+ next_bkt[i]->signatures[j].alt = bkt->signatures[i].current;
+ next_bkt[i]->signatures[j].current = bkt->signatures[i].alt;
+ next_bkt[i]->key_idx[j] = bkt->key_idx[i];
+ return i;
+ }
+
+ /* Pick entry that has not been pushed yet */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++)
+ if (bkt->flag[i] == 0)
+ break;
+
+ /* All entries have been pushed, so entry cannot be added */
+ if (i == RTE_HASH_BUCKET_ENTRIES) {
+ /* Reset flag */
+ bkt->flag[i] = 0;
+ return -ENOSPC;
+ }
+
+ /* Set flag to indicate that this entry is going to be pushed */
+ bkt->flag[i] = 1;
+ /* Need room in alternative bucket to insert the pushed entry */
+ ret = make_space_bucket(h, next_bkt[i]);
+ /*
+ * After recursive function.
+ * Clear flags and insert the pushed entry
+ * in its alternative location if successful,
+ * or return error
+ */
+ bkt->flag[i] = 0;
+ if (ret >= 0) {
+ next_bkt[i]->signatures[ret].alt = bkt->signatures[i].current;
+ next_bkt[i]->signatures[ret].current = bkt->signatures[i].alt;
+ next_bkt[i]->key_idx[ret] = bkt->key_idx[i];
+ return i;
+ } else
+ return ret;
+
+}
+
+static inline int32_t
+__rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ hash_sig_t alt_hash;
+ uint32_t prim_bucket_idx, sec_bucket_idx;
+ unsigned i;
+ struct rte_hash_bucket *prim_bkt, *sec_bkt;
+ void *new_k, *k, *keys = h->key_store;
+ void *slot_id;
+ uint32_t new_idx;
+ int ret;
+
+ prim_bucket_idx = sig & h->bucket_bitmask;
+ prim_bkt = &h->buckets[prim_bucket_idx];
+ rte_prefetch0(prim_bkt);
+
+ alt_hash = rte_hash_secondary_hash(sig);
+ sec_bucket_idx = alt_hash & h->bucket_bitmask;
+ sec_bkt = &h->buckets[sec_bucket_idx];
+ rte_prefetch0(sec_bkt);
+
+ /* Get a new slot for storing the new key */
+ if (rte_ring_sc_dequeue(h->free_slots, &slot_id) != 0)
+ return -ENOSPC;
+ new_k = RTE_PTR_ADD(keys, (uintptr_t)slot_id * h->key_entry_size);
+ rte_prefetch0(new_k);
+ new_idx = (uint32_t)((uintptr_t) slot_id);
+
+ /* Check if key is already inserted in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (prim_bkt->signatures[i].current == sig &&
+ prim_bkt->signatures[i].alt == alt_hash) {
+ k = (char *)keys + prim_bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (prim_bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Check if key is already inserted in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (sec_bkt->signatures[i].alt == sig &&
+ sec_bkt->signatures[i].current == alt_hash) {
+ k = (char *)keys + sec_bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (sec_bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Copy key */
+ rte_memcpy(new_k, key, h->key_len);
+
+ /* Insert new entry is there is room in the primary bucket */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ /* Check if slot is available */
+ if (likely(prim_bkt->signatures[i].sig == NULL_SIGNATURE)) {
+ prim_bkt->signatures[i].current = sig;
+ prim_bkt->signatures[i].alt = alt_hash;
+ prim_bkt->key_idx[i] = new_idx;
+ return new_idx - 1;
+ }
+ }
+
+ /* Primary bucket is full, so we need to make space for new entry */
+ ret = make_space_bucket(h, prim_bkt);
+ /*
+ * After recursive function.
+ * Insert the new entry in the position of the pushed entry
+ * if successful or return error and
+ * store the new slot back in the ring
+ */
+ if (ret >= 0) {
+ prim_bkt->signatures[ret].current = sig;
+ prim_bkt->signatures[ret].alt = alt_hash;
+ prim_bkt->key_idx[ret] = new_idx;
+ return (new_idx - 1);
+ }
+
+ /* Error in addition, store new slot back in the ring and return error */
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t) new_idx));
+ return ret;
+
+}
+
+int32_t
+rte_hash_add_key_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_add_key_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_add_key(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+static inline int32_t
+__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ uint32_t bucket_idx;
+ hash_sig_t alt_hash;
+ unsigned i;
+ struct rte_hash_bucket *bkt;
+ void *k, *keys = h->key_store;
+
+ bucket_idx = sig & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == sig &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+
+ /* Calculate secondary hash */
+ alt_hash = rte_hash_secondary_hash(sig);
+ bucket_idx = alt_hash & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == alt_hash &&
+ bkt->signatures[i].alt == sig) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+
+ return -ENOENT;
+}
+
+int32_t
+rte_hash_lookup_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_lookup(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+static inline int32_t
+__rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ uint32_t bucket_idx;
+ hash_sig_t alt_hash;
+ unsigned i;
+ struct rte_hash_bucket *bkt;
+ void *k, *keys = h->key_store;
+
+ bucket_idx = sig & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == sig &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ bkt->signatures[i].sig = NULL_SIGNATURE;
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t)bkt->key_idx[i]));
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Calculate secondary hash */
+ alt_hash = rte_hash_secondary_hash(sig);
+ bucket_idx = alt_hash & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == alt_hash &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ bkt->signatures[i].sig = NULL_SIGNATURE;
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t)bkt->key_idx[i]));
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ return -ENOENT;
+}
+
+int32_t
+rte_hash_del_key_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_del_key_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_del_key(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_del_key_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+/* Lookup bulk stage 0: Prefetch input key */
+static inline void
+lookup_stage0(unsigned *idx, uint64_t *lookup_mask,
+ const void * const *keys)
+{
+ *idx = __builtin_ctzl(*lookup_mask);
+ if (*lookup_mask == 0)
+ *idx = 0;
+
+ rte_prefetch0(keys[*idx]);
+ *lookup_mask &= ~(1llu << *idx);
+}
+
+/*
+ * Lookup bulk stage 1: Calculate primary/secondary hashes
+ * and prefetch primary/secondary buckets
+ */
+static inline void
+lookup_stage1(unsigned idx, hash_sig_t *prim_hash, hash_sig_t *sec_hash,
+ const struct rte_hash_bucket **primary_bkt,
+ const struct rte_hash_bucket **secondary_bkt,
+ hash_sig_t *hash_vals, const void * const *keys,
+ const struct rte_hash *h)
+{
+ *prim_hash = rte_hash_hash(h, keys[idx]);
+ hash_vals[idx] = *prim_hash;
+ *sec_hash = rte_hash_secondary_hash(*prim_hash);
+
+ *primary_bkt = &h->buckets[*prim_hash & h->bucket_bitmask];
+ *secondary_bkt = &h->buckets[*sec_hash & h->bucket_bitmask];
+
+ rte_prefetch0(*primary_bkt);
+ rte_prefetch0(*secondary_bkt);
+}
+
+/*
+ * Lookup bulk stage 2: Search for match hashes in primary/secondary locations
+ * and prefetch first key slot
+ */
+static inline void
+lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
+ const struct rte_hash_bucket *prim_bkt,
+ const struct rte_hash_bucket *sec_bkt,
+ const void **key_slot, int32_t *positions,
+ uint64_t *extra_hits_mask, const void *keys,
+ const struct rte_hash *h)
+{
+ unsigned prim_hash_matches, sec_hash_matches, key_idx, i;
+ unsigned total_hash_matches;
+
+ prim_hash_matches = 1 << RTE_HASH_BUCKET_ENTRIES;
+ sec_hash_matches = 1 << RTE_HASH_BUCKET_ENTRIES;
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ prim_hash_matches |= ((prim_hash == prim_bkt->signatures[i].current) << i);
+ sec_hash_matches |= ((sec_hash == sec_bkt->signatures[i].current) << i);
+ }
+
+ key_idx = prim_bkt->key_idx[__builtin_ctzl(prim_hash_matches)];
+ if (key_idx == 0)
+ key_idx = sec_bkt->key_idx[__builtin_ctzl(sec_hash_matches)];
+
+ total_hash_matches = (prim_hash_matches |
+ (sec_hash_matches << (RTE_HASH_BUCKET_ENTRIES + 1)));
+ *key_slot = (const char *)keys + key_idx * h->key_entry_size;
+
+ rte_prefetch0(*key_slot);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ positions[idx] = (key_idx - 1);
+
+ *extra_hits_mask |= (uint64_t)(__builtin_popcount(total_hash_matches) > 3) << idx;
+
+}
+
+
+/* Lookup bulk stage 3: Check if key matches, update hit mask */
+static inline void
+lookup_stage3(unsigned idx, const void *key_slot, const void * const *keys,
+ uint64_t *hits, const struct rte_hash *h)
+{
+ unsigned hit;
+
+ hit = !h->rte_hash_cmp_eq(key_slot, keys[idx], h->key_len);
+ *hits |= (uint64_t)(hit) << idx;
+}
+
+static inline void
+__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, int32_t *positions)
+{
+ uint64_t hits = 0;
+ uint64_t extra_hits_mask = 0;
+ uint64_t lookup_mask, miss_mask;
+ unsigned idx;
+ const void *key_store = h->key_store;
+ hash_sig_t hash_vals[RTE_HASH_LOOKUP_BULK_MAX];
+
+ unsigned idx00, idx01, idx10, idx11, idx20, idx21, idx30, idx31;
+ const struct rte_hash_bucket *primary_bkt10, *primary_bkt11;
+ const struct rte_hash_bucket *secondary_bkt10, *secondary_bkt11;
+ const struct rte_hash_bucket *primary_bkt20, *primary_bkt21;
+ const struct rte_hash_bucket *secondary_bkt20, *secondary_bkt21;
+ const void *k_slot20, *k_slot21, *k_slot30, *k_slot31;
+ hash_sig_t primary_hash10, primary_hash11;
+ hash_sig_t secondary_hash10, secondary_hash11;
+ hash_sig_t primary_hash20, primary_hash21;
+ hash_sig_t secondary_hash20, secondary_hash21;
+
+ lookup_mask = (uint64_t) -1 >> (64 - num_keys);
+ miss_mask = lookup_mask;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+
+ while (lookup_mask) {
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20,
+ primary_bkt20, secondary_bkt20, &k_slot20, positions,
+ &extra_hits_mask, key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21,
+ primary_bkt21, secondary_bkt21, &k_slot21, positions,
+ &extra_hits_mask, key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ }
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+
+ /* ignore any items we have already found */
+ extra_hits_mask &= ~hits;
+
+ if (unlikely(extra_hits_mask)) {
+ /* run a single search for each remaining item */
+ do {
+ idx = __builtin_ctzl(extra_hits_mask);
+ positions[idx] = rte_hash_lookup_with_hash(h, keys[idx],
+ hash_vals[idx]);
+ extra_hits_mask &= ~(1llu << idx);
+ if (positions[idx] >= 0)
+ hits |= 1llu << idx;
+ } while (extra_hits_mask);
+ }
+
+ miss_mask &= ~hits;
+ if (unlikely(miss_mask)) {
+ do {
+ idx = __builtin_ctzl(miss_mask);
+ positions[idx] = -ENOENT;
+ miss_mask &= ~(1llu << idx);
+ } while (miss_mask);
+ }
+}
+
+int
+rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, int32_t *positions)
+{
+ RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
+ (num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
+ (positions == NULL)), -EINVAL);
+
+ __rte_hash_lookup_bulk(h, keys, num_keys, positions);
+ return 0;
+}
+
+/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
+static int
+rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
+{
+ const __m128i k1 = _mm_loadu_si128((const __m128i *) key1);
+ const __m128i k2 = _mm_loadu_si128((const __m128i *) key2);
+ const __m128i x = _mm_xor_si128(k1, k2);
+
+ return !_mm_test_all_zeros(x, x);
+}
+
+static int
+rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k16_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 16,
+ (const char *) key2 + 16, key_len);
+}
+
+static int
+rte_hash_k48_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k16_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 16,
+ (const char *) key2 + 16, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 32,
+ (const char *) key2 + 32, key_len);
+}
+
+static int
+rte_hash_k64_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k32_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 32,
+ (const char *) key2 + 32, key_len);
+}
+
+static int
+rte_hash_k80_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
+
+static int
+rte_hash_k96_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
+
+static int
+rte_hash_k112_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 96,
+ (const char *) key2 + 96, key_len);
+}
+
+static int
+rte_hash_k128_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k64_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
diff --git a/lib/librte_hash/rte_hash.c b/lib/librte_hash/rte_hash.c
deleted file mode 100644
index 5100a75..0000000
--- a/lib/librte_hash/rte_hash.c
+++ /dev/null
@@ -1,499 +0,0 @@
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <string.h>
-#include <stdint.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <sys/queue.h>
-
-#include <rte_common.h>
-#include <rte_memory.h> /* for definition of RTE_CACHE_LINE_SIZE */
-#include <rte_log.h>
-#include <rte_memcpy.h>
-#include <rte_prefetch.h>
-#include <rte_branch_prediction.h>
-#include <rte_memzone.h>
-#include <rte_malloc.h>
-#include <rte_eal.h>
-#include <rte_eal_memconfig.h>
-#include <rte_per_lcore.h>
-#include <rte_errno.h>
-#include <rte_string_fns.h>
-#include <rte_cpuflags.h>
-#include <rte_log.h>
-#include <rte_rwlock.h>
-#include <rte_spinlock.h>
-
-#include "rte_hash.h"
-
-TAILQ_HEAD(rte_hash_list, rte_tailq_entry);
-
-static struct rte_tailq_elem rte_hash_tailq = {
- .name = "RTE_HASH",
-};
-EAL_REGISTER_TAILQ(rte_hash_tailq)
-
-/* Macro to enable/disable run-time checking of function parameters */
-#if defined(RTE_LIBRTE_HASH_DEBUG)
-#define RETURN_IF_TRUE(cond, retval) do { \
- if (cond) return (retval); \
-} while (0)
-#else
-#define RETURN_IF_TRUE(cond, retval)
-#endif
-
-/* Hash function used if none is specified */
-#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
-#include <rte_hash_crc.h>
-#define DEFAULT_HASH_FUNC rte_hash_crc
-#else
-#include <rte_jhash.h>
-#define DEFAULT_HASH_FUNC rte_jhash
-#endif
-
-/* Signature bucket size is a multiple of this value */
-#define SIG_BUCKET_ALIGNMENT 16
-
-/* Stoered key size is a multiple of this value */
-#define KEY_ALIGNMENT 16
-
-/* The high bit is always set in real signatures */
-#define NULL_SIGNATURE 0
-
-struct rte_hash {
- char name[RTE_HASH_NAMESIZE]; /**< Name of the hash. */
- uint32_t entries; /**< Total table entries. */
- uint32_t bucket_entries; /**< Bucket entries. */
- uint32_t key_len; /**< Length of hash key. */
- rte_hash_function hash_func; /**< Function used to calculate hash. */
- uint32_t hash_func_init_val; /**< Init value used by hash_func. */
- uint32_t num_buckets; /**< Number of buckets in table. */
- uint32_t bucket_bitmask; /**< Bitmask for getting bucket index
- from hash signature. */
- hash_sig_t sig_msb; /**< MSB is always set in valid signatures. */
- uint8_t *sig_tbl; /**< Flat array of hash signature buckets. */
- uint32_t sig_tbl_bucket_size; /**< Signature buckets may be padded for
- alignment reasons, and this is the
- bucket size used by sig_tbl. */
- uint8_t *key_tbl; /**< Flat array of key value buckets. */
- uint32_t key_tbl_key_size; /**< Keys may be padded for alignment
- reasons, and this is the key size
- used by key_tbl. */
-};
-
-/* Returns a pointer to the first signature in specified bucket. */
-static inline hash_sig_t *
-get_sig_tbl_bucket(const struct rte_hash *h, uint32_t bucket_index)
-{
- return RTE_PTR_ADD(h->sig_tbl, (bucket_index *
- h->sig_tbl_bucket_size));
-}
-
-/* Returns a pointer to the first key in specified bucket. */
-static inline uint8_t *
-get_key_tbl_bucket(const struct rte_hash *h, uint32_t bucket_index)
-{
- return RTE_PTR_ADD(h->key_tbl, (bucket_index * h->bucket_entries *
- h->key_tbl_key_size));
-}
-
-/* Returns a pointer to a key at a specific position in a specified bucket. */
-static inline void *
-get_key_from_bucket(const struct rte_hash *h, uint8_t *bkt, uint32_t pos)
-{
- return RTE_PTR_ADD(bkt, pos * h->key_tbl_key_size);
-}
-
-/* Does integer division with rounding-up of result. */
-static inline uint32_t
-div_roundup(uint32_t numerator, uint32_t denominator)
-{
- return (numerator + denominator - 1) / denominator;
-}
-
-/* Increases a size (if needed) to a multiple of alignment. */
-static inline uint32_t
-align_size(uint32_t val, uint32_t alignment)
-{
- return alignment * div_roundup(val, alignment);
-}
-
-/* Returns the index into the bucket of the first occurrence of a signature. */
-static inline int
-find_first(uint32_t sig, const uint32_t *sig_bucket, uint32_t num_sigs)
-{
- uint32_t i;
- for (i = 0; i < num_sigs; i++) {
- if (sig == sig_bucket[i])
- return i;
- }
- return -1;
-}
-
-struct rte_hash *
-rte_hash_find_existing(const char *name)
-{
- struct rte_hash *h = NULL;
- struct rte_tailq_entry *te;
- struct rte_hash_list *hash_list;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK);
- TAILQ_FOREACH(te, hash_list, next) {
- h = (struct rte_hash *) te->data;
- if (strncmp(name, h->name, RTE_HASH_NAMESIZE) == 0)
- break;
- }
- rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- if (te == NULL) {
- rte_errno = ENOENT;
- return NULL;
- }
- return h;
-}
-
-struct rte_hash *
-rte_hash_create(const struct rte_hash_parameters *params)
-{
- struct rte_hash *h = NULL;
- struct rte_tailq_entry *te;
- uint32_t num_buckets, sig_bucket_size, key_size,
- hash_tbl_size, sig_tbl_size, key_tbl_size, mem_size;
- char hash_name[RTE_HASH_NAMESIZE];
- struct rte_hash_list *hash_list;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- /* Check for valid parameters */
- if ((params == NULL) ||
- (params->entries > RTE_HASH_ENTRIES_MAX) ||
- (params->bucket_entries > RTE_HASH_BUCKET_ENTRIES_MAX) ||
- (params->entries < params->bucket_entries) ||
- !rte_is_power_of_2(params->entries) ||
- !rte_is_power_of_2(params->bucket_entries) ||
- (params->key_len == 0) ||
- (params->key_len > RTE_HASH_KEY_LENGTH_MAX)) {
- rte_errno = EINVAL;
- RTE_LOG(ERR, HASH, "rte_hash_create has invalid parameters\n");
- return NULL;
- }
-
- snprintf(hash_name, sizeof(hash_name), "HT_%s", params->name);
-
- /* Calculate hash dimensions */
- num_buckets = params->entries / params->bucket_entries;
- sig_bucket_size = align_size(params->bucket_entries *
- sizeof(hash_sig_t), SIG_BUCKET_ALIGNMENT);
- key_size = align_size(params->key_len, KEY_ALIGNMENT);
-
- hash_tbl_size = align_size(sizeof(struct rte_hash), RTE_CACHE_LINE_SIZE);
- sig_tbl_size = align_size(num_buckets * sig_bucket_size,
- RTE_CACHE_LINE_SIZE);
- key_tbl_size = align_size(num_buckets * key_size *
- params->bucket_entries, RTE_CACHE_LINE_SIZE);
-
- /* Total memory required for hash context */
- mem_size = hash_tbl_size + sig_tbl_size + key_tbl_size;
-
- rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
-
- /* guarantee there's no existing */
- TAILQ_FOREACH(te, hash_list, next) {
- h = (struct rte_hash *) te->data;
- if (strncmp(params->name, h->name, RTE_HASH_NAMESIZE) == 0)
- break;
- }
- if (te != NULL)
- goto exit;
-
- te = rte_zmalloc("HASH_TAILQ_ENTRY", sizeof(*te), 0);
- if (te == NULL) {
- RTE_LOG(ERR, HASH, "tailq entry allocation failed\n");
- goto exit;
- }
-
- h = (struct rte_hash *)rte_zmalloc_socket(hash_name, mem_size,
- RTE_CACHE_LINE_SIZE, params->socket_id);
- if (h == NULL) {
- RTE_LOG(ERR, HASH, "memory allocation failed\n");
- rte_free(te);
- goto exit;
- }
-
- /* Setup hash context */
- snprintf(h->name, sizeof(h->name), "%s", params->name);
- h->entries = params->entries;
- h->bucket_entries = params->bucket_entries;
- h->key_len = params->key_len;
- h->hash_func_init_val = params->hash_func_init_val;
- h->num_buckets = num_buckets;
- h->bucket_bitmask = h->num_buckets - 1;
- h->sig_msb = 1 << (sizeof(hash_sig_t) * 8 - 1);
- h->sig_tbl = (uint8_t *)h + hash_tbl_size;
- h->sig_tbl_bucket_size = sig_bucket_size;
- h->key_tbl = h->sig_tbl + sig_tbl_size;
- h->key_tbl_key_size = key_size;
- h->hash_func = (params->hash_func == NULL) ?
- DEFAULT_HASH_FUNC : params->hash_func;
-
- te->data = (void *) h;
-
- TAILQ_INSERT_TAIL(hash_list, te, next);
-
-exit:
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- return h;
-}
-
-void
-rte_hash_free(struct rte_hash *h)
-{
- struct rte_tailq_entry *te;
- struct rte_hash_list *hash_list;
-
- if (h == NULL)
- return;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
-
- /* find out tailq entry */
- TAILQ_FOREACH(te, hash_list, next) {
- if (te->data == (void *) h)
- break;
- }
-
- if (te == NULL) {
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
- return;
- }
-
- TAILQ_REMOVE(hash_list, te, next);
-
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- rte_free(h);
- rte_free(te);
-}
-
-hash_sig_t
-rte_hash_hash(const struct rte_hash *h, const void *key)
-{
- /* calc hash result by key */
- return h->hash_func(key, h->key_len, h->hash_func_init_val);
-}
-
-static inline int32_t
-__rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
- int32_t pos;
-
- /* Get the hash signature and bucket index */
- sig |= h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- /* Check if any free slot within the bucket to add the new key */
- pos = find_first(NULL_SIGNATURE, sig_bucket, h->bucket_entries);
-
- if (unlikely(pos < 0))
- return -ENOSPC;
-
- /* Add the new key to the bucket */
- sig_bucket[pos] = sig;
- rte_memcpy(get_key_from_bucket(h, key_bucket, pos), key, h->key_len);
- return bucket_index * h->bucket_entries + pos;
-}
-
-int32_t
-rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_add_key(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-static inline int32_t
-__rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
-
- /* Get the hash signature and bucket index */
- sig = sig | h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- sig_bucket[i] = NULL_SIGNATURE;
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- return -ENOENT;
-}
-
-int32_t
-rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_del_key_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_del_key(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_del_key_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-static inline int32_t
-__rte_hash_lookup_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
-
- /* Get the hash signature and bucket index */
- sig |= h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- return -ENOENT;
-}
-
-int32_t
-rte_hash_lookup_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_lookup(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-int
-rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
- uint32_t num_keys, int32_t *positions)
-{
- uint32_t i, j, bucket_index;
- hash_sig_t sigs[RTE_HASH_LOOKUP_BULK_MAX];
-
- RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
- (num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
- (positions == NULL)), -EINVAL);
-
- /* Get the hash signature and bucket index */
- for (i = 0; i < num_keys; i++) {
- sigs[i] = h->hash_func(keys[i], h->key_len,
- h->hash_func_init_val) | h->sig_msb;
- bucket_index = sigs[i] & h->bucket_bitmask;
-
- /* Pre-fetch relevant buckets */
- rte_prefetch1((void *) get_sig_tbl_bucket(h, bucket_index));
- rte_prefetch1((void *) get_key_tbl_bucket(h, bucket_index));
- }
-
- /* Check if key is already present in the hash */
- for (i = 0; i < num_keys; i++) {
- bucket_index = sigs[i] & h->bucket_bitmask;
- hash_sig_t *sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- uint8_t *key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- positions[i] = -ENOENT;
-
- for (j = 0; j < h->bucket_entries; j++) {
- if ((sigs[i] == sig_bucket[j]) &&
- likely(memcmp(keys[i],
- get_key_from_bucket(h, key_bucket, j),
- h->key_len) == 0)) {
- positions[i] = bucket_index *
- h->bucket_entries + j;
- break;
- }
- }
- }
-
- return 0;
-}
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index da0a00a..936d170 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -47,21 +47,21 @@ extern "C" {
#endif
/** Maximum size of hash table that can be created. */
-#define RTE_HASH_ENTRIES_MAX (1 << 26)
+#define RTE_HASH_ENTRIES_MAX (1 << 30)
-/** Maximum bucket size that can be created. */
-#define RTE_HASH_BUCKET_ENTRIES_MAX 16
+/** @deprecated Maximum bucket size that can be created. */
+#define RTE_HASH_BUCKET_ENTRIES_MAX 4
-/** Maximum length of key that can be used. */
+/** @deprecated Maximum length of key that can be used. */
#define RTE_HASH_KEY_LENGTH_MAX 64
-/** Max number of keys that can be searched for using rte_hash_lookup_multi. */
-#define RTE_HASH_LOOKUP_BULK_MAX 16
-#define RTE_HASH_LOOKUP_MULTI_MAX RTE_HASH_LOOKUP_BULK_MAX
-
-/** Max number of characters in hash name.*/
+/** Maximum number of characters in hash name.*/
#define RTE_HASH_NAMESIZE 32
+/** Maximum number of keys that can be searched for using rte_hash_lookup_bulk. */
+#define RTE_HASH_LOOKUP_BULK_MAX 64
+#define RTE_HASH_LOOKUP_MULTI_MAX RTE_HASH_LOOKUP_BULK_MAX
+
/** Signature of key that is stored internally. */
typedef uint32_t hash_sig_t;
@@ -70,15 +70,14 @@ typedef uint32_t (*rte_hash_function)(const void *key, uint32_t key_len,
uint32_t init_val);
/**
- * Parameters used when creating the hash table. The total table entries and
- * bucket entries must be a power of 2.
+ * Parameters used when creating the hash table.
*/
struct rte_hash_parameters {
const char *name; /**< Name of the hash. */
uint32_t entries; /**< Total hash table entries. */
- uint32_t bucket_entries; /**< Bucket entries. */
+ uint32_t bucket_entries; /**< Bucket entries. */
uint32_t key_len; /**< Length of hash key. */
- rte_hash_function hash_func; /**< Function used to calculate hash. */
+ rte_hash_function hash_func; /**< Primary Hash function used to calculate hash. */
uint32_t hash_func_init_val; /**< Init value used by hash_func. */
int socket_id; /**< NUMA Socket ID for memory. */
};
@@ -86,6 +85,7 @@ struct rte_hash_parameters {
/** @internal A hash table structure. */
struct rte_hash;
+
/**
* Create a new hash table.
*
@@ -106,7 +106,6 @@ struct rte_hash;
struct rte_hash *
rte_hash_create(const struct rte_hash_parameters *params);
-
/**
* Find an existing hash table object and return a pointer to it.
*
@@ -129,7 +128,8 @@ void
rte_hash_free(struct rte_hash *h);
/**
- * Add a key to an existing hash table. This operation is not multi-thread safe
+ * Add a key to an existing hash table.
+ * This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
@@ -146,7 +146,8 @@ int32_t
rte_hash_add_key(const struct rte_hash *h, const void *key);
/**
- * Add a key to an existing hash table. This operation is not multi-thread safe
+ * Add a key to an existing hash table.
+ * This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
@@ -154,7 +155,7 @@ rte_hash_add_key(const struct rte_hash *h, const void *key);
* @param key
* Key to add to the hash table.
* @param sig
- * Hash value to add to the hash table.
+ * Precomputed hash value for 'key'.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOSPC if there is no space in the hash for this key.
@@ -162,12 +163,12 @@ rte_hash_add_key(const struct rte_hash *h, const void *key);
* array of user data. This value is unique for this key.
*/
int32_t
-rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig);
+rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
/**
- * Remove a key from an existing hash table. This operation is not multi-thread
- * safe and should only be called from one thread.
+ * Remove a key from an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
*
* @param h
* Hash table to remove the key from.
@@ -184,15 +185,16 @@ int32_t
rte_hash_del_key(const struct rte_hash *h, const void *key);
/**
- * Remove a key from an existing hash table. This operation is not multi-thread
- * safe and should only be called from one thread.
+ * Remove a key from an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
*
* @param h
* Hash table to remove the key from.
* @param key
* Key to remove from the hash table.
* @param sig
- * Hash value to remove from the hash table.
+ * Precomputed hash value for 'key'.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOENT if the key is not found.
@@ -201,12 +203,11 @@ rte_hash_del_key(const struct rte_hash *h, const void *key);
* value that was returned when the key was added.
*/
int32_t
-rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig);
-
+rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
/**
- * Find a key in the hash table. This operation is multi-thread safe.
+ * Find a key in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
@@ -223,14 +224,15 @@ int32_t
rte_hash_lookup(const struct rte_hash *h, const void *key);
/**
- * Find a key in the hash table. This operation is multi-thread safe.
+ * Find a key in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
* @param key
* Key to find.
* @param sig
- * Hash value to find.
+ * Hash value to remove from the hash table.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOENT if the key is not found.
@@ -243,7 +245,8 @@ rte_hash_lookup_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig);
/**
- * Calc a hash value by key. This operation is not multi-process safe.
+ * Calc a hash value by key.
+ * This operation is not multi-thread safe.
*
* @param h
* Hash table to look in.
@@ -257,7 +260,8 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
#define rte_hash_lookup_multi rte_hash_lookup_bulk
/**
- * Find multiple keys in the hash table. This operation is multi-thread safe.
+ * Find multiple keys in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
@@ -277,6 +281,7 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions);
+
#ifdef __cplusplus
}
#endif
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v4 2/7] hash: add new function rte_hash_reset
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of " Pablo de Lara
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 1/7] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
@ 2015-07-10 17:24 ` Pablo de Lara
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 3/7] hash: add new functionality to store data in hash table Pablo de Lara
` (6 subsequent siblings)
8 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 17:24 UTC (permalink / raw)
To: dev
Added reset function to be able to empty the table,
without having to destroy and create it again.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 4 +---
app/test/test_hash_perf.c | 12 +++---------
lib/librte_hash/rte_cuckoo_hash.c | 21 +++++++++++++++++++++
lib/librte_hash/rte_hash.h | 11 +++++++++--
lib/librte_hash/rte_hash_version.map | 8 ++++++++
5 files changed, 42 insertions(+), 14 deletions(-)
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index e70d859..448586c 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -1132,9 +1132,7 @@ static int test_average_table_utilization(void)
average_keys_added += added_keys;
/* Reset the table */
- rte_hash_free(handle);
- handle = rte_hash_create(&ut_params);
- RETURN_IF_ERROR(handle == NULL, "hash creation failed");
+ rte_hash_reset(handle);
/* Print a dot to show progress on operations */
printf(".");
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index a54f3c1..b01b040 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -382,14 +382,10 @@ free_table(unsigned table_index)
rte_hash_free(h[table_index]);
}
-static int
+static void
reset_table(unsigned table_index)
{
- free_table(table_index);
- if (create_table(table_index) != 0)
- return -1;
-
- return 0;
+ rte_hash_reset(h[table_index]);
}
static int
@@ -421,13 +417,11 @@ run_all_tbl_perf_tests(unsigned with_pushes)
if (timed_deletes(with_hash, i) < 0)
return -1;
- if (reset_table(i) < 0)
- return -1;
+ reset_table(i);
/* Print a dot to show progress on operations */
printf(".");
fflush(stdout);
-
}
free_table(i);
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index 50e3acd..15c5da5 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -371,6 +371,27 @@ rte_hash_secondary_hash(const hash_sig_t primary_hash)
return (primary_hash ^ ((tag + 1) * alt_bits_xor));
}
+void
+rte_hash_reset(struct rte_hash *h)
+{
+ void *ptr;
+ unsigned i;
+
+ if (h == NULL)
+ return;
+
+ memset(h->buckets, 0, h->num_buckets * sizeof(struct rte_hash_bucket));
+ memset(h->key_store, 0, h->key_entry_size * (h->entries + 1));
+
+ /* clear the free ring */
+ while (rte_ring_dequeue(h->free_slots, &ptr) == 0)
+ rte_pause();
+
+ /* Repopulate the free slots ring. Entry zero is reserved for key misses */
+ for (i = 1; i < h->entries + 1; i++)
+ rte_ring_sp_enqueue(h->free_slots, (void *)((uintptr_t) i));
+}
+
/* Search for an entry that can be pushed to its alternative location */
static inline int
make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt)
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index 936d170..8bbc9f0 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -128,8 +128,15 @@ void
rte_hash_free(struct rte_hash *h);
/**
- * Add a key to an existing hash table.
- * This operation is not multi-thread safe
+ * Reset all hash structure, by zeroing all entries
+ * @param h
+ * Hash table to reset
+ */
+void
+rte_hash_reset(struct rte_hash *h);
+
+/**
+ * Add a key to an existing hash table. This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index 94a0fec..d5f5af5 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -18,3 +18,11 @@ DPDK_2.0 {
local: *;
};
+
+DPDK_2.1 {
+ global:
+
+ rte_hash_reset;
+
+ local: *;
+} DPDK_2.0;
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v4 3/7] hash: add new functionality to store data in hash table
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of " Pablo de Lara
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 1/7] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 2/7] hash: add new function rte_hash_reset Pablo de Lara
@ 2015-07-10 17:24 ` Pablo de Lara
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 4/7] hash: add iterate function Pablo de Lara
` (5 subsequent siblings)
8 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 17:24 UTC (permalink / raw)
To: dev
Usually hash tables not only store keys, but also data associated
to them. In order to maintain the existing API, the old functions
will still return the index where the key was stored.
The new functions will return the data associated to that key.
In the case of the lookup_bulk function, it will return also
the number of entries found and a bitmask of which entries
were found.
Unit tests have been updated to use these new functions.
As a final point, a flag has been added in rte_hash_parameters
to indicate if there are new parameters for future versions,
so there is no need to maintain multiple versions
of the existing functions in the future.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash_perf.c | 254 +++++++++++++++++++++++++----------
lib/librte_hash/rte_cuckoo_hash.c | 193 ++++++++++++++++++++------
lib/librte_hash/rte_hash.h | 108 ++++++++++++++-
lib/librte_hash/rte_hash_version.map | 6 +
4 files changed, 441 insertions(+), 120 deletions(-)
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index b01b040..7a6dbf7 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -55,6 +55,7 @@
#define NUM_KEYSIZES 10
#define NUM_SHUFFLES 10
#define BURST_SIZE 16
+#define DATA_MULTIPLIER 13 /* Used for storing data */
enum operations {
ADD = 0,
@@ -83,7 +84,7 @@ struct rte_hash *h[NUM_KEYSIZES];
uint8_t slot_taken[MAX_ENTRIES];
/* Array to store number of cycles per operation */
-uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS][2];
+uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS][2][2];
/* Array to store all input keys */
uint8_t keys[KEYS_TO_ADD][MAX_KEYSIZE];
@@ -106,11 +107,16 @@ static struct rte_hash_parameters ut_params = {
};
static int
-create_table(unsigned table_index)
+create_table(unsigned with_data, unsigned table_index)
{
char name[RTE_HASH_NAMESIZE];
- sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
+ if (with_data)
+ /* Table will store 8-byte data */
+ sprintf(name, "test_hash%d_data", hashtest_key_lens[table_index]);
+ else
+ sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
+
ut_params.name = name;
ut_params.key_len = hashtest_key_lens[table_index];
ut_params.socket_id = rte_socket_id();
@@ -234,6 +240,7 @@ get_input_keys(unsigned with_pushes, unsigned table_index)
else {
/* Store the returned position and mark slot as taken */
slot_taken[ret] = 1;
+ positions[i] = ret;
buckets[bucket_idx]++;
success = 1;
i++;
@@ -249,54 +256,116 @@ get_input_keys(unsigned with_pushes, unsigned table_index)
}
static int
-timed_adds(unsigned with_hash, unsigned table_index)
+timed_adds(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i;
const uint64_t start_tsc = rte_rdtsc();
+ uintptr_t data;
int32_t ret;
for (i = 0; i < KEYS_TO_ADD; i++) {
- if (with_hash)
+ data = (uint64_t) signatures[i] * DATA_MULTIPLIER;
+ if (with_hash && with_data) {
+ ret = rte_hash_add_key_with_hash_data(h[table_index],
+ (const void *) keys[i],
+ signatures[i], data);
+ if (ret < 0) {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
+ } else if (with_hash && !with_data) {
ret = rte_hash_add_key_with_hash(h[table_index],
(const void *) keys[i],
signatures[i]);
- else
+ if (ret >= 0)
+ positions[i] = ret;
+ else {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
+ } else if (!with_hash && with_data) {
+ ret = rte_hash_add_key_data(h[table_index],
+ (const void *) keys[i],
+ data);
+ if (ret < 0) {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
+ } else {
ret = rte_hash_add_key(h[table_index], keys[i]);
-
- if (ret >= 0)
- positions[i] = ret;
- else {
- printf("Failed to add key number %u\n", ret);
- return -1;
+ if (ret >= 0)
+ positions[i] = ret;
+ else {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
}
}
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][ADD][with_hash] = time_taken/KEYS_TO_ADD;
+ cycles[table_index][ADD][with_hash][with_data] = time_taken/KEYS_TO_ADD;
+
return 0;
}
static int
-timed_lookups(unsigned with_hash, unsigned table_index)
+timed_lookups(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i, j;
const uint64_t start_tsc = rte_rdtsc();
+ uintptr_t ret_data;
+ uintptr_t expected_data;
int32_t ret;
for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
for (j = 0; j < KEYS_TO_ADD; j++) {
- if (with_hash)
+ if (with_hash && with_data) {
+ ret = rte_hash_lookup_with_hash_data(h[table_index],
+ (const void *) keys[j],
+ signatures[j], &ret_data);
+ if (ret < 0) {
+ printf("Key number %u was not found\n", j);
+ return -1;
+ }
+ expected_data = (uint64_t) signatures[j] * DATA_MULTIPLIER;
+ if (ret_data != expected_data) {
+ printf("Data returned for key number %u is %"PRIx64","
+ " but should be %"PRIx64"\n", j, ret_data,
+ expected_data);
+ return -1;
+ }
+ } else if (with_hash && !with_data) {
ret = rte_hash_lookup_with_hash(h[table_index],
(const void *) keys[j],
signatures[j]);
- else
+ if (ret < 0 || ret != positions[j]) {
+ printf("Key looked up in %d, should be in %d\n",
+ ret, positions[j]);
+ return -1;
+ }
+ } else if (!with_hash && with_data) {
+ ret = rte_hash_lookup_data(h[table_index],
+ (const void *) keys[j], &ret_data);
+ if (ret < 0) {
+ printf("Key number %u was not found\n", j);
+ return -1;
+ }
+ expected_data = (uint64_t) signatures[j] * DATA_MULTIPLIER;
+ if (ret_data != expected_data) {
+ printf("Data returned for key number %u is %"PRIx64","
+ " but should be %"PRIx64"\n", j, ret_data,
+ expected_data);
+ return -1;
+ }
+ } else {
ret = rte_hash_lookup(h[table_index], keys[j]);
- if (ret < 0 || ret != positions[j]) {
- printf("Key looked up in %d, should be in %d\n",
- ret, positions[j]);
- return -1;
+ if (ret < 0 || ret != positions[j]) {
+ printf("Key looked up in %d, should be in %d\n",
+ ret, positions[j]);
+ return -1;
+ }
}
}
}
@@ -304,34 +373,66 @@ timed_lookups(unsigned with_hash, unsigned table_index)
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][LOOKUP][with_hash] = time_taken/NUM_LOOKUPS;
+ cycles[table_index][LOOKUP][with_hash][with_data] = time_taken/NUM_LOOKUPS;
return 0;
}
static int
-timed_lookups_multi(unsigned table_index)
+timed_lookups_multi(unsigned with_data, unsigned table_index)
{
unsigned i, j, k;
int32_t positions_burst[BURST_SIZE];
const void *keys_burst[BURST_SIZE];
+ uintptr_t expected_data[BURST_SIZE];
+ uintptr_t ret_data[BURST_SIZE];
+ uint64_t hit_mask;
+ int ret;
+
const uint64_t start_tsc = rte_rdtsc();
for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
for (j = 0; j < KEYS_TO_ADD/BURST_SIZE; j++) {
for (k = 0; k < BURST_SIZE; k++)
keys_burst[k] = keys[j * BURST_SIZE + k];
-
- rte_hash_lookup_bulk(h[table_index],
+ if (with_data) {
+ ret = rte_hash_lookup_bulk_data(h[table_index],
+ (const void **) keys_burst,
+ BURST_SIZE,
+ &hit_mask,
+ ret_data);
+ if (ret != BURST_SIZE) {
+ printf("Expect to find %u keys,"
+ " but found %d\n", BURST_SIZE, ret);
+ return -1;
+ }
+ for (k = 0; k < BURST_SIZE; k++) {
+ if ((hit_mask & (1ULL << k)) == 0) {
+ printf("Key number %u not found\n",
+ j * BURST_SIZE + k);
+ return -1;
+ }
+ expected_data[k] = (uint64_t)signatures[j * BURST_SIZE + k]
+ * DATA_MULTIPLIER;
+ if (ret_data[k] != expected_data[k]) {
+ printf("Data returned for key number %u is %"PRIx64","
+ " but should be %"PRIx64"\n", j * BURST_SIZE + k,
+ ret_data[k], expected_data[k]);
+ return -1;
+ }
+ }
+ } else {
+ rte_hash_lookup_bulk(h[table_index],
(const void **) keys_burst,
BURST_SIZE,
positions_burst);
- for (k = 0; k < BURST_SIZE; k++) {
- if (positions_burst[k] != positions[j * BURST_SIZE + k]) {
- printf("Key looked up in %d, should be in %d\n",
- positions_burst[k],
- positions[j * BURST_SIZE + k]);
- return -1;
+ for (k = 0; k < BURST_SIZE; k++) {
+ if (positions_burst[k] != positions[j * BURST_SIZE + k]) {
+ printf("Key looked up in %d, should be in %d\n",
+ positions_burst[k],
+ positions[j * BURST_SIZE + k]);
+ return -1;
+ }
}
}
}
@@ -340,19 +441,20 @@ timed_lookups_multi(unsigned table_index)
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][LOOKUP_MULTI][0] = time_taken/NUM_LOOKUPS;
+ cycles[table_index][LOOKUP_MULTI][0][with_data] = time_taken/NUM_LOOKUPS;
return 0;
}
static int
-timed_deletes(unsigned with_hash, unsigned table_index)
+timed_deletes(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i;
const uint64_t start_tsc = rte_rdtsc();
int32_t ret;
for (i = 0; i < KEYS_TO_ADD; i++) {
+ /* There are no delete functions with data, so just call two functions */
if (with_hash)
ret = rte_hash_del_key_with_hash(h[table_index],
(const void *) keys[i],
@@ -371,7 +473,7 @@ timed_deletes(unsigned with_hash, unsigned table_index)
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][DELETE][with_hash] = time_taken/KEYS_TO_ADD;
+ cycles[table_index][DELETE][with_hash][with_data] = time_taken/KEYS_TO_ADD;
return 0;
}
@@ -391,62 +493,67 @@ reset_table(unsigned table_index)
static int
run_all_tbl_perf_tests(unsigned with_pushes)
{
- unsigned i, j, with_hash;
+ unsigned i, j, with_data, with_hash;
printf("Measuring performance, please wait");
fflush(stdout);
- for (i = 0; i < NUM_KEYSIZES; i++) {
- if (create_table(i) < 0)
- return -1;
- if (get_input_keys(with_pushes, i) < 0)
- return -1;
- for (with_hash = 0; with_hash <= 1; with_hash++) {
- if (timed_adds(with_hash, i) < 0)
+ for (with_data = 0; with_data <= 1; with_data++) {
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ if (create_table(with_data, i) < 0)
return -1;
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
-
- if (timed_lookups(with_hash, i) < 0)
+ if (get_input_keys(with_pushes, i) < 0)
return -1;
+ for (with_hash = 0; with_hash <= 1; with_hash++) {
+ if (timed_adds(with_hash, with_data, i) < 0)
+ return -1;
- if (timed_lookups_multi(i) < 0)
- return -1;
+ for (j = 0; j < NUM_SHUFFLES; j++)
+ shuffle_input_keys(i);
- if (timed_deletes(with_hash, i) < 0)
- return -1;
+ if (timed_lookups(with_hash, with_data, i) < 0)
+ return -1;
- reset_table(i);
+ if (timed_lookups_multi(with_data, i) < 0)
+ return -1;
- /* Print a dot to show progress on operations */
- printf(".");
- fflush(stdout);
- }
+ if (timed_deletes(with_hash, with_data, i) < 0)
+ return -1;
- free_table(i);
+ /* Print a dot to show progress on operations */
+ printf(".");
+ fflush(stdout);
+
+ reset_table(i);
+ }
+ free_table(i);
+ }
}
+
printf("\nResults (in CPU cycles/operation)\n");
- printf("---------------------------------\n");
- printf("\nWithout pre-computed hash values\n");
- printf("\n%-18s%-18s%-18s%-18s%-18s\n",
- "Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
- for (i = 0; i < NUM_KEYSIZES; i++) {
- printf("%-18d", hashtest_key_lens[i]);
- for (j = 0; j < NUM_OPERATIONS; j++)
- printf("%-18"PRIu64, cycles[i][j][0]);
- printf("\n");
- }
- printf("\nWith pre-computed hash values\n");
- printf("\n%-18s%-18s%-18s%-18s%-18s\n",
+ printf("-----------------------------------\n");
+ for (with_data = 0; with_data <= 1; with_data++) {
+ if (with_data)
+ printf("\n Operations with 8-byte data\n");
+ else
+ printf("\n Operations without data\n");
+ for (with_hash = 0; with_hash <= 1; with_hash++) {
+ if (with_hash)
+ printf("\nWith pre-computed hash values\n");
+ else
+ printf("\nWithout pre-computed hash values\n");
+
+ printf("\n%-18s%-18s%-18s%-18s%-18s\n",
"Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
- for (i = 0; i < NUM_KEYSIZES; i++) {
- printf("%-18d", hashtest_key_lens[i]);
- for (j = 0; j < NUM_OPERATIONS; j++)
- printf("%-18"PRIu64, cycles[i][j][1]);
- printf("\n");
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ printf("%-18d", hashtest_key_lens[i]);
+ for (j = 0; j < NUM_OPERATIONS; j++)
+ printf("%-18"PRIu64, cycles[i][j][with_hash][with_data]);
+ printf("\n");
+ }
+ }
}
-
return 0;
}
@@ -546,7 +653,6 @@ test_hash_perf(void)
if (run_all_tbl_perf_tests(with_pushes) < 0)
return -1;
}
-
if (fbk_hash_perf_test() < 0)
return -1;
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index 15c5da5..cd91a7d 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -56,6 +56,7 @@
#include <rte_rwlock.h>
#include <rte_spinlock.h>
#include <rte_ring.h>
+#include <rte_compat.h>
#include "rte_hash.h"
@@ -90,6 +91,8 @@ EAL_REGISTER_TAILQ(rte_hash_tailq)
#define NULL_SIGNATURE 0
+#define KEY_ALIGNMENT 16
+
typedef int (*rte_hash_cmp_eq_t)(const void *key1, const void *key2, size_t key_len);
static int rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len);
static int rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len);
@@ -132,6 +135,16 @@ struct rte_hash_signatures {
};
};
+/* Structure that stores key-value pair */
+struct rte_hash_key {
+ union {
+ uintptr_t idata;
+ void *pdata;
+ };
+ /* Variable key size */
+ char key[];
+} __attribute__((aligned(KEY_ALIGNMENT)));
+
/** Bucket structure */
struct rte_hash_bucket {
struct rte_hash_signatures signatures[RTE_HASH_BUCKET_ENTRIES];
@@ -227,7 +240,8 @@ rte_hash_create(const struct rte_hash_parameters *params)
goto err;
}
- const uint32_t key_entry_size = params->key_len;
+ const uint32_t key_entry_size = sizeof(struct rte_hash_key) + params->key_len;
+
/* Store all keys and leave the first entry as a dummy entry for lookup_bulk */
const uint64_t key_tbl_size = key_entry_size * (params->entries + 1);
@@ -461,13 +475,13 @@ make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt)
static inline int32_t
__rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
- hash_sig_t sig)
+ hash_sig_t sig, uintptr_t data)
{
hash_sig_t alt_hash;
uint32_t prim_bucket_idx, sec_bucket_idx;
unsigned i;
struct rte_hash_bucket *prim_bkt, *sec_bkt;
- void *new_k, *k, *keys = h->key_store;
+ struct rte_hash_key *new_k, *k, *keys = h->key_store;
void *slot_id;
uint32_t new_idx;
int ret;
@@ -492,9 +506,12 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (prim_bkt->signatures[i].current == sig &&
prim_bkt->signatures[i].alt == alt_hash) {
- k = (char *)keys + prim_bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ prim_bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /* Update data */
+ k->idata = data;
/*
* Return index where key is stored,
* substracting the first dummy index
@@ -508,9 +525,12 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (sec_bkt->signatures[i].alt == sig &&
sec_bkt->signatures[i].current == alt_hash) {
- k = (char *)keys + sec_bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ sec_bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /* Update data */
+ k->idata = data;
/*
* Return index where key is stored,
* substracting the first dummy index
@@ -521,7 +541,8 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
}
/* Copy key */
- rte_memcpy(new_k, key, h->key_len);
+ rte_memcpy(new_k->key, key, h->key_len);
+ new_k->idata = data;
/* Insert new entry is there is room in the primary bucket */
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
@@ -561,25 +582,52 @@ rte_hash_add_key_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, sig);
+ return __rte_hash_add_key_with_hash(h, key, sig, NULL);
}
int32_t
rte_hash_add_key(const struct rte_hash *h, const void *key)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
+ return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key), NULL);
+}
+
+int
+rte_hash_add_key_with_hash_data(const struct rte_hash *h,
+ const void *key, hash_sig_t sig, uintptr_t data)
+{
+ int ret;
+
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ ret = __rte_hash_add_key_with_hash(h, key, sig, data);
+ if (ret >= 0)
+ return 0;
+ else
+ return ret;
}
+int
+rte_hash_add_key_data(const struct rte_hash *h, const void *key, uintptr_t data)
+{
+ int ret;
+
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+
+ ret = __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key), data);
+ if (ret >= 0)
+ return 0;
+ else
+ return ret;
+}
static inline int32_t
__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
- hash_sig_t sig)
+ hash_sig_t sig, uintptr_t *data)
{
uint32_t bucket_idx;
hash_sig_t alt_hash;
unsigned i;
struct rte_hash_bucket *bkt;
- void *k, *keys = h->key_store;
+ struct rte_hash_key *k, *keys = h->key_store;
bucket_idx = sig & h->bucket_bitmask;
bkt = &h->buckets[bucket_idx];
@@ -588,13 +636,17 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == sig &&
bkt->signatures[i].sig != NULL_SIGNATURE) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
+ if (data != NULL)
+ *data = k->idata;
/*
* Return index where key is stored,
* substracting the first dummy index
*/
return (bkt->key_idx[i] - 1);
+ }
}
}
@@ -607,13 +659,17 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == alt_hash &&
bkt->signatures[i].alt == sig) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
+ if (data != NULL)
+ *data = k->idata;
/*
* Return index where key is stored,
* substracting the first dummy index
*/
return (bkt->key_idx[i] - 1);
+ }
}
}
@@ -625,14 +681,29 @@ rte_hash_lookup_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, sig);
+ return __rte_hash_lookup_with_hash(h, key, sig, NULL);
}
int32_t
rte_hash_lookup(const struct rte_hash *h, const void *key)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key), NULL);
+}
+
+int
+rte_hash_lookup_with_hash_data(const struct rte_hash *h,
+ const void *key, hash_sig_t sig, uintptr_t *data)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, sig, data);
+}
+
+int
+rte_hash_lookup_data(const struct rte_hash *h, const void *key, uintptr_t *data)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key), data);
}
static inline int32_t
@@ -643,7 +714,7 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
hash_sig_t alt_hash;
unsigned i;
struct rte_hash_bucket *bkt;
- void *k, *keys = h->key_store;
+ struct rte_hash_key *k, *keys = h->key_store;
bucket_idx = sig & h->bucket_bitmask;
bkt = &h->buckets[bucket_idx];
@@ -652,8 +723,9 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == sig &&
bkt->signatures[i].sig != NULL_SIGNATURE) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
bkt->signatures[i].sig = NULL_SIGNATURE;
rte_ring_sp_enqueue(h->free_slots,
(void *)((uintptr_t)bkt->key_idx[i]));
@@ -675,8 +747,9 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == alt_hash &&
bkt->signatures[i].sig != NULL_SIGNATURE) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
bkt->signatures[i].sig = NULL_SIGNATURE;
rte_ring_sp_enqueue(h->free_slots,
(void *)((uintptr_t)bkt->key_idx[i]));
@@ -750,7 +823,7 @@ static inline void
lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
const struct rte_hash_bucket *prim_bkt,
const struct rte_hash_bucket *sec_bkt,
- const void **key_slot, int32_t *positions,
+ const struct rte_hash_key **key_slot, int32_t *positions,
uint64_t *extra_hits_mask, const void *keys,
const struct rte_hash *h)
{
@@ -770,7 +843,8 @@ lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
total_hash_matches = (prim_hash_matches |
(sec_hash_matches << (RTE_HASH_BUCKET_ENTRIES + 1)));
- *key_slot = (const char *)keys + key_idx * h->key_entry_size;
+ *key_slot = (const struct rte_hash_key *) ((const char *)keys +
+ key_idx * h->key_entry_size);
rte_prefetch0(*key_slot);
/*
@@ -784,26 +858,31 @@ lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
}
-/* Lookup bulk stage 3: Check if key matches, update hit mask */
+/* Lookup bulk stage 3: Check if key matches, update hit mask and return data */
static inline void
-lookup_stage3(unsigned idx, const void *key_slot, const void * const *keys,
- uint64_t *hits, const struct rte_hash *h)
+lookup_stage3(unsigned idx, const struct rte_hash_key *key_slot, const void * const *keys,
+ uintptr_t data[], uint64_t *hits, const struct rte_hash *h)
{
unsigned hit;
- hit = !h->rte_hash_cmp_eq(key_slot, keys[idx], h->key_len);
+ hit = !h->rte_hash_cmp_eq(key_slot->key, keys[idx], h->key_len);
+ if (data != NULL)
+ data[idx] = key_slot->idata;
+
*hits |= (uint64_t)(hit) << idx;
}
static inline void
__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
- uint32_t num_keys, int32_t *positions)
+ uint32_t num_keys, int32_t *positions,
+ uint64_t *hit_mask, uintptr_t data[])
{
uint64_t hits = 0;
uint64_t extra_hits_mask = 0;
uint64_t lookup_mask, miss_mask;
unsigned idx;
const void *key_store = h->key_store;
+ int ret;
hash_sig_t hash_vals[RTE_HASH_LOOKUP_BULK_MAX];
unsigned idx00, idx01, idx10, idx11, idx20, idx21, idx30, idx31;
@@ -811,7 +890,7 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
const struct rte_hash_bucket *secondary_bkt10, *secondary_bkt11;
const struct rte_hash_bucket *primary_bkt20, *primary_bkt21;
const struct rte_hash_bucket *secondary_bkt20, *secondary_bkt21;
- const void *k_slot20, *k_slot21, *k_slot30, *k_slot31;
+ const struct rte_hash_key *k_slot20, *k_slot21, *k_slot30, *k_slot31;
hash_sig_t primary_hash10, primary_hash11;
hash_sig_t secondary_hash10, secondary_hash11;
hash_sig_t primary_hash20, primary_hash21;
@@ -882,8 +961,8 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21,
primary_bkt21, secondary_bkt21, &k_slot21, positions,
&extra_hits_mask, key_store, h);
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
}
k_slot30 = k_slot20, k_slot31 = k_slot21;
@@ -909,8 +988,8 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
key_store, h);
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
k_slot30 = k_slot20, k_slot31 = k_slot21;
idx30 = idx20, idx31 = idx21;
@@ -930,14 +1009,14 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
key_store, h);
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
k_slot30 = k_slot20, k_slot31 = k_slot21;
idx30 = idx20, idx31 = idx21;
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
/* ignore any items we have already found */
extra_hits_mask &= ~hits;
@@ -946,11 +1025,18 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
/* run a single search for each remaining item */
do {
idx = __builtin_ctzl(extra_hits_mask);
- positions[idx] = rte_hash_lookup_with_hash(h, keys[idx],
- hash_vals[idx]);
+ if (data != NULL) {
+ ret = rte_hash_lookup_with_hash_data(h,
+ keys[idx], hash_vals[idx], &data[idx]);
+ if (ret >= 0)
+ hits |= 1ULL << idx;
+ } else {
+ positions[idx] = rte_hash_lookup_with_hash(h,
+ keys[idx], hash_vals[idx]);
+ if (positions[idx] >= 0)
+ hits |= 1llu << idx;
+ }
extra_hits_mask &= ~(1llu << idx);
- if (positions[idx] >= 0)
- hits |= 1llu << idx;
} while (extra_hits_mask);
}
@@ -962,6 +1048,9 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
miss_mask &= ~(1llu << idx);
} while (miss_mask);
}
+
+ if (hit_mask != NULL)
+ *hit_mask = hits;
}
int
@@ -972,10 +1061,26 @@ rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
(num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
(positions == NULL)), -EINVAL);
- __rte_hash_lookup_bulk(h, keys, num_keys, positions);
+ __rte_hash_lookup_bulk(h, keys, num_keys, positions, NULL, NULL);
return 0;
}
+int
+rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, uint64_t *hit_mask, uintptr_t data[])
+{
+ RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
+ (num_keys > RTE_HASH_LOOKUP_BULK_MAX),
+ (hit_mask == NULL)), -EINVAL);
+
+ int32_t positions[num_keys];
+
+ __rte_hash_lookup_bulk(h, keys, num_keys, positions, hit_mask, data);
+
+ /* Return number of hits */
+ return __builtin_popcountl(*hit_mask);
+}
+
/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
static int
rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index 8bbc9f0..15f8b2f 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -80,12 +80,12 @@ struct rte_hash_parameters {
rte_hash_function hash_func; /**< Primary Hash function used to calculate hash. */
uint32_t hash_func_init_val; /**< Init value used by hash_func. */
int socket_id; /**< NUMA Socket ID for memory. */
+ uint8_t extra_flag; /**< Indicate if additional parameters are present. */
};
/** @internal A hash table structure. */
struct rte_hash;
-
/**
* Create a new hash table.
*
@@ -136,6 +136,48 @@ void
rte_hash_reset(struct rte_hash *h);
/**
+ * Add a key-value pair to an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
+ *
+ * @param h
+ * Hash table to add the key to.
+ * @param key
+ * Key to add to the hash table.
+ * @param data
+ * Data to add to the hash table.
+ * @return
+ * - 0 if added successfully
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOSPC if there is no space in the hash for this key.
+ */
+int
+rte_hash_add_key_data(const struct rte_hash *h, const void *key, uintptr_t data);
+
+/**
+ * Add a key-value pair with a pre-computed hash value
+ * to an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
+ *
+ * @param h
+ * Hash table to add the key to.
+ * @param key
+ * Key to add to the hash table.
+ * @param sig
+ * Precomputed hash value for 'key'
+ * @param data
+ * Data to add to the hash table.
+ * @return
+ * - 0 if added successfully
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOSPC if there is no space in the hash for this key.
+ */
+int32_t
+rte_hash_add_key_with_hash_data(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, uintptr_t data);
+
+/**
* Add a key to an existing hash table. This operation is not multi-thread safe
* and should only be called from one thread.
*
@@ -212,6 +254,47 @@ rte_hash_del_key(const struct rte_hash *h, const void *key);
int32_t
rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
+
+/**
+ * Find a key-value pair in the hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param key
+ * Key to find.
+ * @param data
+ * Output with pointer to data returned from the hash table.
+ * @return
+ * 0 if successful lookup
+ * - EINVAL if the parameters are invalid.
+ * - ENOENT if the key is not found.
+ */
+int
+rte_hash_lookup_data(const struct rte_hash *h, const void *key, uintptr_t *data);
+
+/**
+ * Find a key-value pair with a pre-computed hash value
+ * to an existing hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param key
+ * Key to find.
+ * @param sig
+ * Precomputed hash value for 'key'
+ * @param data
+ * Output with pointer to data returned from the hash table.
+ * @return
+ * 0 if successful lookup
+ * - EINVAL if the parameters are invalid.
+ * - ENOENT if the key is not found.
+ */
+int
+rte_hash_lookup_with_hash_data(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, uintptr_t *data);
+
/**
* Find a key in the hash table.
* This operation is multi-thread safe.
@@ -266,6 +349,28 @@ hash_sig_t
rte_hash_hash(const struct rte_hash *h, const void *key);
#define rte_hash_lookup_multi rte_hash_lookup_bulk
+#define rte_hash_lookup_multi_data rte_hash_lookup_bulk_data
+/**
+ * Find multiple keys in the hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param keys
+ * A pointer to a list of keys to look for.
+ * @param num_keys
+ * How many keys are in the keys list (less than RTE_HASH_LOOKUP_BULK_MAX).
+ * @param hit_mask
+ * Output containing a bitmask with all successful lookups.
+ * @param data
+ * Output containing array of data returned from all the successful lookups.
+ * @return
+ * -EINVAL if there's an error, otherwise number of successful lookups.
+ */
+int
+rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, uint64_t *hit_mask, uintptr_t data[]);
+
/**
* Find multiple keys in the hash table.
* This operation is multi-thread safe.
@@ -288,7 +393,6 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions);
-
#ifdef __cplusplus
}
#endif
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index d5f5af5..a97eac1 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -22,6 +22,12 @@ DPDK_2.0 {
DPDK_2.1 {
global:
+ rte_hash_add_key_data;
+ rte_hash_add_key_with_hash_data;
+ rte_hash_create;
+ rte_hash_lookup_bulk_data;
+ rte_hash_lookup_data;
+ rte_hash_lookup_with_hash_data;
rte_hash_reset;
local: *;
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v4 4/7] hash: add iterate function
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of " Pablo de Lara
` (2 preceding siblings ...)
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 3/7] hash: add new functionality to store data in hash table Pablo de Lara
@ 2015-07-10 17:24 ` Pablo de Lara
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 5/7] MAINTAINERS: claim responsability for hash library Pablo de Lara
` (4 subsequent siblings)
8 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 17:24 UTC (permalink / raw)
To: dev
Since now rte_hash structure is private, a new function
has been added to let the user iterate through the hash table,
returning next key and data associated on each iteration,
plus the position where they were stored.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 66 ++++++++++++++++++++++++++++++++++++
lib/librte_hash/rte_cuckoo_hash.c | 41 ++++++++++++++++++++++
lib/librte_hash/rte_hash.h | 22 ++++++++++++
lib/librte_hash/rte_hash_version.map | 1 +
4 files changed, 130 insertions(+)
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index 448586c..f5277c9 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -1149,6 +1149,70 @@ static int test_average_table_utilization(void)
return 0;
}
+#define NUM_ENTRIES 1024
+static int test_hash_iteration(void)
+{
+ struct rte_hash *handle;
+ unsigned i;
+ uint8_t keys[NUM_ENTRIES][RTE_HASH_KEY_LENGTH_MAX];
+ const void *next_key;
+ uintptr_t next_data;
+ uintptr_t data[NUM_ENTRIES];
+ unsigned added_keys;
+ uint32_t iter = 0;
+ int ret = 0;
+
+ ut_params.entries = NUM_ENTRIES;
+ ut_params.name = "test_hash_iteration";
+ ut_params.hash_func = rte_jhash;
+ ut_params.key_len = 16;
+ handle = rte_hash_create(&ut_params);
+ RETURN_IF_ERROR(handle == NULL, "hash creation failed");
+
+ /* Add random entries until key cannot be added */
+ for (added_keys = 0; added_keys < NUM_ENTRIES; added_keys++) {
+ data[added_keys] = rte_rand();
+ for (i = 0; i < ut_params.key_len; i++)
+ keys[added_keys][i] = rte_rand() % 255;
+ ret = rte_hash_add_key_data(handle, keys[added_keys], data[added_keys]);
+ if (ret < 0)
+ break;
+ }
+
+ /* Iterate through the hash table */
+ while (rte_hash_iterate(handle, &next_key, &next_data, &iter) >= 0) {
+ /* Search for the key in the list of keys added */
+ for (i = 0; i < NUM_ENTRIES; i++) {
+ if (memcmp(next_key, keys[i], ut_params.key_len) == 0) {
+ if (next_data != data[i]) {
+ printf("Data found in the hash table is"
+ "not the data added with the key\n");
+ goto err;
+ }
+ added_keys--;
+ break;
+ }
+ }
+ if (i == NUM_ENTRIES) {
+ printf("Key found in the hash table was not added\n");
+ goto err;
+ }
+ }
+
+ /* Check if all keys have been iterated */
+ if (added_keys != 0) {
+ printf("There were still %u keys to iterate\n", added_keys);
+ goto err;
+ }
+
+ rte_hash_free(handle);
+ return 0;
+
+err:
+ rte_hash_free(handle);
+ return -1;
+}
+
static uint8_t key[16] = {0x00, 0x01, 0x02, 0x03,
0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b,
@@ -1408,6 +1472,8 @@ test_hash(void)
return -1;
if (test_average_table_utilization() < 0)
return -1;
+ if (test_hash_iteration() < 0)
+ return -1;
run_hash_func_tests();
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index cd91a7d..7384512 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -1081,6 +1081,47 @@ rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
return __builtin_popcountl(*hit_mask);
}
+int32_t
+rte_hash_iterate(const struct rte_hash *h, const void **key, uintptr_t *data, uint32_t *next)
+{
+ uint32_t bucket_idx, idx, position;
+ struct rte_hash_key *next_key;
+
+ RETURN_IF_TRUE(((h == NULL) || (next == NULL)), -EINVAL);
+
+ const uint32_t total_entries = h->num_buckets * RTE_HASH_BUCKET_ENTRIES;
+ /* Out of bounds */
+ if (*next >= total_entries)
+ return -ENOENT;
+
+ /* Calculate bucket and index of current iterator */
+ bucket_idx = *next / RTE_HASH_BUCKET_ENTRIES;
+ idx = *next % RTE_HASH_BUCKET_ENTRIES;
+
+ /* If current position is empty, go to the next one */
+ while (h->buckets[bucket_idx].signatures[idx].sig == NULL_SIGNATURE) {
+ (*next)++;
+ /* End of table */
+ if (*next == total_entries)
+ return -ENOENT;
+ bucket_idx = *next / RTE_HASH_BUCKET_ENTRIES;
+ idx = *next % RTE_HASH_BUCKET_ENTRIES;
+ }
+
+ /* Get position of entry in key table */
+ position = h->buckets[bucket_idx].key_idx[idx];
+ next_key = (struct rte_hash_key *) ((char *)h->key_store +
+ position * h->key_entry_size);
+ /* Return key and data */
+ *key = next_key->key;
+ *data = next_key->idata;
+
+ /* Increment iterator */
+ (*next)++;
+
+ return (position - 1);
+}
+
/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
static int
rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index 15f8b2f..a2e7f74 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -393,6 +393,28 @@ rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions);
+
+/**
+ * Iterate through the hash table, returning key-value pairs.
+ *
+ * @param h
+ * Hash table to iterate
+ * @param key
+ * Output containing the key where current iterator
+ * was pointing at
+ * @param data
+ * Output containing the data associated with key.
+ * Returns NULL if data was not stored.
+ * @param next
+ * Pointer to iterator. Should be 0 to start iterating the hash table.
+ * Iterator is incremented after each call of this function.
+ * @return
+ * Position where key was stored, if successful.
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOENT if end of the hash table.
+ */
+int32_t
+rte_hash_iterate(const struct rte_hash *h, const void **key, uintptr_t *data, uint32_t *next);
#ifdef __cplusplus
}
#endif
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index a97eac1..5653cb7 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -25,6 +25,7 @@ DPDK_2.1 {
rte_hash_add_key_data;
rte_hash_add_key_with_hash_data;
rte_hash_create;
+ rte_hash_iterate;
rte_hash_lookup_bulk_data;
rte_hash_lookup_data;
rte_hash_lookup_with_hash_data;
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v4 5/7] MAINTAINERS: claim responsability for hash library
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of " Pablo de Lara
` (3 preceding siblings ...)
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 4/7] hash: add iterate function Pablo de Lara
@ 2015-07-10 17:24 ` Pablo de Lara
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 6/7] doc: announce ABI change of librte_hash Pablo de Lara
` (3 subsequent siblings)
8 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 17:24 UTC (permalink / raw)
To: dev
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
MAINTAINERS | 1 +
1 file changed, 1 insertion(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 5476a73..75a27e9 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -345,6 +345,7 @@ F: doc/guides/sample_app_ug/l3_forward_access_ctrl.rst
Hashes
M: Bruce Richardson <bruce.richardson@intel.com>
+M: Pablo de Lara <pablo.de.lara.guarch@intel.com>
F: lib/librte_hash/
F: doc/guides/prog_guide/hash_lib.rst
F: app/test/test_*hash*
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v4 6/7] doc: announce ABI change of librte_hash
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of " Pablo de Lara
` (4 preceding siblings ...)
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 5/7] MAINTAINERS: claim responsability for hash library Pablo de Lara
@ 2015-07-10 17:24 ` Pablo de Lara
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 7/7] doc: update hash documentation Pablo de Lara
` (2 subsequent siblings)
8 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 17:24 UTC (permalink / raw)
To: dev
Two of the macros in rte_hash.h are now deprecated, so this patch
adds notice that they will be removed in 2.2.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
doc/guides/rel_notes/abi.rst | 1 +
1 file changed, 1 insertion(+)
diff --git a/doc/guides/rel_notes/abi.rst b/doc/guides/rel_notes/abi.rst
index 110c486..312348e 100644
--- a/doc/guides/rel_notes/abi.rst
+++ b/doc/guides/rel_notes/abi.rst
@@ -12,3 +12,4 @@ Examples of Deprecation Notices
Deprecation Notices
-------------------
+* The Macros #RTE_HASH_BUCKET_ENTRIES_MAX and #RTE_HASH_KEY_LENGTH_MAX are deprecated and will be removed with version 2.2.
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v4 7/7] doc: update hash documentation
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of " Pablo de Lara
` (5 preceding siblings ...)
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 6/7] doc: announce ABI change of librte_hash Pablo de Lara
@ 2015-07-10 17:24 ` Pablo de Lara
2015-07-10 20:52 ` [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of Cuckoo hash Bruce Richardson
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 " Pablo de Lara
8 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 17:24 UTC (permalink / raw)
To: dev
Updates hash library documentation, reflecting
the new implementation changes.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
doc/guides/prog_guide/hash_lib.rst | 138 +++++++++++++++++++++++++++++++++----
doc/guides/prog_guide/index.rst | 4 ++
2 files changed, 129 insertions(+), 13 deletions(-)
diff --git a/doc/guides/prog_guide/hash_lib.rst b/doc/guides/prog_guide/hash_lib.rst
index 9b83835..193dd53 100644
--- a/doc/guides/prog_guide/hash_lib.rst
+++ b/doc/guides/prog_guide/hash_lib.rst
@@ -1,5 +1,5 @@
.. BSD LICENSE
- Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
All rights reserved.
Redistribution and use in source and binary forms, with or without
@@ -50,8 +50,6 @@ The hash also allows the configuration of some low-level implementation related
* Hash function to translate the key into a bucket index
-* Number of entries per bucket
-
The main methods exported by the hash are:
* Add entry with key: The key is provided as input. If a new entry is successfully added to the hash for the specified key,
@@ -65,10 +63,26 @@ The main methods exported by the hash are:
* Lookup for entry with key: The key is provided as input. If an entry with the specified key is found in the hash (lookup hit),
then the position of the entry is returned, otherwise (lookup miss) a negative value is returned.
-The current hash implementation handles the key management only.
-The actual data associated with each key has to be managed by the user using a separate table that
+Apart from these method explained above, the API allows the user three more options:
+
+* Add / lookup / delete with key and precomputed hash: Both the key and its precomputed hash are provided as input. This allows
+ the user to perform these operations faster, as hash is already computed.
+
+* Add / lookup with key and data: A pair of key-value is provided as input. This allows the user to store
+ not only the key, but also data which may be either a 8-byte integer or a pointer to external data (if data size is more than 8 bytes).
+
+* Combination of the two options above: User can provide key, precomputed hash and data.
+
+Also, the API contains a method to allow the user to look up entries in bursts, achieving higher performance
+than looking up individual entries, as the function prefetches next entries at the time it is operating
+with the first ones, which reduces significantly the impact of the necessary memory accesses.
+Notice that this method uses a pipeline of 8 entries (4 stages of 2 entries), so it is highly recommended
+to use at least 8 entries per burst.
+
+The actual data associated with each key can be either managed by the user using a separate table that
mirrors the hash in terms of number of entries and position of each entry,
-as shown in the Flow Classification use case describes in the following sections.
+as shown in the Flow Classification use case describes in the following sections,
+or stored in the hash table itself.
The example hash tables in the L2/L3 Forwarding sample applications defines which port to forward a packet to based on a packet flow identified by the five-tuple lookup.
However, this table could also be used for more sophisticated features and provide many other functions and actions that could be performed on the packets and flows.
@@ -76,17 +90,26 @@ However, this table could also be used for more sophisticated features and provi
Implementation Details
----------------------
-The hash table is implemented as an array of entries which is further divided into buckets,
-with the same number of consecutive array entries in each bucket.
-For any input key, there is always a single bucket where that key can be stored in the hash,
-therefore only the entries within that bucket need to be examined when the key is looked up.
+The hash table has two main tables:
+
+* First table is an array of entries which is further divided into buckets,
+ with the same number of consecutive array entries in each bucket. Each entry contains the computed primary
+ and secondary hashes of a given key (explained below), and an index to the second table.
+
+* The second table is an array of all the keys stored in the hash table and its data associated to each key.
+
+The hash library uses the cuckoo hash method to resolve collisions.
+For any input key, there are two possible buckets (primary and secondary/alternative location)
+where that key can be stored in the hash, therefore only the entries within those bucket need to be examined
+when the key is looked up.
The lookup speed is achieved by reducing the number of entries to be scanned from the total
-number of hash entries down to the number of entries in a hash bucket,
+number of hash entries down to the number of entries in the two hash buckets,
as opposed to the basic method of linearly scanning all the entries in the array.
The hash uses a hash function (configurable) to translate the input key into a 4-byte key signature.
The bucket index is the key signature modulo the number of hash buckets.
-Once the bucket is identified, the scope of the hash add,
-delete and lookup operations is reduced to the entries in that bucket.
+
+Once the buckets are identified, the scope of the hash add,
+delete and lookup operations is reduced to the entries in those buckets (it is very likely that entries are in the primary bucket).
To speed up the search logic within the bucket, each hash entry stores the 4-byte key signature together with the full key for each hash entry.
For large key sizes, comparing the input key against a key from the bucket can take significantly more time than
@@ -95,6 +118,95 @@ Therefore, the signature comparison is done first and the full key comparison do
The full key comparison is still necessary, as two input keys from the same bucket can still potentially have the same 4-byte hash signature,
although this event is relatively rare for hash functions providing good uniform distributions for the set of input keys.
+Example of lookup:
+
+First of all, the primary bucket is identified and entry is likely to be stored there.
+If signature was stored there, we compare its key against the one provided and return the position
+where it was stored and/or the data associated to that key if there is a match.
+If signature is not in the primary bucket, the secondary bucket is looked up, where same procedure
+is carried out. If there is no match there either, key is considered not to be in the table.
+
+Example of addition:
+
+Like lookup, the primary and secondary buckets are indentified. If there is an empty slot in
+the primary bucket, primary and secondary signatures are stored in that slot, key and data (if any) are added to
+the second table and an index to the position in the second table is stored in the slot of the first table.
+If there is no space in the primary bucket, one of the entries on that bucket is pushed to its alternative location,
+and the key to be added is inserted in its position.
+To know where the alternative bucket of the evicted entry is, the secondary signature is looked up and alternative bucket index
+is calculated from doing the modulo, as seen above. If there is room in the alternative bucket, the evicted entry
+is stored in it. If not, same process is repeated (one of the entries gets pushed) until a non full bucket is found.
+Notice that despite all the entry movement in the first table, the second table is not touched, which would impact
+greatly in performance.
+
+In the very unlikely event that table enters in a loop where same entries are being evicted indefinitely,
+key is considered not able to be stored.
+With random keys, this method allows the user to get around 90% of the table utilization, without
+having to drop any stored entry (LRU) or allocate more memory (extended buckets).
+
+Entry distribution in hash table
+--------------------------------
+
+As mentioned above, Cuckoo hash implementation pushes elements out of their bucket,
+if there is a new entry to be added which primary location coincides with their current bucket,
+being pushed to their alternative location.
+Therefore, as user adds more entries to the hash table, distribution of the hash values
+in the buckets will change, being most of them in their primary location and a few in
+their secondary location, which the later will increase, as table gets busier.
+This information is quite useful, as performance will be lower as more entries
+are evicted to their secondary location.
+
+See the tables below showing entry distribution as table utilization increases.
+
+.. _table_hash_lib_1:
+
+.. table:: Entry distribution with 1024 entries
+
+ +--------------+-----------------------+-------------------------+
+ | % Table used | % In Primary location | % In Secondary location |
+ +==============+=======================+=========================+
+ | 25 | 100 | 0 |
+ +--------------+-----------------------+-------------------------+
+ | 50 | 96.1 | 3.9 |
+ +--------------+-----------------------+-------------------------+
+ | 75 | 88.2 | 11.8 |
+ +--------------+-----------------------+-------------------------+
+ | 80 | 86.3 | 13.7 |
+ +--------------+-----------------------+-------------------------+
+ | 85 | 83.1 | 16.9 |
+ +--------------+-----------------------+-------------------------+
+ | 90 | 77.3 | 22.7 |
+ +--------------+-----------------------+-------------------------+
+ | 95.8 | 64.5 | 35.5 |
+ +--------------+-----------------------+-------------------------+
+
+|
+
+.. _table_hash_lib_2:
+
+.. table:: Entry distribution with 1 million entries
+
+ +--------------+-----------------------+-------------------------+
+ | % Table used | % In Primary location | % In Secondary location |
+ +==============+=======================+=========================+
+ | 50 | 96 | 4 |
+ +--------------+-----------------------+-------------------------+
+ | 75 | 86.9 | 13.1 |
+ +--------------+-----------------------+-------------------------+
+ | 80 | 83.9 | 16.1 |
+ +--------------+-----------------------+-------------------------+
+ | 85 | 80.1 | 19.9 |
+ +--------------+-----------------------+-------------------------+
+ | 90 | 74.8 | 25.2 |
+ +--------------+-----------------------+-------------------------+
+ | 94.5 | 67.4 | 32.6 |
+ +--------------+-----------------------+-------------------------+
+
+.. note::
+
+ Last values on the tables above are the average maximum table
+ utilization with random keys and using Jenkins hash function.
+
Use Case: Flow Classification
-----------------------------
diff --git a/doc/guides/prog_guide/index.rst b/doc/guides/prog_guide/index.rst
index 3295661..036640c 100644
--- a/doc/guides/prog_guide/index.rst
+++ b/doc/guides/prog_guide/index.rst
@@ -241,3 +241,7 @@ Programmer's Guide
:numref:`table_qos_33` :ref:`table_qos_33`
:numref:`table_qos_34` :ref:`table_qos_34`
+
+:numref:`table_hash_lib_1` :ref:`table_hash_lib_1`
+
+:numref:`table_hash_lib_2` :ref:`table_hash_lib_2`
--
2.4.2
^ permalink raw reply [flat|nested] 92+ messages in thread* Re: [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of Cuckoo hash
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of " Pablo de Lara
` (6 preceding siblings ...)
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 7/7] doc: update hash documentation Pablo de Lara
@ 2015-07-10 20:52 ` Bruce Richardson
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 " Pablo de Lara
8 siblings, 0 replies; 92+ messages in thread
From: Bruce Richardson @ 2015-07-10 20:52 UTC (permalink / raw)
To: Pablo de Lara; +Cc: dev
On Fri, Jul 10, 2015 at 06:24:17PM +0100, Pablo de Lara wrote:
> This patchset is to replace the existing hash library with
> a more efficient and functional approach, using the Cuckoo hash
> method to deal with collisions. This method is based on using
> two different hash functions to have two possible locations
> in the hash table where an entry can be.
> So, if a bucket is full, a new entry can push one of the items
> in that bucket to its alternative location, making space for itself.
>
> Advantages
> ~~~~~
> - Offers the option to store more entries when the target bucket is full
> (unlike the previous implementation)
> - Memory efficient: for storing those entries, it is not necessary to
> request new memory, as the entries will be stored in the same table
> - Constant worst lookup time: in worst case scenario, it always takes
> the same time to look up an entry, as there are only two possible locations
> where an entry can be.
> - Storing data: user can store data in the hash table, unlike the
> previous implementation, but he can still use the old API
>
> This implementation tipically offers over 90% utilization.
> Notice that API has been extended, but old API remains.
> Check documentation included to know more about this new implementation
> (including how entries are distributed as table utilization increases).
>
> Changes in v4:
> - Unit tests enhancements are not part of this patchset anymore.
> - rte_hash structure has been made internal in another patch,
> so it is not part of this patchset anymore.
> - Add function to iterate through the hash table, as rte_hash
> structure has been made private.
> - Added extra_flag parameter in rte_hash_parameter to be able
> to add new parameters in the future without breaking the ABI
> - Remove proposed lookup_bulk_with_hash function, as it is
> not of much use with the existing hash functions
> (there are no vector hash functions).
> - User can store 8-byte integer or pointer as data, instead
> of variable size data, as discussed in the mailing list.
>
Hi Pablo,
I'm getting some compile errors with this code, perhaps you could recheck e.g
32-bit and with clang.
On the plus side, I like the docs included with this set.
Regards,
/Bruce
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v5 0/7] Cuckoo hash - part 3 of Cuckoo hash
2015-07-10 17:24 ` [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of " Pablo de Lara
` (7 preceding siblings ...)
2015-07-10 20:52 ` [dpdk-dev] [PATCH v4 0/7] Cuckoo hash - part 3 of Cuckoo hash Bruce Richardson
@ 2015-07-10 21:57 ` Pablo de Lara
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 1/7] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
` (8 more replies)
8 siblings, 9 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 21:57 UTC (permalink / raw)
To: dev
This patchset is to replace the existing hash library with
a more efficient and functional approach, using the Cuckoo hash
method to deal with collisions. This method is based on using
two different hash functions to have two possible locations
in the hash table where an entry can be.
So, if a bucket is full, a new entry can push one of the items
in that bucket to its alternative location, making space for itself.
Advantages
~~~~
- Offers the option to store more entries when the target bucket is full
(unlike the previous implementation)
- Memory efficient: for storing those entries, it is not necessary to
request new memory, as the entries will be stored in the same table
- Constant worst lookup time: in worst case scenario, it always takes
the same time to look up an entry, as there are only two possible locations
where an entry can be.
- Storing data: user can store data in the hash table, unlike the
previous implementation, but he can still use the old API
This implementation tipically offers over 90% utilization.
Notice that API has been extended, but old API remains.
Check documentation included to know more about this new implementation
(including how entries are distributed as table utilization increases).
Changes in v5:
- Fix documentation
- Fix 32-bit compilation issues
Changes in v4:
- Unit tests enhancements are not part of this patchset anymore.
- rte_hash structure has been made internal in another patch,
so it is not part of this patchset anymore.
- Add function to iterate through the hash table, as rte_hash
structure has been made private.
- Added extra_flag parameter in rte_hash_parameter to be able
to add new parameters in the future without breaking the ABI
- Remove proposed lookup_bulk_with_hash function, as it is
not of much use with the existing hash functions
(there are no vector hash functions).
- User can store 8-byte integer or pointer as data, instead
of variable size data, as discussed in the mailing list.
Changes in v3:
- Now user can store variable size data, instead of 32 or 64-bit size data,
using the new parameter "data_len" in rte_hash_parameters
- Add lookup_bulk_with_hash function in performance unit tests
- Add new functions that handle data in performance unit tests
- Remove duplicates in performance unit tests
- Fix rte_hash_reset, which was not reseting the last entry
Changes in v2:
- Fixed issue where table could not store maximum number of entries
- Fixed issue where lookup burst could not be more than 32 (instead of 64)
- Remove unnecessary macros and add other useful ones
- Added missing library dependencies
- Used directly rte_hash_secondary instead of rte_hash_alt
- Renamed rte_hash.c to rte_cuckoo_hash.c to ease the view of the new library
- Renamed test_hash_perf.c temporarily to ease the view of the improved unit test
- Moved rte_hash, rte_bucket and rte_hash_key structures to rte_cuckoo_hash.c to
make them private
- Corrected copyright dates
- Added an optimized function to compare keys that are multiple of 16 bytes
- Improved the way to use primary/secondary signatures. Now both are stored in
the bucket, so there is no need to calculate them if required.
Also, there is no need to use the MSB of a signature to differenciate between
an empty entry and signature 0, since we are storing both signatures,
which cannot be both 0.
- Removed rte_hash_rehash, as it was a very expensive operation.
Therefore, the add function returns now -ENOSPC if key cannot be added
because of a loop.
- Prefetched new slot for new key in add function to improve performance.
- Made doxygen comments more clear.
- Removed unnecessary rte_hash_del_key_data and rte_hash_del_key_with_data,
as we can use the lookup functions if we want to get the data before deleting.
- Removed some unnecessary includes in rte_hash.h
- Removed some unnecessary variables in rte_cuckoo_hash.c
- Removed some unnecessary checks before creating a new hash table
- Added documentation (in release notes and programmers guide)
- Added new unit tests and replaced the performance one for hash tables
Pablo de Lara (7):
hash: replace existing hash library with cuckoo hash implementation
hash: add new function rte_hash_reset
hash: add new functionality to store data in hash table
hash: add iterate function
MAINTAINERS: claim responsability for hash library
doc: announce ABI change of librte_hash
doc: update hash documentation
MAINTAINERS | 1 +
app/test/test_hash.c | 189 +++---
app/test/test_hash_perf.c | 305 ++++++---
doc/guides/prog_guide/hash_lib.rst | 138 +++-
doc/guides/prog_guide/index.rst | 4 +
doc/guides/rel_notes/abi.rst | 1 +
lib/librte_hash/Makefile | 8 +-
lib/librte_hash/rte_cuckoo_hash.c | 1194 ++++++++++++++++++++++++++++++++++
lib/librte_hash/rte_hash.c | 499 --------------
lib/librte_hash/rte_hash.h | 198 +++++-
lib/librte_hash/rte_hash_version.map | 15 +
11 files changed, 1812 insertions(+), 740 deletions(-)
create mode 100644 lib/librte_hash/rte_cuckoo_hash.c
delete mode 100644 lib/librte_hash/rte_hash.c
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v5 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 " Pablo de Lara
@ 2015-07-10 21:57 ` Pablo de Lara
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 2/7] hash: add new function rte_hash_reset Pablo de Lara
` (7 subsequent siblings)
8 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 21:57 UTC (permalink / raw)
To: dev
This patch replaces the existing hash library with another approach,
using the Cuckoo Hash method to resolve collisions (open addressing),
which pushes items from a full bucket when a new entry tries
to be added in it, storing the evicted entry in an alternative location,
using a secondary hash function.
This gives the user the ability to store more entries when a bucket
is full, in comparison with the previous implementation.
Therefore, the unit test has been updated, as some scenarios have changed
(such as the previous removed restriction).
Also note that the API has not been changed, although new fields
have been added in the rte_hash structure (structure is internal now).
The main change when creating a new table is that the number of entries
per bucket is fixed now, so its parameter is ignored now
(still there to maintain the same parameters structure).
The hash unit test has been updated to reflect these changes.
As a last note, the maximum burst size in lookup_burst function
hash been increased to 64, to improve performance.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 119 ++---
app/test/test_hash_perf.c | 79 ++-
lib/librte_hash/Makefile | 8 +-
lib/librte_hash/rte_cuckoo_hash.c | 1027 +++++++++++++++++++++++++++++++++++++
lib/librte_hash/rte_hash.c | 499 ------------------
lib/librte_hash/rte_hash.h | 69 +--
6 files changed, 1138 insertions(+), 663 deletions(-)
create mode 100644 lib/librte_hash/rte_cuckoo_hash.c
delete mode 100644 lib/librte_hash/rte_hash.c
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index 7c71ed6..e70d859 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -169,7 +169,6 @@ static struct flow_key keys[5] = { {
/* Parameters used for hash table in unit test functions. Name set later. */
static struct rte_hash_parameters ut_params = {
.entries = 64,
- .bucket_entries = 4,
.key_len = sizeof(struct flow_key), /* 13 */
.hash_func = rte_jhash,
.hash_func_init_val = 0,
@@ -516,9 +515,18 @@ static int test_five_keys(void)
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != -ENOENT,
- "failed to find key (pos[%u]=%d)", i, pos[i]);
+ "found non-existent key (pos[%u]=%d)", i, pos[i]);
}
+ /* Lookup multi */
+ ret = rte_hash_lookup_multi(handle, &key_array[0], 5, (int32_t *)pos);
+ if (ret == 0)
+ for (i = 0; i < 5; i++) {
+ print_key_info("Lkp", key_array[i], pos[i]);
+ RETURN_IF_ERROR(pos[i] != -ENOENT,
+ "found not-existent key (pos[%u]=%d)", i, pos[i]);
+ }
+
rte_hash_free(handle);
return 0;
@@ -527,21 +535,18 @@ static int test_five_keys(void)
/*
* Add keys to the same bucket until bucket full.
* - add 5 keys to the same bucket (hash created with 4 keys per bucket):
- * first 4 successful, 5th unsuccessful
- * - lookup the 5 keys: 4 hits, 1 miss
- * - add the 5 keys again: 4 OK, one error as bucket is full
- * - lookup the 5 keys: 4 hits (updated data), 1 miss
- * - delete the 5 keys: 5 OK (even if the 5th is not in the table)
+ * first 4 successful, 5th successful, pushing existing item in bucket
+ * - lookup the 5 keys: 5 hits
+ * - add the 5 keys again: 5 OK
+ * - lookup the 5 keys: 5 hits (updated data)
+ * - delete the 5 keys: 5 OK
* - lookup the 5 keys: 5 misses
- * - add the 5th key: OK
- * - lookup the 5th key: hit
*/
static int test_full_bucket(void)
{
struct rte_hash_parameters params_pseudo_hash = {
.name = "test4",
.entries = 64,
- .bucket_entries = 4,
.key_len = sizeof(struct flow_key), /* 13 */
.hash_func = pseudo_hash,
.hash_func_init_val = 0,
@@ -555,7 +560,7 @@ static int test_full_bucket(void)
handle = rte_hash_create(¶ms_pseudo_hash);
RETURN_IF_ERROR(handle == NULL, "hash creation failed");
- /* Fill bucket*/
+ /* Fill bucket */
for (i = 0; i < 4; i++) {
pos[i] = rte_hash_add_key(handle, &keys[i]);
print_key_info("Add", &keys[i], pos[i]);
@@ -563,47 +568,39 @@ static int test_full_bucket(void)
"failed to add key (pos[%u]=%d)", i, pos[i]);
expected_pos[i] = pos[i];
}
- /* This shouldn't work because the bucket is full */
+ /*
+ * This should work and will push one of the items
+ * in the bucket because it is full
+ */
pos[4] = rte_hash_add_key(handle, &keys[4]);
print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOSPC,
- "fail: added key to full bucket (pos[4]=%d)", pos[4]);
+ RETURN_IF_ERROR(pos[4] < 0,
+ "failed to add key (pos[4]=%d)", pos[4]);
+ expected_pos[4] = pos[4];
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to find key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: found non-existent key (pos[4]=%d)", pos[4]);
/* Add - update */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_add_key(handle, &keys[i]);
print_key_info("Add", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to add key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_add_key(handle, &keys[4]);
- print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOSPC,
- "fail: added key to full bucket (pos[4]=%d)", pos[4]);
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to find key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: found non-existent key (pos[4]=%d)", pos[4]);
/* Delete 1 key, check other keys are still found */
pos[1] = rte_hash_del_key(handle, &keys[1]);
@@ -623,35 +620,21 @@ static int test_full_bucket(void)
RETURN_IF_ERROR(pos[1] < 0, "failed to add key (pos[1]=%d)", pos[1]);
/* Delete */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_del_key(handle, &keys[i]);
print_key_info("Del", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to delete key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_del_key(handle, &keys[4]);
- print_key_info("Del", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: deleted non-existent key (pos[4]=%d)", pos[4]);
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != -ENOENT,
"fail: found non-existent key (pos[%u]=%d)", i, pos[i]);
}
- /* Add and lookup the 5th key */
- pos[4] = rte_hash_add_key(handle, &keys[4]);
- print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] < 0, "failed to add key (pos[4]=%d)", pos[4]);
- expected_pos[4] = pos[4];
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != expected_pos[4],
- "failed to find key (pos[4]=%d)", pos[4]);
-
rte_hash_free(handle);
/* Cover the NULL case. */
@@ -991,6 +974,7 @@ static int test_fbk_hash_find_existing(void)
return 0;
}
+#define BUCKET_ENTRIES 4
/*
* Do tests for hash creation with bad parameters.
*/
@@ -1017,18 +1001,8 @@ static int test_hash_creation_with_bad_parameters(void)
}
memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_1";
- params.bucket_entries = RTE_HASH_BUCKET_ENTRIES_MAX + 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully with bucket_entries in parameter exceeded\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
params.name = "creation_with_bad_parameters_2";
- params.entries = params.bucket_entries - 1;
+ params.entries = BUCKET_ENTRIES - 1;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
rte_hash_free(handle);
@@ -1038,26 +1012,6 @@ static int test_hash_creation_with_bad_parameters(void)
memcpy(¶ms, &ut_params, sizeof(params));
params.name = "creation_with_bad_parameters_3";
- params.entries = params.entries - 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully if entries in parameter is not power of 2\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_4";
- params.bucket_entries = params.bucket_entries - 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully if bucket_entries in parameter is not power of 2\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_5";
params.key_len = 0;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
@@ -1067,17 +1021,7 @@ static int test_hash_creation_with_bad_parameters(void)
}
memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_6";
- params.key_len = RTE_HASH_KEY_LENGTH_MAX + 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully if key_len is greater than the maximum\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_7";
+ params.name = "creation_with_bad_parameters_4";
params.socket_id = RTE_MAX_NUMA_NODES + 1;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
@@ -1214,7 +1158,6 @@ static uint8_t key[16] = {0x00, 0x01, 0x02, 0x03,
static struct rte_hash_parameters hash_params_ex = {
.name = NULL,
.entries = 64,
- .bucket_entries = 4,
.key_len = 0,
.hash_func = NULL,
.hash_func_init_val = 0,
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index a3876c1..a54f3c1 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -162,7 +162,7 @@ shuffle_input_keys(unsigned table_index)
* ALL can fit in hash table (no errors)
*/
static int
-get_input_keys(unsigned table_index)
+get_input_keys(unsigned with_pushes, unsigned table_index)
{
unsigned i, j;
unsigned bucket_idx, incr, success = 1;
@@ -216,9 +216,14 @@ get_input_keys(unsigned table_index)
success = 0;
signatures[i] = rte_hash_hash(h[table_index], keys[i]);
bucket_idx = signatures[i] & bucket_bitmask;
- /* If bucket is full, do not try to insert the key */
- if (buckets[bucket_idx] == BUCKET_SIZE)
- continue;
+ /*
+ * If we are not inserting keys in secondary location,
+ * when bucket is full, do not try to insert the key
+ */
+ if (with_pushes == 0)
+ if (buckets[bucket_idx] == BUCKET_SIZE)
+ continue;
+
/* If key can be added, leave in successful key arrays "keys" */
ret = rte_hash_add_key_with_hash(h[table_index], keys[i],
signatures[i]);
@@ -388,9 +393,9 @@ reset_table(unsigned table_index)
}
static int
-run_all_tbl_perf_tests(void)
+run_all_tbl_perf_tests(unsigned with_pushes)
{
- unsigned i, j;
+ unsigned i, j, with_hash;
printf("Measuring performance, please wait");
fflush(stdout);
@@ -398,46 +403,32 @@ run_all_tbl_perf_tests(void)
if (create_table(i) < 0)
return -1;
- if (get_input_keys(i) < 0)
- return -1;
-
- if (timed_adds(0, i) < 0)
- return -1;
-
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
-
- if (timed_lookups(0, i) < 0)
- return -1;
-
- if (timed_lookups_multi(i) < 0)
- return -1;
-
- if (timed_deletes(0, i) < 0)
+ if (get_input_keys(with_pushes, i) < 0)
return -1;
+ for (with_hash = 0; with_hash <= 1; with_hash++) {
+ if (timed_adds(with_hash, i) < 0)
+ return -1;
- /* Print a dot to show progress on operations */
- printf(".");
- fflush(stdout);
+ for (j = 0; j < NUM_SHUFFLES; j++)
+ shuffle_input_keys(i);
- if (reset_table(i) < 0)
- return -1;
+ if (timed_lookups(with_hash, i) < 0)
+ return -1;
- if (timed_adds(1, i) < 0)
- return -1;
+ if (timed_lookups_multi(i) < 0)
+ return -1;
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
+ if (timed_deletes(with_hash, i) < 0)
+ return -1;
- if (timed_lookups(1, i) < 0)
- return -1;
+ if (reset_table(i) < 0)
+ return -1;
- if (timed_deletes(1, i) < 0)
- return -1;
+ /* Print a dot to show progress on operations */
+ printf(".");
+ fflush(stdout);
- /* Print a dot to show progress on operations */
- printf(".");
- fflush(stdout);
+ }
free_table(i);
}
@@ -551,8 +542,16 @@ fbk_hash_perf_test(void)
static int
test_hash_perf(void)
{
- if (run_all_tbl_perf_tests() < 0)
- return -1;
+ unsigned with_pushes;
+
+ for (with_pushes = 0; with_pushes <= 1; with_pushes++) {
+ if (with_pushes == 0)
+ printf("\nALL ELEMENTS IN PRIMARY LOCATION\n");
+ else
+ printf("\nELEMENTS IN PRIMARY OR SECONDARY LOCATION\n");
+ if (run_all_tbl_perf_tests(with_pushes) < 0)
+ return -1;
+ }
if (fbk_hash_perf_test() < 0)
return -1;
diff --git a/lib/librte_hash/Makefile b/lib/librte_hash/Makefile
index 981230b..039da46 100644
--- a/lib/librte_hash/Makefile
+++ b/lib/librte_hash/Makefile
@@ -1,6 +1,6 @@
# BSD LICENSE
#
-# Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+# Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@@ -42,7 +42,7 @@ EXPORT_MAP := rte_hash_version.map
LIBABIVER := 1
# all source are stored in SRCS-y
-SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_hash.c
+SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c
# install this header file
@@ -52,7 +52,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_thash.h
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h
-# this lib needs eal
-DEPDIRS-$(CONFIG_RTE_LIBRTE_HASH) += lib/librte_eal lib/librte_malloc
+# this lib needs eal and ring
+DEPDIRS-$(CONFIG_RTE_LIBRTE_HASH) += lib/librte_eal lib/librte_malloc lib/librte_ring
include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
new file mode 100644
index 0000000..50e3acd
--- /dev/null
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -0,0 +1,1027 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <sys/queue.h>
+
+#include <rte_common.h>
+#include <rte_memory.h> /* for definition of RTE_CACHE_LINE_SIZE */
+#include <rte_log.h>
+#include <rte_memcpy.h>
+#include <rte_prefetch.h>
+#include <rte_branch_prediction.h>
+#include <rte_memzone.h>
+#include <rte_malloc.h>
+#include <rte_eal.h>
+#include <rte_eal_memconfig.h>
+#include <rte_per_lcore.h>
+#include <rte_errno.h>
+#include <rte_string_fns.h>
+#include <rte_cpuflags.h>
+#include <rte_log.h>
+#include <rte_rwlock.h>
+#include <rte_spinlock.h>
+#include <rte_ring.h>
+
+#include "rte_hash.h"
+
+TAILQ_HEAD(rte_hash_list, rte_tailq_entry);
+
+static struct rte_tailq_elem rte_hash_tailq = {
+ .name = "RTE_HASH",
+};
+EAL_REGISTER_TAILQ(rte_hash_tailq)
+
+/* Macro to enable/disable run-time checking of function parameters */
+#if defined(RTE_LIBRTE_HASH_DEBUG)
+#define RETURN_IF_TRUE(cond, retval) do { \
+ if (cond) \
+ return retval; \
+} while (0)
+#else
+#define RETURN_IF_TRUE(cond, retval)
+#endif
+
+/* Hash function used if none is specified */
+#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
+#include <rte_hash_crc.h>
+#define DEFAULT_HASH_FUNC rte_hash_crc
+#else
+#include <rte_jhash.h>
+#define DEFAULT_HASH_FUNC rte_jhash
+#endif
+
+/** Number of items per bucket. */
+#define RTE_HASH_BUCKET_ENTRIES 4
+
+#define NULL_SIGNATURE 0
+
+typedef int (*rte_hash_cmp_eq_t)(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k48_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k64_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k80_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k96_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k112_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k128_cmp_eq(const void *key1, const void *key2, size_t key_len);
+
+/** A hash table structure. */
+struct rte_hash {
+ char name[RTE_HASH_NAMESIZE]; /**< Name of the hash. */
+ uint32_t entries; /**< Total table entries. */
+ uint32_t num_buckets; /**< Number of buckets in table. */
+ uint32_t key_len; /**< Length of hash key. */
+ rte_hash_function hash_func; /**< Function used to calculate hash. */
+ rte_hash_cmp_eq_t rte_hash_cmp_eq; /**< Function used to compare keys. */
+ uint32_t hash_func_init_val; /**< Init value used by hash_func. */
+ uint32_t bucket_bitmask; /**< Bitmask for getting bucket index
+ from hash signature. */
+ uint32_t key_entry_size; /**< Size of each key entry. */
+
+ struct rte_ring *free_slots; /**< Ring that stores all indexes
+ of the free slots in the key table */
+ void *key_store; /**< Table storing all keys and data */
+ struct rte_hash_bucket *buckets; /**< Table with buckets storing all the
+ hash values and key indexes
+ to the key table*/
+} __rte_cache_aligned;
+
+/* Structure storing both primary and secondary hashes */
+struct rte_hash_signatures {
+ union {
+ struct {
+ hash_sig_t current;
+ hash_sig_t alt;
+ };
+ uint64_t sig;
+ };
+};
+
+/** Bucket structure */
+struct rte_hash_bucket {
+ struct rte_hash_signatures signatures[RTE_HASH_BUCKET_ENTRIES];
+ /* Includes dummy key index that always contains index 0 */
+ uint32_t key_idx[RTE_HASH_BUCKET_ENTRIES + 1];
+ uint8_t flag[RTE_HASH_BUCKET_ENTRIES];
+} __rte_cache_aligned;
+
+struct rte_hash *
+rte_hash_find_existing(const char *name)
+{
+ struct rte_hash *h = NULL;
+ struct rte_tailq_entry *te;
+ struct rte_hash_list *hash_list;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK);
+ TAILQ_FOREACH(te, hash_list, next) {
+ h = (struct rte_hash *) te->data;
+ if (strncmp(name, h->name, RTE_HASH_NAMESIZE) == 0)
+ break;
+ }
+ rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ if (te == NULL) {
+ rte_errno = ENOENT;
+ return NULL;
+ }
+ return h;
+}
+
+struct rte_hash *
+rte_hash_create(const struct rte_hash_parameters *params)
+{
+ struct rte_hash *h = NULL;
+ struct rte_tailq_entry *te = NULL;
+ struct rte_hash_list *hash_list;
+ struct rte_ring *r = NULL;
+ char hash_name[RTE_HASH_NAMESIZE];
+ void *ptr, *k = NULL;
+ void *buckets = NULL;
+ char ring_name[RTE_RING_NAMESIZE];
+ unsigned i;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ if (params == NULL) {
+ RTE_LOG(ERR, HASH, "rte_hash_create has no parameters\n");
+ return NULL;
+ }
+
+ /* Check for valid parameters */
+ if ((params->entries > RTE_HASH_ENTRIES_MAX) ||
+ (params->entries < RTE_HASH_BUCKET_ENTRIES) ||
+ !rte_is_power_of_2(RTE_HASH_BUCKET_ENTRIES) ||
+ (params->key_len == 0)) {
+ rte_errno = EINVAL;
+ RTE_LOG(ERR, HASH, "rte_hash_create has invalid parameters\n");
+ return NULL;
+ }
+
+ snprintf(hash_name, sizeof(hash_name), "HT_%s", params->name);
+
+ /* Guarantee there's no existing */
+ h = rte_hash_find_existing(params->name);
+ if (h != NULL)
+ return h;
+
+ te = rte_zmalloc("HASH_TAILQ_ENTRY", sizeof(*te), 0);
+ if (te == NULL) {
+ RTE_LOG(ERR, HASH, "tailq entry allocation failed\n");
+ goto err;
+ }
+
+ h = (struct rte_hash *)rte_zmalloc_socket(hash_name, sizeof(struct rte_hash),
+ RTE_CACHE_LINE_SIZE, params->socket_id);
+
+ if (h == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ const uint32_t num_buckets = rte_align32pow2(params->entries)
+ / RTE_HASH_BUCKET_ENTRIES;
+
+ buckets = rte_zmalloc_socket(NULL,
+ num_buckets * sizeof(struct rte_hash_bucket),
+ RTE_CACHE_LINE_SIZE, params->socket_id);
+
+ if (buckets == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ const uint32_t key_entry_size = params->key_len;
+ /* Store all keys and leave the first entry as a dummy entry for lookup_bulk */
+ const uint64_t key_tbl_size = key_entry_size * (params->entries + 1);
+
+ k = rte_zmalloc_socket(NULL, key_tbl_size,
+ RTE_CACHE_LINE_SIZE, params->socket_id);
+
+ if (k == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ /* Select function to compare keys */
+ switch (params->key_len) {
+ case 16:
+ h->rte_hash_cmp_eq = rte_hash_k16_cmp_eq;
+ break;
+ case 32:
+ h->rte_hash_cmp_eq = rte_hash_k32_cmp_eq;
+ break;
+ case 48:
+ h->rte_hash_cmp_eq = rte_hash_k48_cmp_eq;
+ break;
+ case 64:
+ h->rte_hash_cmp_eq = rte_hash_k64_cmp_eq;
+ break;
+ case 80:
+ h->rte_hash_cmp_eq = rte_hash_k80_cmp_eq;
+ break;
+ case 96:
+ h->rte_hash_cmp_eq = rte_hash_k96_cmp_eq;
+ break;
+ case 112:
+ h->rte_hash_cmp_eq = rte_hash_k112_cmp_eq;
+ break;
+ case 128:
+ h->rte_hash_cmp_eq = rte_hash_k128_cmp_eq;
+ break;
+ default:
+ /* If key is not multiple of 16, use generic memcmp */
+ h->rte_hash_cmp_eq = memcmp;
+ }
+
+ snprintf(ring_name, sizeof(ring_name), "HT_%s", params->name);
+ r = rte_ring_lookup(ring_name);
+ if (r != NULL) {
+ /* clear the free ring */
+ while (rte_ring_dequeue(r, &ptr) == 0)
+ rte_pause();
+ } else
+ r = rte_ring_create(ring_name, rte_align32pow2(params->entries + 1),
+ params->socket_id, 0);
+ if (r == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ /* Setup hash context */
+ snprintf(h->name, sizeof(h->name), "%s", params->name);
+ h->entries = params->entries;
+ h->key_len = params->key_len;
+ h->key_entry_size = key_entry_size;
+ h->hash_func_init_val = params->hash_func_init_val;
+
+ h->num_buckets = num_buckets;
+ h->bucket_bitmask = h->num_buckets - 1;
+ h->buckets = buckets;
+ h->hash_func = (params->hash_func == NULL) ?
+ DEFAULT_HASH_FUNC : params->hash_func;
+
+ h->key_store = k;
+ h->free_slots = r;
+
+ /* populate the free slots ring. Entry zero is reserved for key misses */
+ for (i = 1; i < params->entries + 1; i++)
+ rte_ring_sp_enqueue(r, (void *)((uintptr_t) i));
+
+ rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
+ te->data = (void *) h;
+ TAILQ_INSERT_TAIL(hash_list, te, next);
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ return h;
+err:
+ rte_free(te);
+ rte_free(h);
+ rte_free(buckets);
+ rte_free(k);
+ return NULL;
+}
+
+void
+rte_hash_free(struct rte_hash *h)
+{
+ struct rte_tailq_entry *te;
+ struct rte_hash_list *hash_list;
+
+ if (h == NULL)
+ return;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
+
+ /* find out tailq entry */
+ TAILQ_FOREACH(te, hash_list, next) {
+ if (te->data == (void *) h)
+ break;
+ }
+
+ if (te == NULL) {
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+ return;
+ }
+
+ TAILQ_REMOVE(hash_list, te, next);
+
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ rte_free(h->key_store);
+ rte_free(h->buckets);
+ rte_free(h);
+ rte_free(te);
+}
+
+hash_sig_t
+rte_hash_hash(const struct rte_hash *h, const void *key)
+{
+ /* calc hash result by key */
+ return h->hash_func(key, h->key_len, h->hash_func_init_val);
+}
+
+/* Calc the secondary hash value from the primary hash value of a given key */
+static inline hash_sig_t
+rte_hash_secondary_hash(const hash_sig_t primary_hash)
+{
+ static const unsigned all_bits_shift = 12;
+ static const unsigned alt_bits_xor = 0x5bd1e995;
+
+ uint32_t tag = primary_hash >> all_bits_shift;
+
+ return (primary_hash ^ ((tag + 1) * alt_bits_xor));
+}
+
+/* Search for an entry that can be pushed to its alternative location */
+static inline int
+make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt)
+{
+ unsigned i, j;
+ int ret;
+ uint32_t next_bucket_idx;
+ struct rte_hash_bucket *next_bkt[RTE_HASH_BUCKET_ENTRIES];
+
+ /*
+ * Push existing item (search for bucket with space in
+ * alternative locations) to its alternative location
+ */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ /* Search for space in alternative locations */
+ next_bucket_idx = bkt->signatures[i].alt & h->bucket_bitmask;
+ next_bkt[i] = &h->buckets[next_bucket_idx];
+ for (j = 0; j < RTE_HASH_BUCKET_ENTRIES; j++) {
+ if (next_bkt[i]->signatures[j].sig == NULL_SIGNATURE)
+ break;
+ }
+
+ if (j != RTE_HASH_BUCKET_ENTRIES)
+ break;
+ }
+
+ /* Alternative location has spare room (end of recursive function) */
+ if (i != RTE_HASH_BUCKET_ENTRIES) {
+ next_bkt[i]->signatures[j].alt = bkt->signatures[i].current;
+ next_bkt[i]->signatures[j].current = bkt->signatures[i].alt;
+ next_bkt[i]->key_idx[j] = bkt->key_idx[i];
+ return i;
+ }
+
+ /* Pick entry that has not been pushed yet */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++)
+ if (bkt->flag[i] == 0)
+ break;
+
+ /* All entries have been pushed, so entry cannot be added */
+ if (i == RTE_HASH_BUCKET_ENTRIES) {
+ /* Reset flag */
+ bkt->flag[i] = 0;
+ return -ENOSPC;
+ }
+
+ /* Set flag to indicate that this entry is going to be pushed */
+ bkt->flag[i] = 1;
+ /* Need room in alternative bucket to insert the pushed entry */
+ ret = make_space_bucket(h, next_bkt[i]);
+ /*
+ * After recursive function.
+ * Clear flags and insert the pushed entry
+ * in its alternative location if successful,
+ * or return error
+ */
+ bkt->flag[i] = 0;
+ if (ret >= 0) {
+ next_bkt[i]->signatures[ret].alt = bkt->signatures[i].current;
+ next_bkt[i]->signatures[ret].current = bkt->signatures[i].alt;
+ next_bkt[i]->key_idx[ret] = bkt->key_idx[i];
+ return i;
+ } else
+ return ret;
+
+}
+
+static inline int32_t
+__rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ hash_sig_t alt_hash;
+ uint32_t prim_bucket_idx, sec_bucket_idx;
+ unsigned i;
+ struct rte_hash_bucket *prim_bkt, *sec_bkt;
+ void *new_k, *k, *keys = h->key_store;
+ void *slot_id;
+ uint32_t new_idx;
+ int ret;
+
+ prim_bucket_idx = sig & h->bucket_bitmask;
+ prim_bkt = &h->buckets[prim_bucket_idx];
+ rte_prefetch0(prim_bkt);
+
+ alt_hash = rte_hash_secondary_hash(sig);
+ sec_bucket_idx = alt_hash & h->bucket_bitmask;
+ sec_bkt = &h->buckets[sec_bucket_idx];
+ rte_prefetch0(sec_bkt);
+
+ /* Get a new slot for storing the new key */
+ if (rte_ring_sc_dequeue(h->free_slots, &slot_id) != 0)
+ return -ENOSPC;
+ new_k = RTE_PTR_ADD(keys, (uintptr_t)slot_id * h->key_entry_size);
+ rte_prefetch0(new_k);
+ new_idx = (uint32_t)((uintptr_t) slot_id);
+
+ /* Check if key is already inserted in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (prim_bkt->signatures[i].current == sig &&
+ prim_bkt->signatures[i].alt == alt_hash) {
+ k = (char *)keys + prim_bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (prim_bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Check if key is already inserted in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (sec_bkt->signatures[i].alt == sig &&
+ sec_bkt->signatures[i].current == alt_hash) {
+ k = (char *)keys + sec_bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (sec_bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Copy key */
+ rte_memcpy(new_k, key, h->key_len);
+
+ /* Insert new entry is there is room in the primary bucket */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ /* Check if slot is available */
+ if (likely(prim_bkt->signatures[i].sig == NULL_SIGNATURE)) {
+ prim_bkt->signatures[i].current = sig;
+ prim_bkt->signatures[i].alt = alt_hash;
+ prim_bkt->key_idx[i] = new_idx;
+ return new_idx - 1;
+ }
+ }
+
+ /* Primary bucket is full, so we need to make space for new entry */
+ ret = make_space_bucket(h, prim_bkt);
+ /*
+ * After recursive function.
+ * Insert the new entry in the position of the pushed entry
+ * if successful or return error and
+ * store the new slot back in the ring
+ */
+ if (ret >= 0) {
+ prim_bkt->signatures[ret].current = sig;
+ prim_bkt->signatures[ret].alt = alt_hash;
+ prim_bkt->key_idx[ret] = new_idx;
+ return (new_idx - 1);
+ }
+
+ /* Error in addition, store new slot back in the ring and return error */
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t) new_idx));
+ return ret;
+
+}
+
+int32_t
+rte_hash_add_key_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_add_key_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_add_key(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+static inline int32_t
+__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ uint32_t bucket_idx;
+ hash_sig_t alt_hash;
+ unsigned i;
+ struct rte_hash_bucket *bkt;
+ void *k, *keys = h->key_store;
+
+ bucket_idx = sig & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == sig &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+
+ /* Calculate secondary hash */
+ alt_hash = rte_hash_secondary_hash(sig);
+ bucket_idx = alt_hash & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == alt_hash &&
+ bkt->signatures[i].alt == sig) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+
+ return -ENOENT;
+}
+
+int32_t
+rte_hash_lookup_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_lookup(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+static inline int32_t
+__rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ uint32_t bucket_idx;
+ hash_sig_t alt_hash;
+ unsigned i;
+ struct rte_hash_bucket *bkt;
+ void *k, *keys = h->key_store;
+
+ bucket_idx = sig & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == sig &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ bkt->signatures[i].sig = NULL_SIGNATURE;
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t)bkt->key_idx[i]));
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Calculate secondary hash */
+ alt_hash = rte_hash_secondary_hash(sig);
+ bucket_idx = alt_hash & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == alt_hash &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ bkt->signatures[i].sig = NULL_SIGNATURE;
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t)bkt->key_idx[i]));
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ return -ENOENT;
+}
+
+int32_t
+rte_hash_del_key_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_del_key_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_del_key(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_del_key_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+/* Lookup bulk stage 0: Prefetch input key */
+static inline void
+lookup_stage0(unsigned *idx, uint64_t *lookup_mask,
+ const void * const *keys)
+{
+ *idx = __builtin_ctzl(*lookup_mask);
+ if (*lookup_mask == 0)
+ *idx = 0;
+
+ rte_prefetch0(keys[*idx]);
+ *lookup_mask &= ~(1llu << *idx);
+}
+
+/*
+ * Lookup bulk stage 1: Calculate primary/secondary hashes
+ * and prefetch primary/secondary buckets
+ */
+static inline void
+lookup_stage1(unsigned idx, hash_sig_t *prim_hash, hash_sig_t *sec_hash,
+ const struct rte_hash_bucket **primary_bkt,
+ const struct rte_hash_bucket **secondary_bkt,
+ hash_sig_t *hash_vals, const void * const *keys,
+ const struct rte_hash *h)
+{
+ *prim_hash = rte_hash_hash(h, keys[idx]);
+ hash_vals[idx] = *prim_hash;
+ *sec_hash = rte_hash_secondary_hash(*prim_hash);
+
+ *primary_bkt = &h->buckets[*prim_hash & h->bucket_bitmask];
+ *secondary_bkt = &h->buckets[*sec_hash & h->bucket_bitmask];
+
+ rte_prefetch0(*primary_bkt);
+ rte_prefetch0(*secondary_bkt);
+}
+
+/*
+ * Lookup bulk stage 2: Search for match hashes in primary/secondary locations
+ * and prefetch first key slot
+ */
+static inline void
+lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
+ const struct rte_hash_bucket *prim_bkt,
+ const struct rte_hash_bucket *sec_bkt,
+ const void **key_slot, int32_t *positions,
+ uint64_t *extra_hits_mask, const void *keys,
+ const struct rte_hash *h)
+{
+ unsigned prim_hash_matches, sec_hash_matches, key_idx, i;
+ unsigned total_hash_matches;
+
+ prim_hash_matches = 1 << RTE_HASH_BUCKET_ENTRIES;
+ sec_hash_matches = 1 << RTE_HASH_BUCKET_ENTRIES;
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ prim_hash_matches |= ((prim_hash == prim_bkt->signatures[i].current) << i);
+ sec_hash_matches |= ((sec_hash == sec_bkt->signatures[i].current) << i);
+ }
+
+ key_idx = prim_bkt->key_idx[__builtin_ctzl(prim_hash_matches)];
+ if (key_idx == 0)
+ key_idx = sec_bkt->key_idx[__builtin_ctzl(sec_hash_matches)];
+
+ total_hash_matches = (prim_hash_matches |
+ (sec_hash_matches << (RTE_HASH_BUCKET_ENTRIES + 1)));
+ *key_slot = (const char *)keys + key_idx * h->key_entry_size;
+
+ rte_prefetch0(*key_slot);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ positions[idx] = (key_idx - 1);
+
+ *extra_hits_mask |= (uint64_t)(__builtin_popcount(total_hash_matches) > 3) << idx;
+
+}
+
+
+/* Lookup bulk stage 3: Check if key matches, update hit mask */
+static inline void
+lookup_stage3(unsigned idx, const void *key_slot, const void * const *keys,
+ uint64_t *hits, const struct rte_hash *h)
+{
+ unsigned hit;
+
+ hit = !h->rte_hash_cmp_eq(key_slot, keys[idx], h->key_len);
+ *hits |= (uint64_t)(hit) << idx;
+}
+
+static inline void
+__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, int32_t *positions)
+{
+ uint64_t hits = 0;
+ uint64_t extra_hits_mask = 0;
+ uint64_t lookup_mask, miss_mask;
+ unsigned idx;
+ const void *key_store = h->key_store;
+ hash_sig_t hash_vals[RTE_HASH_LOOKUP_BULK_MAX];
+
+ unsigned idx00, idx01, idx10, idx11, idx20, idx21, idx30, idx31;
+ const struct rte_hash_bucket *primary_bkt10, *primary_bkt11;
+ const struct rte_hash_bucket *secondary_bkt10, *secondary_bkt11;
+ const struct rte_hash_bucket *primary_bkt20, *primary_bkt21;
+ const struct rte_hash_bucket *secondary_bkt20, *secondary_bkt21;
+ const void *k_slot20, *k_slot21, *k_slot30, *k_slot31;
+ hash_sig_t primary_hash10, primary_hash11;
+ hash_sig_t secondary_hash10, secondary_hash11;
+ hash_sig_t primary_hash20, primary_hash21;
+ hash_sig_t secondary_hash20, secondary_hash21;
+
+ lookup_mask = (uint64_t) -1 >> (64 - num_keys);
+ miss_mask = lookup_mask;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+
+ while (lookup_mask) {
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20,
+ primary_bkt20, secondary_bkt20, &k_slot20, positions,
+ &extra_hits_mask, key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21,
+ primary_bkt21, secondary_bkt21, &k_slot21, positions,
+ &extra_hits_mask, key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ }
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+
+ /* ignore any items we have already found */
+ extra_hits_mask &= ~hits;
+
+ if (unlikely(extra_hits_mask)) {
+ /* run a single search for each remaining item */
+ do {
+ idx = __builtin_ctzl(extra_hits_mask);
+ positions[idx] = rte_hash_lookup_with_hash(h, keys[idx],
+ hash_vals[idx]);
+ extra_hits_mask &= ~(1llu << idx);
+ if (positions[idx] >= 0)
+ hits |= 1llu << idx;
+ } while (extra_hits_mask);
+ }
+
+ miss_mask &= ~hits;
+ if (unlikely(miss_mask)) {
+ do {
+ idx = __builtin_ctzl(miss_mask);
+ positions[idx] = -ENOENT;
+ miss_mask &= ~(1llu << idx);
+ } while (miss_mask);
+ }
+}
+
+int
+rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, int32_t *positions)
+{
+ RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
+ (num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
+ (positions == NULL)), -EINVAL);
+
+ __rte_hash_lookup_bulk(h, keys, num_keys, positions);
+ return 0;
+}
+
+/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
+static int
+rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
+{
+ const __m128i k1 = _mm_loadu_si128((const __m128i *) key1);
+ const __m128i k2 = _mm_loadu_si128((const __m128i *) key2);
+ const __m128i x = _mm_xor_si128(k1, k2);
+
+ return !_mm_test_all_zeros(x, x);
+}
+
+static int
+rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k16_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 16,
+ (const char *) key2 + 16, key_len);
+}
+
+static int
+rte_hash_k48_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k16_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 16,
+ (const char *) key2 + 16, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 32,
+ (const char *) key2 + 32, key_len);
+}
+
+static int
+rte_hash_k64_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k32_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 32,
+ (const char *) key2 + 32, key_len);
+}
+
+static int
+rte_hash_k80_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
+
+static int
+rte_hash_k96_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
+
+static int
+rte_hash_k112_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 96,
+ (const char *) key2 + 96, key_len);
+}
+
+static int
+rte_hash_k128_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k64_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
diff --git a/lib/librte_hash/rte_hash.c b/lib/librte_hash/rte_hash.c
deleted file mode 100644
index 5100a75..0000000
--- a/lib/librte_hash/rte_hash.c
+++ /dev/null
@@ -1,499 +0,0 @@
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <string.h>
-#include <stdint.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <sys/queue.h>
-
-#include <rte_common.h>
-#include <rte_memory.h> /* for definition of RTE_CACHE_LINE_SIZE */
-#include <rte_log.h>
-#include <rte_memcpy.h>
-#include <rte_prefetch.h>
-#include <rte_branch_prediction.h>
-#include <rte_memzone.h>
-#include <rte_malloc.h>
-#include <rte_eal.h>
-#include <rte_eal_memconfig.h>
-#include <rte_per_lcore.h>
-#include <rte_errno.h>
-#include <rte_string_fns.h>
-#include <rte_cpuflags.h>
-#include <rte_log.h>
-#include <rte_rwlock.h>
-#include <rte_spinlock.h>
-
-#include "rte_hash.h"
-
-TAILQ_HEAD(rte_hash_list, rte_tailq_entry);
-
-static struct rte_tailq_elem rte_hash_tailq = {
- .name = "RTE_HASH",
-};
-EAL_REGISTER_TAILQ(rte_hash_tailq)
-
-/* Macro to enable/disable run-time checking of function parameters */
-#if defined(RTE_LIBRTE_HASH_DEBUG)
-#define RETURN_IF_TRUE(cond, retval) do { \
- if (cond) return (retval); \
-} while (0)
-#else
-#define RETURN_IF_TRUE(cond, retval)
-#endif
-
-/* Hash function used if none is specified */
-#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
-#include <rte_hash_crc.h>
-#define DEFAULT_HASH_FUNC rte_hash_crc
-#else
-#include <rte_jhash.h>
-#define DEFAULT_HASH_FUNC rte_jhash
-#endif
-
-/* Signature bucket size is a multiple of this value */
-#define SIG_BUCKET_ALIGNMENT 16
-
-/* Stoered key size is a multiple of this value */
-#define KEY_ALIGNMENT 16
-
-/* The high bit is always set in real signatures */
-#define NULL_SIGNATURE 0
-
-struct rte_hash {
- char name[RTE_HASH_NAMESIZE]; /**< Name of the hash. */
- uint32_t entries; /**< Total table entries. */
- uint32_t bucket_entries; /**< Bucket entries. */
- uint32_t key_len; /**< Length of hash key. */
- rte_hash_function hash_func; /**< Function used to calculate hash. */
- uint32_t hash_func_init_val; /**< Init value used by hash_func. */
- uint32_t num_buckets; /**< Number of buckets in table. */
- uint32_t bucket_bitmask; /**< Bitmask for getting bucket index
- from hash signature. */
- hash_sig_t sig_msb; /**< MSB is always set in valid signatures. */
- uint8_t *sig_tbl; /**< Flat array of hash signature buckets. */
- uint32_t sig_tbl_bucket_size; /**< Signature buckets may be padded for
- alignment reasons, and this is the
- bucket size used by sig_tbl. */
- uint8_t *key_tbl; /**< Flat array of key value buckets. */
- uint32_t key_tbl_key_size; /**< Keys may be padded for alignment
- reasons, and this is the key size
- used by key_tbl. */
-};
-
-/* Returns a pointer to the first signature in specified bucket. */
-static inline hash_sig_t *
-get_sig_tbl_bucket(const struct rte_hash *h, uint32_t bucket_index)
-{
- return RTE_PTR_ADD(h->sig_tbl, (bucket_index *
- h->sig_tbl_bucket_size));
-}
-
-/* Returns a pointer to the first key in specified bucket. */
-static inline uint8_t *
-get_key_tbl_bucket(const struct rte_hash *h, uint32_t bucket_index)
-{
- return RTE_PTR_ADD(h->key_tbl, (bucket_index * h->bucket_entries *
- h->key_tbl_key_size));
-}
-
-/* Returns a pointer to a key at a specific position in a specified bucket. */
-static inline void *
-get_key_from_bucket(const struct rte_hash *h, uint8_t *bkt, uint32_t pos)
-{
- return RTE_PTR_ADD(bkt, pos * h->key_tbl_key_size);
-}
-
-/* Does integer division with rounding-up of result. */
-static inline uint32_t
-div_roundup(uint32_t numerator, uint32_t denominator)
-{
- return (numerator + denominator - 1) / denominator;
-}
-
-/* Increases a size (if needed) to a multiple of alignment. */
-static inline uint32_t
-align_size(uint32_t val, uint32_t alignment)
-{
- return alignment * div_roundup(val, alignment);
-}
-
-/* Returns the index into the bucket of the first occurrence of a signature. */
-static inline int
-find_first(uint32_t sig, const uint32_t *sig_bucket, uint32_t num_sigs)
-{
- uint32_t i;
- for (i = 0; i < num_sigs; i++) {
- if (sig == sig_bucket[i])
- return i;
- }
- return -1;
-}
-
-struct rte_hash *
-rte_hash_find_existing(const char *name)
-{
- struct rte_hash *h = NULL;
- struct rte_tailq_entry *te;
- struct rte_hash_list *hash_list;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK);
- TAILQ_FOREACH(te, hash_list, next) {
- h = (struct rte_hash *) te->data;
- if (strncmp(name, h->name, RTE_HASH_NAMESIZE) == 0)
- break;
- }
- rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- if (te == NULL) {
- rte_errno = ENOENT;
- return NULL;
- }
- return h;
-}
-
-struct rte_hash *
-rte_hash_create(const struct rte_hash_parameters *params)
-{
- struct rte_hash *h = NULL;
- struct rte_tailq_entry *te;
- uint32_t num_buckets, sig_bucket_size, key_size,
- hash_tbl_size, sig_tbl_size, key_tbl_size, mem_size;
- char hash_name[RTE_HASH_NAMESIZE];
- struct rte_hash_list *hash_list;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- /* Check for valid parameters */
- if ((params == NULL) ||
- (params->entries > RTE_HASH_ENTRIES_MAX) ||
- (params->bucket_entries > RTE_HASH_BUCKET_ENTRIES_MAX) ||
- (params->entries < params->bucket_entries) ||
- !rte_is_power_of_2(params->entries) ||
- !rte_is_power_of_2(params->bucket_entries) ||
- (params->key_len == 0) ||
- (params->key_len > RTE_HASH_KEY_LENGTH_MAX)) {
- rte_errno = EINVAL;
- RTE_LOG(ERR, HASH, "rte_hash_create has invalid parameters\n");
- return NULL;
- }
-
- snprintf(hash_name, sizeof(hash_name), "HT_%s", params->name);
-
- /* Calculate hash dimensions */
- num_buckets = params->entries / params->bucket_entries;
- sig_bucket_size = align_size(params->bucket_entries *
- sizeof(hash_sig_t), SIG_BUCKET_ALIGNMENT);
- key_size = align_size(params->key_len, KEY_ALIGNMENT);
-
- hash_tbl_size = align_size(sizeof(struct rte_hash), RTE_CACHE_LINE_SIZE);
- sig_tbl_size = align_size(num_buckets * sig_bucket_size,
- RTE_CACHE_LINE_SIZE);
- key_tbl_size = align_size(num_buckets * key_size *
- params->bucket_entries, RTE_CACHE_LINE_SIZE);
-
- /* Total memory required for hash context */
- mem_size = hash_tbl_size + sig_tbl_size + key_tbl_size;
-
- rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
-
- /* guarantee there's no existing */
- TAILQ_FOREACH(te, hash_list, next) {
- h = (struct rte_hash *) te->data;
- if (strncmp(params->name, h->name, RTE_HASH_NAMESIZE) == 0)
- break;
- }
- if (te != NULL)
- goto exit;
-
- te = rte_zmalloc("HASH_TAILQ_ENTRY", sizeof(*te), 0);
- if (te == NULL) {
- RTE_LOG(ERR, HASH, "tailq entry allocation failed\n");
- goto exit;
- }
-
- h = (struct rte_hash *)rte_zmalloc_socket(hash_name, mem_size,
- RTE_CACHE_LINE_SIZE, params->socket_id);
- if (h == NULL) {
- RTE_LOG(ERR, HASH, "memory allocation failed\n");
- rte_free(te);
- goto exit;
- }
-
- /* Setup hash context */
- snprintf(h->name, sizeof(h->name), "%s", params->name);
- h->entries = params->entries;
- h->bucket_entries = params->bucket_entries;
- h->key_len = params->key_len;
- h->hash_func_init_val = params->hash_func_init_val;
- h->num_buckets = num_buckets;
- h->bucket_bitmask = h->num_buckets - 1;
- h->sig_msb = 1 << (sizeof(hash_sig_t) * 8 - 1);
- h->sig_tbl = (uint8_t *)h + hash_tbl_size;
- h->sig_tbl_bucket_size = sig_bucket_size;
- h->key_tbl = h->sig_tbl + sig_tbl_size;
- h->key_tbl_key_size = key_size;
- h->hash_func = (params->hash_func == NULL) ?
- DEFAULT_HASH_FUNC : params->hash_func;
-
- te->data = (void *) h;
-
- TAILQ_INSERT_TAIL(hash_list, te, next);
-
-exit:
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- return h;
-}
-
-void
-rte_hash_free(struct rte_hash *h)
-{
- struct rte_tailq_entry *te;
- struct rte_hash_list *hash_list;
-
- if (h == NULL)
- return;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
-
- /* find out tailq entry */
- TAILQ_FOREACH(te, hash_list, next) {
- if (te->data == (void *) h)
- break;
- }
-
- if (te == NULL) {
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
- return;
- }
-
- TAILQ_REMOVE(hash_list, te, next);
-
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- rte_free(h);
- rte_free(te);
-}
-
-hash_sig_t
-rte_hash_hash(const struct rte_hash *h, const void *key)
-{
- /* calc hash result by key */
- return h->hash_func(key, h->key_len, h->hash_func_init_val);
-}
-
-static inline int32_t
-__rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
- int32_t pos;
-
- /* Get the hash signature and bucket index */
- sig |= h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- /* Check if any free slot within the bucket to add the new key */
- pos = find_first(NULL_SIGNATURE, sig_bucket, h->bucket_entries);
-
- if (unlikely(pos < 0))
- return -ENOSPC;
-
- /* Add the new key to the bucket */
- sig_bucket[pos] = sig;
- rte_memcpy(get_key_from_bucket(h, key_bucket, pos), key, h->key_len);
- return bucket_index * h->bucket_entries + pos;
-}
-
-int32_t
-rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_add_key(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-static inline int32_t
-__rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
-
- /* Get the hash signature and bucket index */
- sig = sig | h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- sig_bucket[i] = NULL_SIGNATURE;
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- return -ENOENT;
-}
-
-int32_t
-rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_del_key_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_del_key(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_del_key_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-static inline int32_t
-__rte_hash_lookup_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
-
- /* Get the hash signature and bucket index */
- sig |= h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- return -ENOENT;
-}
-
-int32_t
-rte_hash_lookup_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_lookup(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-int
-rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
- uint32_t num_keys, int32_t *positions)
-{
- uint32_t i, j, bucket_index;
- hash_sig_t sigs[RTE_HASH_LOOKUP_BULK_MAX];
-
- RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
- (num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
- (positions == NULL)), -EINVAL);
-
- /* Get the hash signature and bucket index */
- for (i = 0; i < num_keys; i++) {
- sigs[i] = h->hash_func(keys[i], h->key_len,
- h->hash_func_init_val) | h->sig_msb;
- bucket_index = sigs[i] & h->bucket_bitmask;
-
- /* Pre-fetch relevant buckets */
- rte_prefetch1((void *) get_sig_tbl_bucket(h, bucket_index));
- rte_prefetch1((void *) get_key_tbl_bucket(h, bucket_index));
- }
-
- /* Check if key is already present in the hash */
- for (i = 0; i < num_keys; i++) {
- bucket_index = sigs[i] & h->bucket_bitmask;
- hash_sig_t *sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- uint8_t *key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- positions[i] = -ENOENT;
-
- for (j = 0; j < h->bucket_entries; j++) {
- if ((sigs[i] == sig_bucket[j]) &&
- likely(memcmp(keys[i],
- get_key_from_bucket(h, key_bucket, j),
- h->key_len) == 0)) {
- positions[i] = bucket_index *
- h->bucket_entries + j;
- break;
- }
- }
- }
-
- return 0;
-}
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index da0a00a..936d170 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -47,21 +47,21 @@ extern "C" {
#endif
/** Maximum size of hash table that can be created. */
-#define RTE_HASH_ENTRIES_MAX (1 << 26)
+#define RTE_HASH_ENTRIES_MAX (1 << 30)
-/** Maximum bucket size that can be created. */
-#define RTE_HASH_BUCKET_ENTRIES_MAX 16
+/** @deprecated Maximum bucket size that can be created. */
+#define RTE_HASH_BUCKET_ENTRIES_MAX 4
-/** Maximum length of key that can be used. */
+/** @deprecated Maximum length of key that can be used. */
#define RTE_HASH_KEY_LENGTH_MAX 64
-/** Max number of keys that can be searched for using rte_hash_lookup_multi. */
-#define RTE_HASH_LOOKUP_BULK_MAX 16
-#define RTE_HASH_LOOKUP_MULTI_MAX RTE_HASH_LOOKUP_BULK_MAX
-
-/** Max number of characters in hash name.*/
+/** Maximum number of characters in hash name.*/
#define RTE_HASH_NAMESIZE 32
+/** Maximum number of keys that can be searched for using rte_hash_lookup_bulk. */
+#define RTE_HASH_LOOKUP_BULK_MAX 64
+#define RTE_HASH_LOOKUP_MULTI_MAX RTE_HASH_LOOKUP_BULK_MAX
+
/** Signature of key that is stored internally. */
typedef uint32_t hash_sig_t;
@@ -70,15 +70,14 @@ typedef uint32_t (*rte_hash_function)(const void *key, uint32_t key_len,
uint32_t init_val);
/**
- * Parameters used when creating the hash table. The total table entries and
- * bucket entries must be a power of 2.
+ * Parameters used when creating the hash table.
*/
struct rte_hash_parameters {
const char *name; /**< Name of the hash. */
uint32_t entries; /**< Total hash table entries. */
- uint32_t bucket_entries; /**< Bucket entries. */
+ uint32_t bucket_entries; /**< Bucket entries. */
uint32_t key_len; /**< Length of hash key. */
- rte_hash_function hash_func; /**< Function used to calculate hash. */
+ rte_hash_function hash_func; /**< Primary Hash function used to calculate hash. */
uint32_t hash_func_init_val; /**< Init value used by hash_func. */
int socket_id; /**< NUMA Socket ID for memory. */
};
@@ -86,6 +85,7 @@ struct rte_hash_parameters {
/** @internal A hash table structure. */
struct rte_hash;
+
/**
* Create a new hash table.
*
@@ -106,7 +106,6 @@ struct rte_hash;
struct rte_hash *
rte_hash_create(const struct rte_hash_parameters *params);
-
/**
* Find an existing hash table object and return a pointer to it.
*
@@ -129,7 +128,8 @@ void
rte_hash_free(struct rte_hash *h);
/**
- * Add a key to an existing hash table. This operation is not multi-thread safe
+ * Add a key to an existing hash table.
+ * This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
@@ -146,7 +146,8 @@ int32_t
rte_hash_add_key(const struct rte_hash *h, const void *key);
/**
- * Add a key to an existing hash table. This operation is not multi-thread safe
+ * Add a key to an existing hash table.
+ * This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
@@ -154,7 +155,7 @@ rte_hash_add_key(const struct rte_hash *h, const void *key);
* @param key
* Key to add to the hash table.
* @param sig
- * Hash value to add to the hash table.
+ * Precomputed hash value for 'key'.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOSPC if there is no space in the hash for this key.
@@ -162,12 +163,12 @@ rte_hash_add_key(const struct rte_hash *h, const void *key);
* array of user data. This value is unique for this key.
*/
int32_t
-rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig);
+rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
/**
- * Remove a key from an existing hash table. This operation is not multi-thread
- * safe and should only be called from one thread.
+ * Remove a key from an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
*
* @param h
* Hash table to remove the key from.
@@ -184,15 +185,16 @@ int32_t
rte_hash_del_key(const struct rte_hash *h, const void *key);
/**
- * Remove a key from an existing hash table. This operation is not multi-thread
- * safe and should only be called from one thread.
+ * Remove a key from an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
*
* @param h
* Hash table to remove the key from.
* @param key
* Key to remove from the hash table.
* @param sig
- * Hash value to remove from the hash table.
+ * Precomputed hash value for 'key'.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOENT if the key is not found.
@@ -201,12 +203,11 @@ rte_hash_del_key(const struct rte_hash *h, const void *key);
* value that was returned when the key was added.
*/
int32_t
-rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig);
-
+rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
/**
- * Find a key in the hash table. This operation is multi-thread safe.
+ * Find a key in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
@@ -223,14 +224,15 @@ int32_t
rte_hash_lookup(const struct rte_hash *h, const void *key);
/**
- * Find a key in the hash table. This operation is multi-thread safe.
+ * Find a key in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
* @param key
* Key to find.
* @param sig
- * Hash value to find.
+ * Hash value to remove from the hash table.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOENT if the key is not found.
@@ -243,7 +245,8 @@ rte_hash_lookup_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig);
/**
- * Calc a hash value by key. This operation is not multi-process safe.
+ * Calc a hash value by key.
+ * This operation is not multi-thread safe.
*
* @param h
* Hash table to look in.
@@ -257,7 +260,8 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
#define rte_hash_lookup_multi rte_hash_lookup_bulk
/**
- * Find multiple keys in the hash table. This operation is multi-thread safe.
+ * Find multiple keys in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
@@ -277,6 +281,7 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions);
+
#ifdef __cplusplus
}
#endif
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v5 2/7] hash: add new function rte_hash_reset
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 " Pablo de Lara
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 1/7] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
@ 2015-07-10 21:57 ` Pablo de Lara
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 3/7] hash: add new functionality to store data in hash table Pablo de Lara
` (6 subsequent siblings)
8 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 21:57 UTC (permalink / raw)
To: dev
Added reset function to be able to empty the table,
without having to destroy and create it again.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 4 +---
app/test/test_hash_perf.c | 12 +++---------
lib/librte_hash/rte_cuckoo_hash.c | 21 +++++++++++++++++++++
lib/librte_hash/rte_hash.h | 11 +++++++++--
lib/librte_hash/rte_hash_version.map | 8 ++++++++
5 files changed, 42 insertions(+), 14 deletions(-)
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index e70d859..448586c 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -1132,9 +1132,7 @@ static int test_average_table_utilization(void)
average_keys_added += added_keys;
/* Reset the table */
- rte_hash_free(handle);
- handle = rte_hash_create(&ut_params);
- RETURN_IF_ERROR(handle == NULL, "hash creation failed");
+ rte_hash_reset(handle);
/* Print a dot to show progress on operations */
printf(".");
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index a54f3c1..b01b040 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -382,14 +382,10 @@ free_table(unsigned table_index)
rte_hash_free(h[table_index]);
}
-static int
+static void
reset_table(unsigned table_index)
{
- free_table(table_index);
- if (create_table(table_index) != 0)
- return -1;
-
- return 0;
+ rte_hash_reset(h[table_index]);
}
static int
@@ -421,13 +417,11 @@ run_all_tbl_perf_tests(unsigned with_pushes)
if (timed_deletes(with_hash, i) < 0)
return -1;
- if (reset_table(i) < 0)
- return -1;
+ reset_table(i);
/* Print a dot to show progress on operations */
printf(".");
fflush(stdout);
-
}
free_table(i);
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index 50e3acd..15c5da5 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -371,6 +371,27 @@ rte_hash_secondary_hash(const hash_sig_t primary_hash)
return (primary_hash ^ ((tag + 1) * alt_bits_xor));
}
+void
+rte_hash_reset(struct rte_hash *h)
+{
+ void *ptr;
+ unsigned i;
+
+ if (h == NULL)
+ return;
+
+ memset(h->buckets, 0, h->num_buckets * sizeof(struct rte_hash_bucket));
+ memset(h->key_store, 0, h->key_entry_size * (h->entries + 1));
+
+ /* clear the free ring */
+ while (rte_ring_dequeue(h->free_slots, &ptr) == 0)
+ rte_pause();
+
+ /* Repopulate the free slots ring. Entry zero is reserved for key misses */
+ for (i = 1; i < h->entries + 1; i++)
+ rte_ring_sp_enqueue(h->free_slots, (void *)((uintptr_t) i));
+}
+
/* Search for an entry that can be pushed to its alternative location */
static inline int
make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt)
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index 936d170..8bbc9f0 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -128,8 +128,15 @@ void
rte_hash_free(struct rte_hash *h);
/**
- * Add a key to an existing hash table.
- * This operation is not multi-thread safe
+ * Reset all hash structure, by zeroing all entries
+ * @param h
+ * Hash table to reset
+ */
+void
+rte_hash_reset(struct rte_hash *h);
+
+/**
+ * Add a key to an existing hash table. This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index 94a0fec..d5f5af5 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -18,3 +18,11 @@ DPDK_2.0 {
local: *;
};
+
+DPDK_2.1 {
+ global:
+
+ rte_hash_reset;
+
+ local: *;
+} DPDK_2.0;
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v5 3/7] hash: add new functionality to store data in hash table
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 " Pablo de Lara
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 1/7] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 2/7] hash: add new function rte_hash_reset Pablo de Lara
@ 2015-07-10 21:57 ` Pablo de Lara
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 4/7] hash: add iterate function Pablo de Lara
` (5 subsequent siblings)
8 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 21:57 UTC (permalink / raw)
To: dev
Usually hash tables not only store keys, but also data associated
to them. In order to maintain the existing API, the old functions
will still return the index where the key was stored.
The new functions will return the data associated to that key.
In the case of the lookup_bulk function, it will return also
the number of entries found and a bitmask of which entries
were found.
Unit tests have been updated to use these new functions.
As a final point, a flag has been added in rte_hash_parameters
to indicate if there are new parameters for future versions,
so there is no need to maintain multiple versions
of the existing functions in the future.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash_perf.c | 254 +++++++++++++++++++++++++----------
lib/librte_hash/rte_cuckoo_hash.c | 193 ++++++++++++++++++++------
lib/librte_hash/rte_hash.h | 108 ++++++++++++++-
lib/librte_hash/rte_hash_version.map | 6 +
4 files changed, 441 insertions(+), 120 deletions(-)
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index b01b040..e09f45d 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -55,6 +55,7 @@
#define NUM_KEYSIZES 10
#define NUM_SHUFFLES 10
#define BURST_SIZE 16
+#define DATA_MULTIPLIER 13 /* Used for storing data */
enum operations {
ADD = 0,
@@ -83,7 +84,7 @@ struct rte_hash *h[NUM_KEYSIZES];
uint8_t slot_taken[MAX_ENTRIES];
/* Array to store number of cycles per operation */
-uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS][2];
+uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS][2][2];
/* Array to store all input keys */
uint8_t keys[KEYS_TO_ADD][MAX_KEYSIZE];
@@ -106,11 +107,16 @@ static struct rte_hash_parameters ut_params = {
};
static int
-create_table(unsigned table_index)
+create_table(unsigned with_data, unsigned table_index)
{
char name[RTE_HASH_NAMESIZE];
- sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
+ if (with_data)
+ /* Table will store 8-byte data */
+ sprintf(name, "test_hash%d_data", hashtest_key_lens[table_index]);
+ else
+ sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
+
ut_params.name = name;
ut_params.key_len = hashtest_key_lens[table_index];
ut_params.socket_id = rte_socket_id();
@@ -234,6 +240,7 @@ get_input_keys(unsigned with_pushes, unsigned table_index)
else {
/* Store the returned position and mark slot as taken */
slot_taken[ret] = 1;
+ positions[i] = ret;
buckets[bucket_idx]++;
success = 1;
i++;
@@ -249,54 +256,116 @@ get_input_keys(unsigned with_pushes, unsigned table_index)
}
static int
-timed_adds(unsigned with_hash, unsigned table_index)
+timed_adds(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i;
const uint64_t start_tsc = rte_rdtsc();
+ uintptr_t data;
int32_t ret;
for (i = 0; i < KEYS_TO_ADD; i++) {
- if (with_hash)
+ data = (uint64_t) signatures[i] * DATA_MULTIPLIER;
+ if (with_hash && with_data) {
+ ret = rte_hash_add_key_with_hash_data(h[table_index],
+ (const void *) keys[i],
+ signatures[i], data);
+ if (ret < 0) {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
+ } else if (with_hash && !with_data) {
ret = rte_hash_add_key_with_hash(h[table_index],
(const void *) keys[i],
signatures[i]);
- else
+ if (ret >= 0)
+ positions[i] = ret;
+ else {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
+ } else if (!with_hash && with_data) {
+ ret = rte_hash_add_key_data(h[table_index],
+ (const void *) keys[i],
+ data);
+ if (ret < 0) {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
+ } else {
ret = rte_hash_add_key(h[table_index], keys[i]);
-
- if (ret >= 0)
- positions[i] = ret;
- else {
- printf("Failed to add key number %u\n", ret);
- return -1;
+ if (ret >= 0)
+ positions[i] = ret;
+ else {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
}
}
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][ADD][with_hash] = time_taken/KEYS_TO_ADD;
+ cycles[table_index][ADD][with_hash][with_data] = time_taken/KEYS_TO_ADD;
+
return 0;
}
static int
-timed_lookups(unsigned with_hash, unsigned table_index)
+timed_lookups(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i, j;
const uint64_t start_tsc = rte_rdtsc();
+ uintptr_t ret_data;
+ uintptr_t expected_data;
int32_t ret;
for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
for (j = 0; j < KEYS_TO_ADD; j++) {
- if (with_hash)
+ if (with_hash && with_data) {
+ ret = rte_hash_lookup_with_hash_data(h[table_index],
+ (const void *) keys[j],
+ signatures[j], &ret_data);
+ if (ret < 0) {
+ printf("Key number %u was not found\n", j);
+ return -1;
+ }
+ expected_data = (uint64_t) signatures[j] * DATA_MULTIPLIER;
+ if (ret_data != expected_data) {
+ printf("Data returned for key number %u is %"PRIxPTR","
+ " but should be %"PRIxPTR"\n", j, ret_data,
+ expected_data);
+ return -1;
+ }
+ } else if (with_hash && !with_data) {
ret = rte_hash_lookup_with_hash(h[table_index],
(const void *) keys[j],
signatures[j]);
- else
+ if (ret < 0 || ret != positions[j]) {
+ printf("Key looked up in %d, should be in %d\n",
+ ret, positions[j]);
+ return -1;
+ }
+ } else if (!with_hash && with_data) {
+ ret = rte_hash_lookup_data(h[table_index],
+ (const void *) keys[j], &ret_data);
+ if (ret < 0) {
+ printf("Key number %u was not found\n", j);
+ return -1;
+ }
+ expected_data = (uint64_t) signatures[j] * DATA_MULTIPLIER;
+ if (ret_data != expected_data) {
+ printf("Data returned for key number %u is %"PRIxPTR","
+ " but should be %"PRIxPTR"\n", j, ret_data,
+ expected_data);
+ return -1;
+ }
+ } else {
ret = rte_hash_lookup(h[table_index], keys[j]);
- if (ret < 0 || ret != positions[j]) {
- printf("Key looked up in %d, should be in %d\n",
- ret, positions[j]);
- return -1;
+ if (ret < 0 || ret != positions[j]) {
+ printf("Key looked up in %d, should be in %d\n",
+ ret, positions[j]);
+ return -1;
+ }
}
}
}
@@ -304,34 +373,66 @@ timed_lookups(unsigned with_hash, unsigned table_index)
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][LOOKUP][with_hash] = time_taken/NUM_LOOKUPS;
+ cycles[table_index][LOOKUP][with_hash][with_data] = time_taken/NUM_LOOKUPS;
return 0;
}
static int
-timed_lookups_multi(unsigned table_index)
+timed_lookups_multi(unsigned with_data, unsigned table_index)
{
unsigned i, j, k;
int32_t positions_burst[BURST_SIZE];
const void *keys_burst[BURST_SIZE];
+ uintptr_t expected_data[BURST_SIZE];
+ uintptr_t ret_data[BURST_SIZE];
+ uint64_t hit_mask;
+ int ret;
+
const uint64_t start_tsc = rte_rdtsc();
for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
for (j = 0; j < KEYS_TO_ADD/BURST_SIZE; j++) {
for (k = 0; k < BURST_SIZE; k++)
keys_burst[k] = keys[j * BURST_SIZE + k];
-
- rte_hash_lookup_bulk(h[table_index],
+ if (with_data) {
+ ret = rte_hash_lookup_bulk_data(h[table_index],
+ (const void **) keys_burst,
+ BURST_SIZE,
+ &hit_mask,
+ ret_data);
+ if (ret != BURST_SIZE) {
+ printf("Expect to find %u keys,"
+ " but found %d\n", BURST_SIZE, ret);
+ return -1;
+ }
+ for (k = 0; k < BURST_SIZE; k++) {
+ if ((hit_mask & (1ULL << k)) == 0) {
+ printf("Key number %u not found\n",
+ j * BURST_SIZE + k);
+ return -1;
+ }
+ expected_data[k] = (uint64_t)signatures[j * BURST_SIZE + k]
+ * DATA_MULTIPLIER;
+ if (ret_data[k] != expected_data[k]) {
+ printf("Data returned for key number %u is %"PRIxPTR","
+ " but should be %"PRIxPTR"\n", j * BURST_SIZE + k,
+ ret_data[k], expected_data[k]);
+ return -1;
+ }
+ }
+ } else {
+ rte_hash_lookup_bulk(h[table_index],
(const void **) keys_burst,
BURST_SIZE,
positions_burst);
- for (k = 0; k < BURST_SIZE; k++) {
- if (positions_burst[k] != positions[j * BURST_SIZE + k]) {
- printf("Key looked up in %d, should be in %d\n",
- positions_burst[k],
- positions[j * BURST_SIZE + k]);
- return -1;
+ for (k = 0; k < BURST_SIZE; k++) {
+ if (positions_burst[k] != positions[j * BURST_SIZE + k]) {
+ printf("Key looked up in %d, should be in %d\n",
+ positions_burst[k],
+ positions[j * BURST_SIZE + k]);
+ return -1;
+ }
}
}
}
@@ -340,19 +441,20 @@ timed_lookups_multi(unsigned table_index)
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][LOOKUP_MULTI][0] = time_taken/NUM_LOOKUPS;
+ cycles[table_index][LOOKUP_MULTI][0][with_data] = time_taken/NUM_LOOKUPS;
return 0;
}
static int
-timed_deletes(unsigned with_hash, unsigned table_index)
+timed_deletes(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i;
const uint64_t start_tsc = rte_rdtsc();
int32_t ret;
for (i = 0; i < KEYS_TO_ADD; i++) {
+ /* There are no delete functions with data, so just call two functions */
if (with_hash)
ret = rte_hash_del_key_with_hash(h[table_index],
(const void *) keys[i],
@@ -371,7 +473,7 @@ timed_deletes(unsigned with_hash, unsigned table_index)
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][DELETE][with_hash] = time_taken/KEYS_TO_ADD;
+ cycles[table_index][DELETE][with_hash][with_data] = time_taken/KEYS_TO_ADD;
return 0;
}
@@ -391,62 +493,67 @@ reset_table(unsigned table_index)
static int
run_all_tbl_perf_tests(unsigned with_pushes)
{
- unsigned i, j, with_hash;
+ unsigned i, j, with_data, with_hash;
printf("Measuring performance, please wait");
fflush(stdout);
- for (i = 0; i < NUM_KEYSIZES; i++) {
- if (create_table(i) < 0)
- return -1;
- if (get_input_keys(with_pushes, i) < 0)
- return -1;
- for (with_hash = 0; with_hash <= 1; with_hash++) {
- if (timed_adds(with_hash, i) < 0)
+ for (with_data = 0; with_data <= 1; with_data++) {
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ if (create_table(with_data, i) < 0)
return -1;
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
-
- if (timed_lookups(with_hash, i) < 0)
+ if (get_input_keys(with_pushes, i) < 0)
return -1;
+ for (with_hash = 0; with_hash <= 1; with_hash++) {
+ if (timed_adds(with_hash, with_data, i) < 0)
+ return -1;
- if (timed_lookups_multi(i) < 0)
- return -1;
+ for (j = 0; j < NUM_SHUFFLES; j++)
+ shuffle_input_keys(i);
- if (timed_deletes(with_hash, i) < 0)
- return -1;
+ if (timed_lookups(with_hash, with_data, i) < 0)
+ return -1;
- reset_table(i);
+ if (timed_lookups_multi(with_data, i) < 0)
+ return -1;
- /* Print a dot to show progress on operations */
- printf(".");
- fflush(stdout);
- }
+ if (timed_deletes(with_hash, with_data, i) < 0)
+ return -1;
- free_table(i);
+ /* Print a dot to show progress on operations */
+ printf(".");
+ fflush(stdout);
+
+ reset_table(i);
+ }
+ free_table(i);
+ }
}
+
printf("\nResults (in CPU cycles/operation)\n");
- printf("---------------------------------\n");
- printf("\nWithout pre-computed hash values\n");
- printf("\n%-18s%-18s%-18s%-18s%-18s\n",
- "Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
- for (i = 0; i < NUM_KEYSIZES; i++) {
- printf("%-18d", hashtest_key_lens[i]);
- for (j = 0; j < NUM_OPERATIONS; j++)
- printf("%-18"PRIu64, cycles[i][j][0]);
- printf("\n");
- }
- printf("\nWith pre-computed hash values\n");
- printf("\n%-18s%-18s%-18s%-18s%-18s\n",
+ printf("-----------------------------------\n");
+ for (with_data = 0; with_data <= 1; with_data++) {
+ if (with_data)
+ printf("\n Operations with 8-byte data\n");
+ else
+ printf("\n Operations without data\n");
+ for (with_hash = 0; with_hash <= 1; with_hash++) {
+ if (with_hash)
+ printf("\nWith pre-computed hash values\n");
+ else
+ printf("\nWithout pre-computed hash values\n");
+
+ printf("\n%-18s%-18s%-18s%-18s%-18s\n",
"Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
- for (i = 0; i < NUM_KEYSIZES; i++) {
- printf("%-18d", hashtest_key_lens[i]);
- for (j = 0; j < NUM_OPERATIONS; j++)
- printf("%-18"PRIu64, cycles[i][j][1]);
- printf("\n");
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ printf("%-18d", hashtest_key_lens[i]);
+ for (j = 0; j < NUM_OPERATIONS; j++)
+ printf("%-18"PRIu64, cycles[i][j][with_hash][with_data]);
+ printf("\n");
+ }
+ }
}
-
return 0;
}
@@ -546,7 +653,6 @@ test_hash_perf(void)
if (run_all_tbl_perf_tests(with_pushes) < 0)
return -1;
}
-
if (fbk_hash_perf_test() < 0)
return -1;
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index 15c5da5..5777036 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -56,6 +56,7 @@
#include <rte_rwlock.h>
#include <rte_spinlock.h>
#include <rte_ring.h>
+#include <rte_compat.h>
#include "rte_hash.h"
@@ -90,6 +91,8 @@ EAL_REGISTER_TAILQ(rte_hash_tailq)
#define NULL_SIGNATURE 0
+#define KEY_ALIGNMENT 16
+
typedef int (*rte_hash_cmp_eq_t)(const void *key1, const void *key2, size_t key_len);
static int rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len);
static int rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len);
@@ -132,6 +135,16 @@ struct rte_hash_signatures {
};
};
+/* Structure that stores key-value pair */
+struct rte_hash_key {
+ union {
+ uintptr_t idata;
+ void *pdata;
+ };
+ /* Variable key size */
+ char key[];
+} __attribute__((aligned(KEY_ALIGNMENT)));
+
/** Bucket structure */
struct rte_hash_bucket {
struct rte_hash_signatures signatures[RTE_HASH_BUCKET_ENTRIES];
@@ -227,7 +240,8 @@ rte_hash_create(const struct rte_hash_parameters *params)
goto err;
}
- const uint32_t key_entry_size = params->key_len;
+ const uint32_t key_entry_size = sizeof(struct rte_hash_key) + params->key_len;
+
/* Store all keys and leave the first entry as a dummy entry for lookup_bulk */
const uint64_t key_tbl_size = key_entry_size * (params->entries + 1);
@@ -461,13 +475,13 @@ make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt)
static inline int32_t
__rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
- hash_sig_t sig)
+ hash_sig_t sig, uintptr_t data)
{
hash_sig_t alt_hash;
uint32_t prim_bucket_idx, sec_bucket_idx;
unsigned i;
struct rte_hash_bucket *prim_bkt, *sec_bkt;
- void *new_k, *k, *keys = h->key_store;
+ struct rte_hash_key *new_k, *k, *keys = h->key_store;
void *slot_id;
uint32_t new_idx;
int ret;
@@ -492,9 +506,12 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (prim_bkt->signatures[i].current == sig &&
prim_bkt->signatures[i].alt == alt_hash) {
- k = (char *)keys + prim_bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ prim_bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /* Update data */
+ k->idata = data;
/*
* Return index where key is stored,
* substracting the first dummy index
@@ -508,9 +525,12 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (sec_bkt->signatures[i].alt == sig &&
sec_bkt->signatures[i].current == alt_hash) {
- k = (char *)keys + sec_bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ sec_bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /* Update data */
+ k->idata = data;
/*
* Return index where key is stored,
* substracting the first dummy index
@@ -521,7 +541,8 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
}
/* Copy key */
- rte_memcpy(new_k, key, h->key_len);
+ rte_memcpy(new_k->key, key, h->key_len);
+ new_k->idata = data;
/* Insert new entry is there is room in the primary bucket */
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
@@ -561,25 +582,52 @@ rte_hash_add_key_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, sig);
+ return __rte_hash_add_key_with_hash(h, key, sig, 0);
}
int32_t
rte_hash_add_key(const struct rte_hash *h, const void *key)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
+ return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key), 0);
+}
+
+int
+rte_hash_add_key_with_hash_data(const struct rte_hash *h,
+ const void *key, hash_sig_t sig, uintptr_t data)
+{
+ int ret;
+
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ ret = __rte_hash_add_key_with_hash(h, key, sig, data);
+ if (ret >= 0)
+ return 0;
+ else
+ return ret;
}
+int
+rte_hash_add_key_data(const struct rte_hash *h, const void *key, uintptr_t data)
+{
+ int ret;
+
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+
+ ret = __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key), data);
+ if (ret >= 0)
+ return 0;
+ else
+ return ret;
+}
static inline int32_t
__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
- hash_sig_t sig)
+ hash_sig_t sig, uintptr_t *data)
{
uint32_t bucket_idx;
hash_sig_t alt_hash;
unsigned i;
struct rte_hash_bucket *bkt;
- void *k, *keys = h->key_store;
+ struct rte_hash_key *k, *keys = h->key_store;
bucket_idx = sig & h->bucket_bitmask;
bkt = &h->buckets[bucket_idx];
@@ -588,13 +636,17 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == sig &&
bkt->signatures[i].sig != NULL_SIGNATURE) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
+ if (data != NULL)
+ *data = k->idata;
/*
* Return index where key is stored,
* substracting the first dummy index
*/
return (bkt->key_idx[i] - 1);
+ }
}
}
@@ -607,13 +659,17 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == alt_hash &&
bkt->signatures[i].alt == sig) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
+ if (data != NULL)
+ *data = k->idata;
/*
* Return index where key is stored,
* substracting the first dummy index
*/
return (bkt->key_idx[i] - 1);
+ }
}
}
@@ -625,14 +681,29 @@ rte_hash_lookup_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, sig);
+ return __rte_hash_lookup_with_hash(h, key, sig, NULL);
}
int32_t
rte_hash_lookup(const struct rte_hash *h, const void *key)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key), NULL);
+}
+
+int
+rte_hash_lookup_with_hash_data(const struct rte_hash *h,
+ const void *key, hash_sig_t sig, uintptr_t *data)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, sig, data);
+}
+
+int
+rte_hash_lookup_data(const struct rte_hash *h, const void *key, uintptr_t *data)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key), data);
}
static inline int32_t
@@ -643,7 +714,7 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
hash_sig_t alt_hash;
unsigned i;
struct rte_hash_bucket *bkt;
- void *k, *keys = h->key_store;
+ struct rte_hash_key *k, *keys = h->key_store;
bucket_idx = sig & h->bucket_bitmask;
bkt = &h->buckets[bucket_idx];
@@ -652,8 +723,9 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == sig &&
bkt->signatures[i].sig != NULL_SIGNATURE) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
bkt->signatures[i].sig = NULL_SIGNATURE;
rte_ring_sp_enqueue(h->free_slots,
(void *)((uintptr_t)bkt->key_idx[i]));
@@ -675,8 +747,9 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == alt_hash &&
bkt->signatures[i].sig != NULL_SIGNATURE) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
bkt->signatures[i].sig = NULL_SIGNATURE;
rte_ring_sp_enqueue(h->free_slots,
(void *)((uintptr_t)bkt->key_idx[i]));
@@ -750,7 +823,7 @@ static inline void
lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
const struct rte_hash_bucket *prim_bkt,
const struct rte_hash_bucket *sec_bkt,
- const void **key_slot, int32_t *positions,
+ const struct rte_hash_key **key_slot, int32_t *positions,
uint64_t *extra_hits_mask, const void *keys,
const struct rte_hash *h)
{
@@ -770,7 +843,8 @@ lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
total_hash_matches = (prim_hash_matches |
(sec_hash_matches << (RTE_HASH_BUCKET_ENTRIES + 1)));
- *key_slot = (const char *)keys + key_idx * h->key_entry_size;
+ *key_slot = (const struct rte_hash_key *) ((const char *)keys +
+ key_idx * h->key_entry_size);
rte_prefetch0(*key_slot);
/*
@@ -784,26 +858,31 @@ lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
}
-/* Lookup bulk stage 3: Check if key matches, update hit mask */
+/* Lookup bulk stage 3: Check if key matches, update hit mask and return data */
static inline void
-lookup_stage3(unsigned idx, const void *key_slot, const void * const *keys,
- uint64_t *hits, const struct rte_hash *h)
+lookup_stage3(unsigned idx, const struct rte_hash_key *key_slot, const void * const *keys,
+ uintptr_t data[], uint64_t *hits, const struct rte_hash *h)
{
unsigned hit;
- hit = !h->rte_hash_cmp_eq(key_slot, keys[idx], h->key_len);
+ hit = !h->rte_hash_cmp_eq(key_slot->key, keys[idx], h->key_len);
+ if (data != NULL)
+ data[idx] = key_slot->idata;
+
*hits |= (uint64_t)(hit) << idx;
}
static inline void
__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
- uint32_t num_keys, int32_t *positions)
+ uint32_t num_keys, int32_t *positions,
+ uint64_t *hit_mask, uintptr_t data[])
{
uint64_t hits = 0;
uint64_t extra_hits_mask = 0;
uint64_t lookup_mask, miss_mask;
unsigned idx;
const void *key_store = h->key_store;
+ int ret;
hash_sig_t hash_vals[RTE_HASH_LOOKUP_BULK_MAX];
unsigned idx00, idx01, idx10, idx11, idx20, idx21, idx30, idx31;
@@ -811,7 +890,7 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
const struct rte_hash_bucket *secondary_bkt10, *secondary_bkt11;
const struct rte_hash_bucket *primary_bkt20, *primary_bkt21;
const struct rte_hash_bucket *secondary_bkt20, *secondary_bkt21;
- const void *k_slot20, *k_slot21, *k_slot30, *k_slot31;
+ const struct rte_hash_key *k_slot20, *k_slot21, *k_slot30, *k_slot31;
hash_sig_t primary_hash10, primary_hash11;
hash_sig_t secondary_hash10, secondary_hash11;
hash_sig_t primary_hash20, primary_hash21;
@@ -882,8 +961,8 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21,
primary_bkt21, secondary_bkt21, &k_slot21, positions,
&extra_hits_mask, key_store, h);
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
}
k_slot30 = k_slot20, k_slot31 = k_slot21;
@@ -909,8 +988,8 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
key_store, h);
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
k_slot30 = k_slot20, k_slot31 = k_slot21;
idx30 = idx20, idx31 = idx21;
@@ -930,14 +1009,14 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
key_store, h);
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
k_slot30 = k_slot20, k_slot31 = k_slot21;
idx30 = idx20, idx31 = idx21;
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
/* ignore any items we have already found */
extra_hits_mask &= ~hits;
@@ -946,11 +1025,18 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
/* run a single search for each remaining item */
do {
idx = __builtin_ctzl(extra_hits_mask);
- positions[idx] = rte_hash_lookup_with_hash(h, keys[idx],
- hash_vals[idx]);
+ if (data != NULL) {
+ ret = rte_hash_lookup_with_hash_data(h,
+ keys[idx], hash_vals[idx], &data[idx]);
+ if (ret >= 0)
+ hits |= 1ULL << idx;
+ } else {
+ positions[idx] = rte_hash_lookup_with_hash(h,
+ keys[idx], hash_vals[idx]);
+ if (positions[idx] >= 0)
+ hits |= 1llu << idx;
+ }
extra_hits_mask &= ~(1llu << idx);
- if (positions[idx] >= 0)
- hits |= 1llu << idx;
} while (extra_hits_mask);
}
@@ -962,6 +1048,9 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
miss_mask &= ~(1llu << idx);
} while (miss_mask);
}
+
+ if (hit_mask != NULL)
+ *hit_mask = hits;
}
int
@@ -972,10 +1061,26 @@ rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
(num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
(positions == NULL)), -EINVAL);
- __rte_hash_lookup_bulk(h, keys, num_keys, positions);
+ __rte_hash_lookup_bulk(h, keys, num_keys, positions, NULL, NULL);
return 0;
}
+int
+rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, uint64_t *hit_mask, uintptr_t data[])
+{
+ RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
+ (num_keys > RTE_HASH_LOOKUP_BULK_MAX),
+ (hit_mask == NULL)), -EINVAL);
+
+ int32_t positions[num_keys];
+
+ __rte_hash_lookup_bulk(h, keys, num_keys, positions, hit_mask, data);
+
+ /* Return number of hits */
+ return __builtin_popcountl(*hit_mask);
+}
+
/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
static int
rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index 8bbc9f0..15f8b2f 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -80,12 +80,12 @@ struct rte_hash_parameters {
rte_hash_function hash_func; /**< Primary Hash function used to calculate hash. */
uint32_t hash_func_init_val; /**< Init value used by hash_func. */
int socket_id; /**< NUMA Socket ID for memory. */
+ uint8_t extra_flag; /**< Indicate if additional parameters are present. */
};
/** @internal A hash table structure. */
struct rte_hash;
-
/**
* Create a new hash table.
*
@@ -136,6 +136,48 @@ void
rte_hash_reset(struct rte_hash *h);
/**
+ * Add a key-value pair to an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
+ *
+ * @param h
+ * Hash table to add the key to.
+ * @param key
+ * Key to add to the hash table.
+ * @param data
+ * Data to add to the hash table.
+ * @return
+ * - 0 if added successfully
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOSPC if there is no space in the hash for this key.
+ */
+int
+rte_hash_add_key_data(const struct rte_hash *h, const void *key, uintptr_t data);
+
+/**
+ * Add a key-value pair with a pre-computed hash value
+ * to an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
+ *
+ * @param h
+ * Hash table to add the key to.
+ * @param key
+ * Key to add to the hash table.
+ * @param sig
+ * Precomputed hash value for 'key'
+ * @param data
+ * Data to add to the hash table.
+ * @return
+ * - 0 if added successfully
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOSPC if there is no space in the hash for this key.
+ */
+int32_t
+rte_hash_add_key_with_hash_data(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, uintptr_t data);
+
+/**
* Add a key to an existing hash table. This operation is not multi-thread safe
* and should only be called from one thread.
*
@@ -212,6 +254,47 @@ rte_hash_del_key(const struct rte_hash *h, const void *key);
int32_t
rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
+
+/**
+ * Find a key-value pair in the hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param key
+ * Key to find.
+ * @param data
+ * Output with pointer to data returned from the hash table.
+ * @return
+ * 0 if successful lookup
+ * - EINVAL if the parameters are invalid.
+ * - ENOENT if the key is not found.
+ */
+int
+rte_hash_lookup_data(const struct rte_hash *h, const void *key, uintptr_t *data);
+
+/**
+ * Find a key-value pair with a pre-computed hash value
+ * to an existing hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param key
+ * Key to find.
+ * @param sig
+ * Precomputed hash value for 'key'
+ * @param data
+ * Output with pointer to data returned from the hash table.
+ * @return
+ * 0 if successful lookup
+ * - EINVAL if the parameters are invalid.
+ * - ENOENT if the key is not found.
+ */
+int
+rte_hash_lookup_with_hash_data(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, uintptr_t *data);
+
/**
* Find a key in the hash table.
* This operation is multi-thread safe.
@@ -266,6 +349,28 @@ hash_sig_t
rte_hash_hash(const struct rte_hash *h, const void *key);
#define rte_hash_lookup_multi rte_hash_lookup_bulk
+#define rte_hash_lookup_multi_data rte_hash_lookup_bulk_data
+/**
+ * Find multiple keys in the hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param keys
+ * A pointer to a list of keys to look for.
+ * @param num_keys
+ * How many keys are in the keys list (less than RTE_HASH_LOOKUP_BULK_MAX).
+ * @param hit_mask
+ * Output containing a bitmask with all successful lookups.
+ * @param data
+ * Output containing array of data returned from all the successful lookups.
+ * @return
+ * -EINVAL if there's an error, otherwise number of successful lookups.
+ */
+int
+rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, uint64_t *hit_mask, uintptr_t data[]);
+
/**
* Find multiple keys in the hash table.
* This operation is multi-thread safe.
@@ -288,7 +393,6 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions);
-
#ifdef __cplusplus
}
#endif
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index d5f5af5..a97eac1 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -22,6 +22,12 @@ DPDK_2.0 {
DPDK_2.1 {
global:
+ rte_hash_add_key_data;
+ rte_hash_add_key_with_hash_data;
+ rte_hash_create;
+ rte_hash_lookup_bulk_data;
+ rte_hash_lookup_data;
+ rte_hash_lookup_with_hash_data;
rte_hash_reset;
local: *;
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v5 4/7] hash: add iterate function
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 " Pablo de Lara
` (2 preceding siblings ...)
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 3/7] hash: add new functionality to store data in hash table Pablo de Lara
@ 2015-07-10 21:57 ` Pablo de Lara
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 5/7] MAINTAINERS: claim responsability for hash library Pablo de Lara
` (4 subsequent siblings)
8 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 21:57 UTC (permalink / raw)
To: dev
Since now rte_hash structure is private, a new function
has been added to let the user iterate through the hash table,
returning next key and data associated on each iteration,
plus the position where they were stored.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 66 ++++++++++++++++++++++++++++++++++++
lib/librte_hash/rte_cuckoo_hash.c | 41 ++++++++++++++++++++++
lib/librte_hash/rte_hash.h | 22 ++++++++++++
lib/librte_hash/rte_hash_version.map | 1 +
4 files changed, 130 insertions(+)
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index 448586c..f5277c9 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -1149,6 +1149,70 @@ static int test_average_table_utilization(void)
return 0;
}
+#define NUM_ENTRIES 1024
+static int test_hash_iteration(void)
+{
+ struct rte_hash *handle;
+ unsigned i;
+ uint8_t keys[NUM_ENTRIES][RTE_HASH_KEY_LENGTH_MAX];
+ const void *next_key;
+ uintptr_t next_data;
+ uintptr_t data[NUM_ENTRIES];
+ unsigned added_keys;
+ uint32_t iter = 0;
+ int ret = 0;
+
+ ut_params.entries = NUM_ENTRIES;
+ ut_params.name = "test_hash_iteration";
+ ut_params.hash_func = rte_jhash;
+ ut_params.key_len = 16;
+ handle = rte_hash_create(&ut_params);
+ RETURN_IF_ERROR(handle == NULL, "hash creation failed");
+
+ /* Add random entries until key cannot be added */
+ for (added_keys = 0; added_keys < NUM_ENTRIES; added_keys++) {
+ data[added_keys] = rte_rand();
+ for (i = 0; i < ut_params.key_len; i++)
+ keys[added_keys][i] = rte_rand() % 255;
+ ret = rte_hash_add_key_data(handle, keys[added_keys], data[added_keys]);
+ if (ret < 0)
+ break;
+ }
+
+ /* Iterate through the hash table */
+ while (rte_hash_iterate(handle, &next_key, &next_data, &iter) >= 0) {
+ /* Search for the key in the list of keys added */
+ for (i = 0; i < NUM_ENTRIES; i++) {
+ if (memcmp(next_key, keys[i], ut_params.key_len) == 0) {
+ if (next_data != data[i]) {
+ printf("Data found in the hash table is"
+ "not the data added with the key\n");
+ goto err;
+ }
+ added_keys--;
+ break;
+ }
+ }
+ if (i == NUM_ENTRIES) {
+ printf("Key found in the hash table was not added\n");
+ goto err;
+ }
+ }
+
+ /* Check if all keys have been iterated */
+ if (added_keys != 0) {
+ printf("There were still %u keys to iterate\n", added_keys);
+ goto err;
+ }
+
+ rte_hash_free(handle);
+ return 0;
+
+err:
+ rte_hash_free(handle);
+ return -1;
+}
+
static uint8_t key[16] = {0x00, 0x01, 0x02, 0x03,
0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b,
@@ -1408,6 +1472,8 @@ test_hash(void)
return -1;
if (test_average_table_utilization() < 0)
return -1;
+ if (test_hash_iteration() < 0)
+ return -1;
run_hash_func_tests();
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index 5777036..5796ef4 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -1081,6 +1081,47 @@ rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
return __builtin_popcountl(*hit_mask);
}
+int32_t
+rte_hash_iterate(const struct rte_hash *h, const void **key, uintptr_t *data, uint32_t *next)
+{
+ uint32_t bucket_idx, idx, position;
+ struct rte_hash_key *next_key;
+
+ RETURN_IF_TRUE(((h == NULL) || (next == NULL)), -EINVAL);
+
+ const uint32_t total_entries = h->num_buckets * RTE_HASH_BUCKET_ENTRIES;
+ /* Out of bounds */
+ if (*next >= total_entries)
+ return -ENOENT;
+
+ /* Calculate bucket and index of current iterator */
+ bucket_idx = *next / RTE_HASH_BUCKET_ENTRIES;
+ idx = *next % RTE_HASH_BUCKET_ENTRIES;
+
+ /* If current position is empty, go to the next one */
+ while (h->buckets[bucket_idx].signatures[idx].sig == NULL_SIGNATURE) {
+ (*next)++;
+ /* End of table */
+ if (*next == total_entries)
+ return -ENOENT;
+ bucket_idx = *next / RTE_HASH_BUCKET_ENTRIES;
+ idx = *next % RTE_HASH_BUCKET_ENTRIES;
+ }
+
+ /* Get position of entry in key table */
+ position = h->buckets[bucket_idx].key_idx[idx];
+ next_key = (struct rte_hash_key *) ((char *)h->key_store +
+ position * h->key_entry_size);
+ /* Return key and data */
+ *key = next_key->key;
+ *data = next_key->idata;
+
+ /* Increment iterator */
+ (*next)++;
+
+ return (position - 1);
+}
+
/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
static int
rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index 15f8b2f..a2e7f74 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -393,6 +393,28 @@ rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions);
+
+/**
+ * Iterate through the hash table, returning key-value pairs.
+ *
+ * @param h
+ * Hash table to iterate
+ * @param key
+ * Output containing the key where current iterator
+ * was pointing at
+ * @param data
+ * Output containing the data associated with key.
+ * Returns NULL if data was not stored.
+ * @param next
+ * Pointer to iterator. Should be 0 to start iterating the hash table.
+ * Iterator is incremented after each call of this function.
+ * @return
+ * Position where key was stored, if successful.
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOENT if end of the hash table.
+ */
+int32_t
+rte_hash_iterate(const struct rte_hash *h, const void **key, uintptr_t *data, uint32_t *next);
#ifdef __cplusplus
}
#endif
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index a97eac1..5653cb7 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -25,6 +25,7 @@ DPDK_2.1 {
rte_hash_add_key_data;
rte_hash_add_key_with_hash_data;
rte_hash_create;
+ rte_hash_iterate;
rte_hash_lookup_bulk_data;
rte_hash_lookup_data;
rte_hash_lookup_with_hash_data;
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v5 5/7] MAINTAINERS: claim responsability for hash library
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 " Pablo de Lara
` (3 preceding siblings ...)
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 4/7] hash: add iterate function Pablo de Lara
@ 2015-07-10 21:57 ` Pablo de Lara
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 6/7] doc: announce ABI change of librte_hash Pablo de Lara
` (3 subsequent siblings)
8 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 21:57 UTC (permalink / raw)
To: dev
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
MAINTAINERS | 1 +
1 file changed, 1 insertion(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 5476a73..75a27e9 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -345,6 +345,7 @@ F: doc/guides/sample_app_ug/l3_forward_access_ctrl.rst
Hashes
M: Bruce Richardson <bruce.richardson@intel.com>
+M: Pablo de Lara <pablo.de.lara.guarch@intel.com>
F: lib/librte_hash/
F: doc/guides/prog_guide/hash_lib.rst
F: app/test/test_*hash*
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v5 6/7] doc: announce ABI change of librte_hash
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 " Pablo de Lara
` (4 preceding siblings ...)
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 5/7] MAINTAINERS: claim responsability for hash library Pablo de Lara
@ 2015-07-10 21:57 ` Pablo de Lara
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 7/7] doc: update hash documentation Pablo de Lara
` (2 subsequent siblings)
8 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 21:57 UTC (permalink / raw)
To: dev
Two of the macros in rte_hash.h are now deprecated, so this patch
adds notice that they will be removed in 2.2.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
doc/guides/rel_notes/abi.rst | 1 +
1 file changed, 1 insertion(+)
diff --git a/doc/guides/rel_notes/abi.rst b/doc/guides/rel_notes/abi.rst
index 110c486..312348e 100644
--- a/doc/guides/rel_notes/abi.rst
+++ b/doc/guides/rel_notes/abi.rst
@@ -12,3 +12,4 @@ Examples of Deprecation Notices
Deprecation Notices
-------------------
+* The Macros #RTE_HASH_BUCKET_ENTRIES_MAX and #RTE_HASH_KEY_LENGTH_MAX are deprecated and will be removed with version 2.2.
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v5 7/7] doc: update hash documentation
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 " Pablo de Lara
` (5 preceding siblings ...)
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 6/7] doc: announce ABI change of librte_hash Pablo de Lara
@ 2015-07-10 21:57 ` Pablo de Lara
2015-07-10 22:52 ` [dpdk-dev] [PATCH v5 0/7] Cuckoo hash - part 3 of Cuckoo hash Bruce Richardson
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 " Pablo de Lara
8 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 21:57 UTC (permalink / raw)
To: dev
Updates hash library documentation, reflecting
the new implementation changes.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
doc/guides/prog_guide/hash_lib.rst | 138 +++++++++++++++++++++++++++++++++----
doc/guides/prog_guide/index.rst | 4 ++
2 files changed, 129 insertions(+), 13 deletions(-)
diff --git a/doc/guides/prog_guide/hash_lib.rst b/doc/guides/prog_guide/hash_lib.rst
index 9b83835..193dd53 100644
--- a/doc/guides/prog_guide/hash_lib.rst
+++ b/doc/guides/prog_guide/hash_lib.rst
@@ -1,5 +1,5 @@
.. BSD LICENSE
- Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
All rights reserved.
Redistribution and use in source and binary forms, with or without
@@ -50,8 +50,6 @@ The hash also allows the configuration of some low-level implementation related
* Hash function to translate the key into a bucket index
-* Number of entries per bucket
-
The main methods exported by the hash are:
* Add entry with key: The key is provided as input. If a new entry is successfully added to the hash for the specified key,
@@ -65,10 +63,26 @@ The main methods exported by the hash are:
* Lookup for entry with key: The key is provided as input. If an entry with the specified key is found in the hash (lookup hit),
then the position of the entry is returned, otherwise (lookup miss) a negative value is returned.
-The current hash implementation handles the key management only.
-The actual data associated with each key has to be managed by the user using a separate table that
+Apart from these method explained above, the API allows the user three more options:
+
+* Add / lookup / delete with key and precomputed hash: Both the key and its precomputed hash are provided as input. This allows
+ the user to perform these operations faster, as hash is already computed.
+
+* Add / lookup with key and data: A pair of key-value is provided as input. This allows the user to store
+ not only the key, but also data which may be either a 8-byte integer or a pointer to external data (if data size is more than 8 bytes).
+
+* Combination of the two options above: User can provide key, precomputed hash and data.
+
+Also, the API contains a method to allow the user to look up entries in bursts, achieving higher performance
+than looking up individual entries, as the function prefetches next entries at the time it is operating
+with the first ones, which reduces significantly the impact of the necessary memory accesses.
+Notice that this method uses a pipeline of 8 entries (4 stages of 2 entries), so it is highly recommended
+to use at least 8 entries per burst.
+
+The actual data associated with each key can be either managed by the user using a separate table that
mirrors the hash in terms of number of entries and position of each entry,
-as shown in the Flow Classification use case describes in the following sections.
+as shown in the Flow Classification use case describes in the following sections,
+or stored in the hash table itself.
The example hash tables in the L2/L3 Forwarding sample applications defines which port to forward a packet to based on a packet flow identified by the five-tuple lookup.
However, this table could also be used for more sophisticated features and provide many other functions and actions that could be performed on the packets and flows.
@@ -76,17 +90,26 @@ However, this table could also be used for more sophisticated features and provi
Implementation Details
----------------------
-The hash table is implemented as an array of entries which is further divided into buckets,
-with the same number of consecutive array entries in each bucket.
-For any input key, there is always a single bucket where that key can be stored in the hash,
-therefore only the entries within that bucket need to be examined when the key is looked up.
+The hash table has two main tables:
+
+* First table is an array of entries which is further divided into buckets,
+ with the same number of consecutive array entries in each bucket. Each entry contains the computed primary
+ and secondary hashes of a given key (explained below), and an index to the second table.
+
+* The second table is an array of all the keys stored in the hash table and its data associated to each key.
+
+The hash library uses the cuckoo hash method to resolve collisions.
+For any input key, there are two possible buckets (primary and secondary/alternative location)
+where that key can be stored in the hash, therefore only the entries within those bucket need to be examined
+when the key is looked up.
The lookup speed is achieved by reducing the number of entries to be scanned from the total
-number of hash entries down to the number of entries in a hash bucket,
+number of hash entries down to the number of entries in the two hash buckets,
as opposed to the basic method of linearly scanning all the entries in the array.
The hash uses a hash function (configurable) to translate the input key into a 4-byte key signature.
The bucket index is the key signature modulo the number of hash buckets.
-Once the bucket is identified, the scope of the hash add,
-delete and lookup operations is reduced to the entries in that bucket.
+
+Once the buckets are identified, the scope of the hash add,
+delete and lookup operations is reduced to the entries in those buckets (it is very likely that entries are in the primary bucket).
To speed up the search logic within the bucket, each hash entry stores the 4-byte key signature together with the full key for each hash entry.
For large key sizes, comparing the input key against a key from the bucket can take significantly more time than
@@ -95,6 +118,95 @@ Therefore, the signature comparison is done first and the full key comparison do
The full key comparison is still necessary, as two input keys from the same bucket can still potentially have the same 4-byte hash signature,
although this event is relatively rare for hash functions providing good uniform distributions for the set of input keys.
+Example of lookup:
+
+First of all, the primary bucket is identified and entry is likely to be stored there.
+If signature was stored there, we compare its key against the one provided and return the position
+where it was stored and/or the data associated to that key if there is a match.
+If signature is not in the primary bucket, the secondary bucket is looked up, where same procedure
+is carried out. If there is no match there either, key is considered not to be in the table.
+
+Example of addition:
+
+Like lookup, the primary and secondary buckets are indentified. If there is an empty slot in
+the primary bucket, primary and secondary signatures are stored in that slot, key and data (if any) are added to
+the second table and an index to the position in the second table is stored in the slot of the first table.
+If there is no space in the primary bucket, one of the entries on that bucket is pushed to its alternative location,
+and the key to be added is inserted in its position.
+To know where the alternative bucket of the evicted entry is, the secondary signature is looked up and alternative bucket index
+is calculated from doing the modulo, as seen above. If there is room in the alternative bucket, the evicted entry
+is stored in it. If not, same process is repeated (one of the entries gets pushed) until a non full bucket is found.
+Notice that despite all the entry movement in the first table, the second table is not touched, which would impact
+greatly in performance.
+
+In the very unlikely event that table enters in a loop where same entries are being evicted indefinitely,
+key is considered not able to be stored.
+With random keys, this method allows the user to get around 90% of the table utilization, without
+having to drop any stored entry (LRU) or allocate more memory (extended buckets).
+
+Entry distribution in hash table
+--------------------------------
+
+As mentioned above, Cuckoo hash implementation pushes elements out of their bucket,
+if there is a new entry to be added which primary location coincides with their current bucket,
+being pushed to their alternative location.
+Therefore, as user adds more entries to the hash table, distribution of the hash values
+in the buckets will change, being most of them in their primary location and a few in
+their secondary location, which the later will increase, as table gets busier.
+This information is quite useful, as performance will be lower as more entries
+are evicted to their secondary location.
+
+See the tables below showing entry distribution as table utilization increases.
+
+.. _table_hash_lib_1:
+
+.. table:: Entry distribution with 1024 entries
+
+ +--------------+-----------------------+-------------------------+
+ | % Table used | % In Primary location | % In Secondary location |
+ +==============+=======================+=========================+
+ | 25 | 100 | 0 |
+ +--------------+-----------------------+-------------------------+
+ | 50 | 96.1 | 3.9 |
+ +--------------+-----------------------+-------------------------+
+ | 75 | 88.2 | 11.8 |
+ +--------------+-----------------------+-------------------------+
+ | 80 | 86.3 | 13.7 |
+ +--------------+-----------------------+-------------------------+
+ | 85 | 83.1 | 16.9 |
+ +--------------+-----------------------+-------------------------+
+ | 90 | 77.3 | 22.7 |
+ +--------------+-----------------------+-------------------------+
+ | 95.8 | 64.5 | 35.5 |
+ +--------------+-----------------------+-------------------------+
+
+|
+
+.. _table_hash_lib_2:
+
+.. table:: Entry distribution with 1 million entries
+
+ +--------------+-----------------------+-------------------------+
+ | % Table used | % In Primary location | % In Secondary location |
+ +==============+=======================+=========================+
+ | 50 | 96 | 4 |
+ +--------------+-----------------------+-------------------------+
+ | 75 | 86.9 | 13.1 |
+ +--------------+-----------------------+-------------------------+
+ | 80 | 83.9 | 16.1 |
+ +--------------+-----------------------+-------------------------+
+ | 85 | 80.1 | 19.9 |
+ +--------------+-----------------------+-------------------------+
+ | 90 | 74.8 | 25.2 |
+ +--------------+-----------------------+-------------------------+
+ | 94.5 | 67.4 | 32.6 |
+ +--------------+-----------------------+-------------------------+
+
+.. note::
+
+ Last values on the tables above are the average maximum table
+ utilization with random keys and using Jenkins hash function.
+
Use Case: Flow Classification
-----------------------------
diff --git a/doc/guides/prog_guide/index.rst b/doc/guides/prog_guide/index.rst
index 3295661..036640c 100644
--- a/doc/guides/prog_guide/index.rst
+++ b/doc/guides/prog_guide/index.rst
@@ -241,3 +241,7 @@ Programmer's Guide
:numref:`table_qos_33` :ref:`table_qos_33`
:numref:`table_qos_34` :ref:`table_qos_34`
+
+:numref:`table_hash_lib_1` :ref:`table_hash_lib_1`
+
+:numref:`table_hash_lib_2` :ref:`table_hash_lib_2`
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* Re: [dpdk-dev] [PATCH v5 0/7] Cuckoo hash - part 3 of Cuckoo hash
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 " Pablo de Lara
` (6 preceding siblings ...)
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 7/7] doc: update hash documentation Pablo de Lara
@ 2015-07-10 22:52 ` Bruce Richardson
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 " Pablo de Lara
8 siblings, 0 replies; 92+ messages in thread
From: Bruce Richardson @ 2015-07-10 22:52 UTC (permalink / raw)
To: Pablo de Lara; +Cc: dev
On Fri, Jul 10, 2015 at 10:57:40PM +0100, Pablo de Lara wrote:
> This patchset is to replace the existing hash library with
> a more efficient and functional approach, using the Cuckoo hash
> method to deal with collisions. This method is based on using
> two different hash functions to have two possible locations
> in the hash table where an entry can be.
> So, if a bucket is full, a new entry can push one of the items
> in that bucket to its alternative location, making space for itself.
>
> Advantages
> ~~~~
> - Offers the option to store more entries when the target bucket is full
> (unlike the previous implementation)
> - Memory efficient: for storing those entries, it is not necessary to
> request new memory, as the entries will be stored in the same table
> - Constant worst lookup time: in worst case scenario, it always takes
> the same time to look up an entry, as there are only two possible locations
> where an entry can be.
> - Storing data: user can store data in the hash table, unlike the
> previous implementation, but he can still use the old API
>
> This implementation tipically offers over 90% utilization.
> Notice that API has been extended, but old API remains.
> Check documentation included to know more about this new implementation
> (including how entries are distributed as table utilization increases).
>
Only final quibble I have it that I think the data parameter stored should be void *
rather than uintptr_t, though both can be cast to each other easily enough.
Otherwise the code compiles ok now, and the unit tests pass fine.
Series Acked-by: Bruce Richardson <bruce.richardson@intel.com>
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v6 0/7] Cuckoo hash - part 3 of Cuckoo hash
2015-07-10 21:57 ` [dpdk-dev] [PATCH v5 " Pablo de Lara
` (7 preceding siblings ...)
2015-07-10 22:52 ` [dpdk-dev] [PATCH v5 0/7] Cuckoo hash - part 3 of Cuckoo hash Bruce Richardson
@ 2015-07-10 23:30 ` Pablo de Lara
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 1/7] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
` (7 more replies)
8 siblings, 8 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 23:30 UTC (permalink / raw)
To: dev
This patchset is to replace the existing hash library with
a more efficient and functional approach, using the Cuckoo hash
method to deal with collisions. This method is based on using
two different hash functions to have two possible locations
in the hash table where an entry can be.
So, if a bucket is full, a new entry can push one of the items
in that bucket to its alternative location, making space for itself.
Advantages
~~~
- Offers the option to store more entries when the target bucket is full
(unlike the previous implementation)
- Memory efficient: for storing those entries, it is not necessary to
request new memory, as the entries will be stored in the same table
- Constant worst lookup time: in worst case scenario, it always takes
the same time to look up an entry, as there are only two possible locations
where an entry can be.
- Storing data: user can store data in the hash table, unlike the
previous implementation, but he can still use the old API
This implementation typically offers over 90% utilization.
Notice that API has been extended, but old API remains.
Check documentation included to know more about this new implementation
(including how entries are distributed as table utilization increases).
Changes in v6:
- Replace datatype for functions from uintptr_t to void *
Changes in v5:
- Fix documentation
- Fix 32-bit compilation issues
Changes in v4:
- Unit tests enhancements are not part of this patchset anymore.
- rte_hash structure has been made internal in another patch,
so it is not part of this patchset anymore.
- Add function to iterate through the hash table, as rte_hash
structure has been made private.
- Added extra_flag parameter in rte_hash_parameter to be able
to add new parameters in the future without breaking the ABI
- Remove proposed lookup_bulk_with_hash function, as it is
not of much use with the existing hash functions
(there are no vector hash functions).
- User can store 8-byte integer or pointer as data, instead
of variable size data, as discussed in the mailing list.
Changes in v3:
- Now user can store variable size data, instead of 32 or 64-bit size data,
using the new parameter "data_len" in rte_hash_parameters
- Add lookup_bulk_with_hash function in performance unit tests
- Add new functions that handle data in performance unit tests
- Remove duplicates in performance unit tests
- Fix rte_hash_reset, which was not resetting the last entry
Changes in v2:
- Fixed issue where table could not store maximum number of entries
- Fixed issue where lookup burst could not be more than 32 (instead of 64)
- Remove unnecessary macros and add other useful ones
- Added missing library dependencies
- Used directly rte_hash_secondary instead of rte_hash_alt
- Renamed rte_hash.c to rte_cuckoo_hash.c to ease the view of the new library
- Renamed test_hash_perf.c temporarily to ease the view of the improved unit test
- Moved rte_hash, rte_bucket and rte_hash_key structures to rte_cuckoo_hash.c to
make them private
- Corrected copyright dates
- Added an optimized function to compare keys that are multiple of 16 bytes
- Improved the way to use primary/secondary signatures. Now both are stored in
the bucket, so there is no need to calculate them if required.
Also, there is no need to use the MSB of a signature to differenciate between
an empty entry and signature 0, since we are storing both signatures,
which cannot be both 0.
- Removed rte_hash_rehash, as it was a very expensive operation.
Therefore, the add function returns now -ENOSPC if key cannot be added
because of a loop.
- Prefetched new slot for new key in add function to improve performance.
- Made doxygen comments more clear.
- Removed unnecessary rte_hash_del_key_data and rte_hash_del_key_with_data,
as we can use the lookup functions if we want to get the data before deleting.
- Removed some unnecessary includes in rte_hash.h
- Removed some unnecessary variables in rte_cuckoo_hash.c
- Removed some unnecessary checks before creating a new hash table
- Added documentation (in release notes and programmers guide)
- Added new unit tests and replaced the performance one for hash tables
Series Acked-by: Bruce Richardson <bruce.richardson@intel.com>
Pablo de Lara (7):
hash: replace existing hash library with cuckoo hash implementation
hash: add new function rte_hash_reset
hash: add new functionality to store data in hash table
hash: add iterate function
MAINTAINERS: claim responsability for hash library
doc: announce ABI change of librte_hash
doc: update hash documentation
MAINTAINERS | 1 +
app/test/test_hash.c | 189 +++---
app/test/test_hash_perf.c | 303 ++++++---
doc/guides/prog_guide/hash_lib.rst | 138 +++-
doc/guides/prog_guide/index.rst | 4 +
doc/guides/rel_notes/abi.rst | 1 +
lib/librte_hash/Makefile | 8 +-
lib/librte_hash/rte_cuckoo_hash.c | 1194 ++++++++++++++++++++++++++++++++++
lib/librte_hash/rte_hash.c | 499 --------------
lib/librte_hash/rte_hash.h | 198 +++++-
lib/librte_hash/rte_hash_version.map | 15 +
11 files changed, 1810 insertions(+), 740 deletions(-)
create mode 100644 lib/librte_hash/rte_cuckoo_hash.c
delete mode 100644 lib/librte_hash/rte_hash.c
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v6 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 " Pablo de Lara
@ 2015-07-10 23:30 ` Pablo de Lara
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 2/7] hash: add new function rte_hash_reset Pablo de Lara
` (6 subsequent siblings)
7 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 23:30 UTC (permalink / raw)
To: dev
This patch replaces the existing hash library with another approach,
using the Cuckoo Hash method to resolve collisions (open addressing),
which pushes items from a full bucket when a new entry tries
to be added in it, storing the evicted entry in an alternative location,
using a secondary hash function.
This gives the user the ability to store more entries when a bucket
is full, in comparison with the previous implementation.
Therefore, the unit test has been updated, as some scenarios have changed
(such as the previous removed restriction).
Also note that the API has not been changed, although new fields
have been added in the rte_hash structure (structure is internal now).
The main change when creating a new table is that the number of entries
per bucket is fixed now, so its parameter is ignored now
(still there to maintain the same parameters structure).
The hash unit test has been updated to reflect these changes.
As a last note, the maximum burst size in lookup_burst function
hash been increased to 64, to improve performance.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 119 ++---
app/test/test_hash_perf.c | 79 ++-
lib/librte_hash/Makefile | 8 +-
lib/librte_hash/rte_cuckoo_hash.c | 1027 +++++++++++++++++++++++++++++++++++++
lib/librte_hash/rte_hash.c | 499 ------------------
lib/librte_hash/rte_hash.h | 69 +--
6 files changed, 1138 insertions(+), 663 deletions(-)
create mode 100644 lib/librte_hash/rte_cuckoo_hash.c
delete mode 100644 lib/librte_hash/rte_hash.c
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index 7c71ed6..e70d859 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -169,7 +169,6 @@ static struct flow_key keys[5] = { {
/* Parameters used for hash table in unit test functions. Name set later. */
static struct rte_hash_parameters ut_params = {
.entries = 64,
- .bucket_entries = 4,
.key_len = sizeof(struct flow_key), /* 13 */
.hash_func = rte_jhash,
.hash_func_init_val = 0,
@@ -516,9 +515,18 @@ static int test_five_keys(void)
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != -ENOENT,
- "failed to find key (pos[%u]=%d)", i, pos[i]);
+ "found non-existent key (pos[%u]=%d)", i, pos[i]);
}
+ /* Lookup multi */
+ ret = rte_hash_lookup_multi(handle, &key_array[0], 5, (int32_t *)pos);
+ if (ret == 0)
+ for (i = 0; i < 5; i++) {
+ print_key_info("Lkp", key_array[i], pos[i]);
+ RETURN_IF_ERROR(pos[i] != -ENOENT,
+ "found not-existent key (pos[%u]=%d)", i, pos[i]);
+ }
+
rte_hash_free(handle);
return 0;
@@ -527,21 +535,18 @@ static int test_five_keys(void)
/*
* Add keys to the same bucket until bucket full.
* - add 5 keys to the same bucket (hash created with 4 keys per bucket):
- * first 4 successful, 5th unsuccessful
- * - lookup the 5 keys: 4 hits, 1 miss
- * - add the 5 keys again: 4 OK, one error as bucket is full
- * - lookup the 5 keys: 4 hits (updated data), 1 miss
- * - delete the 5 keys: 5 OK (even if the 5th is not in the table)
+ * first 4 successful, 5th successful, pushing existing item in bucket
+ * - lookup the 5 keys: 5 hits
+ * - add the 5 keys again: 5 OK
+ * - lookup the 5 keys: 5 hits (updated data)
+ * - delete the 5 keys: 5 OK
* - lookup the 5 keys: 5 misses
- * - add the 5th key: OK
- * - lookup the 5th key: hit
*/
static int test_full_bucket(void)
{
struct rte_hash_parameters params_pseudo_hash = {
.name = "test4",
.entries = 64,
- .bucket_entries = 4,
.key_len = sizeof(struct flow_key), /* 13 */
.hash_func = pseudo_hash,
.hash_func_init_val = 0,
@@ -555,7 +560,7 @@ static int test_full_bucket(void)
handle = rte_hash_create(¶ms_pseudo_hash);
RETURN_IF_ERROR(handle == NULL, "hash creation failed");
- /* Fill bucket*/
+ /* Fill bucket */
for (i = 0; i < 4; i++) {
pos[i] = rte_hash_add_key(handle, &keys[i]);
print_key_info("Add", &keys[i], pos[i]);
@@ -563,47 +568,39 @@ static int test_full_bucket(void)
"failed to add key (pos[%u]=%d)", i, pos[i]);
expected_pos[i] = pos[i];
}
- /* This shouldn't work because the bucket is full */
+ /*
+ * This should work and will push one of the items
+ * in the bucket because it is full
+ */
pos[4] = rte_hash_add_key(handle, &keys[4]);
print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOSPC,
- "fail: added key to full bucket (pos[4]=%d)", pos[4]);
+ RETURN_IF_ERROR(pos[4] < 0,
+ "failed to add key (pos[4]=%d)", pos[4]);
+ expected_pos[4] = pos[4];
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to find key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: found non-existent key (pos[4]=%d)", pos[4]);
/* Add - update */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_add_key(handle, &keys[i]);
print_key_info("Add", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to add key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_add_key(handle, &keys[4]);
- print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOSPC,
- "fail: added key to full bucket (pos[4]=%d)", pos[4]);
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to find key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: found non-existent key (pos[4]=%d)", pos[4]);
/* Delete 1 key, check other keys are still found */
pos[1] = rte_hash_del_key(handle, &keys[1]);
@@ -623,35 +620,21 @@ static int test_full_bucket(void)
RETURN_IF_ERROR(pos[1] < 0, "failed to add key (pos[1]=%d)", pos[1]);
/* Delete */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_del_key(handle, &keys[i]);
print_key_info("Del", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to delete key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_del_key(handle, &keys[4]);
- print_key_info("Del", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: deleted non-existent key (pos[4]=%d)", pos[4]);
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != -ENOENT,
"fail: found non-existent key (pos[%u]=%d)", i, pos[i]);
}
- /* Add and lookup the 5th key */
- pos[4] = rte_hash_add_key(handle, &keys[4]);
- print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] < 0, "failed to add key (pos[4]=%d)", pos[4]);
- expected_pos[4] = pos[4];
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != expected_pos[4],
- "failed to find key (pos[4]=%d)", pos[4]);
-
rte_hash_free(handle);
/* Cover the NULL case. */
@@ -991,6 +974,7 @@ static int test_fbk_hash_find_existing(void)
return 0;
}
+#define BUCKET_ENTRIES 4
/*
* Do tests for hash creation with bad parameters.
*/
@@ -1017,18 +1001,8 @@ static int test_hash_creation_with_bad_parameters(void)
}
memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_1";
- params.bucket_entries = RTE_HASH_BUCKET_ENTRIES_MAX + 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully with bucket_entries in parameter exceeded\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
params.name = "creation_with_bad_parameters_2";
- params.entries = params.bucket_entries - 1;
+ params.entries = BUCKET_ENTRIES - 1;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
rte_hash_free(handle);
@@ -1038,26 +1012,6 @@ static int test_hash_creation_with_bad_parameters(void)
memcpy(¶ms, &ut_params, sizeof(params));
params.name = "creation_with_bad_parameters_3";
- params.entries = params.entries - 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully if entries in parameter is not power of 2\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_4";
- params.bucket_entries = params.bucket_entries - 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully if bucket_entries in parameter is not power of 2\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_5";
params.key_len = 0;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
@@ -1067,17 +1021,7 @@ static int test_hash_creation_with_bad_parameters(void)
}
memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_6";
- params.key_len = RTE_HASH_KEY_LENGTH_MAX + 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully if key_len is greater than the maximum\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_7";
+ params.name = "creation_with_bad_parameters_4";
params.socket_id = RTE_MAX_NUMA_NODES + 1;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
@@ -1214,7 +1158,6 @@ static uint8_t key[16] = {0x00, 0x01, 0x02, 0x03,
static struct rte_hash_parameters hash_params_ex = {
.name = NULL,
.entries = 64,
- .bucket_entries = 4,
.key_len = 0,
.hash_func = NULL,
.hash_func_init_val = 0,
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index a3876c1..a54f3c1 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -162,7 +162,7 @@ shuffle_input_keys(unsigned table_index)
* ALL can fit in hash table (no errors)
*/
static int
-get_input_keys(unsigned table_index)
+get_input_keys(unsigned with_pushes, unsigned table_index)
{
unsigned i, j;
unsigned bucket_idx, incr, success = 1;
@@ -216,9 +216,14 @@ get_input_keys(unsigned table_index)
success = 0;
signatures[i] = rte_hash_hash(h[table_index], keys[i]);
bucket_idx = signatures[i] & bucket_bitmask;
- /* If bucket is full, do not try to insert the key */
- if (buckets[bucket_idx] == BUCKET_SIZE)
- continue;
+ /*
+ * If we are not inserting keys in secondary location,
+ * when bucket is full, do not try to insert the key
+ */
+ if (with_pushes == 0)
+ if (buckets[bucket_idx] == BUCKET_SIZE)
+ continue;
+
/* If key can be added, leave in successful key arrays "keys" */
ret = rte_hash_add_key_with_hash(h[table_index], keys[i],
signatures[i]);
@@ -388,9 +393,9 @@ reset_table(unsigned table_index)
}
static int
-run_all_tbl_perf_tests(void)
+run_all_tbl_perf_tests(unsigned with_pushes)
{
- unsigned i, j;
+ unsigned i, j, with_hash;
printf("Measuring performance, please wait");
fflush(stdout);
@@ -398,46 +403,32 @@ run_all_tbl_perf_tests(void)
if (create_table(i) < 0)
return -1;
- if (get_input_keys(i) < 0)
- return -1;
-
- if (timed_adds(0, i) < 0)
- return -1;
-
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
-
- if (timed_lookups(0, i) < 0)
- return -1;
-
- if (timed_lookups_multi(i) < 0)
- return -1;
-
- if (timed_deletes(0, i) < 0)
+ if (get_input_keys(with_pushes, i) < 0)
return -1;
+ for (with_hash = 0; with_hash <= 1; with_hash++) {
+ if (timed_adds(with_hash, i) < 0)
+ return -1;
- /* Print a dot to show progress on operations */
- printf(".");
- fflush(stdout);
+ for (j = 0; j < NUM_SHUFFLES; j++)
+ shuffle_input_keys(i);
- if (reset_table(i) < 0)
- return -1;
+ if (timed_lookups(with_hash, i) < 0)
+ return -1;
- if (timed_adds(1, i) < 0)
- return -1;
+ if (timed_lookups_multi(i) < 0)
+ return -1;
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
+ if (timed_deletes(with_hash, i) < 0)
+ return -1;
- if (timed_lookups(1, i) < 0)
- return -1;
+ if (reset_table(i) < 0)
+ return -1;
- if (timed_deletes(1, i) < 0)
- return -1;
+ /* Print a dot to show progress on operations */
+ printf(".");
+ fflush(stdout);
- /* Print a dot to show progress on operations */
- printf(".");
- fflush(stdout);
+ }
free_table(i);
}
@@ -551,8 +542,16 @@ fbk_hash_perf_test(void)
static int
test_hash_perf(void)
{
- if (run_all_tbl_perf_tests() < 0)
- return -1;
+ unsigned with_pushes;
+
+ for (with_pushes = 0; with_pushes <= 1; with_pushes++) {
+ if (with_pushes == 0)
+ printf("\nALL ELEMENTS IN PRIMARY LOCATION\n");
+ else
+ printf("\nELEMENTS IN PRIMARY OR SECONDARY LOCATION\n");
+ if (run_all_tbl_perf_tests(with_pushes) < 0)
+ return -1;
+ }
if (fbk_hash_perf_test() < 0)
return -1;
diff --git a/lib/librte_hash/Makefile b/lib/librte_hash/Makefile
index 981230b..039da46 100644
--- a/lib/librte_hash/Makefile
+++ b/lib/librte_hash/Makefile
@@ -1,6 +1,6 @@
# BSD LICENSE
#
-# Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+# Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@@ -42,7 +42,7 @@ EXPORT_MAP := rte_hash_version.map
LIBABIVER := 1
# all source are stored in SRCS-y
-SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_hash.c
+SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c
# install this header file
@@ -52,7 +52,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_thash.h
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h
-# this lib needs eal
-DEPDIRS-$(CONFIG_RTE_LIBRTE_HASH) += lib/librte_eal lib/librte_malloc
+# this lib needs eal and ring
+DEPDIRS-$(CONFIG_RTE_LIBRTE_HASH) += lib/librte_eal lib/librte_malloc lib/librte_ring
include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
new file mode 100644
index 0000000..50e3acd
--- /dev/null
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -0,0 +1,1027 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <sys/queue.h>
+
+#include <rte_common.h>
+#include <rte_memory.h> /* for definition of RTE_CACHE_LINE_SIZE */
+#include <rte_log.h>
+#include <rte_memcpy.h>
+#include <rte_prefetch.h>
+#include <rte_branch_prediction.h>
+#include <rte_memzone.h>
+#include <rte_malloc.h>
+#include <rte_eal.h>
+#include <rte_eal_memconfig.h>
+#include <rte_per_lcore.h>
+#include <rte_errno.h>
+#include <rte_string_fns.h>
+#include <rte_cpuflags.h>
+#include <rte_log.h>
+#include <rte_rwlock.h>
+#include <rte_spinlock.h>
+#include <rte_ring.h>
+
+#include "rte_hash.h"
+
+TAILQ_HEAD(rte_hash_list, rte_tailq_entry);
+
+static struct rte_tailq_elem rte_hash_tailq = {
+ .name = "RTE_HASH",
+};
+EAL_REGISTER_TAILQ(rte_hash_tailq)
+
+/* Macro to enable/disable run-time checking of function parameters */
+#if defined(RTE_LIBRTE_HASH_DEBUG)
+#define RETURN_IF_TRUE(cond, retval) do { \
+ if (cond) \
+ return retval; \
+} while (0)
+#else
+#define RETURN_IF_TRUE(cond, retval)
+#endif
+
+/* Hash function used if none is specified */
+#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
+#include <rte_hash_crc.h>
+#define DEFAULT_HASH_FUNC rte_hash_crc
+#else
+#include <rte_jhash.h>
+#define DEFAULT_HASH_FUNC rte_jhash
+#endif
+
+/** Number of items per bucket. */
+#define RTE_HASH_BUCKET_ENTRIES 4
+
+#define NULL_SIGNATURE 0
+
+typedef int (*rte_hash_cmp_eq_t)(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k48_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k64_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k80_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k96_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k112_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k128_cmp_eq(const void *key1, const void *key2, size_t key_len);
+
+/** A hash table structure. */
+struct rte_hash {
+ char name[RTE_HASH_NAMESIZE]; /**< Name of the hash. */
+ uint32_t entries; /**< Total table entries. */
+ uint32_t num_buckets; /**< Number of buckets in table. */
+ uint32_t key_len; /**< Length of hash key. */
+ rte_hash_function hash_func; /**< Function used to calculate hash. */
+ rte_hash_cmp_eq_t rte_hash_cmp_eq; /**< Function used to compare keys. */
+ uint32_t hash_func_init_val; /**< Init value used by hash_func. */
+ uint32_t bucket_bitmask; /**< Bitmask for getting bucket index
+ from hash signature. */
+ uint32_t key_entry_size; /**< Size of each key entry. */
+
+ struct rte_ring *free_slots; /**< Ring that stores all indexes
+ of the free slots in the key table */
+ void *key_store; /**< Table storing all keys and data */
+ struct rte_hash_bucket *buckets; /**< Table with buckets storing all the
+ hash values and key indexes
+ to the key table*/
+} __rte_cache_aligned;
+
+/* Structure storing both primary and secondary hashes */
+struct rte_hash_signatures {
+ union {
+ struct {
+ hash_sig_t current;
+ hash_sig_t alt;
+ };
+ uint64_t sig;
+ };
+};
+
+/** Bucket structure */
+struct rte_hash_bucket {
+ struct rte_hash_signatures signatures[RTE_HASH_BUCKET_ENTRIES];
+ /* Includes dummy key index that always contains index 0 */
+ uint32_t key_idx[RTE_HASH_BUCKET_ENTRIES + 1];
+ uint8_t flag[RTE_HASH_BUCKET_ENTRIES];
+} __rte_cache_aligned;
+
+struct rte_hash *
+rte_hash_find_existing(const char *name)
+{
+ struct rte_hash *h = NULL;
+ struct rte_tailq_entry *te;
+ struct rte_hash_list *hash_list;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK);
+ TAILQ_FOREACH(te, hash_list, next) {
+ h = (struct rte_hash *) te->data;
+ if (strncmp(name, h->name, RTE_HASH_NAMESIZE) == 0)
+ break;
+ }
+ rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ if (te == NULL) {
+ rte_errno = ENOENT;
+ return NULL;
+ }
+ return h;
+}
+
+struct rte_hash *
+rte_hash_create(const struct rte_hash_parameters *params)
+{
+ struct rte_hash *h = NULL;
+ struct rte_tailq_entry *te = NULL;
+ struct rte_hash_list *hash_list;
+ struct rte_ring *r = NULL;
+ char hash_name[RTE_HASH_NAMESIZE];
+ void *ptr, *k = NULL;
+ void *buckets = NULL;
+ char ring_name[RTE_RING_NAMESIZE];
+ unsigned i;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ if (params == NULL) {
+ RTE_LOG(ERR, HASH, "rte_hash_create has no parameters\n");
+ return NULL;
+ }
+
+ /* Check for valid parameters */
+ if ((params->entries > RTE_HASH_ENTRIES_MAX) ||
+ (params->entries < RTE_HASH_BUCKET_ENTRIES) ||
+ !rte_is_power_of_2(RTE_HASH_BUCKET_ENTRIES) ||
+ (params->key_len == 0)) {
+ rte_errno = EINVAL;
+ RTE_LOG(ERR, HASH, "rte_hash_create has invalid parameters\n");
+ return NULL;
+ }
+
+ snprintf(hash_name, sizeof(hash_name), "HT_%s", params->name);
+
+ /* Guarantee there's no existing */
+ h = rte_hash_find_existing(params->name);
+ if (h != NULL)
+ return h;
+
+ te = rte_zmalloc("HASH_TAILQ_ENTRY", sizeof(*te), 0);
+ if (te == NULL) {
+ RTE_LOG(ERR, HASH, "tailq entry allocation failed\n");
+ goto err;
+ }
+
+ h = (struct rte_hash *)rte_zmalloc_socket(hash_name, sizeof(struct rte_hash),
+ RTE_CACHE_LINE_SIZE, params->socket_id);
+
+ if (h == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ const uint32_t num_buckets = rte_align32pow2(params->entries)
+ / RTE_HASH_BUCKET_ENTRIES;
+
+ buckets = rte_zmalloc_socket(NULL,
+ num_buckets * sizeof(struct rte_hash_bucket),
+ RTE_CACHE_LINE_SIZE, params->socket_id);
+
+ if (buckets == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ const uint32_t key_entry_size = params->key_len;
+ /* Store all keys and leave the first entry as a dummy entry for lookup_bulk */
+ const uint64_t key_tbl_size = key_entry_size * (params->entries + 1);
+
+ k = rte_zmalloc_socket(NULL, key_tbl_size,
+ RTE_CACHE_LINE_SIZE, params->socket_id);
+
+ if (k == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ /* Select function to compare keys */
+ switch (params->key_len) {
+ case 16:
+ h->rte_hash_cmp_eq = rte_hash_k16_cmp_eq;
+ break;
+ case 32:
+ h->rte_hash_cmp_eq = rte_hash_k32_cmp_eq;
+ break;
+ case 48:
+ h->rte_hash_cmp_eq = rte_hash_k48_cmp_eq;
+ break;
+ case 64:
+ h->rte_hash_cmp_eq = rte_hash_k64_cmp_eq;
+ break;
+ case 80:
+ h->rte_hash_cmp_eq = rte_hash_k80_cmp_eq;
+ break;
+ case 96:
+ h->rte_hash_cmp_eq = rte_hash_k96_cmp_eq;
+ break;
+ case 112:
+ h->rte_hash_cmp_eq = rte_hash_k112_cmp_eq;
+ break;
+ case 128:
+ h->rte_hash_cmp_eq = rte_hash_k128_cmp_eq;
+ break;
+ default:
+ /* If key is not multiple of 16, use generic memcmp */
+ h->rte_hash_cmp_eq = memcmp;
+ }
+
+ snprintf(ring_name, sizeof(ring_name), "HT_%s", params->name);
+ r = rte_ring_lookup(ring_name);
+ if (r != NULL) {
+ /* clear the free ring */
+ while (rte_ring_dequeue(r, &ptr) == 0)
+ rte_pause();
+ } else
+ r = rte_ring_create(ring_name, rte_align32pow2(params->entries + 1),
+ params->socket_id, 0);
+ if (r == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ /* Setup hash context */
+ snprintf(h->name, sizeof(h->name), "%s", params->name);
+ h->entries = params->entries;
+ h->key_len = params->key_len;
+ h->key_entry_size = key_entry_size;
+ h->hash_func_init_val = params->hash_func_init_val;
+
+ h->num_buckets = num_buckets;
+ h->bucket_bitmask = h->num_buckets - 1;
+ h->buckets = buckets;
+ h->hash_func = (params->hash_func == NULL) ?
+ DEFAULT_HASH_FUNC : params->hash_func;
+
+ h->key_store = k;
+ h->free_slots = r;
+
+ /* populate the free slots ring. Entry zero is reserved for key misses */
+ for (i = 1; i < params->entries + 1; i++)
+ rte_ring_sp_enqueue(r, (void *)((uintptr_t) i));
+
+ rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
+ te->data = (void *) h;
+ TAILQ_INSERT_TAIL(hash_list, te, next);
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ return h;
+err:
+ rte_free(te);
+ rte_free(h);
+ rte_free(buckets);
+ rte_free(k);
+ return NULL;
+}
+
+void
+rte_hash_free(struct rte_hash *h)
+{
+ struct rte_tailq_entry *te;
+ struct rte_hash_list *hash_list;
+
+ if (h == NULL)
+ return;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
+
+ /* find out tailq entry */
+ TAILQ_FOREACH(te, hash_list, next) {
+ if (te->data == (void *) h)
+ break;
+ }
+
+ if (te == NULL) {
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+ return;
+ }
+
+ TAILQ_REMOVE(hash_list, te, next);
+
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ rte_free(h->key_store);
+ rte_free(h->buckets);
+ rte_free(h);
+ rte_free(te);
+}
+
+hash_sig_t
+rte_hash_hash(const struct rte_hash *h, const void *key)
+{
+ /* calc hash result by key */
+ return h->hash_func(key, h->key_len, h->hash_func_init_val);
+}
+
+/* Calc the secondary hash value from the primary hash value of a given key */
+static inline hash_sig_t
+rte_hash_secondary_hash(const hash_sig_t primary_hash)
+{
+ static const unsigned all_bits_shift = 12;
+ static const unsigned alt_bits_xor = 0x5bd1e995;
+
+ uint32_t tag = primary_hash >> all_bits_shift;
+
+ return (primary_hash ^ ((tag + 1) * alt_bits_xor));
+}
+
+/* Search for an entry that can be pushed to its alternative location */
+static inline int
+make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt)
+{
+ unsigned i, j;
+ int ret;
+ uint32_t next_bucket_idx;
+ struct rte_hash_bucket *next_bkt[RTE_HASH_BUCKET_ENTRIES];
+
+ /*
+ * Push existing item (search for bucket with space in
+ * alternative locations) to its alternative location
+ */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ /* Search for space in alternative locations */
+ next_bucket_idx = bkt->signatures[i].alt & h->bucket_bitmask;
+ next_bkt[i] = &h->buckets[next_bucket_idx];
+ for (j = 0; j < RTE_HASH_BUCKET_ENTRIES; j++) {
+ if (next_bkt[i]->signatures[j].sig == NULL_SIGNATURE)
+ break;
+ }
+
+ if (j != RTE_HASH_BUCKET_ENTRIES)
+ break;
+ }
+
+ /* Alternative location has spare room (end of recursive function) */
+ if (i != RTE_HASH_BUCKET_ENTRIES) {
+ next_bkt[i]->signatures[j].alt = bkt->signatures[i].current;
+ next_bkt[i]->signatures[j].current = bkt->signatures[i].alt;
+ next_bkt[i]->key_idx[j] = bkt->key_idx[i];
+ return i;
+ }
+
+ /* Pick entry that has not been pushed yet */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++)
+ if (bkt->flag[i] == 0)
+ break;
+
+ /* All entries have been pushed, so entry cannot be added */
+ if (i == RTE_HASH_BUCKET_ENTRIES) {
+ /* Reset flag */
+ bkt->flag[i] = 0;
+ return -ENOSPC;
+ }
+
+ /* Set flag to indicate that this entry is going to be pushed */
+ bkt->flag[i] = 1;
+ /* Need room in alternative bucket to insert the pushed entry */
+ ret = make_space_bucket(h, next_bkt[i]);
+ /*
+ * After recursive function.
+ * Clear flags and insert the pushed entry
+ * in its alternative location if successful,
+ * or return error
+ */
+ bkt->flag[i] = 0;
+ if (ret >= 0) {
+ next_bkt[i]->signatures[ret].alt = bkt->signatures[i].current;
+ next_bkt[i]->signatures[ret].current = bkt->signatures[i].alt;
+ next_bkt[i]->key_idx[ret] = bkt->key_idx[i];
+ return i;
+ } else
+ return ret;
+
+}
+
+static inline int32_t
+__rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ hash_sig_t alt_hash;
+ uint32_t prim_bucket_idx, sec_bucket_idx;
+ unsigned i;
+ struct rte_hash_bucket *prim_bkt, *sec_bkt;
+ void *new_k, *k, *keys = h->key_store;
+ void *slot_id;
+ uint32_t new_idx;
+ int ret;
+
+ prim_bucket_idx = sig & h->bucket_bitmask;
+ prim_bkt = &h->buckets[prim_bucket_idx];
+ rte_prefetch0(prim_bkt);
+
+ alt_hash = rte_hash_secondary_hash(sig);
+ sec_bucket_idx = alt_hash & h->bucket_bitmask;
+ sec_bkt = &h->buckets[sec_bucket_idx];
+ rte_prefetch0(sec_bkt);
+
+ /* Get a new slot for storing the new key */
+ if (rte_ring_sc_dequeue(h->free_slots, &slot_id) != 0)
+ return -ENOSPC;
+ new_k = RTE_PTR_ADD(keys, (uintptr_t)slot_id * h->key_entry_size);
+ rte_prefetch0(new_k);
+ new_idx = (uint32_t)((uintptr_t) slot_id);
+
+ /* Check if key is already inserted in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (prim_bkt->signatures[i].current == sig &&
+ prim_bkt->signatures[i].alt == alt_hash) {
+ k = (char *)keys + prim_bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (prim_bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Check if key is already inserted in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (sec_bkt->signatures[i].alt == sig &&
+ sec_bkt->signatures[i].current == alt_hash) {
+ k = (char *)keys + sec_bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (sec_bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Copy key */
+ rte_memcpy(new_k, key, h->key_len);
+
+ /* Insert new entry is there is room in the primary bucket */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ /* Check if slot is available */
+ if (likely(prim_bkt->signatures[i].sig == NULL_SIGNATURE)) {
+ prim_bkt->signatures[i].current = sig;
+ prim_bkt->signatures[i].alt = alt_hash;
+ prim_bkt->key_idx[i] = new_idx;
+ return new_idx - 1;
+ }
+ }
+
+ /* Primary bucket is full, so we need to make space for new entry */
+ ret = make_space_bucket(h, prim_bkt);
+ /*
+ * After recursive function.
+ * Insert the new entry in the position of the pushed entry
+ * if successful or return error and
+ * store the new slot back in the ring
+ */
+ if (ret >= 0) {
+ prim_bkt->signatures[ret].current = sig;
+ prim_bkt->signatures[ret].alt = alt_hash;
+ prim_bkt->key_idx[ret] = new_idx;
+ return (new_idx - 1);
+ }
+
+ /* Error in addition, store new slot back in the ring and return error */
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t) new_idx));
+ return ret;
+
+}
+
+int32_t
+rte_hash_add_key_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_add_key_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_add_key(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+static inline int32_t
+__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ uint32_t bucket_idx;
+ hash_sig_t alt_hash;
+ unsigned i;
+ struct rte_hash_bucket *bkt;
+ void *k, *keys = h->key_store;
+
+ bucket_idx = sig & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == sig &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+
+ /* Calculate secondary hash */
+ alt_hash = rte_hash_secondary_hash(sig);
+ bucket_idx = alt_hash & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == alt_hash &&
+ bkt->signatures[i].alt == sig) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+
+ return -ENOENT;
+}
+
+int32_t
+rte_hash_lookup_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_lookup(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+static inline int32_t
+__rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ uint32_t bucket_idx;
+ hash_sig_t alt_hash;
+ unsigned i;
+ struct rte_hash_bucket *bkt;
+ void *k, *keys = h->key_store;
+
+ bucket_idx = sig & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == sig &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ bkt->signatures[i].sig = NULL_SIGNATURE;
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t)bkt->key_idx[i]));
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Calculate secondary hash */
+ alt_hash = rte_hash_secondary_hash(sig);
+ bucket_idx = alt_hash & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == alt_hash &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ bkt->signatures[i].sig = NULL_SIGNATURE;
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t)bkt->key_idx[i]));
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ return -ENOENT;
+}
+
+int32_t
+rte_hash_del_key_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_del_key_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_del_key(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_del_key_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+/* Lookup bulk stage 0: Prefetch input key */
+static inline void
+lookup_stage0(unsigned *idx, uint64_t *lookup_mask,
+ const void * const *keys)
+{
+ *idx = __builtin_ctzl(*lookup_mask);
+ if (*lookup_mask == 0)
+ *idx = 0;
+
+ rte_prefetch0(keys[*idx]);
+ *lookup_mask &= ~(1llu << *idx);
+}
+
+/*
+ * Lookup bulk stage 1: Calculate primary/secondary hashes
+ * and prefetch primary/secondary buckets
+ */
+static inline void
+lookup_stage1(unsigned idx, hash_sig_t *prim_hash, hash_sig_t *sec_hash,
+ const struct rte_hash_bucket **primary_bkt,
+ const struct rte_hash_bucket **secondary_bkt,
+ hash_sig_t *hash_vals, const void * const *keys,
+ const struct rte_hash *h)
+{
+ *prim_hash = rte_hash_hash(h, keys[idx]);
+ hash_vals[idx] = *prim_hash;
+ *sec_hash = rte_hash_secondary_hash(*prim_hash);
+
+ *primary_bkt = &h->buckets[*prim_hash & h->bucket_bitmask];
+ *secondary_bkt = &h->buckets[*sec_hash & h->bucket_bitmask];
+
+ rte_prefetch0(*primary_bkt);
+ rte_prefetch0(*secondary_bkt);
+}
+
+/*
+ * Lookup bulk stage 2: Search for match hashes in primary/secondary locations
+ * and prefetch first key slot
+ */
+static inline void
+lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
+ const struct rte_hash_bucket *prim_bkt,
+ const struct rte_hash_bucket *sec_bkt,
+ const void **key_slot, int32_t *positions,
+ uint64_t *extra_hits_mask, const void *keys,
+ const struct rte_hash *h)
+{
+ unsigned prim_hash_matches, sec_hash_matches, key_idx, i;
+ unsigned total_hash_matches;
+
+ prim_hash_matches = 1 << RTE_HASH_BUCKET_ENTRIES;
+ sec_hash_matches = 1 << RTE_HASH_BUCKET_ENTRIES;
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ prim_hash_matches |= ((prim_hash == prim_bkt->signatures[i].current) << i);
+ sec_hash_matches |= ((sec_hash == sec_bkt->signatures[i].current) << i);
+ }
+
+ key_idx = prim_bkt->key_idx[__builtin_ctzl(prim_hash_matches)];
+ if (key_idx == 0)
+ key_idx = sec_bkt->key_idx[__builtin_ctzl(sec_hash_matches)];
+
+ total_hash_matches = (prim_hash_matches |
+ (sec_hash_matches << (RTE_HASH_BUCKET_ENTRIES + 1)));
+ *key_slot = (const char *)keys + key_idx * h->key_entry_size;
+
+ rte_prefetch0(*key_slot);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ positions[idx] = (key_idx - 1);
+
+ *extra_hits_mask |= (uint64_t)(__builtin_popcount(total_hash_matches) > 3) << idx;
+
+}
+
+
+/* Lookup bulk stage 3: Check if key matches, update hit mask */
+static inline void
+lookup_stage3(unsigned idx, const void *key_slot, const void * const *keys,
+ uint64_t *hits, const struct rte_hash *h)
+{
+ unsigned hit;
+
+ hit = !h->rte_hash_cmp_eq(key_slot, keys[idx], h->key_len);
+ *hits |= (uint64_t)(hit) << idx;
+}
+
+static inline void
+__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, int32_t *positions)
+{
+ uint64_t hits = 0;
+ uint64_t extra_hits_mask = 0;
+ uint64_t lookup_mask, miss_mask;
+ unsigned idx;
+ const void *key_store = h->key_store;
+ hash_sig_t hash_vals[RTE_HASH_LOOKUP_BULK_MAX];
+
+ unsigned idx00, idx01, idx10, idx11, idx20, idx21, idx30, idx31;
+ const struct rte_hash_bucket *primary_bkt10, *primary_bkt11;
+ const struct rte_hash_bucket *secondary_bkt10, *secondary_bkt11;
+ const struct rte_hash_bucket *primary_bkt20, *primary_bkt21;
+ const struct rte_hash_bucket *secondary_bkt20, *secondary_bkt21;
+ const void *k_slot20, *k_slot21, *k_slot30, *k_slot31;
+ hash_sig_t primary_hash10, primary_hash11;
+ hash_sig_t secondary_hash10, secondary_hash11;
+ hash_sig_t primary_hash20, primary_hash21;
+ hash_sig_t secondary_hash20, secondary_hash21;
+
+ lookup_mask = (uint64_t) -1 >> (64 - num_keys);
+ miss_mask = lookup_mask;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+
+ while (lookup_mask) {
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20,
+ primary_bkt20, secondary_bkt20, &k_slot20, positions,
+ &extra_hits_mask, key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21,
+ primary_bkt21, secondary_bkt21, &k_slot21, positions,
+ &extra_hits_mask, key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ }
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+
+ /* ignore any items we have already found */
+ extra_hits_mask &= ~hits;
+
+ if (unlikely(extra_hits_mask)) {
+ /* run a single search for each remaining item */
+ do {
+ idx = __builtin_ctzl(extra_hits_mask);
+ positions[idx] = rte_hash_lookup_with_hash(h, keys[idx],
+ hash_vals[idx]);
+ extra_hits_mask &= ~(1llu << idx);
+ if (positions[idx] >= 0)
+ hits |= 1llu << idx;
+ } while (extra_hits_mask);
+ }
+
+ miss_mask &= ~hits;
+ if (unlikely(miss_mask)) {
+ do {
+ idx = __builtin_ctzl(miss_mask);
+ positions[idx] = -ENOENT;
+ miss_mask &= ~(1llu << idx);
+ } while (miss_mask);
+ }
+}
+
+int
+rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, int32_t *positions)
+{
+ RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
+ (num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
+ (positions == NULL)), -EINVAL);
+
+ __rte_hash_lookup_bulk(h, keys, num_keys, positions);
+ return 0;
+}
+
+/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
+static int
+rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
+{
+ const __m128i k1 = _mm_loadu_si128((const __m128i *) key1);
+ const __m128i k2 = _mm_loadu_si128((const __m128i *) key2);
+ const __m128i x = _mm_xor_si128(k1, k2);
+
+ return !_mm_test_all_zeros(x, x);
+}
+
+static int
+rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k16_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 16,
+ (const char *) key2 + 16, key_len);
+}
+
+static int
+rte_hash_k48_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k16_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 16,
+ (const char *) key2 + 16, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 32,
+ (const char *) key2 + 32, key_len);
+}
+
+static int
+rte_hash_k64_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k32_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 32,
+ (const char *) key2 + 32, key_len);
+}
+
+static int
+rte_hash_k80_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
+
+static int
+rte_hash_k96_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
+
+static int
+rte_hash_k112_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 96,
+ (const char *) key2 + 96, key_len);
+}
+
+static int
+rte_hash_k128_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k64_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
diff --git a/lib/librte_hash/rte_hash.c b/lib/librte_hash/rte_hash.c
deleted file mode 100644
index 5100a75..0000000
--- a/lib/librte_hash/rte_hash.c
+++ /dev/null
@@ -1,499 +0,0 @@
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <string.h>
-#include <stdint.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <sys/queue.h>
-
-#include <rte_common.h>
-#include <rte_memory.h> /* for definition of RTE_CACHE_LINE_SIZE */
-#include <rte_log.h>
-#include <rte_memcpy.h>
-#include <rte_prefetch.h>
-#include <rte_branch_prediction.h>
-#include <rte_memzone.h>
-#include <rte_malloc.h>
-#include <rte_eal.h>
-#include <rte_eal_memconfig.h>
-#include <rte_per_lcore.h>
-#include <rte_errno.h>
-#include <rte_string_fns.h>
-#include <rte_cpuflags.h>
-#include <rte_log.h>
-#include <rte_rwlock.h>
-#include <rte_spinlock.h>
-
-#include "rte_hash.h"
-
-TAILQ_HEAD(rte_hash_list, rte_tailq_entry);
-
-static struct rte_tailq_elem rte_hash_tailq = {
- .name = "RTE_HASH",
-};
-EAL_REGISTER_TAILQ(rte_hash_tailq)
-
-/* Macro to enable/disable run-time checking of function parameters */
-#if defined(RTE_LIBRTE_HASH_DEBUG)
-#define RETURN_IF_TRUE(cond, retval) do { \
- if (cond) return (retval); \
-} while (0)
-#else
-#define RETURN_IF_TRUE(cond, retval)
-#endif
-
-/* Hash function used if none is specified */
-#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
-#include <rte_hash_crc.h>
-#define DEFAULT_HASH_FUNC rte_hash_crc
-#else
-#include <rte_jhash.h>
-#define DEFAULT_HASH_FUNC rte_jhash
-#endif
-
-/* Signature bucket size is a multiple of this value */
-#define SIG_BUCKET_ALIGNMENT 16
-
-/* Stoered key size is a multiple of this value */
-#define KEY_ALIGNMENT 16
-
-/* The high bit is always set in real signatures */
-#define NULL_SIGNATURE 0
-
-struct rte_hash {
- char name[RTE_HASH_NAMESIZE]; /**< Name of the hash. */
- uint32_t entries; /**< Total table entries. */
- uint32_t bucket_entries; /**< Bucket entries. */
- uint32_t key_len; /**< Length of hash key. */
- rte_hash_function hash_func; /**< Function used to calculate hash. */
- uint32_t hash_func_init_val; /**< Init value used by hash_func. */
- uint32_t num_buckets; /**< Number of buckets in table. */
- uint32_t bucket_bitmask; /**< Bitmask for getting bucket index
- from hash signature. */
- hash_sig_t sig_msb; /**< MSB is always set in valid signatures. */
- uint8_t *sig_tbl; /**< Flat array of hash signature buckets. */
- uint32_t sig_tbl_bucket_size; /**< Signature buckets may be padded for
- alignment reasons, and this is the
- bucket size used by sig_tbl. */
- uint8_t *key_tbl; /**< Flat array of key value buckets. */
- uint32_t key_tbl_key_size; /**< Keys may be padded for alignment
- reasons, and this is the key size
- used by key_tbl. */
-};
-
-/* Returns a pointer to the first signature in specified bucket. */
-static inline hash_sig_t *
-get_sig_tbl_bucket(const struct rte_hash *h, uint32_t bucket_index)
-{
- return RTE_PTR_ADD(h->sig_tbl, (bucket_index *
- h->sig_tbl_bucket_size));
-}
-
-/* Returns a pointer to the first key in specified bucket. */
-static inline uint8_t *
-get_key_tbl_bucket(const struct rte_hash *h, uint32_t bucket_index)
-{
- return RTE_PTR_ADD(h->key_tbl, (bucket_index * h->bucket_entries *
- h->key_tbl_key_size));
-}
-
-/* Returns a pointer to a key at a specific position in a specified bucket. */
-static inline void *
-get_key_from_bucket(const struct rte_hash *h, uint8_t *bkt, uint32_t pos)
-{
- return RTE_PTR_ADD(bkt, pos * h->key_tbl_key_size);
-}
-
-/* Does integer division with rounding-up of result. */
-static inline uint32_t
-div_roundup(uint32_t numerator, uint32_t denominator)
-{
- return (numerator + denominator - 1) / denominator;
-}
-
-/* Increases a size (if needed) to a multiple of alignment. */
-static inline uint32_t
-align_size(uint32_t val, uint32_t alignment)
-{
- return alignment * div_roundup(val, alignment);
-}
-
-/* Returns the index into the bucket of the first occurrence of a signature. */
-static inline int
-find_first(uint32_t sig, const uint32_t *sig_bucket, uint32_t num_sigs)
-{
- uint32_t i;
- for (i = 0; i < num_sigs; i++) {
- if (sig == sig_bucket[i])
- return i;
- }
- return -1;
-}
-
-struct rte_hash *
-rte_hash_find_existing(const char *name)
-{
- struct rte_hash *h = NULL;
- struct rte_tailq_entry *te;
- struct rte_hash_list *hash_list;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK);
- TAILQ_FOREACH(te, hash_list, next) {
- h = (struct rte_hash *) te->data;
- if (strncmp(name, h->name, RTE_HASH_NAMESIZE) == 0)
- break;
- }
- rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- if (te == NULL) {
- rte_errno = ENOENT;
- return NULL;
- }
- return h;
-}
-
-struct rte_hash *
-rte_hash_create(const struct rte_hash_parameters *params)
-{
- struct rte_hash *h = NULL;
- struct rte_tailq_entry *te;
- uint32_t num_buckets, sig_bucket_size, key_size,
- hash_tbl_size, sig_tbl_size, key_tbl_size, mem_size;
- char hash_name[RTE_HASH_NAMESIZE];
- struct rte_hash_list *hash_list;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- /* Check for valid parameters */
- if ((params == NULL) ||
- (params->entries > RTE_HASH_ENTRIES_MAX) ||
- (params->bucket_entries > RTE_HASH_BUCKET_ENTRIES_MAX) ||
- (params->entries < params->bucket_entries) ||
- !rte_is_power_of_2(params->entries) ||
- !rte_is_power_of_2(params->bucket_entries) ||
- (params->key_len == 0) ||
- (params->key_len > RTE_HASH_KEY_LENGTH_MAX)) {
- rte_errno = EINVAL;
- RTE_LOG(ERR, HASH, "rte_hash_create has invalid parameters\n");
- return NULL;
- }
-
- snprintf(hash_name, sizeof(hash_name), "HT_%s", params->name);
-
- /* Calculate hash dimensions */
- num_buckets = params->entries / params->bucket_entries;
- sig_bucket_size = align_size(params->bucket_entries *
- sizeof(hash_sig_t), SIG_BUCKET_ALIGNMENT);
- key_size = align_size(params->key_len, KEY_ALIGNMENT);
-
- hash_tbl_size = align_size(sizeof(struct rte_hash), RTE_CACHE_LINE_SIZE);
- sig_tbl_size = align_size(num_buckets * sig_bucket_size,
- RTE_CACHE_LINE_SIZE);
- key_tbl_size = align_size(num_buckets * key_size *
- params->bucket_entries, RTE_CACHE_LINE_SIZE);
-
- /* Total memory required for hash context */
- mem_size = hash_tbl_size + sig_tbl_size + key_tbl_size;
-
- rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
-
- /* guarantee there's no existing */
- TAILQ_FOREACH(te, hash_list, next) {
- h = (struct rte_hash *) te->data;
- if (strncmp(params->name, h->name, RTE_HASH_NAMESIZE) == 0)
- break;
- }
- if (te != NULL)
- goto exit;
-
- te = rte_zmalloc("HASH_TAILQ_ENTRY", sizeof(*te), 0);
- if (te == NULL) {
- RTE_LOG(ERR, HASH, "tailq entry allocation failed\n");
- goto exit;
- }
-
- h = (struct rte_hash *)rte_zmalloc_socket(hash_name, mem_size,
- RTE_CACHE_LINE_SIZE, params->socket_id);
- if (h == NULL) {
- RTE_LOG(ERR, HASH, "memory allocation failed\n");
- rte_free(te);
- goto exit;
- }
-
- /* Setup hash context */
- snprintf(h->name, sizeof(h->name), "%s", params->name);
- h->entries = params->entries;
- h->bucket_entries = params->bucket_entries;
- h->key_len = params->key_len;
- h->hash_func_init_val = params->hash_func_init_val;
- h->num_buckets = num_buckets;
- h->bucket_bitmask = h->num_buckets - 1;
- h->sig_msb = 1 << (sizeof(hash_sig_t) * 8 - 1);
- h->sig_tbl = (uint8_t *)h + hash_tbl_size;
- h->sig_tbl_bucket_size = sig_bucket_size;
- h->key_tbl = h->sig_tbl + sig_tbl_size;
- h->key_tbl_key_size = key_size;
- h->hash_func = (params->hash_func == NULL) ?
- DEFAULT_HASH_FUNC : params->hash_func;
-
- te->data = (void *) h;
-
- TAILQ_INSERT_TAIL(hash_list, te, next);
-
-exit:
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- return h;
-}
-
-void
-rte_hash_free(struct rte_hash *h)
-{
- struct rte_tailq_entry *te;
- struct rte_hash_list *hash_list;
-
- if (h == NULL)
- return;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
-
- /* find out tailq entry */
- TAILQ_FOREACH(te, hash_list, next) {
- if (te->data == (void *) h)
- break;
- }
-
- if (te == NULL) {
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
- return;
- }
-
- TAILQ_REMOVE(hash_list, te, next);
-
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- rte_free(h);
- rte_free(te);
-}
-
-hash_sig_t
-rte_hash_hash(const struct rte_hash *h, const void *key)
-{
- /* calc hash result by key */
- return h->hash_func(key, h->key_len, h->hash_func_init_val);
-}
-
-static inline int32_t
-__rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
- int32_t pos;
-
- /* Get the hash signature and bucket index */
- sig |= h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- /* Check if any free slot within the bucket to add the new key */
- pos = find_first(NULL_SIGNATURE, sig_bucket, h->bucket_entries);
-
- if (unlikely(pos < 0))
- return -ENOSPC;
-
- /* Add the new key to the bucket */
- sig_bucket[pos] = sig;
- rte_memcpy(get_key_from_bucket(h, key_bucket, pos), key, h->key_len);
- return bucket_index * h->bucket_entries + pos;
-}
-
-int32_t
-rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_add_key(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-static inline int32_t
-__rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
-
- /* Get the hash signature and bucket index */
- sig = sig | h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- sig_bucket[i] = NULL_SIGNATURE;
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- return -ENOENT;
-}
-
-int32_t
-rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_del_key_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_del_key(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_del_key_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-static inline int32_t
-__rte_hash_lookup_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
-
- /* Get the hash signature and bucket index */
- sig |= h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- return -ENOENT;
-}
-
-int32_t
-rte_hash_lookup_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_lookup(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-int
-rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
- uint32_t num_keys, int32_t *positions)
-{
- uint32_t i, j, bucket_index;
- hash_sig_t sigs[RTE_HASH_LOOKUP_BULK_MAX];
-
- RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
- (num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
- (positions == NULL)), -EINVAL);
-
- /* Get the hash signature and bucket index */
- for (i = 0; i < num_keys; i++) {
- sigs[i] = h->hash_func(keys[i], h->key_len,
- h->hash_func_init_val) | h->sig_msb;
- bucket_index = sigs[i] & h->bucket_bitmask;
-
- /* Pre-fetch relevant buckets */
- rte_prefetch1((void *) get_sig_tbl_bucket(h, bucket_index));
- rte_prefetch1((void *) get_key_tbl_bucket(h, bucket_index));
- }
-
- /* Check if key is already present in the hash */
- for (i = 0; i < num_keys; i++) {
- bucket_index = sigs[i] & h->bucket_bitmask;
- hash_sig_t *sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- uint8_t *key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- positions[i] = -ENOENT;
-
- for (j = 0; j < h->bucket_entries; j++) {
- if ((sigs[i] == sig_bucket[j]) &&
- likely(memcmp(keys[i],
- get_key_from_bucket(h, key_bucket, j),
- h->key_len) == 0)) {
- positions[i] = bucket_index *
- h->bucket_entries + j;
- break;
- }
- }
- }
-
- return 0;
-}
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index da0a00a..936d170 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -47,21 +47,21 @@ extern "C" {
#endif
/** Maximum size of hash table that can be created. */
-#define RTE_HASH_ENTRIES_MAX (1 << 26)
+#define RTE_HASH_ENTRIES_MAX (1 << 30)
-/** Maximum bucket size that can be created. */
-#define RTE_HASH_BUCKET_ENTRIES_MAX 16
+/** @deprecated Maximum bucket size that can be created. */
+#define RTE_HASH_BUCKET_ENTRIES_MAX 4
-/** Maximum length of key that can be used. */
+/** @deprecated Maximum length of key that can be used. */
#define RTE_HASH_KEY_LENGTH_MAX 64
-/** Max number of keys that can be searched for using rte_hash_lookup_multi. */
-#define RTE_HASH_LOOKUP_BULK_MAX 16
-#define RTE_HASH_LOOKUP_MULTI_MAX RTE_HASH_LOOKUP_BULK_MAX
-
-/** Max number of characters in hash name.*/
+/** Maximum number of characters in hash name.*/
#define RTE_HASH_NAMESIZE 32
+/** Maximum number of keys that can be searched for using rte_hash_lookup_bulk. */
+#define RTE_HASH_LOOKUP_BULK_MAX 64
+#define RTE_HASH_LOOKUP_MULTI_MAX RTE_HASH_LOOKUP_BULK_MAX
+
/** Signature of key that is stored internally. */
typedef uint32_t hash_sig_t;
@@ -70,15 +70,14 @@ typedef uint32_t (*rte_hash_function)(const void *key, uint32_t key_len,
uint32_t init_val);
/**
- * Parameters used when creating the hash table. The total table entries and
- * bucket entries must be a power of 2.
+ * Parameters used when creating the hash table.
*/
struct rte_hash_parameters {
const char *name; /**< Name of the hash. */
uint32_t entries; /**< Total hash table entries. */
- uint32_t bucket_entries; /**< Bucket entries. */
+ uint32_t bucket_entries; /**< Bucket entries. */
uint32_t key_len; /**< Length of hash key. */
- rte_hash_function hash_func; /**< Function used to calculate hash. */
+ rte_hash_function hash_func; /**< Primary Hash function used to calculate hash. */
uint32_t hash_func_init_val; /**< Init value used by hash_func. */
int socket_id; /**< NUMA Socket ID for memory. */
};
@@ -86,6 +85,7 @@ struct rte_hash_parameters {
/** @internal A hash table structure. */
struct rte_hash;
+
/**
* Create a new hash table.
*
@@ -106,7 +106,6 @@ struct rte_hash;
struct rte_hash *
rte_hash_create(const struct rte_hash_parameters *params);
-
/**
* Find an existing hash table object and return a pointer to it.
*
@@ -129,7 +128,8 @@ void
rte_hash_free(struct rte_hash *h);
/**
- * Add a key to an existing hash table. This operation is not multi-thread safe
+ * Add a key to an existing hash table.
+ * This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
@@ -146,7 +146,8 @@ int32_t
rte_hash_add_key(const struct rte_hash *h, const void *key);
/**
- * Add a key to an existing hash table. This operation is not multi-thread safe
+ * Add a key to an existing hash table.
+ * This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
@@ -154,7 +155,7 @@ rte_hash_add_key(const struct rte_hash *h, const void *key);
* @param key
* Key to add to the hash table.
* @param sig
- * Hash value to add to the hash table.
+ * Precomputed hash value for 'key'.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOSPC if there is no space in the hash for this key.
@@ -162,12 +163,12 @@ rte_hash_add_key(const struct rte_hash *h, const void *key);
* array of user data. This value is unique for this key.
*/
int32_t
-rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig);
+rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
/**
- * Remove a key from an existing hash table. This operation is not multi-thread
- * safe and should only be called from one thread.
+ * Remove a key from an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
*
* @param h
* Hash table to remove the key from.
@@ -184,15 +185,16 @@ int32_t
rte_hash_del_key(const struct rte_hash *h, const void *key);
/**
- * Remove a key from an existing hash table. This operation is not multi-thread
- * safe and should only be called from one thread.
+ * Remove a key from an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
*
* @param h
* Hash table to remove the key from.
* @param key
* Key to remove from the hash table.
* @param sig
- * Hash value to remove from the hash table.
+ * Precomputed hash value for 'key'.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOENT if the key is not found.
@@ -201,12 +203,11 @@ rte_hash_del_key(const struct rte_hash *h, const void *key);
* value that was returned when the key was added.
*/
int32_t
-rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig);
-
+rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
/**
- * Find a key in the hash table. This operation is multi-thread safe.
+ * Find a key in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
@@ -223,14 +224,15 @@ int32_t
rte_hash_lookup(const struct rte_hash *h, const void *key);
/**
- * Find a key in the hash table. This operation is multi-thread safe.
+ * Find a key in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
* @param key
* Key to find.
* @param sig
- * Hash value to find.
+ * Hash value to remove from the hash table.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOENT if the key is not found.
@@ -243,7 +245,8 @@ rte_hash_lookup_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig);
/**
- * Calc a hash value by key. This operation is not multi-process safe.
+ * Calc a hash value by key.
+ * This operation is not multi-thread safe.
*
* @param h
* Hash table to look in.
@@ -257,7 +260,8 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
#define rte_hash_lookup_multi rte_hash_lookup_bulk
/**
- * Find multiple keys in the hash table. This operation is multi-thread safe.
+ * Find multiple keys in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
@@ -277,6 +281,7 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions);
+
#ifdef __cplusplus
}
#endif
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v6 2/7] hash: add new function rte_hash_reset
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 " Pablo de Lara
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 1/7] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
@ 2015-07-10 23:30 ` Pablo de Lara
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 3/7] hash: add new functionality to store data in hash table Pablo de Lara
` (5 subsequent siblings)
7 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 23:30 UTC (permalink / raw)
To: dev
Added reset function to be able to empty the table,
without having to destroy and create it again.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 4 +---
app/test/test_hash_perf.c | 12 +++---------
lib/librte_hash/rte_cuckoo_hash.c | 21 +++++++++++++++++++++
lib/librte_hash/rte_hash.h | 11 +++++++++--
lib/librte_hash/rte_hash_version.map | 8 ++++++++
5 files changed, 42 insertions(+), 14 deletions(-)
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index e70d859..448586c 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -1132,9 +1132,7 @@ static int test_average_table_utilization(void)
average_keys_added += added_keys;
/* Reset the table */
- rte_hash_free(handle);
- handle = rte_hash_create(&ut_params);
- RETURN_IF_ERROR(handle == NULL, "hash creation failed");
+ rte_hash_reset(handle);
/* Print a dot to show progress on operations */
printf(".");
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index a54f3c1..b01b040 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -382,14 +382,10 @@ free_table(unsigned table_index)
rte_hash_free(h[table_index]);
}
-static int
+static void
reset_table(unsigned table_index)
{
- free_table(table_index);
- if (create_table(table_index) != 0)
- return -1;
-
- return 0;
+ rte_hash_reset(h[table_index]);
}
static int
@@ -421,13 +417,11 @@ run_all_tbl_perf_tests(unsigned with_pushes)
if (timed_deletes(with_hash, i) < 0)
return -1;
- if (reset_table(i) < 0)
- return -1;
+ reset_table(i);
/* Print a dot to show progress on operations */
printf(".");
fflush(stdout);
-
}
free_table(i);
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index 50e3acd..15c5da5 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -371,6 +371,27 @@ rte_hash_secondary_hash(const hash_sig_t primary_hash)
return (primary_hash ^ ((tag + 1) * alt_bits_xor));
}
+void
+rte_hash_reset(struct rte_hash *h)
+{
+ void *ptr;
+ unsigned i;
+
+ if (h == NULL)
+ return;
+
+ memset(h->buckets, 0, h->num_buckets * sizeof(struct rte_hash_bucket));
+ memset(h->key_store, 0, h->key_entry_size * (h->entries + 1));
+
+ /* clear the free ring */
+ while (rte_ring_dequeue(h->free_slots, &ptr) == 0)
+ rte_pause();
+
+ /* Repopulate the free slots ring. Entry zero is reserved for key misses */
+ for (i = 1; i < h->entries + 1; i++)
+ rte_ring_sp_enqueue(h->free_slots, (void *)((uintptr_t) i));
+}
+
/* Search for an entry that can be pushed to its alternative location */
static inline int
make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt)
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index 936d170..8bbc9f0 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -128,8 +128,15 @@ void
rte_hash_free(struct rte_hash *h);
/**
- * Add a key to an existing hash table.
- * This operation is not multi-thread safe
+ * Reset all hash structure, by zeroing all entries
+ * @param h
+ * Hash table to reset
+ */
+void
+rte_hash_reset(struct rte_hash *h);
+
+/**
+ * Add a key to an existing hash table. This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index 94a0fec..d5f5af5 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -18,3 +18,11 @@ DPDK_2.0 {
local: *;
};
+
+DPDK_2.1 {
+ global:
+
+ rte_hash_reset;
+
+ local: *;
+} DPDK_2.0;
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v6 3/7] hash: add new functionality to store data in hash table
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 " Pablo de Lara
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 1/7] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 2/7] hash: add new function rte_hash_reset Pablo de Lara
@ 2015-07-10 23:30 ` Pablo de Lara
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 4/7] hash: add iterate function Pablo de Lara
` (4 subsequent siblings)
7 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 23:30 UTC (permalink / raw)
To: dev
Usually hash tables not only store keys, but also data associated
to them. In order to maintain the existing API, the old functions
will still return the index where the key was stored.
The new functions will return the data associated to that key.
In the case of the lookup_bulk function, it will return also
the number of entries found and a bitmask of which entries
were found.
Unit tests have been updated to use these new functions.
As a final point, a flag has been added in rte_hash_parameters
to indicate if there are new parameters for future versions,
so there is no need to maintain multiple versions
of the existing functions in the future.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash_perf.c | 252 +++++++++++++++++++++++++----------
lib/librte_hash/rte_cuckoo_hash.c | 193 +++++++++++++++++++++------
lib/librte_hash/rte_hash.h | 108 ++++++++++++++-
lib/librte_hash/rte_hash_version.map | 6 +
4 files changed, 439 insertions(+), 120 deletions(-)
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index b01b040..e9a522b 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -83,7 +83,7 @@ struct rte_hash *h[NUM_KEYSIZES];
uint8_t slot_taken[MAX_ENTRIES];
/* Array to store number of cycles per operation */
-uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS][2];
+uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS][2][2];
/* Array to store all input keys */
uint8_t keys[KEYS_TO_ADD][MAX_KEYSIZE];
@@ -106,11 +106,16 @@ static struct rte_hash_parameters ut_params = {
};
static int
-create_table(unsigned table_index)
+create_table(unsigned with_data, unsigned table_index)
{
char name[RTE_HASH_NAMESIZE];
- sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
+ if (with_data)
+ /* Table will store 8-byte data */
+ sprintf(name, "test_hash%d_data", hashtest_key_lens[table_index]);
+ else
+ sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
+
ut_params.name = name;
ut_params.key_len = hashtest_key_lens[table_index];
ut_params.socket_id = rte_socket_id();
@@ -234,6 +239,7 @@ get_input_keys(unsigned with_pushes, unsigned table_index)
else {
/* Store the returned position and mark slot as taken */
slot_taken[ret] = 1;
+ positions[i] = ret;
buckets[bucket_idx]++;
success = 1;
i++;
@@ -249,54 +255,116 @@ get_input_keys(unsigned with_pushes, unsigned table_index)
}
static int
-timed_adds(unsigned with_hash, unsigned table_index)
+timed_adds(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i;
const uint64_t start_tsc = rte_rdtsc();
+ void *data;
int32_t ret;
for (i = 0; i < KEYS_TO_ADD; i++) {
- if (with_hash)
+ data = (void *) ((uintptr_t) signatures[i]);
+ if (with_hash && with_data) {
+ ret = rte_hash_add_key_with_hash_data(h[table_index],
+ (const void *) keys[i],
+ signatures[i], data);
+ if (ret < 0) {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
+ } else if (with_hash && !with_data) {
ret = rte_hash_add_key_with_hash(h[table_index],
(const void *) keys[i],
signatures[i]);
- else
+ if (ret >= 0)
+ positions[i] = ret;
+ else {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
+ } else if (!with_hash && with_data) {
+ ret = rte_hash_add_key_data(h[table_index],
+ (const void *) keys[i],
+ data);
+ if (ret < 0) {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
+ } else {
ret = rte_hash_add_key(h[table_index], keys[i]);
-
- if (ret >= 0)
- positions[i] = ret;
- else {
- printf("Failed to add key number %u\n", ret);
- return -1;
+ if (ret >= 0)
+ positions[i] = ret;
+ else {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
}
}
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][ADD][with_hash] = time_taken/KEYS_TO_ADD;
+ cycles[table_index][ADD][with_hash][with_data] = time_taken/KEYS_TO_ADD;
+
return 0;
}
static int
-timed_lookups(unsigned with_hash, unsigned table_index)
+timed_lookups(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i, j;
const uint64_t start_tsc = rte_rdtsc();
+ void *ret_data;
+ void *expected_data;
int32_t ret;
for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
for (j = 0; j < KEYS_TO_ADD; j++) {
- if (with_hash)
+ if (with_hash && with_data) {
+ ret = rte_hash_lookup_with_hash_data(h[table_index],
+ (const void *) keys[j],
+ signatures[j], &ret_data);
+ if (ret < 0) {
+ printf("Key number %u was not found\n", j);
+ return -1;
+ }
+ expected_data = (void *) ((uintptr_t) signatures[j]);
+ if (ret_data != expected_data) {
+ printf("Data returned for key number %u is %p,"
+ " but should be %p\n", j, ret_data,
+ expected_data);
+ return -1;
+ }
+ } else if (with_hash && !with_data) {
ret = rte_hash_lookup_with_hash(h[table_index],
(const void *) keys[j],
signatures[j]);
- else
+ if (ret < 0 || ret != positions[j]) {
+ printf("Key looked up in %d, should be in %d\n",
+ ret, positions[j]);
+ return -1;
+ }
+ } else if (!with_hash && with_data) {
+ ret = rte_hash_lookup_data(h[table_index],
+ (const void *) keys[j], &ret_data);
+ if (ret < 0) {
+ printf("Key number %u was not found\n", j);
+ return -1;
+ }
+ expected_data = (void *) ((uintptr_t) signatures[j]);
+ if (ret_data != expected_data) {
+ printf("Data returned for key number %u is %p,"
+ " but should be %p\n", j, ret_data,
+ expected_data);
+ return -1;
+ }
+ } else {
ret = rte_hash_lookup(h[table_index], keys[j]);
- if (ret < 0 || ret != positions[j]) {
- printf("Key looked up in %d, should be in %d\n",
- ret, positions[j]);
- return -1;
+ if (ret < 0 || ret != positions[j]) {
+ printf("Key looked up in %d, should be in %d\n",
+ ret, positions[j]);
+ return -1;
+ }
}
}
}
@@ -304,34 +372,65 @@ timed_lookups(unsigned with_hash, unsigned table_index)
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][LOOKUP][with_hash] = time_taken/NUM_LOOKUPS;
+ cycles[table_index][LOOKUP][with_hash][with_data] = time_taken/NUM_LOOKUPS;
return 0;
}
static int
-timed_lookups_multi(unsigned table_index)
+timed_lookups_multi(unsigned with_data, unsigned table_index)
{
unsigned i, j, k;
int32_t positions_burst[BURST_SIZE];
const void *keys_burst[BURST_SIZE];
+ void *expected_data[BURST_SIZE];
+ void *ret_data[BURST_SIZE];
+ uint64_t hit_mask;
+ int ret;
+
const uint64_t start_tsc = rte_rdtsc();
for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
for (j = 0; j < KEYS_TO_ADD/BURST_SIZE; j++) {
for (k = 0; k < BURST_SIZE; k++)
keys_burst[k] = keys[j * BURST_SIZE + k];
-
- rte_hash_lookup_bulk(h[table_index],
+ if (with_data) {
+ ret = rte_hash_lookup_bulk_data(h[table_index],
+ (const void **) keys_burst,
+ BURST_SIZE,
+ &hit_mask,
+ ret_data);
+ if (ret != BURST_SIZE) {
+ printf("Expect to find %u keys,"
+ " but found %d\n", BURST_SIZE, ret);
+ return -1;
+ }
+ for (k = 0; k < BURST_SIZE; k++) {
+ if ((hit_mask & (1ULL << k)) == 0) {
+ printf("Key number %u not found\n",
+ j * BURST_SIZE + k);
+ return -1;
+ }
+ expected_data[k] = (void *) ((uintptr_t) signatures[j * BURST_SIZE + k]);
+ if (ret_data[k] != expected_data[k]) {
+ printf("Data returned for key number %u is %p,"
+ " but should be %p\n", j * BURST_SIZE + k,
+ ret_data[k], expected_data[k]);
+ return -1;
+ }
+ }
+ } else {
+ rte_hash_lookup_bulk(h[table_index],
(const void **) keys_burst,
BURST_SIZE,
positions_burst);
- for (k = 0; k < BURST_SIZE; k++) {
- if (positions_burst[k] != positions[j * BURST_SIZE + k]) {
- printf("Key looked up in %d, should be in %d\n",
- positions_burst[k],
- positions[j * BURST_SIZE + k]);
- return -1;
+ for (k = 0; k < BURST_SIZE; k++) {
+ if (positions_burst[k] != positions[j * BURST_SIZE + k]) {
+ printf("Key looked up in %d, should be in %d\n",
+ positions_burst[k],
+ positions[j * BURST_SIZE + k]);
+ return -1;
+ }
}
}
}
@@ -340,19 +439,20 @@ timed_lookups_multi(unsigned table_index)
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][LOOKUP_MULTI][0] = time_taken/NUM_LOOKUPS;
+ cycles[table_index][LOOKUP_MULTI][0][with_data] = time_taken/NUM_LOOKUPS;
return 0;
}
static int
-timed_deletes(unsigned with_hash, unsigned table_index)
+timed_deletes(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i;
const uint64_t start_tsc = rte_rdtsc();
int32_t ret;
for (i = 0; i < KEYS_TO_ADD; i++) {
+ /* There are no delete functions with data, so just call two functions */
if (with_hash)
ret = rte_hash_del_key_with_hash(h[table_index],
(const void *) keys[i],
@@ -371,7 +471,7 @@ timed_deletes(unsigned with_hash, unsigned table_index)
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][DELETE][with_hash] = time_taken/KEYS_TO_ADD;
+ cycles[table_index][DELETE][with_hash][with_data] = time_taken/KEYS_TO_ADD;
return 0;
}
@@ -391,62 +491,67 @@ reset_table(unsigned table_index)
static int
run_all_tbl_perf_tests(unsigned with_pushes)
{
- unsigned i, j, with_hash;
+ unsigned i, j, with_data, with_hash;
printf("Measuring performance, please wait");
fflush(stdout);
- for (i = 0; i < NUM_KEYSIZES; i++) {
- if (create_table(i) < 0)
- return -1;
- if (get_input_keys(with_pushes, i) < 0)
- return -1;
- for (with_hash = 0; with_hash <= 1; with_hash++) {
- if (timed_adds(with_hash, i) < 0)
+ for (with_data = 0; with_data <= 1; with_data++) {
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ if (create_table(with_data, i) < 0)
return -1;
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
-
- if (timed_lookups(with_hash, i) < 0)
+ if (get_input_keys(with_pushes, i) < 0)
return -1;
+ for (with_hash = 0; with_hash <= 1; with_hash++) {
+ if (timed_adds(with_hash, with_data, i) < 0)
+ return -1;
- if (timed_lookups_multi(i) < 0)
- return -1;
+ for (j = 0; j < NUM_SHUFFLES; j++)
+ shuffle_input_keys(i);
- if (timed_deletes(with_hash, i) < 0)
- return -1;
+ if (timed_lookups(with_hash, with_data, i) < 0)
+ return -1;
- reset_table(i);
+ if (timed_lookups_multi(with_data, i) < 0)
+ return -1;
- /* Print a dot to show progress on operations */
- printf(".");
- fflush(stdout);
- }
+ if (timed_deletes(with_hash, with_data, i) < 0)
+ return -1;
- free_table(i);
+ /* Print a dot to show progress on operations */
+ printf(".");
+ fflush(stdout);
+
+ reset_table(i);
+ }
+ free_table(i);
+ }
}
+
printf("\nResults (in CPU cycles/operation)\n");
- printf("---------------------------------\n");
- printf("\nWithout pre-computed hash values\n");
- printf("\n%-18s%-18s%-18s%-18s%-18s\n",
- "Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
- for (i = 0; i < NUM_KEYSIZES; i++) {
- printf("%-18d", hashtest_key_lens[i]);
- for (j = 0; j < NUM_OPERATIONS; j++)
- printf("%-18"PRIu64, cycles[i][j][0]);
- printf("\n");
- }
- printf("\nWith pre-computed hash values\n");
- printf("\n%-18s%-18s%-18s%-18s%-18s\n",
+ printf("-----------------------------------\n");
+ for (with_data = 0; with_data <= 1; with_data++) {
+ if (with_data)
+ printf("\n Operations with 8-byte data\n");
+ else
+ printf("\n Operations without data\n");
+ for (with_hash = 0; with_hash <= 1; with_hash++) {
+ if (with_hash)
+ printf("\nWith pre-computed hash values\n");
+ else
+ printf("\nWithout pre-computed hash values\n");
+
+ printf("\n%-18s%-18s%-18s%-18s%-18s\n",
"Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
- for (i = 0; i < NUM_KEYSIZES; i++) {
- printf("%-18d", hashtest_key_lens[i]);
- for (j = 0; j < NUM_OPERATIONS; j++)
- printf("%-18"PRIu64, cycles[i][j][1]);
- printf("\n");
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ printf("%-18d", hashtest_key_lens[i]);
+ for (j = 0; j < NUM_OPERATIONS; j++)
+ printf("%-18"PRIu64, cycles[i][j][with_hash][with_data]);
+ printf("\n");
+ }
+ }
}
-
return 0;
}
@@ -546,7 +651,6 @@ test_hash_perf(void)
if (run_all_tbl_perf_tests(with_pushes) < 0)
return -1;
}
-
if (fbk_hash_perf_test() < 0)
return -1;
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index 15c5da5..cb1ab31 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -56,6 +56,7 @@
#include <rte_rwlock.h>
#include <rte_spinlock.h>
#include <rte_ring.h>
+#include <rte_compat.h>
#include "rte_hash.h"
@@ -90,6 +91,8 @@ EAL_REGISTER_TAILQ(rte_hash_tailq)
#define NULL_SIGNATURE 0
+#define KEY_ALIGNMENT 16
+
typedef int (*rte_hash_cmp_eq_t)(const void *key1, const void *key2, size_t key_len);
static int rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len);
static int rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len);
@@ -132,6 +135,16 @@ struct rte_hash_signatures {
};
};
+/* Structure that stores key-value pair */
+struct rte_hash_key {
+ union {
+ uintptr_t idata;
+ void *pdata;
+ };
+ /* Variable key size */
+ char key[];
+} __attribute__((aligned(KEY_ALIGNMENT)));
+
/** Bucket structure */
struct rte_hash_bucket {
struct rte_hash_signatures signatures[RTE_HASH_BUCKET_ENTRIES];
@@ -227,7 +240,8 @@ rte_hash_create(const struct rte_hash_parameters *params)
goto err;
}
- const uint32_t key_entry_size = params->key_len;
+ const uint32_t key_entry_size = sizeof(struct rte_hash_key) + params->key_len;
+
/* Store all keys and leave the first entry as a dummy entry for lookup_bulk */
const uint64_t key_tbl_size = key_entry_size * (params->entries + 1);
@@ -461,13 +475,13 @@ make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt)
static inline int32_t
__rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
- hash_sig_t sig)
+ hash_sig_t sig, void *data)
{
hash_sig_t alt_hash;
uint32_t prim_bucket_idx, sec_bucket_idx;
unsigned i;
struct rte_hash_bucket *prim_bkt, *sec_bkt;
- void *new_k, *k, *keys = h->key_store;
+ struct rte_hash_key *new_k, *k, *keys = h->key_store;
void *slot_id;
uint32_t new_idx;
int ret;
@@ -492,9 +506,12 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (prim_bkt->signatures[i].current == sig &&
prim_bkt->signatures[i].alt == alt_hash) {
- k = (char *)keys + prim_bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ prim_bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /* Update data */
+ k->pdata = data;
/*
* Return index where key is stored,
* substracting the first dummy index
@@ -508,9 +525,12 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (sec_bkt->signatures[i].alt == sig &&
sec_bkt->signatures[i].current == alt_hash) {
- k = (char *)keys + sec_bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ sec_bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /* Update data */
+ k->pdata = data;
/*
* Return index where key is stored,
* substracting the first dummy index
@@ -521,7 +541,8 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
}
/* Copy key */
- rte_memcpy(new_k, key, h->key_len);
+ rte_memcpy(new_k->key, key, h->key_len);
+ new_k->pdata = data;
/* Insert new entry is there is room in the primary bucket */
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
@@ -561,25 +582,52 @@ rte_hash_add_key_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, sig);
+ return __rte_hash_add_key_with_hash(h, key, sig, 0);
}
int32_t
rte_hash_add_key(const struct rte_hash *h, const void *key)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
+ return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key), 0);
+}
+
+int
+rte_hash_add_key_with_hash_data(const struct rte_hash *h,
+ const void *key, hash_sig_t sig, void *data)
+{
+ int ret;
+
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ ret = __rte_hash_add_key_with_hash(h, key, sig, data);
+ if (ret >= 0)
+ return 0;
+ else
+ return ret;
}
+int
+rte_hash_add_key_data(const struct rte_hash *h, const void *key, void *data)
+{
+ int ret;
+
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+
+ ret = __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key), data);
+ if (ret >= 0)
+ return 0;
+ else
+ return ret;
+}
static inline int32_t
__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
- hash_sig_t sig)
+ hash_sig_t sig, void **data)
{
uint32_t bucket_idx;
hash_sig_t alt_hash;
unsigned i;
struct rte_hash_bucket *bkt;
- void *k, *keys = h->key_store;
+ struct rte_hash_key *k, *keys = h->key_store;
bucket_idx = sig & h->bucket_bitmask;
bkt = &h->buckets[bucket_idx];
@@ -588,13 +636,17 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == sig &&
bkt->signatures[i].sig != NULL_SIGNATURE) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
+ if (data != NULL)
+ *data = k->pdata;
/*
* Return index where key is stored,
* substracting the first dummy index
*/
return (bkt->key_idx[i] - 1);
+ }
}
}
@@ -607,13 +659,17 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == alt_hash &&
bkt->signatures[i].alt == sig) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
+ if (data != NULL)
+ *data = k->pdata;
/*
* Return index where key is stored,
* substracting the first dummy index
*/
return (bkt->key_idx[i] - 1);
+ }
}
}
@@ -625,14 +681,29 @@ rte_hash_lookup_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, sig);
+ return __rte_hash_lookup_with_hash(h, key, sig, NULL);
}
int32_t
rte_hash_lookup(const struct rte_hash *h, const void *key)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key), NULL);
+}
+
+int
+rte_hash_lookup_with_hash_data(const struct rte_hash *h,
+ const void *key, hash_sig_t sig, void **data)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, sig, data);
+}
+
+int
+rte_hash_lookup_data(const struct rte_hash *h, const void *key, void **data)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key), data);
}
static inline int32_t
@@ -643,7 +714,7 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
hash_sig_t alt_hash;
unsigned i;
struct rte_hash_bucket *bkt;
- void *k, *keys = h->key_store;
+ struct rte_hash_key *k, *keys = h->key_store;
bucket_idx = sig & h->bucket_bitmask;
bkt = &h->buckets[bucket_idx];
@@ -652,8 +723,9 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == sig &&
bkt->signatures[i].sig != NULL_SIGNATURE) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
bkt->signatures[i].sig = NULL_SIGNATURE;
rte_ring_sp_enqueue(h->free_slots,
(void *)((uintptr_t)bkt->key_idx[i]));
@@ -675,8 +747,9 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == alt_hash &&
bkt->signatures[i].sig != NULL_SIGNATURE) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
bkt->signatures[i].sig = NULL_SIGNATURE;
rte_ring_sp_enqueue(h->free_slots,
(void *)((uintptr_t)bkt->key_idx[i]));
@@ -750,7 +823,7 @@ static inline void
lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
const struct rte_hash_bucket *prim_bkt,
const struct rte_hash_bucket *sec_bkt,
- const void **key_slot, int32_t *positions,
+ const struct rte_hash_key **key_slot, int32_t *positions,
uint64_t *extra_hits_mask, const void *keys,
const struct rte_hash *h)
{
@@ -770,7 +843,8 @@ lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
total_hash_matches = (prim_hash_matches |
(sec_hash_matches << (RTE_HASH_BUCKET_ENTRIES + 1)));
- *key_slot = (const char *)keys + key_idx * h->key_entry_size;
+ *key_slot = (const struct rte_hash_key *) ((const char *)keys +
+ key_idx * h->key_entry_size);
rte_prefetch0(*key_slot);
/*
@@ -784,26 +858,31 @@ lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
}
-/* Lookup bulk stage 3: Check if key matches, update hit mask */
+/* Lookup bulk stage 3: Check if key matches, update hit mask and return data */
static inline void
-lookup_stage3(unsigned idx, const void *key_slot, const void * const *keys,
- uint64_t *hits, const struct rte_hash *h)
+lookup_stage3(unsigned idx, const struct rte_hash_key *key_slot, const void * const *keys,
+ void *data[], uint64_t *hits, const struct rte_hash *h)
{
unsigned hit;
- hit = !h->rte_hash_cmp_eq(key_slot, keys[idx], h->key_len);
+ hit = !h->rte_hash_cmp_eq(key_slot->key, keys[idx], h->key_len);
+ if (data != NULL)
+ data[idx] = key_slot->pdata;
+
*hits |= (uint64_t)(hit) << idx;
}
static inline void
__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
- uint32_t num_keys, int32_t *positions)
+ uint32_t num_keys, int32_t *positions,
+ uint64_t *hit_mask, void *data[])
{
uint64_t hits = 0;
uint64_t extra_hits_mask = 0;
uint64_t lookup_mask, miss_mask;
unsigned idx;
const void *key_store = h->key_store;
+ int ret;
hash_sig_t hash_vals[RTE_HASH_LOOKUP_BULK_MAX];
unsigned idx00, idx01, idx10, idx11, idx20, idx21, idx30, idx31;
@@ -811,7 +890,7 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
const struct rte_hash_bucket *secondary_bkt10, *secondary_bkt11;
const struct rte_hash_bucket *primary_bkt20, *primary_bkt21;
const struct rte_hash_bucket *secondary_bkt20, *secondary_bkt21;
- const void *k_slot20, *k_slot21, *k_slot30, *k_slot31;
+ const struct rte_hash_key *k_slot20, *k_slot21, *k_slot30, *k_slot31;
hash_sig_t primary_hash10, primary_hash11;
hash_sig_t secondary_hash10, secondary_hash11;
hash_sig_t primary_hash20, primary_hash21;
@@ -882,8 +961,8 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21,
primary_bkt21, secondary_bkt21, &k_slot21, positions,
&extra_hits_mask, key_store, h);
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
}
k_slot30 = k_slot20, k_slot31 = k_slot21;
@@ -909,8 +988,8 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
key_store, h);
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
k_slot30 = k_slot20, k_slot31 = k_slot21;
idx30 = idx20, idx31 = idx21;
@@ -930,14 +1009,14 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
key_store, h);
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
k_slot30 = k_slot20, k_slot31 = k_slot21;
idx30 = idx20, idx31 = idx21;
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
/* ignore any items we have already found */
extra_hits_mask &= ~hits;
@@ -946,11 +1025,18 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
/* run a single search for each remaining item */
do {
idx = __builtin_ctzl(extra_hits_mask);
- positions[idx] = rte_hash_lookup_with_hash(h, keys[idx],
- hash_vals[idx]);
+ if (data != NULL) {
+ ret = rte_hash_lookup_with_hash_data(h,
+ keys[idx], hash_vals[idx], &data[idx]);
+ if (ret >= 0)
+ hits |= 1ULL << idx;
+ } else {
+ positions[idx] = rte_hash_lookup_with_hash(h,
+ keys[idx], hash_vals[idx]);
+ if (positions[idx] >= 0)
+ hits |= 1llu << idx;
+ }
extra_hits_mask &= ~(1llu << idx);
- if (positions[idx] >= 0)
- hits |= 1llu << idx;
} while (extra_hits_mask);
}
@@ -962,6 +1048,9 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
miss_mask &= ~(1llu << idx);
} while (miss_mask);
}
+
+ if (hit_mask != NULL)
+ *hit_mask = hits;
}
int
@@ -972,10 +1061,26 @@ rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
(num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
(positions == NULL)), -EINVAL);
- __rte_hash_lookup_bulk(h, keys, num_keys, positions);
+ __rte_hash_lookup_bulk(h, keys, num_keys, positions, NULL, NULL);
return 0;
}
+int
+rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, uint64_t *hit_mask, void *data[])
+{
+ RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
+ (num_keys > RTE_HASH_LOOKUP_BULK_MAX),
+ (hit_mask == NULL)), -EINVAL);
+
+ int32_t positions[num_keys];
+
+ __rte_hash_lookup_bulk(h, keys, num_keys, positions, hit_mask, data);
+
+ /* Return number of hits */
+ return __builtin_popcountl(*hit_mask);
+}
+
/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
static int
rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index 8bbc9f0..ff75445 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -80,12 +80,12 @@ struct rte_hash_parameters {
rte_hash_function hash_func; /**< Primary Hash function used to calculate hash. */
uint32_t hash_func_init_val; /**< Init value used by hash_func. */
int socket_id; /**< NUMA Socket ID for memory. */
+ uint8_t extra_flag; /**< Indicate if additional parameters are present. */
};
/** @internal A hash table structure. */
struct rte_hash;
-
/**
* Create a new hash table.
*
@@ -136,6 +136,48 @@ void
rte_hash_reset(struct rte_hash *h);
/**
+ * Add a key-value pair to an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
+ *
+ * @param h
+ * Hash table to add the key to.
+ * @param key
+ * Key to add to the hash table.
+ * @param data
+ * Data to add to the hash table.
+ * @return
+ * - 0 if added successfully
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOSPC if there is no space in the hash for this key.
+ */
+int
+rte_hash_add_key_data(const struct rte_hash *h, const void *key, void *data);
+
+/**
+ * Add a key-value pair with a pre-computed hash value
+ * to an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
+ *
+ * @param h
+ * Hash table to add the key to.
+ * @param key
+ * Key to add to the hash table.
+ * @param sig
+ * Precomputed hash value for 'key'
+ * @param data
+ * Data to add to the hash table.
+ * @return
+ * - 0 if added successfully
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOSPC if there is no space in the hash for this key.
+ */
+int32_t
+rte_hash_add_key_with_hash_data(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, void *data);
+
+/**
* Add a key to an existing hash table. This operation is not multi-thread safe
* and should only be called from one thread.
*
@@ -212,6 +254,47 @@ rte_hash_del_key(const struct rte_hash *h, const void *key);
int32_t
rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
+
+/**
+ * Find a key-value pair in the hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param key
+ * Key to find.
+ * @param data
+ * Output with pointer to data returned from the hash table.
+ * @return
+ * 0 if successful lookup
+ * - EINVAL if the parameters are invalid.
+ * - ENOENT if the key is not found.
+ */
+int
+rte_hash_lookup_data(const struct rte_hash *h, const void *key, void **data);
+
+/**
+ * Find a key-value pair with a pre-computed hash value
+ * to an existing hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param key
+ * Key to find.
+ * @param sig
+ * Precomputed hash value for 'key'
+ * @param data
+ * Output with pointer to data returned from the hash table.
+ * @return
+ * 0 if successful lookup
+ * - EINVAL if the parameters are invalid.
+ * - ENOENT if the key is not found.
+ */
+int
+rte_hash_lookup_with_hash_data(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, void **data);
+
/**
* Find a key in the hash table.
* This operation is multi-thread safe.
@@ -266,6 +349,28 @@ hash_sig_t
rte_hash_hash(const struct rte_hash *h, const void *key);
#define rte_hash_lookup_multi rte_hash_lookup_bulk
+#define rte_hash_lookup_multi_data rte_hash_lookup_bulk_data
+/**
+ * Find multiple keys in the hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param keys
+ * A pointer to a list of keys to look for.
+ * @param num_keys
+ * How many keys are in the keys list (less than RTE_HASH_LOOKUP_BULK_MAX).
+ * @param hit_mask
+ * Output containing a bitmask with all successful lookups.
+ * @param data
+ * Output containing array of data returned from all the successful lookups.
+ * @return
+ * -EINVAL if there's an error, otherwise number of successful lookups.
+ */
+int
+rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, uint64_t *hit_mask, void *data[]);
+
/**
* Find multiple keys in the hash table.
* This operation is multi-thread safe.
@@ -288,7 +393,6 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions);
-
#ifdef __cplusplus
}
#endif
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index d5f5af5..a97eac1 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -22,6 +22,12 @@ DPDK_2.0 {
DPDK_2.1 {
global:
+ rte_hash_add_key_data;
+ rte_hash_add_key_with_hash_data;
+ rte_hash_create;
+ rte_hash_lookup_bulk_data;
+ rte_hash_lookup_data;
+ rte_hash_lookup_with_hash_data;
rte_hash_reset;
local: *;
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v6 4/7] hash: add iterate function
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 " Pablo de Lara
` (2 preceding siblings ...)
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 3/7] hash: add new functionality to store data in hash table Pablo de Lara
@ 2015-07-10 23:30 ` Pablo de Lara
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 5/7] MAINTAINERS: claim responsability for hash library Pablo de Lara
` (3 subsequent siblings)
7 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 23:30 UTC (permalink / raw)
To: dev
Since now rte_hash structure is private, a new function
has been added to let the user iterate through the hash table,
returning next key and data associated on each iteration,
plus the position where they were stored.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 66 ++++++++++++++++++++++++++++++++++++
lib/librte_hash/rte_cuckoo_hash.c | 41 ++++++++++++++++++++++
lib/librte_hash/rte_hash.h | 22 ++++++++++++
lib/librte_hash/rte_hash_version.map | 1 +
4 files changed, 130 insertions(+)
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index 448586c..7f8c0d3 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -1149,6 +1149,70 @@ static int test_average_table_utilization(void)
return 0;
}
+#define NUM_ENTRIES 1024
+static int test_hash_iteration(void)
+{
+ struct rte_hash *handle;
+ unsigned i;
+ uint8_t keys[NUM_ENTRIES][RTE_HASH_KEY_LENGTH_MAX];
+ const void *next_key;
+ void *next_data;
+ void *data[NUM_ENTRIES];
+ unsigned added_keys;
+ uint32_t iter = 0;
+ int ret = 0;
+
+ ut_params.entries = NUM_ENTRIES;
+ ut_params.name = "test_hash_iteration";
+ ut_params.hash_func = rte_jhash;
+ ut_params.key_len = 16;
+ handle = rte_hash_create(&ut_params);
+ RETURN_IF_ERROR(handle == NULL, "hash creation failed");
+
+ /* Add random entries until key cannot be added */
+ for (added_keys = 0; added_keys < NUM_ENTRIES; added_keys++) {
+ data[added_keys] = (void *) ((uintptr_t) rte_rand());
+ for (i = 0; i < ut_params.key_len; i++)
+ keys[added_keys][i] = rte_rand() % 255;
+ ret = rte_hash_add_key_data(handle, keys[added_keys], data[added_keys]);
+ if (ret < 0)
+ break;
+ }
+
+ /* Iterate through the hash table */
+ while (rte_hash_iterate(handle, &next_key, &next_data, &iter) >= 0) {
+ /* Search for the key in the list of keys added */
+ for (i = 0; i < NUM_ENTRIES; i++) {
+ if (memcmp(next_key, keys[i], ut_params.key_len) == 0) {
+ if (next_data != data[i]) {
+ printf("Data found in the hash table is"
+ "not the data added with the key\n");
+ goto err;
+ }
+ added_keys--;
+ break;
+ }
+ }
+ if (i == NUM_ENTRIES) {
+ printf("Key found in the hash table was not added\n");
+ goto err;
+ }
+ }
+
+ /* Check if all keys have been iterated */
+ if (added_keys != 0) {
+ printf("There were still %u keys to iterate\n", added_keys);
+ goto err;
+ }
+
+ rte_hash_free(handle);
+ return 0;
+
+err:
+ rte_hash_free(handle);
+ return -1;
+}
+
static uint8_t key[16] = {0x00, 0x01, 0x02, 0x03,
0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b,
@@ -1408,6 +1472,8 @@ test_hash(void)
return -1;
if (test_average_table_utilization() < 0)
return -1;
+ if (test_hash_iteration() < 0)
+ return -1;
run_hash_func_tests();
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index cb1ab31..b0c942b 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -1081,6 +1081,47 @@ rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
return __builtin_popcountl(*hit_mask);
}
+int32_t
+rte_hash_iterate(const struct rte_hash *h, const void **key, void **data, uint32_t *next)
+{
+ uint32_t bucket_idx, idx, position;
+ struct rte_hash_key *next_key;
+
+ RETURN_IF_TRUE(((h == NULL) || (next == NULL)), -EINVAL);
+
+ const uint32_t total_entries = h->num_buckets * RTE_HASH_BUCKET_ENTRIES;
+ /* Out of bounds */
+ if (*next >= total_entries)
+ return -ENOENT;
+
+ /* Calculate bucket and index of current iterator */
+ bucket_idx = *next / RTE_HASH_BUCKET_ENTRIES;
+ idx = *next % RTE_HASH_BUCKET_ENTRIES;
+
+ /* If current position is empty, go to the next one */
+ while (h->buckets[bucket_idx].signatures[idx].sig == NULL_SIGNATURE) {
+ (*next)++;
+ /* End of table */
+ if (*next == total_entries)
+ return -ENOENT;
+ bucket_idx = *next / RTE_HASH_BUCKET_ENTRIES;
+ idx = *next % RTE_HASH_BUCKET_ENTRIES;
+ }
+
+ /* Get position of entry in key table */
+ position = h->buckets[bucket_idx].key_idx[idx];
+ next_key = (struct rte_hash_key *) ((char *)h->key_store +
+ position * h->key_entry_size);
+ /* Return key and data */
+ *key = next_key->key;
+ *data = next_key->pdata;
+
+ /* Increment iterator */
+ (*next)++;
+
+ return (position - 1);
+}
+
/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
static int
rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index ff75445..68109d5 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -393,6 +393,28 @@ rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions);
+
+/**
+ * Iterate through the hash table, returning key-value pairs.
+ *
+ * @param h
+ * Hash table to iterate
+ * @param key
+ * Output containing the key where current iterator
+ * was pointing at
+ * @param data
+ * Output containing the data associated with key.
+ * Returns NULL if data was not stored.
+ * @param next
+ * Pointer to iterator. Should be 0 to start iterating the hash table.
+ * Iterator is incremented after each call of this function.
+ * @return
+ * Position where key was stored, if successful.
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOENT if end of the hash table.
+ */
+int32_t
+rte_hash_iterate(const struct rte_hash *h, const void **key, void **data, uint32_t *next);
#ifdef __cplusplus
}
#endif
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index a97eac1..5653cb7 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -25,6 +25,7 @@ DPDK_2.1 {
rte_hash_add_key_data;
rte_hash_add_key_with_hash_data;
rte_hash_create;
+ rte_hash_iterate;
rte_hash_lookup_bulk_data;
rte_hash_lookup_data;
rte_hash_lookup_with_hash_data;
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v6 5/7] MAINTAINERS: claim responsability for hash library
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 " Pablo de Lara
` (3 preceding siblings ...)
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 4/7] hash: add iterate function Pablo de Lara
@ 2015-07-10 23:30 ` Pablo de Lara
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 6/7] doc: announce ABI change of librte_hash Pablo de Lara
` (2 subsequent siblings)
7 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 23:30 UTC (permalink / raw)
To: dev
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
MAINTAINERS | 1 +
1 file changed, 1 insertion(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 5476a73..75a27e9 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -345,6 +345,7 @@ F: doc/guides/sample_app_ug/l3_forward_access_ctrl.rst
Hashes
M: Bruce Richardson <bruce.richardson@intel.com>
+M: Pablo de Lara <pablo.de.lara.guarch@intel.com>
F: lib/librte_hash/
F: doc/guides/prog_guide/hash_lib.rst
F: app/test/test_*hash*
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v6 6/7] doc: announce ABI change of librte_hash
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 " Pablo de Lara
` (4 preceding siblings ...)
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 5/7] MAINTAINERS: claim responsability for hash library Pablo de Lara
@ 2015-07-10 23:30 ` Pablo de Lara
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 7/7] doc: update hash documentation Pablo de Lara
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 0/7] Cuckoo hash - part 3 of Cuckoo hash Pablo de Lara
7 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 23:30 UTC (permalink / raw)
To: dev
Two of the macros in rte_hash.h are now deprecated, so this patch
adds notice that they will be removed in 2.2.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
doc/guides/rel_notes/abi.rst | 1 +
1 file changed, 1 insertion(+)
diff --git a/doc/guides/rel_notes/abi.rst b/doc/guides/rel_notes/abi.rst
index 9d60a74..b45017c 100644
--- a/doc/guides/rel_notes/abi.rst
+++ b/doc/guides/rel_notes/abi.rst
@@ -21,3 +21,4 @@ Deprecation Notices
1024 queues per port. This change will be in release 2.2.
There is no backward compatibility planned from release 2.2.
All binaries will need to be rebuilt from release 2.2.
+* The Macros #RTE_HASH_BUCKET_ENTRIES_MAX and #RTE_HASH_KEY_LENGTH_MAX are deprecated and will be removed with version 2.2.
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v6 7/7] doc: update hash documentation
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 " Pablo de Lara
` (5 preceding siblings ...)
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 6/7] doc: announce ABI change of librte_hash Pablo de Lara
@ 2015-07-10 23:30 ` Pablo de Lara
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 0/7] Cuckoo hash - part 3 of Cuckoo hash Pablo de Lara
7 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-10 23:30 UTC (permalink / raw)
To: dev
Updates hash library documentation, reflecting
the new implementation changes.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
doc/guides/prog_guide/hash_lib.rst | 138 +++++++++++++++++++++++++++++++++----
doc/guides/prog_guide/index.rst | 4 ++
2 files changed, 129 insertions(+), 13 deletions(-)
diff --git a/doc/guides/prog_guide/hash_lib.rst b/doc/guides/prog_guide/hash_lib.rst
index 9b83835..193dd53 100644
--- a/doc/guides/prog_guide/hash_lib.rst
+++ b/doc/guides/prog_guide/hash_lib.rst
@@ -1,5 +1,5 @@
.. BSD LICENSE
- Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
All rights reserved.
Redistribution and use in source and binary forms, with or without
@@ -50,8 +50,6 @@ The hash also allows the configuration of some low-level implementation related
* Hash function to translate the key into a bucket index
-* Number of entries per bucket
-
The main methods exported by the hash are:
* Add entry with key: The key is provided as input. If a new entry is successfully added to the hash for the specified key,
@@ -65,10 +63,26 @@ The main methods exported by the hash are:
* Lookup for entry with key: The key is provided as input. If an entry with the specified key is found in the hash (lookup hit),
then the position of the entry is returned, otherwise (lookup miss) a negative value is returned.
-The current hash implementation handles the key management only.
-The actual data associated with each key has to be managed by the user using a separate table that
+Apart from these method explained above, the API allows the user three more options:
+
+* Add / lookup / delete with key and precomputed hash: Both the key and its precomputed hash are provided as input. This allows
+ the user to perform these operations faster, as hash is already computed.
+
+* Add / lookup with key and data: A pair of key-value is provided as input. This allows the user to store
+ not only the key, but also data which may be either a 8-byte integer or a pointer to external data (if data size is more than 8 bytes).
+
+* Combination of the two options above: User can provide key, precomputed hash and data.
+
+Also, the API contains a method to allow the user to look up entries in bursts, achieving higher performance
+than looking up individual entries, as the function prefetches next entries at the time it is operating
+with the first ones, which reduces significantly the impact of the necessary memory accesses.
+Notice that this method uses a pipeline of 8 entries (4 stages of 2 entries), so it is highly recommended
+to use at least 8 entries per burst.
+
+The actual data associated with each key can be either managed by the user using a separate table that
mirrors the hash in terms of number of entries and position of each entry,
-as shown in the Flow Classification use case describes in the following sections.
+as shown in the Flow Classification use case describes in the following sections,
+or stored in the hash table itself.
The example hash tables in the L2/L3 Forwarding sample applications defines which port to forward a packet to based on a packet flow identified by the five-tuple lookup.
However, this table could also be used for more sophisticated features and provide many other functions and actions that could be performed on the packets and flows.
@@ -76,17 +90,26 @@ However, this table could also be used for more sophisticated features and provi
Implementation Details
----------------------
-The hash table is implemented as an array of entries which is further divided into buckets,
-with the same number of consecutive array entries in each bucket.
-For any input key, there is always a single bucket where that key can be stored in the hash,
-therefore only the entries within that bucket need to be examined when the key is looked up.
+The hash table has two main tables:
+
+* First table is an array of entries which is further divided into buckets,
+ with the same number of consecutive array entries in each bucket. Each entry contains the computed primary
+ and secondary hashes of a given key (explained below), and an index to the second table.
+
+* The second table is an array of all the keys stored in the hash table and its data associated to each key.
+
+The hash library uses the cuckoo hash method to resolve collisions.
+For any input key, there are two possible buckets (primary and secondary/alternative location)
+where that key can be stored in the hash, therefore only the entries within those bucket need to be examined
+when the key is looked up.
The lookup speed is achieved by reducing the number of entries to be scanned from the total
-number of hash entries down to the number of entries in a hash bucket,
+number of hash entries down to the number of entries in the two hash buckets,
as opposed to the basic method of linearly scanning all the entries in the array.
The hash uses a hash function (configurable) to translate the input key into a 4-byte key signature.
The bucket index is the key signature modulo the number of hash buckets.
-Once the bucket is identified, the scope of the hash add,
-delete and lookup operations is reduced to the entries in that bucket.
+
+Once the buckets are identified, the scope of the hash add,
+delete and lookup operations is reduced to the entries in those buckets (it is very likely that entries are in the primary bucket).
To speed up the search logic within the bucket, each hash entry stores the 4-byte key signature together with the full key for each hash entry.
For large key sizes, comparing the input key against a key from the bucket can take significantly more time than
@@ -95,6 +118,95 @@ Therefore, the signature comparison is done first and the full key comparison do
The full key comparison is still necessary, as two input keys from the same bucket can still potentially have the same 4-byte hash signature,
although this event is relatively rare for hash functions providing good uniform distributions for the set of input keys.
+Example of lookup:
+
+First of all, the primary bucket is identified and entry is likely to be stored there.
+If signature was stored there, we compare its key against the one provided and return the position
+where it was stored and/or the data associated to that key if there is a match.
+If signature is not in the primary bucket, the secondary bucket is looked up, where same procedure
+is carried out. If there is no match there either, key is considered not to be in the table.
+
+Example of addition:
+
+Like lookup, the primary and secondary buckets are indentified. If there is an empty slot in
+the primary bucket, primary and secondary signatures are stored in that slot, key and data (if any) are added to
+the second table and an index to the position in the second table is stored in the slot of the first table.
+If there is no space in the primary bucket, one of the entries on that bucket is pushed to its alternative location,
+and the key to be added is inserted in its position.
+To know where the alternative bucket of the evicted entry is, the secondary signature is looked up and alternative bucket index
+is calculated from doing the modulo, as seen above. If there is room in the alternative bucket, the evicted entry
+is stored in it. If not, same process is repeated (one of the entries gets pushed) until a non full bucket is found.
+Notice that despite all the entry movement in the first table, the second table is not touched, which would impact
+greatly in performance.
+
+In the very unlikely event that table enters in a loop where same entries are being evicted indefinitely,
+key is considered not able to be stored.
+With random keys, this method allows the user to get around 90% of the table utilization, without
+having to drop any stored entry (LRU) or allocate more memory (extended buckets).
+
+Entry distribution in hash table
+--------------------------------
+
+As mentioned above, Cuckoo hash implementation pushes elements out of their bucket,
+if there is a new entry to be added which primary location coincides with their current bucket,
+being pushed to their alternative location.
+Therefore, as user adds more entries to the hash table, distribution of the hash values
+in the buckets will change, being most of them in their primary location and a few in
+their secondary location, which the later will increase, as table gets busier.
+This information is quite useful, as performance will be lower as more entries
+are evicted to their secondary location.
+
+See the tables below showing entry distribution as table utilization increases.
+
+.. _table_hash_lib_1:
+
+.. table:: Entry distribution with 1024 entries
+
+ +--------------+-----------------------+-------------------------+
+ | % Table used | % In Primary location | % In Secondary location |
+ +==============+=======================+=========================+
+ | 25 | 100 | 0 |
+ +--------------+-----------------------+-------------------------+
+ | 50 | 96.1 | 3.9 |
+ +--------------+-----------------------+-------------------------+
+ | 75 | 88.2 | 11.8 |
+ +--------------+-----------------------+-------------------------+
+ | 80 | 86.3 | 13.7 |
+ +--------------+-----------------------+-------------------------+
+ | 85 | 83.1 | 16.9 |
+ +--------------+-----------------------+-------------------------+
+ | 90 | 77.3 | 22.7 |
+ +--------------+-----------------------+-------------------------+
+ | 95.8 | 64.5 | 35.5 |
+ +--------------+-----------------------+-------------------------+
+
+|
+
+.. _table_hash_lib_2:
+
+.. table:: Entry distribution with 1 million entries
+
+ +--------------+-----------------------+-------------------------+
+ | % Table used | % In Primary location | % In Secondary location |
+ +==============+=======================+=========================+
+ | 50 | 96 | 4 |
+ +--------------+-----------------------+-------------------------+
+ | 75 | 86.9 | 13.1 |
+ +--------------+-----------------------+-------------------------+
+ | 80 | 83.9 | 16.1 |
+ +--------------+-----------------------+-------------------------+
+ | 85 | 80.1 | 19.9 |
+ +--------------+-----------------------+-------------------------+
+ | 90 | 74.8 | 25.2 |
+ +--------------+-----------------------+-------------------------+
+ | 94.5 | 67.4 | 32.6 |
+ +--------------+-----------------------+-------------------------+
+
+.. note::
+
+ Last values on the tables above are the average maximum table
+ utilization with random keys and using Jenkins hash function.
+
Use Case: Flow Classification
-----------------------------
diff --git a/doc/guides/prog_guide/index.rst b/doc/guides/prog_guide/index.rst
index 3295661..036640c 100644
--- a/doc/guides/prog_guide/index.rst
+++ b/doc/guides/prog_guide/index.rst
@@ -241,3 +241,7 @@ Programmer's Guide
:numref:`table_qos_33` :ref:`table_qos_33`
:numref:`table_qos_34` :ref:`table_qos_34`
+
+:numref:`table_hash_lib_1` :ref:`table_hash_lib_1`
+
+:numref:`table_hash_lib_2` :ref:`table_hash_lib_2`
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v7 0/7] Cuckoo hash - part 3 of Cuckoo hash
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 " Pablo de Lara
` (6 preceding siblings ...)
2015-07-10 23:30 ` [dpdk-dev] [PATCH v6 7/7] doc: update hash documentation Pablo de Lara
@ 2015-07-11 0:18 ` Pablo de Lara
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
` (7 more replies)
7 siblings, 8 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-11 0:18 UTC (permalink / raw)
To: dev
This patchset is to replace the existing hash library with
a more efficient and functional approach, using the Cuckoo hash
method to deal with collisions. This method is based on using
two different hash functions to have two possible locations
in the hash table where an entry can be.
So, if a bucket is full, a new entry can push one of the items
in that bucket to its alternative location, making space for itself.
Advantages
~~
- Offers the option to store more entries when the target bucket is full
(unlike the previous implementation)
- Memory efficient: for storing those entries, it is not necessary to
request new memory, as the entries will be stored in the same table
- Constant worst lookup time: in worst case scenario, it always takes
the same time to look up an entry, as there are only two possible locations
where an entry can be.
- Storing data: user can store data in the hash table, unlike the
previous implementation, but he can still use the old API
This implementation typically offers over 90% utilization.
Notice that API has been extended, but old API remains.
Check documentation included to know more about this new implementation
(including how entries are distributed as table utilization increases).
Changes in v7:
- Fix inaccurate documentation
Changes in v6:
- Replace datatype for functions from uintptr_t to void *
Changes in v5:
- Fix 32-bit compilation issues
Changes in v4:
- Unit tests enhancements are not part of this patchset anymore.
- rte_hash structure has been made internal in another patch,
so it is not part of this patchset anymore.
- Add function to iterate through the hash table, as rte_hash
structure has been made private.
- Added extra_flag parameter in rte_hash_parameter to be able
to add new parameters in the future without breaking the ABI
- Remove proposed lookup_bulk_with_hash function, as it is
not of much use with the existing hash functions
(there are no vector hash functions).
- User can store 8-byte integer or pointer as data, instead
of variable size data, as discussed in the mailing list.
Changes in v3:
- Now user can store variable size data, instead of 32 or 64-bit size data,
using the new parameter "data_len" in rte_hash_parameters
- Add lookup_bulk_with_hash function in performance unit tests
- Add new functions that handle data in performance unit tests
- Remove duplicates in performance unit tests
- Fix rte_hash_reset, which was not resetting the last entry
Changes in v2:
- Fixed issue where table could not store maximum number of entries
- Fixed issue where lookup burst could not be more than 32 (instead of 64)
- Remove unnecessary macros and add other useful ones
- Added missing library dependencies
- Used directly rte_hash_secondary instead of rte_hash_alt
- Renamed rte_hash.c to rte_cuckoo_hash.c to ease the view of the new library
- Renamed test_hash_perf.c temporarily to ease the view of the improved unit test
- Moved rte_hash, rte_bucket and rte_hash_key structures to rte_cuckoo_hash.c to
make them private
- Corrected copyright dates
- Added an optimized function to compare keys that are multiple of 16 bytes
- Improved the way to use primary/secondary signatures. Now both are stored in
the bucket, so there is no need to calculate them if required.
Also, there is no need to use the MSB of a signature to differenciate between
an empty entry and signature 0, since we are storing both signatures,
which cannot be both 0.
- Removed rte_hash_rehash, as it was a very expensive operation.
Therefore, the add function returns now -ENOSPC if key cannot be added
because of a loop.
- Prefetched new slot for new key in add function to improve performance.
- Made doxygen comments more clear.
- Removed unnecessary rte_hash_del_key_data and rte_hash_del_key_with_data,
as we can use the lookup functions if we want to get the data before deleting.
- Removed some unnecessary includes in rte_hash.h
- Removed some unnecessary variables in rte_cuckoo_hash.c
- Removed some unnecessary checks before creating a new hash table
- Added documentation (in release notes and programmers guide)
- Added new unit tests and replaced the performance one for hash tables
Series Acked-by: Bruce Richardson <bruce.richardson@intel.com>
Pablo de Lara (7):
hash: replace existing hash library with cuckoo hash implementation
hash: add new function rte_hash_reset
hash: add new functionality to store data in hash table
hash: add iterate function
MAINTAINERS: claim responsability for hash library
doc: announce ABI change of librte_hash
doc: update hash documentation
MAINTAINERS | 1 +
app/test/test_hash.c | 189 +++---
app/test/test_hash_perf.c | 303 ++++++---
doc/guides/prog_guide/hash_lib.rst | 138 +++-
doc/guides/prog_guide/index.rst | 4 +
doc/guides/rel_notes/abi.rst | 1 +
lib/librte_hash/Makefile | 8 +-
lib/librte_hash/rte_cuckoo_hash.c | 1194 ++++++++++++++++++++++++++++++++++
lib/librte_hash/rte_hash.c | 499 --------------
lib/librte_hash/rte_hash.h | 198 +++++-
lib/librte_hash/rte_hash_version.map | 15 +
11 files changed, 1810 insertions(+), 740 deletions(-)
create mode 100644 lib/librte_hash/rte_cuckoo_hash.c
delete mode 100644 lib/librte_hash/rte_hash.c
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 0/7] Cuckoo hash - part 3 of Cuckoo hash Pablo de Lara
@ 2015-07-11 0:18 ` Pablo de Lara
2015-07-12 22:29 ` Thomas Monjalon
2015-07-16 9:39 ` Tony Lu
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 2/7] hash: add new function rte_hash_reset Pablo de Lara
` (6 subsequent siblings)
7 siblings, 2 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-11 0:18 UTC (permalink / raw)
To: dev
This patch replaces the existing hash library with another approach,
using the Cuckoo Hash method to resolve collisions (open addressing),
which pushes items from a full bucket when a new entry tries
to be added in it, storing the evicted entry in an alternative location,
using a secondary hash function.
This gives the user the ability to store more entries when a bucket
is full, in comparison with the previous implementation.
Therefore, the unit test has been updated, as some scenarios have changed
(such as the previous removed restriction).
Also note that the API has not been changed, although new fields
have been added in the rte_hash structure (structure is internal now).
The main change when creating a new table is that the number of entries
per bucket is fixed now, so its parameter is ignored now
(still there to maintain the same parameters structure).
The hash unit test has been updated to reflect these changes.
As a last note, the maximum burst size in lookup_burst function
hash been increased to 64, to improve performance.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 119 ++---
app/test/test_hash_perf.c | 79 ++-
lib/librte_hash/Makefile | 8 +-
lib/librte_hash/rte_cuckoo_hash.c | 1027 +++++++++++++++++++++++++++++++++++++
lib/librte_hash/rte_hash.c | 499 ------------------
lib/librte_hash/rte_hash.h | 69 +--
6 files changed, 1138 insertions(+), 663 deletions(-)
create mode 100644 lib/librte_hash/rte_cuckoo_hash.c
delete mode 100644 lib/librte_hash/rte_hash.c
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index 7c71ed6..e70d859 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -169,7 +169,6 @@ static struct flow_key keys[5] = { {
/* Parameters used for hash table in unit test functions. Name set later. */
static struct rte_hash_parameters ut_params = {
.entries = 64,
- .bucket_entries = 4,
.key_len = sizeof(struct flow_key), /* 13 */
.hash_func = rte_jhash,
.hash_func_init_val = 0,
@@ -516,9 +515,18 @@ static int test_five_keys(void)
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != -ENOENT,
- "failed to find key (pos[%u]=%d)", i, pos[i]);
+ "found non-existent key (pos[%u]=%d)", i, pos[i]);
}
+ /* Lookup multi */
+ ret = rte_hash_lookup_multi(handle, &key_array[0], 5, (int32_t *)pos);
+ if (ret == 0)
+ for (i = 0; i < 5; i++) {
+ print_key_info("Lkp", key_array[i], pos[i]);
+ RETURN_IF_ERROR(pos[i] != -ENOENT,
+ "found not-existent key (pos[%u]=%d)", i, pos[i]);
+ }
+
rte_hash_free(handle);
return 0;
@@ -527,21 +535,18 @@ static int test_five_keys(void)
/*
* Add keys to the same bucket until bucket full.
* - add 5 keys to the same bucket (hash created with 4 keys per bucket):
- * first 4 successful, 5th unsuccessful
- * - lookup the 5 keys: 4 hits, 1 miss
- * - add the 5 keys again: 4 OK, one error as bucket is full
- * - lookup the 5 keys: 4 hits (updated data), 1 miss
- * - delete the 5 keys: 5 OK (even if the 5th is not in the table)
+ * first 4 successful, 5th successful, pushing existing item in bucket
+ * - lookup the 5 keys: 5 hits
+ * - add the 5 keys again: 5 OK
+ * - lookup the 5 keys: 5 hits (updated data)
+ * - delete the 5 keys: 5 OK
* - lookup the 5 keys: 5 misses
- * - add the 5th key: OK
- * - lookup the 5th key: hit
*/
static int test_full_bucket(void)
{
struct rte_hash_parameters params_pseudo_hash = {
.name = "test4",
.entries = 64,
- .bucket_entries = 4,
.key_len = sizeof(struct flow_key), /* 13 */
.hash_func = pseudo_hash,
.hash_func_init_val = 0,
@@ -555,7 +560,7 @@ static int test_full_bucket(void)
handle = rte_hash_create(¶ms_pseudo_hash);
RETURN_IF_ERROR(handle == NULL, "hash creation failed");
- /* Fill bucket*/
+ /* Fill bucket */
for (i = 0; i < 4; i++) {
pos[i] = rte_hash_add_key(handle, &keys[i]);
print_key_info("Add", &keys[i], pos[i]);
@@ -563,47 +568,39 @@ static int test_full_bucket(void)
"failed to add key (pos[%u]=%d)", i, pos[i]);
expected_pos[i] = pos[i];
}
- /* This shouldn't work because the bucket is full */
+ /*
+ * This should work and will push one of the items
+ * in the bucket because it is full
+ */
pos[4] = rte_hash_add_key(handle, &keys[4]);
print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOSPC,
- "fail: added key to full bucket (pos[4]=%d)", pos[4]);
+ RETURN_IF_ERROR(pos[4] < 0,
+ "failed to add key (pos[4]=%d)", pos[4]);
+ expected_pos[4] = pos[4];
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to find key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: found non-existent key (pos[4]=%d)", pos[4]);
/* Add - update */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_add_key(handle, &keys[i]);
print_key_info("Add", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to add key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_add_key(handle, &keys[4]);
- print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOSPC,
- "fail: added key to full bucket (pos[4]=%d)", pos[4]);
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to find key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: found non-existent key (pos[4]=%d)", pos[4]);
/* Delete 1 key, check other keys are still found */
pos[1] = rte_hash_del_key(handle, &keys[1]);
@@ -623,35 +620,21 @@ static int test_full_bucket(void)
RETURN_IF_ERROR(pos[1] < 0, "failed to add key (pos[1]=%d)", pos[1]);
/* Delete */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_del_key(handle, &keys[i]);
print_key_info("Del", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != expected_pos[i],
"failed to delete key (pos[%u]=%d)", i, pos[i]);
}
- pos[4] = rte_hash_del_key(handle, &keys[4]);
- print_key_info("Del", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != -ENOENT,
- "fail: deleted non-existent key (pos[4]=%d)", pos[4]);
/* Lookup */
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
pos[i] = rte_hash_lookup(handle, &keys[i]);
print_key_info("Lkp", &keys[i], pos[i]);
RETURN_IF_ERROR(pos[i] != -ENOENT,
"fail: found non-existent key (pos[%u]=%d)", i, pos[i]);
}
- /* Add and lookup the 5th key */
- pos[4] = rte_hash_add_key(handle, &keys[4]);
- print_key_info("Add", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] < 0, "failed to add key (pos[4]=%d)", pos[4]);
- expected_pos[4] = pos[4];
- pos[4] = rte_hash_lookup(handle, &keys[4]);
- print_key_info("Lkp", &keys[4], pos[4]);
- RETURN_IF_ERROR(pos[4] != expected_pos[4],
- "failed to find key (pos[4]=%d)", pos[4]);
-
rte_hash_free(handle);
/* Cover the NULL case. */
@@ -991,6 +974,7 @@ static int test_fbk_hash_find_existing(void)
return 0;
}
+#define BUCKET_ENTRIES 4
/*
* Do tests for hash creation with bad parameters.
*/
@@ -1017,18 +1001,8 @@ static int test_hash_creation_with_bad_parameters(void)
}
memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_1";
- params.bucket_entries = RTE_HASH_BUCKET_ENTRIES_MAX + 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully with bucket_entries in parameter exceeded\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
params.name = "creation_with_bad_parameters_2";
- params.entries = params.bucket_entries - 1;
+ params.entries = BUCKET_ENTRIES - 1;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
rte_hash_free(handle);
@@ -1038,26 +1012,6 @@ static int test_hash_creation_with_bad_parameters(void)
memcpy(¶ms, &ut_params, sizeof(params));
params.name = "creation_with_bad_parameters_3";
- params.entries = params.entries - 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully if entries in parameter is not power of 2\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_4";
- params.bucket_entries = params.bucket_entries - 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully if bucket_entries in parameter is not power of 2\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_5";
params.key_len = 0;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
@@ -1067,17 +1021,7 @@ static int test_hash_creation_with_bad_parameters(void)
}
memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_6";
- params.key_len = RTE_HASH_KEY_LENGTH_MAX + 1;
- handle = rte_hash_create(¶ms);
- if (handle != NULL) {
- rte_hash_free(handle);
- printf("Impossible creating hash sucessfully if key_len is greater than the maximum\n");
- return -1;
- }
-
- memcpy(¶ms, &ut_params, sizeof(params));
- params.name = "creation_with_bad_parameters_7";
+ params.name = "creation_with_bad_parameters_4";
params.socket_id = RTE_MAX_NUMA_NODES + 1;
handle = rte_hash_create(¶ms);
if (handle != NULL) {
@@ -1214,7 +1158,6 @@ static uint8_t key[16] = {0x00, 0x01, 0x02, 0x03,
static struct rte_hash_parameters hash_params_ex = {
.name = NULL,
.entries = 64,
- .bucket_entries = 4,
.key_len = 0,
.hash_func = NULL,
.hash_func_init_val = 0,
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index a3876c1..a54f3c1 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -162,7 +162,7 @@ shuffle_input_keys(unsigned table_index)
* ALL can fit in hash table (no errors)
*/
static int
-get_input_keys(unsigned table_index)
+get_input_keys(unsigned with_pushes, unsigned table_index)
{
unsigned i, j;
unsigned bucket_idx, incr, success = 1;
@@ -216,9 +216,14 @@ get_input_keys(unsigned table_index)
success = 0;
signatures[i] = rte_hash_hash(h[table_index], keys[i]);
bucket_idx = signatures[i] & bucket_bitmask;
- /* If bucket is full, do not try to insert the key */
- if (buckets[bucket_idx] == BUCKET_SIZE)
- continue;
+ /*
+ * If we are not inserting keys in secondary location,
+ * when bucket is full, do not try to insert the key
+ */
+ if (with_pushes == 0)
+ if (buckets[bucket_idx] == BUCKET_SIZE)
+ continue;
+
/* If key can be added, leave in successful key arrays "keys" */
ret = rte_hash_add_key_with_hash(h[table_index], keys[i],
signatures[i]);
@@ -388,9 +393,9 @@ reset_table(unsigned table_index)
}
static int
-run_all_tbl_perf_tests(void)
+run_all_tbl_perf_tests(unsigned with_pushes)
{
- unsigned i, j;
+ unsigned i, j, with_hash;
printf("Measuring performance, please wait");
fflush(stdout);
@@ -398,46 +403,32 @@ run_all_tbl_perf_tests(void)
if (create_table(i) < 0)
return -1;
- if (get_input_keys(i) < 0)
- return -1;
-
- if (timed_adds(0, i) < 0)
- return -1;
-
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
-
- if (timed_lookups(0, i) < 0)
- return -1;
-
- if (timed_lookups_multi(i) < 0)
- return -1;
-
- if (timed_deletes(0, i) < 0)
+ if (get_input_keys(with_pushes, i) < 0)
return -1;
+ for (with_hash = 0; with_hash <= 1; with_hash++) {
+ if (timed_adds(with_hash, i) < 0)
+ return -1;
- /* Print a dot to show progress on operations */
- printf(".");
- fflush(stdout);
+ for (j = 0; j < NUM_SHUFFLES; j++)
+ shuffle_input_keys(i);
- if (reset_table(i) < 0)
- return -1;
+ if (timed_lookups(with_hash, i) < 0)
+ return -1;
- if (timed_adds(1, i) < 0)
- return -1;
+ if (timed_lookups_multi(i) < 0)
+ return -1;
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
+ if (timed_deletes(with_hash, i) < 0)
+ return -1;
- if (timed_lookups(1, i) < 0)
- return -1;
+ if (reset_table(i) < 0)
+ return -1;
- if (timed_deletes(1, i) < 0)
- return -1;
+ /* Print a dot to show progress on operations */
+ printf(".");
+ fflush(stdout);
- /* Print a dot to show progress on operations */
- printf(".");
- fflush(stdout);
+ }
free_table(i);
}
@@ -551,8 +542,16 @@ fbk_hash_perf_test(void)
static int
test_hash_perf(void)
{
- if (run_all_tbl_perf_tests() < 0)
- return -1;
+ unsigned with_pushes;
+
+ for (with_pushes = 0; with_pushes <= 1; with_pushes++) {
+ if (with_pushes == 0)
+ printf("\nALL ELEMENTS IN PRIMARY LOCATION\n");
+ else
+ printf("\nELEMENTS IN PRIMARY OR SECONDARY LOCATION\n");
+ if (run_all_tbl_perf_tests(with_pushes) < 0)
+ return -1;
+ }
if (fbk_hash_perf_test() < 0)
return -1;
diff --git a/lib/librte_hash/Makefile b/lib/librte_hash/Makefile
index 981230b..039da46 100644
--- a/lib/librte_hash/Makefile
+++ b/lib/librte_hash/Makefile
@@ -1,6 +1,6 @@
# BSD LICENSE
#
-# Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+# Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@@ -42,7 +42,7 @@ EXPORT_MAP := rte_hash_version.map
LIBABIVER := 1
# all source are stored in SRCS-y
-SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_hash.c
+SRCS-$(CONFIG_RTE_LIBRTE_HASH) := rte_cuckoo_hash.c
SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c
# install this header file
@@ -52,7 +52,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_thash.h
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h
-# this lib needs eal
-DEPDIRS-$(CONFIG_RTE_LIBRTE_HASH) += lib/librte_eal lib/librte_malloc
+# this lib needs eal and ring
+DEPDIRS-$(CONFIG_RTE_LIBRTE_HASH) += lib/librte_eal lib/librte_malloc lib/librte_ring
include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
new file mode 100644
index 0000000..50e3acd
--- /dev/null
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -0,0 +1,1027 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <sys/queue.h>
+
+#include <rte_common.h>
+#include <rte_memory.h> /* for definition of RTE_CACHE_LINE_SIZE */
+#include <rte_log.h>
+#include <rte_memcpy.h>
+#include <rte_prefetch.h>
+#include <rte_branch_prediction.h>
+#include <rte_memzone.h>
+#include <rte_malloc.h>
+#include <rte_eal.h>
+#include <rte_eal_memconfig.h>
+#include <rte_per_lcore.h>
+#include <rte_errno.h>
+#include <rte_string_fns.h>
+#include <rte_cpuflags.h>
+#include <rte_log.h>
+#include <rte_rwlock.h>
+#include <rte_spinlock.h>
+#include <rte_ring.h>
+
+#include "rte_hash.h"
+
+TAILQ_HEAD(rte_hash_list, rte_tailq_entry);
+
+static struct rte_tailq_elem rte_hash_tailq = {
+ .name = "RTE_HASH",
+};
+EAL_REGISTER_TAILQ(rte_hash_tailq)
+
+/* Macro to enable/disable run-time checking of function parameters */
+#if defined(RTE_LIBRTE_HASH_DEBUG)
+#define RETURN_IF_TRUE(cond, retval) do { \
+ if (cond) \
+ return retval; \
+} while (0)
+#else
+#define RETURN_IF_TRUE(cond, retval)
+#endif
+
+/* Hash function used if none is specified */
+#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
+#include <rte_hash_crc.h>
+#define DEFAULT_HASH_FUNC rte_hash_crc
+#else
+#include <rte_jhash.h>
+#define DEFAULT_HASH_FUNC rte_jhash
+#endif
+
+/** Number of items per bucket. */
+#define RTE_HASH_BUCKET_ENTRIES 4
+
+#define NULL_SIGNATURE 0
+
+typedef int (*rte_hash_cmp_eq_t)(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k48_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k64_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k80_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k96_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k112_cmp_eq(const void *key1, const void *key2, size_t key_len);
+static int rte_hash_k128_cmp_eq(const void *key1, const void *key2, size_t key_len);
+
+/** A hash table structure. */
+struct rte_hash {
+ char name[RTE_HASH_NAMESIZE]; /**< Name of the hash. */
+ uint32_t entries; /**< Total table entries. */
+ uint32_t num_buckets; /**< Number of buckets in table. */
+ uint32_t key_len; /**< Length of hash key. */
+ rte_hash_function hash_func; /**< Function used to calculate hash. */
+ rte_hash_cmp_eq_t rte_hash_cmp_eq; /**< Function used to compare keys. */
+ uint32_t hash_func_init_val; /**< Init value used by hash_func. */
+ uint32_t bucket_bitmask; /**< Bitmask for getting bucket index
+ from hash signature. */
+ uint32_t key_entry_size; /**< Size of each key entry. */
+
+ struct rte_ring *free_slots; /**< Ring that stores all indexes
+ of the free slots in the key table */
+ void *key_store; /**< Table storing all keys and data */
+ struct rte_hash_bucket *buckets; /**< Table with buckets storing all the
+ hash values and key indexes
+ to the key table*/
+} __rte_cache_aligned;
+
+/* Structure storing both primary and secondary hashes */
+struct rte_hash_signatures {
+ union {
+ struct {
+ hash_sig_t current;
+ hash_sig_t alt;
+ };
+ uint64_t sig;
+ };
+};
+
+/** Bucket structure */
+struct rte_hash_bucket {
+ struct rte_hash_signatures signatures[RTE_HASH_BUCKET_ENTRIES];
+ /* Includes dummy key index that always contains index 0 */
+ uint32_t key_idx[RTE_HASH_BUCKET_ENTRIES + 1];
+ uint8_t flag[RTE_HASH_BUCKET_ENTRIES];
+} __rte_cache_aligned;
+
+struct rte_hash *
+rte_hash_find_existing(const char *name)
+{
+ struct rte_hash *h = NULL;
+ struct rte_tailq_entry *te;
+ struct rte_hash_list *hash_list;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK);
+ TAILQ_FOREACH(te, hash_list, next) {
+ h = (struct rte_hash *) te->data;
+ if (strncmp(name, h->name, RTE_HASH_NAMESIZE) == 0)
+ break;
+ }
+ rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ if (te == NULL) {
+ rte_errno = ENOENT;
+ return NULL;
+ }
+ return h;
+}
+
+struct rte_hash *
+rte_hash_create(const struct rte_hash_parameters *params)
+{
+ struct rte_hash *h = NULL;
+ struct rte_tailq_entry *te = NULL;
+ struct rte_hash_list *hash_list;
+ struct rte_ring *r = NULL;
+ char hash_name[RTE_HASH_NAMESIZE];
+ void *ptr, *k = NULL;
+ void *buckets = NULL;
+ char ring_name[RTE_RING_NAMESIZE];
+ unsigned i;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ if (params == NULL) {
+ RTE_LOG(ERR, HASH, "rte_hash_create has no parameters\n");
+ return NULL;
+ }
+
+ /* Check for valid parameters */
+ if ((params->entries > RTE_HASH_ENTRIES_MAX) ||
+ (params->entries < RTE_HASH_BUCKET_ENTRIES) ||
+ !rte_is_power_of_2(RTE_HASH_BUCKET_ENTRIES) ||
+ (params->key_len == 0)) {
+ rte_errno = EINVAL;
+ RTE_LOG(ERR, HASH, "rte_hash_create has invalid parameters\n");
+ return NULL;
+ }
+
+ snprintf(hash_name, sizeof(hash_name), "HT_%s", params->name);
+
+ /* Guarantee there's no existing */
+ h = rte_hash_find_existing(params->name);
+ if (h != NULL)
+ return h;
+
+ te = rte_zmalloc("HASH_TAILQ_ENTRY", sizeof(*te), 0);
+ if (te == NULL) {
+ RTE_LOG(ERR, HASH, "tailq entry allocation failed\n");
+ goto err;
+ }
+
+ h = (struct rte_hash *)rte_zmalloc_socket(hash_name, sizeof(struct rte_hash),
+ RTE_CACHE_LINE_SIZE, params->socket_id);
+
+ if (h == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ const uint32_t num_buckets = rte_align32pow2(params->entries)
+ / RTE_HASH_BUCKET_ENTRIES;
+
+ buckets = rte_zmalloc_socket(NULL,
+ num_buckets * sizeof(struct rte_hash_bucket),
+ RTE_CACHE_LINE_SIZE, params->socket_id);
+
+ if (buckets == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ const uint32_t key_entry_size = params->key_len;
+ /* Store all keys and leave the first entry as a dummy entry for lookup_bulk */
+ const uint64_t key_tbl_size = key_entry_size * (params->entries + 1);
+
+ k = rte_zmalloc_socket(NULL, key_tbl_size,
+ RTE_CACHE_LINE_SIZE, params->socket_id);
+
+ if (k == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ /* Select function to compare keys */
+ switch (params->key_len) {
+ case 16:
+ h->rte_hash_cmp_eq = rte_hash_k16_cmp_eq;
+ break;
+ case 32:
+ h->rte_hash_cmp_eq = rte_hash_k32_cmp_eq;
+ break;
+ case 48:
+ h->rte_hash_cmp_eq = rte_hash_k48_cmp_eq;
+ break;
+ case 64:
+ h->rte_hash_cmp_eq = rte_hash_k64_cmp_eq;
+ break;
+ case 80:
+ h->rte_hash_cmp_eq = rte_hash_k80_cmp_eq;
+ break;
+ case 96:
+ h->rte_hash_cmp_eq = rte_hash_k96_cmp_eq;
+ break;
+ case 112:
+ h->rte_hash_cmp_eq = rte_hash_k112_cmp_eq;
+ break;
+ case 128:
+ h->rte_hash_cmp_eq = rte_hash_k128_cmp_eq;
+ break;
+ default:
+ /* If key is not multiple of 16, use generic memcmp */
+ h->rte_hash_cmp_eq = memcmp;
+ }
+
+ snprintf(ring_name, sizeof(ring_name), "HT_%s", params->name);
+ r = rte_ring_lookup(ring_name);
+ if (r != NULL) {
+ /* clear the free ring */
+ while (rte_ring_dequeue(r, &ptr) == 0)
+ rte_pause();
+ } else
+ r = rte_ring_create(ring_name, rte_align32pow2(params->entries + 1),
+ params->socket_id, 0);
+ if (r == NULL) {
+ RTE_LOG(ERR, HASH, "memory allocation failed\n");
+ goto err;
+ }
+
+ /* Setup hash context */
+ snprintf(h->name, sizeof(h->name), "%s", params->name);
+ h->entries = params->entries;
+ h->key_len = params->key_len;
+ h->key_entry_size = key_entry_size;
+ h->hash_func_init_val = params->hash_func_init_val;
+
+ h->num_buckets = num_buckets;
+ h->bucket_bitmask = h->num_buckets - 1;
+ h->buckets = buckets;
+ h->hash_func = (params->hash_func == NULL) ?
+ DEFAULT_HASH_FUNC : params->hash_func;
+
+ h->key_store = k;
+ h->free_slots = r;
+
+ /* populate the free slots ring. Entry zero is reserved for key misses */
+ for (i = 1; i < params->entries + 1; i++)
+ rte_ring_sp_enqueue(r, (void *)((uintptr_t) i));
+
+ rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
+ te->data = (void *) h;
+ TAILQ_INSERT_TAIL(hash_list, te, next);
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ return h;
+err:
+ rte_free(te);
+ rte_free(h);
+ rte_free(buckets);
+ rte_free(k);
+ return NULL;
+}
+
+void
+rte_hash_free(struct rte_hash *h)
+{
+ struct rte_tailq_entry *te;
+ struct rte_hash_list *hash_list;
+
+ if (h == NULL)
+ return;
+
+ hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
+
+ rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
+
+ /* find out tailq entry */
+ TAILQ_FOREACH(te, hash_list, next) {
+ if (te->data == (void *) h)
+ break;
+ }
+
+ if (te == NULL) {
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+ return;
+ }
+
+ TAILQ_REMOVE(hash_list, te, next);
+
+ rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
+
+ rte_free(h->key_store);
+ rte_free(h->buckets);
+ rte_free(h);
+ rte_free(te);
+}
+
+hash_sig_t
+rte_hash_hash(const struct rte_hash *h, const void *key)
+{
+ /* calc hash result by key */
+ return h->hash_func(key, h->key_len, h->hash_func_init_val);
+}
+
+/* Calc the secondary hash value from the primary hash value of a given key */
+static inline hash_sig_t
+rte_hash_secondary_hash(const hash_sig_t primary_hash)
+{
+ static const unsigned all_bits_shift = 12;
+ static const unsigned alt_bits_xor = 0x5bd1e995;
+
+ uint32_t tag = primary_hash >> all_bits_shift;
+
+ return (primary_hash ^ ((tag + 1) * alt_bits_xor));
+}
+
+/* Search for an entry that can be pushed to its alternative location */
+static inline int
+make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt)
+{
+ unsigned i, j;
+ int ret;
+ uint32_t next_bucket_idx;
+ struct rte_hash_bucket *next_bkt[RTE_HASH_BUCKET_ENTRIES];
+
+ /*
+ * Push existing item (search for bucket with space in
+ * alternative locations) to its alternative location
+ */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ /* Search for space in alternative locations */
+ next_bucket_idx = bkt->signatures[i].alt & h->bucket_bitmask;
+ next_bkt[i] = &h->buckets[next_bucket_idx];
+ for (j = 0; j < RTE_HASH_BUCKET_ENTRIES; j++) {
+ if (next_bkt[i]->signatures[j].sig == NULL_SIGNATURE)
+ break;
+ }
+
+ if (j != RTE_HASH_BUCKET_ENTRIES)
+ break;
+ }
+
+ /* Alternative location has spare room (end of recursive function) */
+ if (i != RTE_HASH_BUCKET_ENTRIES) {
+ next_bkt[i]->signatures[j].alt = bkt->signatures[i].current;
+ next_bkt[i]->signatures[j].current = bkt->signatures[i].alt;
+ next_bkt[i]->key_idx[j] = bkt->key_idx[i];
+ return i;
+ }
+
+ /* Pick entry that has not been pushed yet */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++)
+ if (bkt->flag[i] == 0)
+ break;
+
+ /* All entries have been pushed, so entry cannot be added */
+ if (i == RTE_HASH_BUCKET_ENTRIES) {
+ /* Reset flag */
+ bkt->flag[i] = 0;
+ return -ENOSPC;
+ }
+
+ /* Set flag to indicate that this entry is going to be pushed */
+ bkt->flag[i] = 1;
+ /* Need room in alternative bucket to insert the pushed entry */
+ ret = make_space_bucket(h, next_bkt[i]);
+ /*
+ * After recursive function.
+ * Clear flags and insert the pushed entry
+ * in its alternative location if successful,
+ * or return error
+ */
+ bkt->flag[i] = 0;
+ if (ret >= 0) {
+ next_bkt[i]->signatures[ret].alt = bkt->signatures[i].current;
+ next_bkt[i]->signatures[ret].current = bkt->signatures[i].alt;
+ next_bkt[i]->key_idx[ret] = bkt->key_idx[i];
+ return i;
+ } else
+ return ret;
+
+}
+
+static inline int32_t
+__rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ hash_sig_t alt_hash;
+ uint32_t prim_bucket_idx, sec_bucket_idx;
+ unsigned i;
+ struct rte_hash_bucket *prim_bkt, *sec_bkt;
+ void *new_k, *k, *keys = h->key_store;
+ void *slot_id;
+ uint32_t new_idx;
+ int ret;
+
+ prim_bucket_idx = sig & h->bucket_bitmask;
+ prim_bkt = &h->buckets[prim_bucket_idx];
+ rte_prefetch0(prim_bkt);
+
+ alt_hash = rte_hash_secondary_hash(sig);
+ sec_bucket_idx = alt_hash & h->bucket_bitmask;
+ sec_bkt = &h->buckets[sec_bucket_idx];
+ rte_prefetch0(sec_bkt);
+
+ /* Get a new slot for storing the new key */
+ if (rte_ring_sc_dequeue(h->free_slots, &slot_id) != 0)
+ return -ENOSPC;
+ new_k = RTE_PTR_ADD(keys, (uintptr_t)slot_id * h->key_entry_size);
+ rte_prefetch0(new_k);
+ new_idx = (uint32_t)((uintptr_t) slot_id);
+
+ /* Check if key is already inserted in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (prim_bkt->signatures[i].current == sig &&
+ prim_bkt->signatures[i].alt == alt_hash) {
+ k = (char *)keys + prim_bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (prim_bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Check if key is already inserted in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (sec_bkt->signatures[i].alt == sig &&
+ sec_bkt->signatures[i].current == alt_hash) {
+ k = (char *)keys + sec_bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (sec_bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Copy key */
+ rte_memcpy(new_k, key, h->key_len);
+
+ /* Insert new entry is there is room in the primary bucket */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ /* Check if slot is available */
+ if (likely(prim_bkt->signatures[i].sig == NULL_SIGNATURE)) {
+ prim_bkt->signatures[i].current = sig;
+ prim_bkt->signatures[i].alt = alt_hash;
+ prim_bkt->key_idx[i] = new_idx;
+ return new_idx - 1;
+ }
+ }
+
+ /* Primary bucket is full, so we need to make space for new entry */
+ ret = make_space_bucket(h, prim_bkt);
+ /*
+ * After recursive function.
+ * Insert the new entry in the position of the pushed entry
+ * if successful or return error and
+ * store the new slot back in the ring
+ */
+ if (ret >= 0) {
+ prim_bkt->signatures[ret].current = sig;
+ prim_bkt->signatures[ret].alt = alt_hash;
+ prim_bkt->key_idx[ret] = new_idx;
+ return (new_idx - 1);
+ }
+
+ /* Error in addition, store new slot back in the ring and return error */
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t) new_idx));
+ return ret;
+
+}
+
+int32_t
+rte_hash_add_key_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_add_key_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_add_key(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+static inline int32_t
+__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ uint32_t bucket_idx;
+ hash_sig_t alt_hash;
+ unsigned i;
+ struct rte_hash_bucket *bkt;
+ void *k, *keys = h->key_store;
+
+ bucket_idx = sig & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == sig &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+
+ /* Calculate secondary hash */
+ alt_hash = rte_hash_secondary_hash(sig);
+ bucket_idx = alt_hash & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == alt_hash &&
+ bkt->signatures[i].alt == sig) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+
+ return -ENOENT;
+}
+
+int32_t
+rte_hash_lookup_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_lookup(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+static inline int32_t
+__rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig)
+{
+ uint32_t bucket_idx;
+ hash_sig_t alt_hash;
+ unsigned i;
+ struct rte_hash_bucket *bkt;
+ void *k, *keys = h->key_store;
+
+ bucket_idx = sig & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in primary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == sig &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ bkt->signatures[i].sig = NULL_SIGNATURE;
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t)bkt->key_idx[i]));
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ /* Calculate secondary hash */
+ alt_hash = rte_hash_secondary_hash(sig);
+ bucket_idx = alt_hash & h->bucket_bitmask;
+ bkt = &h->buckets[bucket_idx];
+
+ /* Check if key is in secondary location */
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ if (bkt->signatures[i].current == alt_hash &&
+ bkt->signatures[i].sig != NULL_SIGNATURE) {
+ k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
+ if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ bkt->signatures[i].sig = NULL_SIGNATURE;
+ rte_ring_sp_enqueue(h->free_slots,
+ (void *)((uintptr_t)bkt->key_idx[i]));
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ return (bkt->key_idx[i] - 1);
+ }
+ }
+ }
+
+ return -ENOENT;
+}
+
+int32_t
+rte_hash_del_key_with_hash(const struct rte_hash *h,
+ const void *key, hash_sig_t sig)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_del_key_with_hash(h, key, sig);
+}
+
+int32_t
+rte_hash_del_key(const struct rte_hash *h, const void *key)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_del_key_with_hash(h, key, rte_hash_hash(h, key));
+}
+
+/* Lookup bulk stage 0: Prefetch input key */
+static inline void
+lookup_stage0(unsigned *idx, uint64_t *lookup_mask,
+ const void * const *keys)
+{
+ *idx = __builtin_ctzl(*lookup_mask);
+ if (*lookup_mask == 0)
+ *idx = 0;
+
+ rte_prefetch0(keys[*idx]);
+ *lookup_mask &= ~(1llu << *idx);
+}
+
+/*
+ * Lookup bulk stage 1: Calculate primary/secondary hashes
+ * and prefetch primary/secondary buckets
+ */
+static inline void
+lookup_stage1(unsigned idx, hash_sig_t *prim_hash, hash_sig_t *sec_hash,
+ const struct rte_hash_bucket **primary_bkt,
+ const struct rte_hash_bucket **secondary_bkt,
+ hash_sig_t *hash_vals, const void * const *keys,
+ const struct rte_hash *h)
+{
+ *prim_hash = rte_hash_hash(h, keys[idx]);
+ hash_vals[idx] = *prim_hash;
+ *sec_hash = rte_hash_secondary_hash(*prim_hash);
+
+ *primary_bkt = &h->buckets[*prim_hash & h->bucket_bitmask];
+ *secondary_bkt = &h->buckets[*sec_hash & h->bucket_bitmask];
+
+ rte_prefetch0(*primary_bkt);
+ rte_prefetch0(*secondary_bkt);
+}
+
+/*
+ * Lookup bulk stage 2: Search for match hashes in primary/secondary locations
+ * and prefetch first key slot
+ */
+static inline void
+lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
+ const struct rte_hash_bucket *prim_bkt,
+ const struct rte_hash_bucket *sec_bkt,
+ const void **key_slot, int32_t *positions,
+ uint64_t *extra_hits_mask, const void *keys,
+ const struct rte_hash *h)
+{
+ unsigned prim_hash_matches, sec_hash_matches, key_idx, i;
+ unsigned total_hash_matches;
+
+ prim_hash_matches = 1 << RTE_HASH_BUCKET_ENTRIES;
+ sec_hash_matches = 1 << RTE_HASH_BUCKET_ENTRIES;
+ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
+ prim_hash_matches |= ((prim_hash == prim_bkt->signatures[i].current) << i);
+ sec_hash_matches |= ((sec_hash == sec_bkt->signatures[i].current) << i);
+ }
+
+ key_idx = prim_bkt->key_idx[__builtin_ctzl(prim_hash_matches)];
+ if (key_idx == 0)
+ key_idx = sec_bkt->key_idx[__builtin_ctzl(sec_hash_matches)];
+
+ total_hash_matches = (prim_hash_matches |
+ (sec_hash_matches << (RTE_HASH_BUCKET_ENTRIES + 1)));
+ *key_slot = (const char *)keys + key_idx * h->key_entry_size;
+
+ rte_prefetch0(*key_slot);
+ /*
+ * Return index where key is stored,
+ * substracting the first dummy index
+ */
+ positions[idx] = (key_idx - 1);
+
+ *extra_hits_mask |= (uint64_t)(__builtin_popcount(total_hash_matches) > 3) << idx;
+
+}
+
+
+/* Lookup bulk stage 3: Check if key matches, update hit mask */
+static inline void
+lookup_stage3(unsigned idx, const void *key_slot, const void * const *keys,
+ uint64_t *hits, const struct rte_hash *h)
+{
+ unsigned hit;
+
+ hit = !h->rte_hash_cmp_eq(key_slot, keys[idx], h->key_len);
+ *hits |= (uint64_t)(hit) << idx;
+}
+
+static inline void
+__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, int32_t *positions)
+{
+ uint64_t hits = 0;
+ uint64_t extra_hits_mask = 0;
+ uint64_t lookup_mask, miss_mask;
+ unsigned idx;
+ const void *key_store = h->key_store;
+ hash_sig_t hash_vals[RTE_HASH_LOOKUP_BULK_MAX];
+
+ unsigned idx00, idx01, idx10, idx11, idx20, idx21, idx30, idx31;
+ const struct rte_hash_bucket *primary_bkt10, *primary_bkt11;
+ const struct rte_hash_bucket *secondary_bkt10, *secondary_bkt11;
+ const struct rte_hash_bucket *primary_bkt20, *primary_bkt21;
+ const struct rte_hash_bucket *secondary_bkt20, *secondary_bkt21;
+ const void *k_slot20, *k_slot21, *k_slot30, *k_slot31;
+ hash_sig_t primary_hash10, primary_hash11;
+ hash_sig_t secondary_hash10, secondary_hash11;
+ hash_sig_t primary_hash20, primary_hash21;
+ hash_sig_t secondary_hash20, secondary_hash21;
+
+ lookup_mask = (uint64_t) -1 >> (64 - num_keys);
+ miss_mask = lookup_mask;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+
+ while (lookup_mask) {
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage0(&idx00, &lookup_mask, keys);
+ lookup_stage0(&idx01, &lookup_mask, keys);
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20,
+ primary_bkt20, secondary_bkt20, &k_slot20, positions,
+ &extra_hits_mask, key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21,
+ primary_bkt21, secondary_bkt21, &k_slot21, positions,
+ &extra_hits_mask, key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ }
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+ idx10 = idx00, idx11 = idx01;
+
+ lookup_stage1(idx10, &primary_hash10, &secondary_hash10,
+ &primary_bkt10, &secondary_bkt10, hash_vals, keys, h);
+ lookup_stage1(idx11, &primary_hash11, &secondary_hash11,
+ &primary_bkt11, &secondary_bkt11, hash_vals, keys, h);
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+ primary_bkt20 = primary_bkt10;
+ primary_bkt21 = primary_bkt11;
+ secondary_bkt20 = secondary_bkt10;
+ secondary_bkt21 = secondary_bkt11;
+ primary_hash20 = primary_hash10;
+ primary_hash21 = primary_hash11;
+ secondary_hash20 = secondary_hash10;
+ secondary_hash21 = secondary_hash11;
+ idx20 = idx10, idx21 = idx11;
+
+ lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20,
+ secondary_bkt20, &k_slot20, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
+ secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
+ key_store, h);
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+
+ k_slot30 = k_slot20, k_slot31 = k_slot21;
+ idx30 = idx20, idx31 = idx21;
+
+ lookup_stage3(idx30, k_slot30, keys, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, &hits, h);
+
+ /* ignore any items we have already found */
+ extra_hits_mask &= ~hits;
+
+ if (unlikely(extra_hits_mask)) {
+ /* run a single search for each remaining item */
+ do {
+ idx = __builtin_ctzl(extra_hits_mask);
+ positions[idx] = rte_hash_lookup_with_hash(h, keys[idx],
+ hash_vals[idx]);
+ extra_hits_mask &= ~(1llu << idx);
+ if (positions[idx] >= 0)
+ hits |= 1llu << idx;
+ } while (extra_hits_mask);
+ }
+
+ miss_mask &= ~hits;
+ if (unlikely(miss_mask)) {
+ do {
+ idx = __builtin_ctzl(miss_mask);
+ positions[idx] = -ENOENT;
+ miss_mask &= ~(1llu << idx);
+ } while (miss_mask);
+ }
+}
+
+int
+rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, int32_t *positions)
+{
+ RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
+ (num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
+ (positions == NULL)), -EINVAL);
+
+ __rte_hash_lookup_bulk(h, keys, num_keys, positions);
+ return 0;
+}
+
+/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
+static int
+rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
+{
+ const __m128i k1 = _mm_loadu_si128((const __m128i *) key1);
+ const __m128i k2 = _mm_loadu_si128((const __m128i *) key2);
+ const __m128i x = _mm_xor_si128(k1, k2);
+
+ return !_mm_test_all_zeros(x, x);
+}
+
+static int
+rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k16_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 16,
+ (const char *) key2 + 16, key_len);
+}
+
+static int
+rte_hash_k48_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k16_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 16,
+ (const char *) key2 + 16, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 32,
+ (const char *) key2 + 32, key_len);
+}
+
+static int
+rte_hash_k64_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k32_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 32,
+ (const char *) key2 + 32, key_len);
+}
+
+static int
+rte_hash_k80_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
+
+static int
+rte_hash_k96_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
+
+static int
+rte_hash_k112_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k32_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len) ||
+ rte_hash_k16_cmp_eq((const char *) key1 + 96,
+ (const char *) key2 + 96, key_len);
+}
+
+static int
+rte_hash_k128_cmp_eq(const void *key1, const void *key2, size_t key_len)
+{
+ return rte_hash_k64_cmp_eq(key1, key2, key_len) ||
+ rte_hash_k64_cmp_eq((const char *) key1 + 64,
+ (const char *) key2 + 64, key_len);
+}
diff --git a/lib/librte_hash/rte_hash.c b/lib/librte_hash/rte_hash.c
deleted file mode 100644
index 5100a75..0000000
--- a/lib/librte_hash/rte_hash.c
+++ /dev/null
@@ -1,499 +0,0 @@
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <string.h>
-#include <stdint.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <sys/queue.h>
-
-#include <rte_common.h>
-#include <rte_memory.h> /* for definition of RTE_CACHE_LINE_SIZE */
-#include <rte_log.h>
-#include <rte_memcpy.h>
-#include <rte_prefetch.h>
-#include <rte_branch_prediction.h>
-#include <rte_memzone.h>
-#include <rte_malloc.h>
-#include <rte_eal.h>
-#include <rte_eal_memconfig.h>
-#include <rte_per_lcore.h>
-#include <rte_errno.h>
-#include <rte_string_fns.h>
-#include <rte_cpuflags.h>
-#include <rte_log.h>
-#include <rte_rwlock.h>
-#include <rte_spinlock.h>
-
-#include "rte_hash.h"
-
-TAILQ_HEAD(rte_hash_list, rte_tailq_entry);
-
-static struct rte_tailq_elem rte_hash_tailq = {
- .name = "RTE_HASH",
-};
-EAL_REGISTER_TAILQ(rte_hash_tailq)
-
-/* Macro to enable/disable run-time checking of function parameters */
-#if defined(RTE_LIBRTE_HASH_DEBUG)
-#define RETURN_IF_TRUE(cond, retval) do { \
- if (cond) return (retval); \
-} while (0)
-#else
-#define RETURN_IF_TRUE(cond, retval)
-#endif
-
-/* Hash function used if none is specified */
-#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
-#include <rte_hash_crc.h>
-#define DEFAULT_HASH_FUNC rte_hash_crc
-#else
-#include <rte_jhash.h>
-#define DEFAULT_HASH_FUNC rte_jhash
-#endif
-
-/* Signature bucket size is a multiple of this value */
-#define SIG_BUCKET_ALIGNMENT 16
-
-/* Stoered key size is a multiple of this value */
-#define KEY_ALIGNMENT 16
-
-/* The high bit is always set in real signatures */
-#define NULL_SIGNATURE 0
-
-struct rte_hash {
- char name[RTE_HASH_NAMESIZE]; /**< Name of the hash. */
- uint32_t entries; /**< Total table entries. */
- uint32_t bucket_entries; /**< Bucket entries. */
- uint32_t key_len; /**< Length of hash key. */
- rte_hash_function hash_func; /**< Function used to calculate hash. */
- uint32_t hash_func_init_val; /**< Init value used by hash_func. */
- uint32_t num_buckets; /**< Number of buckets in table. */
- uint32_t bucket_bitmask; /**< Bitmask for getting bucket index
- from hash signature. */
- hash_sig_t sig_msb; /**< MSB is always set in valid signatures. */
- uint8_t *sig_tbl; /**< Flat array of hash signature buckets. */
- uint32_t sig_tbl_bucket_size; /**< Signature buckets may be padded for
- alignment reasons, and this is the
- bucket size used by sig_tbl. */
- uint8_t *key_tbl; /**< Flat array of key value buckets. */
- uint32_t key_tbl_key_size; /**< Keys may be padded for alignment
- reasons, and this is the key size
- used by key_tbl. */
-};
-
-/* Returns a pointer to the first signature in specified bucket. */
-static inline hash_sig_t *
-get_sig_tbl_bucket(const struct rte_hash *h, uint32_t bucket_index)
-{
- return RTE_PTR_ADD(h->sig_tbl, (bucket_index *
- h->sig_tbl_bucket_size));
-}
-
-/* Returns a pointer to the first key in specified bucket. */
-static inline uint8_t *
-get_key_tbl_bucket(const struct rte_hash *h, uint32_t bucket_index)
-{
- return RTE_PTR_ADD(h->key_tbl, (bucket_index * h->bucket_entries *
- h->key_tbl_key_size));
-}
-
-/* Returns a pointer to a key at a specific position in a specified bucket. */
-static inline void *
-get_key_from_bucket(const struct rte_hash *h, uint8_t *bkt, uint32_t pos)
-{
- return RTE_PTR_ADD(bkt, pos * h->key_tbl_key_size);
-}
-
-/* Does integer division with rounding-up of result. */
-static inline uint32_t
-div_roundup(uint32_t numerator, uint32_t denominator)
-{
- return (numerator + denominator - 1) / denominator;
-}
-
-/* Increases a size (if needed) to a multiple of alignment. */
-static inline uint32_t
-align_size(uint32_t val, uint32_t alignment)
-{
- return alignment * div_roundup(val, alignment);
-}
-
-/* Returns the index into the bucket of the first occurrence of a signature. */
-static inline int
-find_first(uint32_t sig, const uint32_t *sig_bucket, uint32_t num_sigs)
-{
- uint32_t i;
- for (i = 0; i < num_sigs; i++) {
- if (sig == sig_bucket[i])
- return i;
- }
- return -1;
-}
-
-struct rte_hash *
-rte_hash_find_existing(const char *name)
-{
- struct rte_hash *h = NULL;
- struct rte_tailq_entry *te;
- struct rte_hash_list *hash_list;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK);
- TAILQ_FOREACH(te, hash_list, next) {
- h = (struct rte_hash *) te->data;
- if (strncmp(name, h->name, RTE_HASH_NAMESIZE) == 0)
- break;
- }
- rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- if (te == NULL) {
- rte_errno = ENOENT;
- return NULL;
- }
- return h;
-}
-
-struct rte_hash *
-rte_hash_create(const struct rte_hash_parameters *params)
-{
- struct rte_hash *h = NULL;
- struct rte_tailq_entry *te;
- uint32_t num_buckets, sig_bucket_size, key_size,
- hash_tbl_size, sig_tbl_size, key_tbl_size, mem_size;
- char hash_name[RTE_HASH_NAMESIZE];
- struct rte_hash_list *hash_list;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- /* Check for valid parameters */
- if ((params == NULL) ||
- (params->entries > RTE_HASH_ENTRIES_MAX) ||
- (params->bucket_entries > RTE_HASH_BUCKET_ENTRIES_MAX) ||
- (params->entries < params->bucket_entries) ||
- !rte_is_power_of_2(params->entries) ||
- !rte_is_power_of_2(params->bucket_entries) ||
- (params->key_len == 0) ||
- (params->key_len > RTE_HASH_KEY_LENGTH_MAX)) {
- rte_errno = EINVAL;
- RTE_LOG(ERR, HASH, "rte_hash_create has invalid parameters\n");
- return NULL;
- }
-
- snprintf(hash_name, sizeof(hash_name), "HT_%s", params->name);
-
- /* Calculate hash dimensions */
- num_buckets = params->entries / params->bucket_entries;
- sig_bucket_size = align_size(params->bucket_entries *
- sizeof(hash_sig_t), SIG_BUCKET_ALIGNMENT);
- key_size = align_size(params->key_len, KEY_ALIGNMENT);
-
- hash_tbl_size = align_size(sizeof(struct rte_hash), RTE_CACHE_LINE_SIZE);
- sig_tbl_size = align_size(num_buckets * sig_bucket_size,
- RTE_CACHE_LINE_SIZE);
- key_tbl_size = align_size(num_buckets * key_size *
- params->bucket_entries, RTE_CACHE_LINE_SIZE);
-
- /* Total memory required for hash context */
- mem_size = hash_tbl_size + sig_tbl_size + key_tbl_size;
-
- rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
-
- /* guarantee there's no existing */
- TAILQ_FOREACH(te, hash_list, next) {
- h = (struct rte_hash *) te->data;
- if (strncmp(params->name, h->name, RTE_HASH_NAMESIZE) == 0)
- break;
- }
- if (te != NULL)
- goto exit;
-
- te = rte_zmalloc("HASH_TAILQ_ENTRY", sizeof(*te), 0);
- if (te == NULL) {
- RTE_LOG(ERR, HASH, "tailq entry allocation failed\n");
- goto exit;
- }
-
- h = (struct rte_hash *)rte_zmalloc_socket(hash_name, mem_size,
- RTE_CACHE_LINE_SIZE, params->socket_id);
- if (h == NULL) {
- RTE_LOG(ERR, HASH, "memory allocation failed\n");
- rte_free(te);
- goto exit;
- }
-
- /* Setup hash context */
- snprintf(h->name, sizeof(h->name), "%s", params->name);
- h->entries = params->entries;
- h->bucket_entries = params->bucket_entries;
- h->key_len = params->key_len;
- h->hash_func_init_val = params->hash_func_init_val;
- h->num_buckets = num_buckets;
- h->bucket_bitmask = h->num_buckets - 1;
- h->sig_msb = 1 << (sizeof(hash_sig_t) * 8 - 1);
- h->sig_tbl = (uint8_t *)h + hash_tbl_size;
- h->sig_tbl_bucket_size = sig_bucket_size;
- h->key_tbl = h->sig_tbl + sig_tbl_size;
- h->key_tbl_key_size = key_size;
- h->hash_func = (params->hash_func == NULL) ?
- DEFAULT_HASH_FUNC : params->hash_func;
-
- te->data = (void *) h;
-
- TAILQ_INSERT_TAIL(hash_list, te, next);
-
-exit:
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- return h;
-}
-
-void
-rte_hash_free(struct rte_hash *h)
-{
- struct rte_tailq_entry *te;
- struct rte_hash_list *hash_list;
-
- if (h == NULL)
- return;
-
- hash_list = RTE_TAILQ_CAST(rte_hash_tailq.head, rte_hash_list);
-
- rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
-
- /* find out tailq entry */
- TAILQ_FOREACH(te, hash_list, next) {
- if (te->data == (void *) h)
- break;
- }
-
- if (te == NULL) {
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
- return;
- }
-
- TAILQ_REMOVE(hash_list, te, next);
-
- rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
-
- rte_free(h);
- rte_free(te);
-}
-
-hash_sig_t
-rte_hash_hash(const struct rte_hash *h, const void *key)
-{
- /* calc hash result by key */
- return h->hash_func(key, h->key_len, h->hash_func_init_val);
-}
-
-static inline int32_t
-__rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
- int32_t pos;
-
- /* Get the hash signature and bucket index */
- sig |= h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- /* Check if any free slot within the bucket to add the new key */
- pos = find_first(NULL_SIGNATURE, sig_bucket, h->bucket_entries);
-
- if (unlikely(pos < 0))
- return -ENOSPC;
-
- /* Add the new key to the bucket */
- sig_bucket[pos] = sig;
- rte_memcpy(get_key_from_bucket(h, key_bucket, pos), key, h->key_len);
- return bucket_index * h->bucket_entries + pos;
-}
-
-int32_t
-rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_add_key(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-static inline int32_t
-__rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
-
- /* Get the hash signature and bucket index */
- sig = sig | h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- sig_bucket[i] = NULL_SIGNATURE;
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- return -ENOENT;
-}
-
-int32_t
-rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_del_key_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_del_key(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_del_key_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-static inline int32_t
-__rte_hash_lookup_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- hash_sig_t *sig_bucket;
- uint8_t *key_bucket;
- uint32_t bucket_index, i;
-
- /* Get the hash signature and bucket index */
- sig |= h->sig_msb;
- bucket_index = sig & h->bucket_bitmask;
- sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- /* Check if key is already present in the hash */
- for (i = 0; i < h->bucket_entries; i++) {
- if ((sig == sig_bucket[i]) &&
- likely(memcmp(key, get_key_from_bucket(h, key_bucket, i),
- h->key_len) == 0)) {
- return bucket_index * h->bucket_entries + i;
- }
- }
-
- return -ENOENT;
-}
-
-int32_t
-rte_hash_lookup_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, sig);
-}
-
-int32_t
-rte_hash_lookup(const struct rte_hash *h, const void *key)
-{
- RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
-}
-
-int
-rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
- uint32_t num_keys, int32_t *positions)
-{
- uint32_t i, j, bucket_index;
- hash_sig_t sigs[RTE_HASH_LOOKUP_BULK_MAX];
-
- RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
- (num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
- (positions == NULL)), -EINVAL);
-
- /* Get the hash signature and bucket index */
- for (i = 0; i < num_keys; i++) {
- sigs[i] = h->hash_func(keys[i], h->key_len,
- h->hash_func_init_val) | h->sig_msb;
- bucket_index = sigs[i] & h->bucket_bitmask;
-
- /* Pre-fetch relevant buckets */
- rte_prefetch1((void *) get_sig_tbl_bucket(h, bucket_index));
- rte_prefetch1((void *) get_key_tbl_bucket(h, bucket_index));
- }
-
- /* Check if key is already present in the hash */
- for (i = 0; i < num_keys; i++) {
- bucket_index = sigs[i] & h->bucket_bitmask;
- hash_sig_t *sig_bucket = get_sig_tbl_bucket(h, bucket_index);
- uint8_t *key_bucket = get_key_tbl_bucket(h, bucket_index);
-
- positions[i] = -ENOENT;
-
- for (j = 0; j < h->bucket_entries; j++) {
- if ((sigs[i] == sig_bucket[j]) &&
- likely(memcmp(keys[i],
- get_key_from_bucket(h, key_bucket, j),
- h->key_len) == 0)) {
- positions[i] = bucket_index *
- h->bucket_entries + j;
- break;
- }
- }
- }
-
- return 0;
-}
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index da0a00a..936d170 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -47,21 +47,21 @@ extern "C" {
#endif
/** Maximum size of hash table that can be created. */
-#define RTE_HASH_ENTRIES_MAX (1 << 26)
+#define RTE_HASH_ENTRIES_MAX (1 << 30)
-/** Maximum bucket size that can be created. */
-#define RTE_HASH_BUCKET_ENTRIES_MAX 16
+/** @deprecated Maximum bucket size that can be created. */
+#define RTE_HASH_BUCKET_ENTRIES_MAX 4
-/** Maximum length of key that can be used. */
+/** @deprecated Maximum length of key that can be used. */
#define RTE_HASH_KEY_LENGTH_MAX 64
-/** Max number of keys that can be searched for using rte_hash_lookup_multi. */
-#define RTE_HASH_LOOKUP_BULK_MAX 16
-#define RTE_HASH_LOOKUP_MULTI_MAX RTE_HASH_LOOKUP_BULK_MAX
-
-/** Max number of characters in hash name.*/
+/** Maximum number of characters in hash name.*/
#define RTE_HASH_NAMESIZE 32
+/** Maximum number of keys that can be searched for using rte_hash_lookup_bulk. */
+#define RTE_HASH_LOOKUP_BULK_MAX 64
+#define RTE_HASH_LOOKUP_MULTI_MAX RTE_HASH_LOOKUP_BULK_MAX
+
/** Signature of key that is stored internally. */
typedef uint32_t hash_sig_t;
@@ -70,15 +70,14 @@ typedef uint32_t (*rte_hash_function)(const void *key, uint32_t key_len,
uint32_t init_val);
/**
- * Parameters used when creating the hash table. The total table entries and
- * bucket entries must be a power of 2.
+ * Parameters used when creating the hash table.
*/
struct rte_hash_parameters {
const char *name; /**< Name of the hash. */
uint32_t entries; /**< Total hash table entries. */
- uint32_t bucket_entries; /**< Bucket entries. */
+ uint32_t bucket_entries; /**< Bucket entries. */
uint32_t key_len; /**< Length of hash key. */
- rte_hash_function hash_func; /**< Function used to calculate hash. */
+ rte_hash_function hash_func; /**< Primary Hash function used to calculate hash. */
uint32_t hash_func_init_val; /**< Init value used by hash_func. */
int socket_id; /**< NUMA Socket ID for memory. */
};
@@ -86,6 +85,7 @@ struct rte_hash_parameters {
/** @internal A hash table structure. */
struct rte_hash;
+
/**
* Create a new hash table.
*
@@ -106,7 +106,6 @@ struct rte_hash;
struct rte_hash *
rte_hash_create(const struct rte_hash_parameters *params);
-
/**
* Find an existing hash table object and return a pointer to it.
*
@@ -129,7 +128,8 @@ void
rte_hash_free(struct rte_hash *h);
/**
- * Add a key to an existing hash table. This operation is not multi-thread safe
+ * Add a key to an existing hash table.
+ * This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
@@ -146,7 +146,8 @@ int32_t
rte_hash_add_key(const struct rte_hash *h, const void *key);
/**
- * Add a key to an existing hash table. This operation is not multi-thread safe
+ * Add a key to an existing hash table.
+ * This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
@@ -154,7 +155,7 @@ rte_hash_add_key(const struct rte_hash *h, const void *key);
* @param key
* Key to add to the hash table.
* @param sig
- * Hash value to add to the hash table.
+ * Precomputed hash value for 'key'.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOSPC if there is no space in the hash for this key.
@@ -162,12 +163,12 @@ rte_hash_add_key(const struct rte_hash *h, const void *key);
* array of user data. This value is unique for this key.
*/
int32_t
-rte_hash_add_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig);
+rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
/**
- * Remove a key from an existing hash table. This operation is not multi-thread
- * safe and should only be called from one thread.
+ * Remove a key from an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
*
* @param h
* Hash table to remove the key from.
@@ -184,15 +185,16 @@ int32_t
rte_hash_del_key(const struct rte_hash *h, const void *key);
/**
- * Remove a key from an existing hash table. This operation is not multi-thread
- * safe and should only be called from one thread.
+ * Remove a key from an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
*
* @param h
* Hash table to remove the key from.
* @param key
* Key to remove from the hash table.
* @param sig
- * Hash value to remove from the hash table.
+ * Precomputed hash value for 'key'.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOENT if the key is not found.
@@ -201,12 +203,11 @@ rte_hash_del_key(const struct rte_hash *h, const void *key);
* value that was returned when the key was added.
*/
int32_t
-rte_hash_del_key_with_hash(const struct rte_hash *h,
- const void *key, hash_sig_t sig);
-
+rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
/**
- * Find a key in the hash table. This operation is multi-thread safe.
+ * Find a key in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
@@ -223,14 +224,15 @@ int32_t
rte_hash_lookup(const struct rte_hash *h, const void *key);
/**
- * Find a key in the hash table. This operation is multi-thread safe.
+ * Find a key in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
* @param key
* Key to find.
* @param sig
- * Hash value to find.
+ * Hash value to remove from the hash table.
* @return
* - -EINVAL if the parameters are invalid.
* - -ENOENT if the key is not found.
@@ -243,7 +245,8 @@ rte_hash_lookup_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig);
/**
- * Calc a hash value by key. This operation is not multi-process safe.
+ * Calc a hash value by key.
+ * This operation is not multi-thread safe.
*
* @param h
* Hash table to look in.
@@ -257,7 +260,8 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
#define rte_hash_lookup_multi rte_hash_lookup_bulk
/**
- * Find multiple keys in the hash table. This operation is multi-thread safe.
+ * Find multiple keys in the hash table.
+ * This operation is multi-thread safe.
*
* @param h
* Hash table to look in.
@@ -277,6 +281,7 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions);
+
#ifdef __cplusplus
}
#endif
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* Re: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
@ 2015-07-12 22:29 ` Thomas Monjalon
2015-07-13 16:11 ` Bruce Richardson
2015-07-16 9:39 ` Tony Lu
1 sibling, 1 reply; 92+ messages in thread
From: Thomas Monjalon @ 2015-07-12 22:29 UTC (permalink / raw)
To: Pablo de Lara; +Cc: dev
2015-07-11 01:18, Pablo de Lara:
> The main change when creating a new table is that the number of entries
> per bucket is fixed now, so its parameter is ignored now
> (still there to maintain the same parameters structure).
Why not rename the "bucket_entries" field to "reserved"?
The API of this field has changed (now ignored) so it should be reflected
without changing the ABI.
^ permalink raw reply [flat|nested] 92+ messages in thread
* Re: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-12 22:29 ` Thomas Monjalon
@ 2015-07-13 16:11 ` Bruce Richardson
2015-07-13 16:14 ` Bruce Richardson
0 siblings, 1 reply; 92+ messages in thread
From: Bruce Richardson @ 2015-07-13 16:11 UTC (permalink / raw)
To: Thomas Monjalon; +Cc: dev
On Mon, Jul 13, 2015 at 12:29:53AM +0200, Thomas Monjalon wrote:
> 2015-07-11 01:18, Pablo de Lara:
> > The main change when creating a new table is that the number of entries
> > per bucket is fixed now, so its parameter is ignored now
> > (still there to maintain the same parameters structure).
>
> Why not rename the "bucket_entries" field to "reserved"?
> The API of this field has changed (now ignored) so it should be reflected
> without changing the ABI.
Since the hash_create function is itself already versionned to take account of the
new struct parameter, there is no reason to keep the field at all, as far as I can see.
We can just drop it, and let the ABI versionning handle the change.
/Bruce
^ permalink raw reply [flat|nested] 92+ messages in thread
* Re: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-13 16:11 ` Bruce Richardson
@ 2015-07-13 16:14 ` Bruce Richardson
2015-07-13 16:20 ` Thomas Monjalon
0 siblings, 1 reply; 92+ messages in thread
From: Bruce Richardson @ 2015-07-13 16:14 UTC (permalink / raw)
To: Thomas Monjalon; +Cc: dev
On Mon, Jul 13, 2015 at 05:11:54PM +0100, Bruce Richardson wrote:
> On Mon, Jul 13, 2015 at 12:29:53AM +0200, Thomas Monjalon wrote:
> > 2015-07-11 01:18, Pablo de Lara:
> > > The main change when creating a new table is that the number of entries
> > > per bucket is fixed now, so its parameter is ignored now
> > > (still there to maintain the same parameters structure).
> >
> > Why not rename the "bucket_entries" field to "reserved"?
> > The API of this field has changed (now ignored) so it should be reflected
> > without changing the ABI.
>
> Since the hash_create function is itself already versionned to take account of the
> new struct parameter, there is no reason to keep the field at all, as far as I can see.
> We can just drop it, and let the ABI versionning handle the change.
>
> /Bruce
Sorry, my mistake. It's no longer versioned in the patchset that was merged, so
the field does need to be kept. :-(
/Bruce
^ permalink raw reply [flat|nested] 92+ messages in thread
* Re: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-13 16:14 ` Bruce Richardson
@ 2015-07-13 16:20 ` Thomas Monjalon
2015-07-13 16:26 ` Bruce Richardson
0 siblings, 1 reply; 92+ messages in thread
From: Thomas Monjalon @ 2015-07-13 16:20 UTC (permalink / raw)
To: Bruce Richardson; +Cc: dev
2015-07-13 17:14, Bruce Richardson:
> On Mon, Jul 13, 2015 at 05:11:54PM +0100, Bruce Richardson wrote:
> > On Mon, Jul 13, 2015 at 12:29:53AM +0200, Thomas Monjalon wrote:
> > > 2015-07-11 01:18, Pablo de Lara:
> > > > The main change when creating a new table is that the number of entries
> > > > per bucket is fixed now, so its parameter is ignored now
> > > > (still there to maintain the same parameters structure).
> > >
> > > Why not rename the "bucket_entries" field to "reserved"?
> > > The API of this field has changed (now ignored) so it should be reflected
> > > without changing the ABI.
> >
> > Since the hash_create function is itself already versionned to take account of the
> > new struct parameter, there is no reason to keep the field at all, as far as I can see.
> > We can just drop it, and let the ABI versionning handle the change.
> >
> > /Bruce
>
> Sorry, my mistake. It's no longer versioned in the patchset that was merged, so
> the field does need to be kept. :-(
So do you agree to submit a patch which rename the unused field?
^ permalink raw reply [flat|nested] 92+ messages in thread
* Re: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-13 16:20 ` Thomas Monjalon
@ 2015-07-13 16:26 ` Bruce Richardson
0 siblings, 0 replies; 92+ messages in thread
From: Bruce Richardson @ 2015-07-13 16:26 UTC (permalink / raw)
To: Thomas Monjalon; +Cc: dev
On Mon, Jul 13, 2015 at 06:20:08PM +0200, Thomas Monjalon wrote:
> 2015-07-13 17:14, Bruce Richardson:
> > On Mon, Jul 13, 2015 at 05:11:54PM +0100, Bruce Richardson wrote:
> > > On Mon, Jul 13, 2015 at 12:29:53AM +0200, Thomas Monjalon wrote:
> > > > 2015-07-11 01:18, Pablo de Lara:
> > > > > The main change when creating a new table is that the number of entries
> > > > > per bucket is fixed now, so its parameter is ignored now
> > > > > (still there to maintain the same parameters structure).
> > > >
> > > > Why not rename the "bucket_entries" field to "reserved"?
> > > > The API of this field has changed (now ignored) so it should be reflected
> > > > without changing the ABI.
> > >
> > > Since the hash_create function is itself already versionned to take account of the
> > > new struct parameter, there is no reason to keep the field at all, as far as I can see.
> > > We can just drop it, and let the ABI versionning handle the change.
> > >
> > > /Bruce
> >
> > Sorry, my mistake. It's no longer versioned in the patchset that was merged, so
> > the field does need to be kept. :-(
>
> So do you agree to submit a patch which rename the unused field?
Yes. It should be in your inbox now... :-)
^ permalink raw reply [flat|nested] 92+ messages in thread
* Re: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
2015-07-12 22:29 ` Thomas Monjalon
@ 2015-07-16 9:39 ` Tony Lu
2015-07-16 20:42 ` De Lara Guarch, Pablo
1 sibling, 1 reply; 92+ messages in thread
From: Tony Lu @ 2015-07-16 9:39 UTC (permalink / raw)
To: 'Pablo de Lara', dev
>diff --git a/lib/librte_hash/rte_cuckoo_hash.c
b/lib/librte_hash/rte_cuckoo_hash.c
>new file mode 100644
>index 0000000..50e3acd
>--- /dev/null
>+++ b/lib/librte_hash/rte_cuckoo_hash.c
>@@ -0,0 +1,1027 @@
...
>+
>+/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
>+static int
>+rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len
>__rte_unused)
>+{
>+ const __m128i k1 = _mm_loadu_si128((const __m128i *) key1);
>+ const __m128i k2 = _mm_loadu_si128((const __m128i *) key2);
>+ const __m128i x = _mm_xor_si128(k1, k2);
>+
>+ return !_mm_test_all_zeros(x, x);
>+}
...
When compiling the latest dev DPDK for non-x86 arch, it fails on the above
code,
as the SSE is x86 specific defined in <emmintrin.h>. Is it possible to
replace this
function with platform independent code?
Thanks
-Zhigang Lu
^ permalink raw reply [flat|nested] 92+ messages in thread* Re: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-16 9:39 ` Tony Lu
@ 2015-07-16 20:42 ` De Lara Guarch, Pablo
2015-07-17 3:34 ` Tony Lu
0 siblings, 1 reply; 92+ messages in thread
From: De Lara Guarch, Pablo @ 2015-07-16 20:42 UTC (permalink / raw)
To: Tony Lu, dev
Hi Tony,
> -----Original Message-----
> From: Tony Lu [mailto:zlu@ezchip.com]
> Sent: Thursday, July 16, 2015 10:40 AM
> To: De Lara Guarch, Pablo; dev@dpdk.org
> Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library
> with cuckoo hash implementation
>
> >diff --git a/lib/librte_hash/rte_cuckoo_hash.c
> b/lib/librte_hash/rte_cuckoo_hash.c
> >new file mode 100644
> >index 0000000..50e3acd
> >--- /dev/null
> >+++ b/lib/librte_hash/rte_cuckoo_hash.c
> >@@ -0,0 +1,1027 @@
> ...
> >+
> >+/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
> >+static int
> >+rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len
> >__rte_unused)
> >+{
> >+ const __m128i k1 = _mm_loadu_si128((const __m128i *) key1);
> >+ const __m128i k2 = _mm_loadu_si128((const __m128i *) key2);
> >+ const __m128i x = _mm_xor_si128(k1, k2);
> >+
> >+ return !_mm_test_all_zeros(x, x);
> >+}
> ...
>
> When compiling the latest dev DPDK for non-x86 arch, it fails on the above
> code,
> as the SSE is x86 specific defined in <emmintrin.h>. Is it possible to
> replace this
> function with platform independent code?
Thanks for spotting this. I just sent a patch that should fix the problem.
Can you check if it works?
Thanks,
Pablo
>
> Thanks
> -Zhigang Lu
^ permalink raw reply [flat|nested] 92+ messages in thread* Re: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-16 20:42 ` De Lara Guarch, Pablo
@ 2015-07-17 3:34 ` Tony Lu
2015-07-17 7:34 ` De Lara Guarch, Pablo
0 siblings, 1 reply; 92+ messages in thread
From: Tony Lu @ 2015-07-17 3:34 UTC (permalink / raw)
To: 'De Lara Guarch, Pablo', dev
Hi, Pablo
>-----Original Message-----
>From: De Lara Guarch, Pablo [mailto:pablo.de.lara.guarch@intel.com]
>Sent: Friday, July 17, 2015 4:42 AM
>To: Tony Lu; dev@dpdk.org
>Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library
with
>cuckoo hash implementation
>
>Hi Tony,
>
>> -----Original Message-----
>> From: Tony Lu [mailto:zlu@ezchip.com]
>> Sent: Thursday, July 16, 2015 10:40 AM
>> To: De Lara Guarch, Pablo; dev@dpdk.org
>> Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash
>> library with cuckoo hash implementation
>>
>> >diff --git a/lib/librte_hash/rte_cuckoo_hash.c
>> b/lib/librte_hash/rte_cuckoo_hash.c
>> >new file mode 100644
>> >index 0000000..50e3acd
>> >--- /dev/null
>> >+++ b/lib/librte_hash/rte_cuckoo_hash.c
>> >@@ -0,0 +1,1027 @@
>> ...
>> >+
>> >+/* Functions to compare multiple of 16 byte keys (up to 128 bytes)
>> >+*/ static int rte_hash_k16_cmp_eq(const void *key1, const void
>> >+*key2, size_t key_len
>> >__rte_unused)
>> >+{
>> >+ const __m128i k1 = _mm_loadu_si128((const __m128i *) key1);
>> >+ const __m128i k2 = _mm_loadu_si128((const __m128i *) key2);
>> >+ const __m128i x = _mm_xor_si128(k1, k2);
>> >+
>> >+ return !_mm_test_all_zeros(x, x);
>> >+}
>> ...
>>
>> When compiling the latest dev DPDK for non-x86 arch, it fails on the
>> above code, as the SSE is x86 specific defined in <emmintrin.h>. Is
>> it possible to replace this function with platform independent code?
>
>Thanks for spotting this. I just sent a patch that should fix the problem.
>Can you check if it works?
Thanks for your quick response, but __m128i and all the _mm_ related
functions
are X86 specific defined in <emmintrin.h>. This header file is only
available in X86
compiler library, but no-X86 archs do not have this file. So if we can
replace all
the X86 specific code in the above function, that would be great.
Thanks
-Tony
>Thanks,
>Pablo
>>
>> Thanks
>> -Zhigang Lu
^ permalink raw reply [flat|nested] 92+ messages in thread* Re: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-17 3:34 ` Tony Lu
@ 2015-07-17 7:34 ` De Lara Guarch, Pablo
2015-07-17 7:58 ` Tony Lu
0 siblings, 1 reply; 92+ messages in thread
From: De Lara Guarch, Pablo @ 2015-07-17 7:34 UTC (permalink / raw)
To: Tony Lu, dev
> -----Original Message-----
> From: Tony Lu [mailto:zlu@ezchip.com]
> Sent: Friday, July 17, 2015 4:35 AM
> To: De Lara Guarch, Pablo; dev@dpdk.org
> Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library
> with cuckoo hash implementation
>
> Hi, Pablo
>
> >-----Original Message-----
> >From: De Lara Guarch, Pablo [mailto:pablo.de.lara.guarch@intel.com]
> >Sent: Friday, July 17, 2015 4:42 AM
> >To: Tony Lu; dev@dpdk.org
> >Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library
> with
> >cuckoo hash implementation
> >
> >Hi Tony,
> >
> >> -----Original Message-----
> >> From: Tony Lu [mailto:zlu@ezchip.com]
> >> Sent: Thursday, July 16, 2015 10:40 AM
> >> To: De Lara Guarch, Pablo; dev@dpdk.org
> >> Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash
> >> library with cuckoo hash implementation
> >>
> >> >diff --git a/lib/librte_hash/rte_cuckoo_hash.c
> >> b/lib/librte_hash/rte_cuckoo_hash.c
> >> >new file mode 100644
> >> >index 0000000..50e3acd
> >> >--- /dev/null
> >> >+++ b/lib/librte_hash/rte_cuckoo_hash.c
> >> >@@ -0,0 +1,1027 @@
> >> ...
> >> >+
> >> >+/* Functions to compare multiple of 16 byte keys (up to 128 bytes)
> >> >+*/ static int rte_hash_k16_cmp_eq(const void *key1, const void
> >> >+*key2, size_t key_len
> >> >__rte_unused)
> >> >+{
> >> >+ const __m128i k1 = _mm_loadu_si128((const __m128i *) key1);
> >> >+ const __m128i k2 = _mm_loadu_si128((const __m128i *) key2);
> >> >+ const __m128i x = _mm_xor_si128(k1, k2);
> >> >+
> >> >+ return !_mm_test_all_zeros(x, x);
> >> >+}
> >> ...
> >>
> >> When compiling the latest dev DPDK for non-x86 arch, it fails on the
> >> above code, as the SSE is x86 specific defined in <emmintrin.h>. Is
> >> it possible to replace this function with platform independent code?
> >
> >Thanks for spotting this. I just sent a patch that should fix the problem.
> >Can you check if it works?
>
> Thanks for your quick response, but __m128i and all the _mm_ related
> functions
> are X86 specific defined in <emmintrin.h>. This header file is only
> available in X86
> compiler library, but no-X86 archs do not have this file. So if we can
> replace all
> the X86 specific code in the above function, that would be great.
>
With the patch that I sent, if you are compiling for a non-x86 arch, you should not have any problem,
since all that code will only be used if using x86 arch. Have you tried compiling DPDK with the patch?
Pablo
> Thanks
> -Tony
>
>
> >Thanks,
> >Pablo
> >>
> >> Thanks
> >> -Zhigang Lu
>
^ permalink raw reply [flat|nested] 92+ messages in thread* Re: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-17 7:34 ` De Lara Guarch, Pablo
@ 2015-07-17 7:58 ` Tony Lu
2015-07-17 9:06 ` De Lara Guarch, Pablo
0 siblings, 1 reply; 92+ messages in thread
From: Tony Lu @ 2015-07-17 7:58 UTC (permalink / raw)
To: 'De Lara Guarch, Pablo', dev
>-----Original Message-----
>From: De Lara Guarch, Pablo [mailto:pablo.de.lara.guarch@intel.com]
>Sent: Friday, July 17, 2015 3:35 PM
>To: Tony Lu; dev@dpdk.org
>Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library
with
>cuckoo hash implementation
>
>
>
>> -----Original Message-----
>> From: Tony Lu [mailto:zlu@ezchip.com]
>> Sent: Friday, July 17, 2015 4:35 AM
>> To: De Lara Guarch, Pablo; dev@dpdk.org
>> Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash
>> library with cuckoo hash implementation
>>
>> Hi, Pablo
>>
>> >-----Original Message-----
>> >From: De Lara Guarch, Pablo [mailto:pablo.de.lara.guarch@intel.com]
>> >Sent: Friday, July 17, 2015 4:42 AM
>> >To: Tony Lu; dev@dpdk.org
>> >Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash
>> >library
>> with
>> >cuckoo hash implementation
>> >
>> >Hi Tony,
>> >
>> >> -----Original Message-----
>> >> From: Tony Lu [mailto:zlu@ezchip.com]
>> >> Sent: Thursday, July 16, 2015 10:40 AM
>> >> To: De Lara Guarch, Pablo; dev@dpdk.org
>> >> Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash
>> >> library with cuckoo hash implementation
>> >>
>> >> >diff --git a/lib/librte_hash/rte_cuckoo_hash.c
>> >> b/lib/librte_hash/rte_cuckoo_hash.c
>> >> >new file mode 100644
>> >> >index 0000000..50e3acd
>> >> >--- /dev/null
>> >> >+++ b/lib/librte_hash/rte_cuckoo_hash.c
>> >> >@@ -0,0 +1,1027 @@
>> >> ...
>> >> >+
>> >> >+/* Functions to compare multiple of 16 byte keys (up to 128
>> >> >+bytes) */ static int rte_hash_k16_cmp_eq(const void *key1, const
>> >> >+void *key2, size_t key_len
>> >> >__rte_unused)
>> >> >+{
>> >> >+ const __m128i k1 = _mm_loadu_si128((const __m128i *) key1);
>> >> >+ const __m128i k2 = _mm_loadu_si128((const __m128i *) key2);
>> >> >+ const __m128i x = _mm_xor_si128(k1, k2);
>> >> >+
>> >> >+ return !_mm_test_all_zeros(x, x); }
>> >> ...
>> >>
>> >> When compiling the latest dev DPDK for non-x86 arch, it fails on
>> >> the above code, as the SSE is x86 specific defined in
>> >> <emmintrin.h>. Is it possible to replace this function with platform
>independent code?
>> >
>> >Thanks for spotting this. I just sent a patch that should fix the
problem.
>> >Can you check if it works?
>>
>> Thanks for your quick response, but __m128i and all the _mm_ related
>> functions are X86 specific defined in <emmintrin.h>. This header file
>> is only available in X86 compiler library, but no-X86 archs do not
>> have this file. So if we can replace all the X86 specific code in the
>> above function, that would be great.
>>
>With the patch that I sent, if you are compiling for a non-x86 arch, you
should
>not have any problem, since all that code will only be used if using x86
arch.
>Have you tried compiling DPDK with the patch?
Yes, I have built the DPDK with your patch, and got the following errors.
This is
because there are no __m128i, _mm_loadu_si128(), _mm_cmpeq_epi32() and
_mm_movemask_epi8() on no-X86 arches.
== Build lib/librte_hash
CC rte_cuckoo_hash.o
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c: In function
'rte_hash_k16_cmp_eq':
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error:
expected '=', ',', ';', 'asm' or '__attribute__' before 'k1'
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error: 'k1'
undeclared (first use in this function)
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error: (Each
undeclared identifier is reported only once
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error: for
each function it appears in.)
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: warning:
implicit declaration of function '_mm_loadu_si128'
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: warning:
nested extern declaration of '_mm_loadu_si128'
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error:
expected ')' before '__m128i'
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: warning:
type defaults to 'int' in declaration of 'type name'
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: warning:
cast from pointer to integer of different size
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: error:
expected '=', ',', ';', 'asm' or '__attribute__' before 'k2'
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: error: 'k2'
undeclared (first use in this function)
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: error:
expected ')' before '__m128i'
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: warning:
type defaults to 'int' in declaration of 'type name'
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: warning:
cast from pointer to integer of different size
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1133: error:
expected '=', ',', ';', 'asm' or '__attribute__' before 'x'
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1133: error: 'x'
undeclared (first use in this function)
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1133: warning:
implicit declaration of function '_mm_cmpeq_epi32'
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1133: warning:
nested extern declaration of '_mm_cmpeq_epi32'
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1135: warning:
implicit declaration of function '_mm_movemask_epi8'
/u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1135: warning:
nested extern declaration of '_mm_movemask_epi8'
make[3]: *** [rte_cuckoo_hash.o] Error 1
make[2]: *** [librte_hash] Error 2
make[1]: *** [lib] Error 2
make: *** [all] Error 2
Thanks
-Tony
>Pablo
>
>> Thanks
>> -Tony
>>
>>
>> >Thanks,
>> >Pablo
>> >>
>> >> Thanks
>> >> -Zhigang Lu
>>
^ permalink raw reply [flat|nested] 92+ messages in thread* Re: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-17 7:58 ` Tony Lu
@ 2015-07-17 9:06 ` De Lara Guarch, Pablo
2015-07-17 14:39 ` Tony Lu
0 siblings, 1 reply; 92+ messages in thread
From: De Lara Guarch, Pablo @ 2015-07-17 9:06 UTC (permalink / raw)
To: Tony Lu, dev
Hi Tony,
> -----Original Message-----
> From: Tony Lu [mailto:zlu@ezchip.com]
> Sent: Friday, July 17, 2015 8:58 AM
> To: De Lara Guarch, Pablo; dev@dpdk.org
> Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library
> with cuckoo hash implementation
>
> >-----Original Message-----
> >From: De Lara Guarch, Pablo [mailto:pablo.de.lara.guarch@intel.com]
> >Sent: Friday, July 17, 2015 3:35 PM
> >To: Tony Lu; dev@dpdk.org
> >Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library
> with
> >cuckoo hash implementation
> >
> >
> >
> >> -----Original Message-----
> >> From: Tony Lu [mailto:zlu@ezchip.com]
> >> Sent: Friday, July 17, 2015 4:35 AM
> >> To: De Lara Guarch, Pablo; dev@dpdk.org
> >> Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash
> >> library with cuckoo hash implementation
> >>
> >> Hi, Pablo
> >>
> >> >-----Original Message-----
> >> >From: De Lara Guarch, Pablo [mailto:pablo.de.lara.guarch@intel.com]
> >> >Sent: Friday, July 17, 2015 4:42 AM
> >> >To: Tony Lu; dev@dpdk.org
> >> >Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash
> >> >library
> >> with
> >> >cuckoo hash implementation
> >> >
> >> >Hi Tony,
> >> >
> >> >> -----Original Message-----
> >> >> From: Tony Lu [mailto:zlu@ezchip.com]
> >> >> Sent: Thursday, July 16, 2015 10:40 AM
> >> >> To: De Lara Guarch, Pablo; dev@dpdk.org
> >> >> Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash
> >> >> library with cuckoo hash implementation
> >> >>
> >> >> >diff --git a/lib/librte_hash/rte_cuckoo_hash.c
> >> >> b/lib/librte_hash/rte_cuckoo_hash.c
> >> >> >new file mode 100644
> >> >> >index 0000000..50e3acd
> >> >> >--- /dev/null
> >> >> >+++ b/lib/librte_hash/rte_cuckoo_hash.c
> >> >> >@@ -0,0 +1,1027 @@
> >> >> ...
> >> >> >+
> >> >> >+/* Functions to compare multiple of 16 byte keys (up to 128
> >> >> >+bytes) */ static int rte_hash_k16_cmp_eq(const void *key1, const
> >> >> >+void *key2, size_t key_len
> >> >> >__rte_unused)
> >> >> >+{
> >> >> >+ const __m128i k1 = _mm_loadu_si128((const __m128i *)
> key1);
> >> >> >+ const __m128i k2 = _mm_loadu_si128((const __m128i *)
> key2);
> >> >> >+ const __m128i x = _mm_xor_si128(k1, k2);
> >> >> >+
> >> >> >+ return !_mm_test_all_zeros(x, x); }
> >> >> ...
> >> >>
> >> >> When compiling the latest dev DPDK for non-x86 arch, it fails on
> >> >> the above code, as the SSE is x86 specific defined in
> >> >> <emmintrin.h>. Is it possible to replace this function with platform
> >independent code?
> >> >
> >> >Thanks for spotting this. I just sent a patch that should fix the
> problem.
> >> >Can you check if it works?
> >>
> >> Thanks for your quick response, but __m128i and all the _mm_ related
> >> functions are X86 specific defined in <emmintrin.h>. This header file
> >> is only available in X86 compiler library, but no-X86 archs do not
> >> have this file. So if we can replace all the X86 specific code in the
> >> above function, that would be great.
> >>
> >With the patch that I sent, if you are compiling for a non-x86 arch, you
> should
> >not have any problem, since all that code will only be used if using x86
> arch.
> >Have you tried compiling DPDK with the patch?
>
> Yes, I have built the DPDK with your patch, and got the following errors.
> This is
> because there are no __m128i, _mm_loadu_si128(), _mm_cmpeq_epi32()
> and
> _mm_movemask_epi8() on no-X86 arches.
>
> == Build lib/librte_hash
> CC rte_cuckoo_hash.o
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c: In function
> 'rte_hash_k16_cmp_eq':
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error:
> expected '=', ',', ';', 'asm' or '__attribute__' before 'k1'
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error: 'k1'
> undeclared (first use in this function)
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error: (Each
> undeclared identifier is reported only once
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error: for
> each function it appears in.)
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: warning:
> implicit declaration of function '_mm_loadu_si128'
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: warning:
> nested extern declaration of '_mm_loadu_si128'
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error:
> expected ')' before '__m128i'
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: warning:
> type defaults to 'int' in declaration of 'type name'
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: warning:
> cast from pointer to integer of different size
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: error:
> expected '=', ',', ';', 'asm' or '__attribute__' before 'k2'
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: error: 'k2'
> undeclared (first use in this function)
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: error:
> expected ')' before '__m128i'
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: warning:
> type defaults to 'int' in declaration of 'type name'
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: warning:
> cast from pointer to integer of different size
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1133: error:
> expected '=', ',', ';', 'asm' or '__attribute__' before 'x'
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1133: error: 'x'
> undeclared (first use in this function)
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1133: warning:
> implicit declaration of function '_mm_cmpeq_epi32'
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1133: warning:
> nested extern declaration of '_mm_cmpeq_epi32'
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1135: warning:
> implicit declaration of function '_mm_movemask_epi8'
> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1135: warning:
> nested extern declaration of '_mm_movemask_epi8'
> make[3]: *** [rte_cuckoo_hash.o] Error 1
> make[2]: *** [librte_hash] Error 2
> make[1]: *** [lib] Error 2
> make: *** [all] Error 2
Looking at the snippet, I would say the patch has not been applied
(did you apply the patch hash: "fix compilation for non-x86 systems"?),
because looking at the lines where it is failing, they are the old lines in rte_cuckoo_hash.c,
which have been moved to another file (and should not be included for you).
Pablo
>
> Thanks
> -Tony
>
> >Pablo
> >
> >> Thanks
> >> -Tony
> >>
> >>
> >> >Thanks,
> >> >Pablo
> >> >>
> >> >> Thanks
> >> >> -Zhigang Lu
> >>
>
^ permalink raw reply [flat|nested] 92+ messages in thread* Re: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation
2015-07-17 9:06 ` De Lara Guarch, Pablo
@ 2015-07-17 14:39 ` Tony Lu
0 siblings, 0 replies; 92+ messages in thread
From: Tony Lu @ 2015-07-17 14:39 UTC (permalink / raw)
To: 'De Lara Guarch, Pablo', dev
Hi, Pablo
The patch "hash: fix compilation for non-x86 systems " works for no-X86
arches.
Thanks for your quick fix.
>-----Original Message-----
>From: De Lara Guarch, Pablo [mailto:pablo.de.lara.guarch@intel.com]
>Sent: Friday, July 17, 2015 5:06 PM
>To: Tony Lu; dev@dpdk.org
>Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library
with
>cuckoo hash implementation
>
>Hi Tony,
>
>> -----Original Message-----
>> From: Tony Lu [mailto:zlu@ezchip.com]
>> Sent: Friday, July 17, 2015 8:58 AM
>> To: De Lara Guarch, Pablo; dev@dpdk.org
>> Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash
>> library with cuckoo hash implementation
>>
>> >-----Original Message-----
>> >From: De Lara Guarch, Pablo [mailto:pablo.de.lara.guarch@intel.com]
>> >Sent: Friday, July 17, 2015 3:35 PM
>> >To: Tony Lu; dev@dpdk.org
>> >Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash
>> >library
>> with
>> >cuckoo hash implementation
>> >
>> >
>> >
>> >> -----Original Message-----
>> >> From: Tony Lu [mailto:zlu@ezchip.com]
>> >> Sent: Friday, July 17, 2015 4:35 AM
>> >> To: De Lara Guarch, Pablo; dev@dpdk.org
>> >> Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash
>> >> library with cuckoo hash implementation
>> >>
>> >> Hi, Pablo
>> >>
>> >> >-----Original Message-----
>> >> >From: De Lara Guarch, Pablo
>> >> >[mailto:pablo.de.lara.guarch@intel.com]
>> >> >Sent: Friday, July 17, 2015 4:42 AM
>> >> >To: Tony Lu; dev@dpdk.org
>> >> >Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash
>> >> >library
>> >> with
>> >> >cuckoo hash implementation
>> >> >
>> >> >Hi Tony,
>> >> >
>> >> >> -----Original Message-----
>> >> >> From: Tony Lu [mailto:zlu@ezchip.com]
>> >> >> Sent: Thursday, July 16, 2015 10:40 AM
>> >> >> To: De Lara Guarch, Pablo; dev@dpdk.org
>> >> >> Subject: RE: [dpdk-dev] [PATCH v7 1/7] hash: replace existing
>> >> >> hash library with cuckoo hash implementation
>> >> >>
>> >> >> >diff --git a/lib/librte_hash/rte_cuckoo_hash.c
>> >> >> b/lib/librte_hash/rte_cuckoo_hash.c
>> >> >> >new file mode 100644
>> >> >> >index 0000000..50e3acd
>> >> >> >--- /dev/null
>> >> >> >+++ b/lib/librte_hash/rte_cuckoo_hash.c
>> >> >> >@@ -0,0 +1,1027 @@
>> >> >> ...
>> >> >> >+
>> >> >> >+/* Functions to compare multiple of 16 byte keys (up to 128
>> >> >> >+bytes) */ static int rte_hash_k16_cmp_eq(const void *key1,
>> >> >> >+const void *key2, size_t key_len
>> >> >> >__rte_unused)
>> >> >> >+{
>> >> >> >+ const __m128i k1 = _mm_loadu_si128((const __m128i *)
>> key1);
>> >> >> >+ const __m128i k2 = _mm_loadu_si128((const __m128i *)
>> key2);
>> >> >> >+ const __m128i x = _mm_xor_si128(k1, k2);
>> >> >> >+
>> >> >> >+ return !_mm_test_all_zeros(x, x); }
>> >> >> ...
>> >> >>
>> >> >> When compiling the latest dev DPDK for non-x86 arch, it fails on
>> >> >> the above code, as the SSE is x86 specific defined in
>> >> >> <emmintrin.h>. Is it possible to replace this function with
>> >> >> platform
>> >independent code?
>> >> >
>> >> >Thanks for spotting this. I just sent a patch that should fix the
>> problem.
>> >> >Can you check if it works?
>> >>
>> >> Thanks for your quick response, but __m128i and all the _mm_
>> >> related functions are X86 specific defined in <emmintrin.h>. This
>> >> header file is only available in X86 compiler library, but no-X86
>> >> archs do not have this file. So if we can replace all the X86
>> >> specific code in the above function, that would be great.
>> >>
>> >With the patch that I sent, if you are compiling for a non-x86 arch,
>> >you
>> should
>> >not have any problem, since all that code will only be used if using
>> >x86
>> arch.
>> >Have you tried compiling DPDK with the patch?
>>
>> Yes, I have built the DPDK with your patch, and got the following errors.
>> This is
>> because there are no __m128i, _mm_loadu_si128(), _mm_cmpeq_epi32() and
>> _mm_movemask_epi8() on no-X86 arches.
>>
>> == Build lib/librte_hash
>> CC rte_cuckoo_hash.o
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c: In function
>> 'rte_hash_k16_cmp_eq':
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error:
>> expected '=', ',', ';', 'asm' or '__attribute__' before 'k1'
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error:
'k1'
>> undeclared (first use in this function)
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error:
>> (Each undeclared identifier is reported only once
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error:
>> for each function it appears in.)
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: warning:
>> implicit declaration of function '_mm_loadu_si128'
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: warning:
>> nested extern declaration of '_mm_loadu_si128'
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: error:
>> expected ')' before '__m128i'
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: warning:
>> type defaults to 'int' in declaration of 'type name'
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1126: warning:
>> cast from pointer to integer of different size
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: error:
>> expected '=', ',', ';', 'asm' or '__attribute__' before 'k2'
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: error:
'k2'
>> undeclared (first use in this function)
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: error:
>> expected ')' before '__m128i'
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: warning:
>> type defaults to 'int' in declaration of 'type name'
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1127: warning:
>> cast from pointer to integer of different size
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1133: error:
>> expected '=', ',', ';', 'asm' or '__attribute__' before 'x'
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1133: error:
'x'
>> undeclared (first use in this function)
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1133: warning:
>> implicit declaration of function '_mm_cmpeq_epi32'
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1133: warning:
>> nested extern declaration of '_mm_cmpeq_epi32'
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1135: warning:
>> implicit declaration of function '_mm_movemask_epi8'
>> /u/zlu.bjg/git/dpdk.org/lib/librte_hash/rte_cuckoo_hash.c:1135: warning:
>> nested extern declaration of '_mm_movemask_epi8'
>> make[3]: *** [rte_cuckoo_hash.o] Error 1
>> make[2]: *** [librte_hash] Error 2
>> make[1]: *** [lib] Error 2
>> make: *** [all] Error 2
>
>Looking at the snippet, I would say the patch has not been applied (did you
apply
>the patch hash: "fix compilation for non-x86 systems"?), because looking at
the
>lines where it is failing, they are the old lines in rte_cuckoo_hash.c,
which have
>been moved to another file (and should not be included for you).
>Pablo
>>
>> Thanks
>> -Tony
>>
>> >Pablo
>> >
>> >> Thanks
>> >> -Tony
>> >>
>> >>
>> >> >Thanks,
>> >> >Pablo
>> >> >>
>> >> >> Thanks
>> >> >> -Zhigang Lu
>> >>
>>
^ permalink raw reply [flat|nested] 92+ messages in thread
* [dpdk-dev] [PATCH v7 2/7] hash: add new function rte_hash_reset
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 0/7] Cuckoo hash - part 3 of Cuckoo hash Pablo de Lara
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
@ 2015-07-11 0:18 ` Pablo de Lara
2015-07-12 22:16 ` Thomas Monjalon
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 3/7] hash: add new functionality to store data in hash table Pablo de Lara
` (5 subsequent siblings)
7 siblings, 1 reply; 92+ messages in thread
From: Pablo de Lara @ 2015-07-11 0:18 UTC (permalink / raw)
To: dev
Added reset function to be able to empty the table,
without having to destroy and create it again.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 4 +---
app/test/test_hash_perf.c | 12 +++---------
lib/librte_hash/rte_cuckoo_hash.c | 21 +++++++++++++++++++++
lib/librte_hash/rte_hash.h | 11 +++++++++--
lib/librte_hash/rte_hash_version.map | 8 ++++++++
5 files changed, 42 insertions(+), 14 deletions(-)
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index e70d859..448586c 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -1132,9 +1132,7 @@ static int test_average_table_utilization(void)
average_keys_added += added_keys;
/* Reset the table */
- rte_hash_free(handle);
- handle = rte_hash_create(&ut_params);
- RETURN_IF_ERROR(handle == NULL, "hash creation failed");
+ rte_hash_reset(handle);
/* Print a dot to show progress on operations */
printf(".");
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index a54f3c1..b01b040 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -382,14 +382,10 @@ free_table(unsigned table_index)
rte_hash_free(h[table_index]);
}
-static int
+static void
reset_table(unsigned table_index)
{
- free_table(table_index);
- if (create_table(table_index) != 0)
- return -1;
-
- return 0;
+ rte_hash_reset(h[table_index]);
}
static int
@@ -421,13 +417,11 @@ run_all_tbl_perf_tests(unsigned with_pushes)
if (timed_deletes(with_hash, i) < 0)
return -1;
- if (reset_table(i) < 0)
- return -1;
+ reset_table(i);
/* Print a dot to show progress on operations */
printf(".");
fflush(stdout);
-
}
free_table(i);
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index 50e3acd..15c5da5 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -371,6 +371,27 @@ rte_hash_secondary_hash(const hash_sig_t primary_hash)
return (primary_hash ^ ((tag + 1) * alt_bits_xor));
}
+void
+rte_hash_reset(struct rte_hash *h)
+{
+ void *ptr;
+ unsigned i;
+
+ if (h == NULL)
+ return;
+
+ memset(h->buckets, 0, h->num_buckets * sizeof(struct rte_hash_bucket));
+ memset(h->key_store, 0, h->key_entry_size * (h->entries + 1));
+
+ /* clear the free ring */
+ while (rte_ring_dequeue(h->free_slots, &ptr) == 0)
+ rte_pause();
+
+ /* Repopulate the free slots ring. Entry zero is reserved for key misses */
+ for (i = 1; i < h->entries + 1; i++)
+ rte_ring_sp_enqueue(h->free_slots, (void *)((uintptr_t) i));
+}
+
/* Search for an entry that can be pushed to its alternative location */
static inline int
make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt)
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index 936d170..8bbc9f0 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -128,8 +128,15 @@ void
rte_hash_free(struct rte_hash *h);
/**
- * Add a key to an existing hash table.
- * This operation is not multi-thread safe
+ * Reset all hash structure, by zeroing all entries
+ * @param h
+ * Hash table to reset
+ */
+void
+rte_hash_reset(struct rte_hash *h);
+
+/**
+ * Add a key to an existing hash table. This operation is not multi-thread safe
* and should only be called from one thread.
*
* @param h
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index 94a0fec..d5f5af5 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -18,3 +18,11 @@ DPDK_2.0 {
local: *;
};
+
+DPDK_2.1 {
+ global:
+
+ rte_hash_reset;
+
+ local: *;
+} DPDK_2.0;
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v7 3/7] hash: add new functionality to store data in hash table
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 0/7] Cuckoo hash - part 3 of Cuckoo hash Pablo de Lara
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 1/7] hash: replace existing hash library with cuckoo hash implementation Pablo de Lara
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 2/7] hash: add new function rte_hash_reset Pablo de Lara
@ 2015-07-11 0:18 ` Pablo de Lara
2015-07-12 22:00 ` Thomas Monjalon
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 4/7] hash: add iterate function Pablo de Lara
` (4 subsequent siblings)
7 siblings, 1 reply; 92+ messages in thread
From: Pablo de Lara @ 2015-07-11 0:18 UTC (permalink / raw)
To: dev
Usually hash tables not only store keys, but also data associated
to them. In order to maintain the existing API, the old functions
will still return the index where the key was stored.
The new functions will return the data associated to that key.
In the case of the lookup_bulk function, it will return also
the number of entries found and a bitmask of which entries
were found.
Unit tests have been updated to use these new functions.
As a final point, a flag has been added in rte_hash_parameters
to indicate if there are new parameters for future versions,
so there is no need to maintain multiple versions
of the existing functions in the future.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash_perf.c | 252 +++++++++++++++++++++++++----------
lib/librte_hash/rte_cuckoo_hash.c | 193 +++++++++++++++++++++------
lib/librte_hash/rte_hash.h | 108 ++++++++++++++-
lib/librte_hash/rte_hash_version.map | 6 +
4 files changed, 439 insertions(+), 120 deletions(-)
diff --git a/app/test/test_hash_perf.c b/app/test/test_hash_perf.c
index b01b040..e9a522b 100644
--- a/app/test/test_hash_perf.c
+++ b/app/test/test_hash_perf.c
@@ -83,7 +83,7 @@ struct rte_hash *h[NUM_KEYSIZES];
uint8_t slot_taken[MAX_ENTRIES];
/* Array to store number of cycles per operation */
-uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS][2];
+uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS][2][2];
/* Array to store all input keys */
uint8_t keys[KEYS_TO_ADD][MAX_KEYSIZE];
@@ -106,11 +106,16 @@ static struct rte_hash_parameters ut_params = {
};
static int
-create_table(unsigned table_index)
+create_table(unsigned with_data, unsigned table_index)
{
char name[RTE_HASH_NAMESIZE];
- sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
+ if (with_data)
+ /* Table will store 8-byte data */
+ sprintf(name, "test_hash%d_data", hashtest_key_lens[table_index]);
+ else
+ sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
+
ut_params.name = name;
ut_params.key_len = hashtest_key_lens[table_index];
ut_params.socket_id = rte_socket_id();
@@ -234,6 +239,7 @@ get_input_keys(unsigned with_pushes, unsigned table_index)
else {
/* Store the returned position and mark slot as taken */
slot_taken[ret] = 1;
+ positions[i] = ret;
buckets[bucket_idx]++;
success = 1;
i++;
@@ -249,54 +255,116 @@ get_input_keys(unsigned with_pushes, unsigned table_index)
}
static int
-timed_adds(unsigned with_hash, unsigned table_index)
+timed_adds(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i;
const uint64_t start_tsc = rte_rdtsc();
+ void *data;
int32_t ret;
for (i = 0; i < KEYS_TO_ADD; i++) {
- if (with_hash)
+ data = (void *) ((uintptr_t) signatures[i]);
+ if (with_hash && with_data) {
+ ret = rte_hash_add_key_with_hash_data(h[table_index],
+ (const void *) keys[i],
+ signatures[i], data);
+ if (ret < 0) {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
+ } else if (with_hash && !with_data) {
ret = rte_hash_add_key_with_hash(h[table_index],
(const void *) keys[i],
signatures[i]);
- else
+ if (ret >= 0)
+ positions[i] = ret;
+ else {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
+ } else if (!with_hash && with_data) {
+ ret = rte_hash_add_key_data(h[table_index],
+ (const void *) keys[i],
+ data);
+ if (ret < 0) {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
+ } else {
ret = rte_hash_add_key(h[table_index], keys[i]);
-
- if (ret >= 0)
- positions[i] = ret;
- else {
- printf("Failed to add key number %u\n", ret);
- return -1;
+ if (ret >= 0)
+ positions[i] = ret;
+ else {
+ printf("Failed to add key number %u\n", ret);
+ return -1;
+ }
}
}
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][ADD][with_hash] = time_taken/KEYS_TO_ADD;
+ cycles[table_index][ADD][with_hash][with_data] = time_taken/KEYS_TO_ADD;
+
return 0;
}
static int
-timed_lookups(unsigned with_hash, unsigned table_index)
+timed_lookups(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i, j;
const uint64_t start_tsc = rte_rdtsc();
+ void *ret_data;
+ void *expected_data;
int32_t ret;
for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
for (j = 0; j < KEYS_TO_ADD; j++) {
- if (with_hash)
+ if (with_hash && with_data) {
+ ret = rte_hash_lookup_with_hash_data(h[table_index],
+ (const void *) keys[j],
+ signatures[j], &ret_data);
+ if (ret < 0) {
+ printf("Key number %u was not found\n", j);
+ return -1;
+ }
+ expected_data = (void *) ((uintptr_t) signatures[j]);
+ if (ret_data != expected_data) {
+ printf("Data returned for key number %u is %p,"
+ " but should be %p\n", j, ret_data,
+ expected_data);
+ return -1;
+ }
+ } else if (with_hash && !with_data) {
ret = rte_hash_lookup_with_hash(h[table_index],
(const void *) keys[j],
signatures[j]);
- else
+ if (ret < 0 || ret != positions[j]) {
+ printf("Key looked up in %d, should be in %d\n",
+ ret, positions[j]);
+ return -1;
+ }
+ } else if (!with_hash && with_data) {
+ ret = rte_hash_lookup_data(h[table_index],
+ (const void *) keys[j], &ret_data);
+ if (ret < 0) {
+ printf("Key number %u was not found\n", j);
+ return -1;
+ }
+ expected_data = (void *) ((uintptr_t) signatures[j]);
+ if (ret_data != expected_data) {
+ printf("Data returned for key number %u is %p,"
+ " but should be %p\n", j, ret_data,
+ expected_data);
+ return -1;
+ }
+ } else {
ret = rte_hash_lookup(h[table_index], keys[j]);
- if (ret < 0 || ret != positions[j]) {
- printf("Key looked up in %d, should be in %d\n",
- ret, positions[j]);
- return -1;
+ if (ret < 0 || ret != positions[j]) {
+ printf("Key looked up in %d, should be in %d\n",
+ ret, positions[j]);
+ return -1;
+ }
}
}
}
@@ -304,34 +372,65 @@ timed_lookups(unsigned with_hash, unsigned table_index)
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][LOOKUP][with_hash] = time_taken/NUM_LOOKUPS;
+ cycles[table_index][LOOKUP][with_hash][with_data] = time_taken/NUM_LOOKUPS;
return 0;
}
static int
-timed_lookups_multi(unsigned table_index)
+timed_lookups_multi(unsigned with_data, unsigned table_index)
{
unsigned i, j, k;
int32_t positions_burst[BURST_SIZE];
const void *keys_burst[BURST_SIZE];
+ void *expected_data[BURST_SIZE];
+ void *ret_data[BURST_SIZE];
+ uint64_t hit_mask;
+ int ret;
+
const uint64_t start_tsc = rte_rdtsc();
for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) {
for (j = 0; j < KEYS_TO_ADD/BURST_SIZE; j++) {
for (k = 0; k < BURST_SIZE; k++)
keys_burst[k] = keys[j * BURST_SIZE + k];
-
- rte_hash_lookup_bulk(h[table_index],
+ if (with_data) {
+ ret = rte_hash_lookup_bulk_data(h[table_index],
+ (const void **) keys_burst,
+ BURST_SIZE,
+ &hit_mask,
+ ret_data);
+ if (ret != BURST_SIZE) {
+ printf("Expect to find %u keys,"
+ " but found %d\n", BURST_SIZE, ret);
+ return -1;
+ }
+ for (k = 0; k < BURST_SIZE; k++) {
+ if ((hit_mask & (1ULL << k)) == 0) {
+ printf("Key number %u not found\n",
+ j * BURST_SIZE + k);
+ return -1;
+ }
+ expected_data[k] = (void *) ((uintptr_t) signatures[j * BURST_SIZE + k]);
+ if (ret_data[k] != expected_data[k]) {
+ printf("Data returned for key number %u is %p,"
+ " but should be %p\n", j * BURST_SIZE + k,
+ ret_data[k], expected_data[k]);
+ return -1;
+ }
+ }
+ } else {
+ rte_hash_lookup_bulk(h[table_index],
(const void **) keys_burst,
BURST_SIZE,
positions_burst);
- for (k = 0; k < BURST_SIZE; k++) {
- if (positions_burst[k] != positions[j * BURST_SIZE + k]) {
- printf("Key looked up in %d, should be in %d\n",
- positions_burst[k],
- positions[j * BURST_SIZE + k]);
- return -1;
+ for (k = 0; k < BURST_SIZE; k++) {
+ if (positions_burst[k] != positions[j * BURST_SIZE + k]) {
+ printf("Key looked up in %d, should be in %d\n",
+ positions_burst[k],
+ positions[j * BURST_SIZE + k]);
+ return -1;
+ }
}
}
}
@@ -340,19 +439,20 @@ timed_lookups_multi(unsigned table_index)
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][LOOKUP_MULTI][0] = time_taken/NUM_LOOKUPS;
+ cycles[table_index][LOOKUP_MULTI][0][with_data] = time_taken/NUM_LOOKUPS;
return 0;
}
static int
-timed_deletes(unsigned with_hash, unsigned table_index)
+timed_deletes(unsigned with_hash, unsigned with_data, unsigned table_index)
{
unsigned i;
const uint64_t start_tsc = rte_rdtsc();
int32_t ret;
for (i = 0; i < KEYS_TO_ADD; i++) {
+ /* There are no delete functions with data, so just call two functions */
if (with_hash)
ret = rte_hash_del_key_with_hash(h[table_index],
(const void *) keys[i],
@@ -371,7 +471,7 @@ timed_deletes(unsigned with_hash, unsigned table_index)
const uint64_t end_tsc = rte_rdtsc();
const uint64_t time_taken = end_tsc - start_tsc;
- cycles[table_index][DELETE][with_hash] = time_taken/KEYS_TO_ADD;
+ cycles[table_index][DELETE][with_hash][with_data] = time_taken/KEYS_TO_ADD;
return 0;
}
@@ -391,62 +491,67 @@ reset_table(unsigned table_index)
static int
run_all_tbl_perf_tests(unsigned with_pushes)
{
- unsigned i, j, with_hash;
+ unsigned i, j, with_data, with_hash;
printf("Measuring performance, please wait");
fflush(stdout);
- for (i = 0; i < NUM_KEYSIZES; i++) {
- if (create_table(i) < 0)
- return -1;
- if (get_input_keys(with_pushes, i) < 0)
- return -1;
- for (with_hash = 0; with_hash <= 1; with_hash++) {
- if (timed_adds(with_hash, i) < 0)
+ for (with_data = 0; with_data <= 1; with_data++) {
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ if (create_table(with_data, i) < 0)
return -1;
- for (j = 0; j < NUM_SHUFFLES; j++)
- shuffle_input_keys(i);
-
- if (timed_lookups(with_hash, i) < 0)
+ if (get_input_keys(with_pushes, i) < 0)
return -1;
+ for (with_hash = 0; with_hash <= 1; with_hash++) {
+ if (timed_adds(with_hash, with_data, i) < 0)
+ return -1;
- if (timed_lookups_multi(i) < 0)
- return -1;
+ for (j = 0; j < NUM_SHUFFLES; j++)
+ shuffle_input_keys(i);
- if (timed_deletes(with_hash, i) < 0)
- return -1;
+ if (timed_lookups(with_hash, with_data, i) < 0)
+ return -1;
- reset_table(i);
+ if (timed_lookups_multi(with_data, i) < 0)
+ return -1;
- /* Print a dot to show progress on operations */
- printf(".");
- fflush(stdout);
- }
+ if (timed_deletes(with_hash, with_data, i) < 0)
+ return -1;
- free_table(i);
+ /* Print a dot to show progress on operations */
+ printf(".");
+ fflush(stdout);
+
+ reset_table(i);
+ }
+ free_table(i);
+ }
}
+
printf("\nResults (in CPU cycles/operation)\n");
- printf("---------------------------------\n");
- printf("\nWithout pre-computed hash values\n");
- printf("\n%-18s%-18s%-18s%-18s%-18s\n",
- "Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
- for (i = 0; i < NUM_KEYSIZES; i++) {
- printf("%-18d", hashtest_key_lens[i]);
- for (j = 0; j < NUM_OPERATIONS; j++)
- printf("%-18"PRIu64, cycles[i][j][0]);
- printf("\n");
- }
- printf("\nWith pre-computed hash values\n");
- printf("\n%-18s%-18s%-18s%-18s%-18s\n",
+ printf("-----------------------------------\n");
+ for (with_data = 0; with_data <= 1; with_data++) {
+ if (with_data)
+ printf("\n Operations with 8-byte data\n");
+ else
+ printf("\n Operations without data\n");
+ for (with_hash = 0; with_hash <= 1; with_hash++) {
+ if (with_hash)
+ printf("\nWith pre-computed hash values\n");
+ else
+ printf("\nWithout pre-computed hash values\n");
+
+ printf("\n%-18s%-18s%-18s%-18s%-18s\n",
"Keysize", "Add", "Lookup", "Lookup_bulk", "Delete");
- for (i = 0; i < NUM_KEYSIZES; i++) {
- printf("%-18d", hashtest_key_lens[i]);
- for (j = 0; j < NUM_OPERATIONS; j++)
- printf("%-18"PRIu64, cycles[i][j][1]);
- printf("\n");
+ for (i = 0; i < NUM_KEYSIZES; i++) {
+ printf("%-18d", hashtest_key_lens[i]);
+ for (j = 0; j < NUM_OPERATIONS; j++)
+ printf("%-18"PRIu64, cycles[i][j][with_hash][with_data]);
+ printf("\n");
+ }
+ }
}
-
return 0;
}
@@ -546,7 +651,6 @@ test_hash_perf(void)
if (run_all_tbl_perf_tests(with_pushes) < 0)
return -1;
}
-
if (fbk_hash_perf_test() < 0)
return -1;
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index 15c5da5..cb1ab31 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -56,6 +56,7 @@
#include <rte_rwlock.h>
#include <rte_spinlock.h>
#include <rte_ring.h>
+#include <rte_compat.h>
#include "rte_hash.h"
@@ -90,6 +91,8 @@ EAL_REGISTER_TAILQ(rte_hash_tailq)
#define NULL_SIGNATURE 0
+#define KEY_ALIGNMENT 16
+
typedef int (*rte_hash_cmp_eq_t)(const void *key1, const void *key2, size_t key_len);
static int rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len);
static int rte_hash_k32_cmp_eq(const void *key1, const void *key2, size_t key_len);
@@ -132,6 +135,16 @@ struct rte_hash_signatures {
};
};
+/* Structure that stores key-value pair */
+struct rte_hash_key {
+ union {
+ uintptr_t idata;
+ void *pdata;
+ };
+ /* Variable key size */
+ char key[];
+} __attribute__((aligned(KEY_ALIGNMENT)));
+
/** Bucket structure */
struct rte_hash_bucket {
struct rte_hash_signatures signatures[RTE_HASH_BUCKET_ENTRIES];
@@ -227,7 +240,8 @@ rte_hash_create(const struct rte_hash_parameters *params)
goto err;
}
- const uint32_t key_entry_size = params->key_len;
+ const uint32_t key_entry_size = sizeof(struct rte_hash_key) + params->key_len;
+
/* Store all keys and leave the first entry as a dummy entry for lookup_bulk */
const uint64_t key_tbl_size = key_entry_size * (params->entries + 1);
@@ -461,13 +475,13 @@ make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt)
static inline int32_t
__rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
- hash_sig_t sig)
+ hash_sig_t sig, void *data)
{
hash_sig_t alt_hash;
uint32_t prim_bucket_idx, sec_bucket_idx;
unsigned i;
struct rte_hash_bucket *prim_bkt, *sec_bkt;
- void *new_k, *k, *keys = h->key_store;
+ struct rte_hash_key *new_k, *k, *keys = h->key_store;
void *slot_id;
uint32_t new_idx;
int ret;
@@ -492,9 +506,12 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (prim_bkt->signatures[i].current == sig &&
prim_bkt->signatures[i].alt == alt_hash) {
- k = (char *)keys + prim_bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ prim_bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /* Update data */
+ k->pdata = data;
/*
* Return index where key is stored,
* substracting the first dummy index
@@ -508,9 +525,12 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (sec_bkt->signatures[i].alt == sig &&
sec_bkt->signatures[i].current == alt_hash) {
- k = (char *)keys + sec_bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ sec_bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
rte_ring_sp_enqueue(h->free_slots, &slot_id);
+ /* Update data */
+ k->pdata = data;
/*
* Return index where key is stored,
* substracting the first dummy index
@@ -521,7 +541,8 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
}
/* Copy key */
- rte_memcpy(new_k, key, h->key_len);
+ rte_memcpy(new_k->key, key, h->key_len);
+ new_k->pdata = data;
/* Insert new entry is there is room in the primary bucket */
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
@@ -561,25 +582,52 @@ rte_hash_add_key_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, sig);
+ return __rte_hash_add_key_with_hash(h, key, sig, 0);
}
int32_t
rte_hash_add_key(const struct rte_hash *h, const void *key)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key));
+ return __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key), 0);
+}
+
+int
+rte_hash_add_key_with_hash_data(const struct rte_hash *h,
+ const void *key, hash_sig_t sig, void *data)
+{
+ int ret;
+
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ ret = __rte_hash_add_key_with_hash(h, key, sig, data);
+ if (ret >= 0)
+ return 0;
+ else
+ return ret;
}
+int
+rte_hash_add_key_data(const struct rte_hash *h, const void *key, void *data)
+{
+ int ret;
+
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+
+ ret = __rte_hash_add_key_with_hash(h, key, rte_hash_hash(h, key), data);
+ if (ret >= 0)
+ return 0;
+ else
+ return ret;
+}
static inline int32_t
__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
- hash_sig_t sig)
+ hash_sig_t sig, void **data)
{
uint32_t bucket_idx;
hash_sig_t alt_hash;
unsigned i;
struct rte_hash_bucket *bkt;
- void *k, *keys = h->key_store;
+ struct rte_hash_key *k, *keys = h->key_store;
bucket_idx = sig & h->bucket_bitmask;
bkt = &h->buckets[bucket_idx];
@@ -588,13 +636,17 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == sig &&
bkt->signatures[i].sig != NULL_SIGNATURE) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
+ if (data != NULL)
+ *data = k->pdata;
/*
* Return index where key is stored,
* substracting the first dummy index
*/
return (bkt->key_idx[i] - 1);
+ }
}
}
@@ -607,13 +659,17 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == alt_hash &&
bkt->signatures[i].alt == sig) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0)
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
+ if (data != NULL)
+ *data = k->pdata;
/*
* Return index where key is stored,
* substracting the first dummy index
*/
return (bkt->key_idx[i] - 1);
+ }
}
}
@@ -625,14 +681,29 @@ rte_hash_lookup_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, sig);
+ return __rte_hash_lookup_with_hash(h, key, sig, NULL);
}
int32_t
rte_hash_lookup(const struct rte_hash *h, const void *key)
{
RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
- return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key));
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key), NULL);
+}
+
+int
+rte_hash_lookup_with_hash_data(const struct rte_hash *h,
+ const void *key, hash_sig_t sig, void **data)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, sig, data);
+}
+
+int
+rte_hash_lookup_data(const struct rte_hash *h, const void *key, void **data)
+{
+ RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL);
+ return __rte_hash_lookup_with_hash(h, key, rte_hash_hash(h, key), data);
}
static inline int32_t
@@ -643,7 +714,7 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
hash_sig_t alt_hash;
unsigned i;
struct rte_hash_bucket *bkt;
- void *k, *keys = h->key_store;
+ struct rte_hash_key *k, *keys = h->key_store;
bucket_idx = sig & h->bucket_bitmask;
bkt = &h->buckets[bucket_idx];
@@ -652,8 +723,9 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == sig &&
bkt->signatures[i].sig != NULL_SIGNATURE) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
bkt->signatures[i].sig = NULL_SIGNATURE;
rte_ring_sp_enqueue(h->free_slots,
(void *)((uintptr_t)bkt->key_idx[i]));
@@ -675,8 +747,9 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
if (bkt->signatures[i].current == alt_hash &&
bkt->signatures[i].sig != NULL_SIGNATURE) {
- k = (char *)keys + bkt->key_idx[i] * h->key_entry_size;
- if (h->rte_hash_cmp_eq(key, k, h->key_len) == 0) {
+ k = (struct rte_hash_key *) ((char *)keys +
+ bkt->key_idx[i] * h->key_entry_size);
+ if (h->rte_hash_cmp_eq(key, k->key, h->key_len) == 0) {
bkt->signatures[i].sig = NULL_SIGNATURE;
rte_ring_sp_enqueue(h->free_slots,
(void *)((uintptr_t)bkt->key_idx[i]));
@@ -750,7 +823,7 @@ static inline void
lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
const struct rte_hash_bucket *prim_bkt,
const struct rte_hash_bucket *sec_bkt,
- const void **key_slot, int32_t *positions,
+ const struct rte_hash_key **key_slot, int32_t *positions,
uint64_t *extra_hits_mask, const void *keys,
const struct rte_hash *h)
{
@@ -770,7 +843,8 @@ lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
total_hash_matches = (prim_hash_matches |
(sec_hash_matches << (RTE_HASH_BUCKET_ENTRIES + 1)));
- *key_slot = (const char *)keys + key_idx * h->key_entry_size;
+ *key_slot = (const struct rte_hash_key *) ((const char *)keys +
+ key_idx * h->key_entry_size);
rte_prefetch0(*key_slot);
/*
@@ -784,26 +858,31 @@ lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash,
}
-/* Lookup bulk stage 3: Check if key matches, update hit mask */
+/* Lookup bulk stage 3: Check if key matches, update hit mask and return data */
static inline void
-lookup_stage3(unsigned idx, const void *key_slot, const void * const *keys,
- uint64_t *hits, const struct rte_hash *h)
+lookup_stage3(unsigned idx, const struct rte_hash_key *key_slot, const void * const *keys,
+ void *data[], uint64_t *hits, const struct rte_hash *h)
{
unsigned hit;
- hit = !h->rte_hash_cmp_eq(key_slot, keys[idx], h->key_len);
+ hit = !h->rte_hash_cmp_eq(key_slot->key, keys[idx], h->key_len);
+ if (data != NULL)
+ data[idx] = key_slot->pdata;
+
*hits |= (uint64_t)(hit) << idx;
}
static inline void
__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
- uint32_t num_keys, int32_t *positions)
+ uint32_t num_keys, int32_t *positions,
+ uint64_t *hit_mask, void *data[])
{
uint64_t hits = 0;
uint64_t extra_hits_mask = 0;
uint64_t lookup_mask, miss_mask;
unsigned idx;
const void *key_store = h->key_store;
+ int ret;
hash_sig_t hash_vals[RTE_HASH_LOOKUP_BULK_MAX];
unsigned idx00, idx01, idx10, idx11, idx20, idx21, idx30, idx31;
@@ -811,7 +890,7 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
const struct rte_hash_bucket *secondary_bkt10, *secondary_bkt11;
const struct rte_hash_bucket *primary_bkt20, *primary_bkt21;
const struct rte_hash_bucket *secondary_bkt20, *secondary_bkt21;
- const void *k_slot20, *k_slot21, *k_slot30, *k_slot31;
+ const struct rte_hash_key *k_slot20, *k_slot21, *k_slot30, *k_slot31;
hash_sig_t primary_hash10, primary_hash11;
hash_sig_t secondary_hash10, secondary_hash11;
hash_sig_t primary_hash20, primary_hash21;
@@ -882,8 +961,8 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21,
primary_bkt21, secondary_bkt21, &k_slot21, positions,
&extra_hits_mask, key_store, h);
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
}
k_slot30 = k_slot20, k_slot31 = k_slot21;
@@ -909,8 +988,8 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
key_store, h);
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
k_slot30 = k_slot20, k_slot31 = k_slot21;
idx30 = idx20, idx31 = idx21;
@@ -930,14 +1009,14 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21,
secondary_bkt21, &k_slot21, positions, &extra_hits_mask,
key_store, h);
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
k_slot30 = k_slot20, k_slot31 = k_slot21;
idx30 = idx20, idx31 = idx21;
- lookup_stage3(idx30, k_slot30, keys, &hits, h);
- lookup_stage3(idx31, k_slot31, keys, &hits, h);
+ lookup_stage3(idx30, k_slot30, keys, data, &hits, h);
+ lookup_stage3(idx31, k_slot31, keys, data, &hits, h);
/* ignore any items we have already found */
extra_hits_mask &= ~hits;
@@ -946,11 +1025,18 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
/* run a single search for each remaining item */
do {
idx = __builtin_ctzl(extra_hits_mask);
- positions[idx] = rte_hash_lookup_with_hash(h, keys[idx],
- hash_vals[idx]);
+ if (data != NULL) {
+ ret = rte_hash_lookup_with_hash_data(h,
+ keys[idx], hash_vals[idx], &data[idx]);
+ if (ret >= 0)
+ hits |= 1ULL << idx;
+ } else {
+ positions[idx] = rte_hash_lookup_with_hash(h,
+ keys[idx], hash_vals[idx]);
+ if (positions[idx] >= 0)
+ hits |= 1llu << idx;
+ }
extra_hits_mask &= ~(1llu << idx);
- if (positions[idx] >= 0)
- hits |= 1llu << idx;
} while (extra_hits_mask);
}
@@ -962,6 +1048,9 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
miss_mask &= ~(1llu << idx);
} while (miss_mask);
}
+
+ if (hit_mask != NULL)
+ *hit_mask = hits;
}
int
@@ -972,10 +1061,26 @@ rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
(num_keys > RTE_HASH_LOOKUP_BULK_MAX) ||
(positions == NULL)), -EINVAL);
- __rte_hash_lookup_bulk(h, keys, num_keys, positions);
+ __rte_hash_lookup_bulk(h, keys, num_keys, positions, NULL, NULL);
return 0;
}
+int
+rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, uint64_t *hit_mask, void *data[])
+{
+ RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
+ (num_keys > RTE_HASH_LOOKUP_BULK_MAX),
+ (hit_mask == NULL)), -EINVAL);
+
+ int32_t positions[num_keys];
+
+ __rte_hash_lookup_bulk(h, keys, num_keys, positions, hit_mask, data);
+
+ /* Return number of hits */
+ return __builtin_popcountl(*hit_mask);
+}
+
/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
static int
rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index 8bbc9f0..ff75445 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -80,12 +80,12 @@ struct rte_hash_parameters {
rte_hash_function hash_func; /**< Primary Hash function used to calculate hash. */
uint32_t hash_func_init_val; /**< Init value used by hash_func. */
int socket_id; /**< NUMA Socket ID for memory. */
+ uint8_t extra_flag; /**< Indicate if additional parameters are present. */
};
/** @internal A hash table structure. */
struct rte_hash;
-
/**
* Create a new hash table.
*
@@ -136,6 +136,48 @@ void
rte_hash_reset(struct rte_hash *h);
/**
+ * Add a key-value pair to an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
+ *
+ * @param h
+ * Hash table to add the key to.
+ * @param key
+ * Key to add to the hash table.
+ * @param data
+ * Data to add to the hash table.
+ * @return
+ * - 0 if added successfully
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOSPC if there is no space in the hash for this key.
+ */
+int
+rte_hash_add_key_data(const struct rte_hash *h, const void *key, void *data);
+
+/**
+ * Add a key-value pair with a pre-computed hash value
+ * to an existing hash table.
+ * This operation is not multi-thread safe
+ * and should only be called from one thread.
+ *
+ * @param h
+ * Hash table to add the key to.
+ * @param key
+ * Key to add to the hash table.
+ * @param sig
+ * Precomputed hash value for 'key'
+ * @param data
+ * Data to add to the hash table.
+ * @return
+ * - 0 if added successfully
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOSPC if there is no space in the hash for this key.
+ */
+int32_t
+rte_hash_add_key_with_hash_data(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, void *data);
+
+/**
* Add a key to an existing hash table. This operation is not multi-thread safe
* and should only be called from one thread.
*
@@ -212,6 +254,47 @@ rte_hash_del_key(const struct rte_hash *h, const void *key);
int32_t
rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t sig);
+
+/**
+ * Find a key-value pair in the hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param key
+ * Key to find.
+ * @param data
+ * Output with pointer to data returned from the hash table.
+ * @return
+ * 0 if successful lookup
+ * - EINVAL if the parameters are invalid.
+ * - ENOENT if the key is not found.
+ */
+int
+rte_hash_lookup_data(const struct rte_hash *h, const void *key, void **data);
+
+/**
+ * Find a key-value pair with a pre-computed hash value
+ * to an existing hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param key
+ * Key to find.
+ * @param sig
+ * Precomputed hash value for 'key'
+ * @param data
+ * Output with pointer to data returned from the hash table.
+ * @return
+ * 0 if successful lookup
+ * - EINVAL if the parameters are invalid.
+ * - ENOENT if the key is not found.
+ */
+int
+rte_hash_lookup_with_hash_data(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, void **data);
+
/**
* Find a key in the hash table.
* This operation is multi-thread safe.
@@ -266,6 +349,28 @@ hash_sig_t
rte_hash_hash(const struct rte_hash *h, const void *key);
#define rte_hash_lookup_multi rte_hash_lookup_bulk
+#define rte_hash_lookup_multi_data rte_hash_lookup_bulk_data
+/**
+ * Find multiple keys in the hash table.
+ * This operation is multi-thread safe.
+ *
+ * @param h
+ * Hash table to look in.
+ * @param keys
+ * A pointer to a list of keys to look for.
+ * @param num_keys
+ * How many keys are in the keys list (less than RTE_HASH_LOOKUP_BULK_MAX).
+ * @param hit_mask
+ * Output containing a bitmask with all successful lookups.
+ * @param data
+ * Output containing array of data returned from all the successful lookups.
+ * @return
+ * -EINVAL if there's an error, otherwise number of successful lookups.
+ */
+int
+rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
+ uint32_t num_keys, uint64_t *hit_mask, void *data[]);
+
/**
* Find multiple keys in the hash table.
* This operation is multi-thread safe.
@@ -288,7 +393,6 @@ rte_hash_hash(const struct rte_hash *h, const void *key);
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions);
-
#ifdef __cplusplus
}
#endif
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index d5f5af5..a97eac1 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -22,6 +22,12 @@ DPDK_2.0 {
DPDK_2.1 {
global:
+ rte_hash_add_key_data;
+ rte_hash_add_key_with_hash_data;
+ rte_hash_create;
+ rte_hash_lookup_bulk_data;
+ rte_hash_lookup_data;
+ rte_hash_lookup_with_hash_data;
rte_hash_reset;
local: *;
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* Re: [dpdk-dev] [PATCH v7 3/7] hash: add new functionality to store data in hash table
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 3/7] hash: add new functionality to store data in hash table Pablo de Lara
@ 2015-07-12 22:00 ` Thomas Monjalon
0 siblings, 0 replies; 92+ messages in thread
From: Thomas Monjalon @ 2015-07-12 22:00 UTC (permalink / raw)
To: Pablo de Lara; +Cc: dev
Hi Pablo,
There is a remaining error below:
2015-07-11 01:18, Pablo de Lara:
> +int
> +rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
> + uint32_t num_keys, uint64_t *hit_mask, void *data[])
> +{
> + RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) ||
> + (num_keys > RTE_HASH_LOOKUP_BULK_MAX),
> + (hit_mask == NULL)), -EINVAL);
>
lib/librte_hash/rte_cuckoo_hash.c: In function ‘rte_hash_lookup_bulk_data’:
lib/librte_hash/rte_cuckoo_hash.c:1073:41: error: left-hand operand of comma expression has no effect [-Werror=unused-value]
(num_keys > RTE_HASH_LOOKUP_BULK_MAX),
^
The comma should be ||
Will be fixed before when applying.
^ permalink raw reply [flat|nested] 92+ messages in thread
* [dpdk-dev] [PATCH v7 4/7] hash: add iterate function
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 0/7] Cuckoo hash - part 3 of Cuckoo hash Pablo de Lara
` (2 preceding siblings ...)
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 3/7] hash: add new functionality to store data in hash table Pablo de Lara
@ 2015-07-11 0:18 ` Pablo de Lara
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 5/7] MAINTAINERS: claim responsability for hash library Pablo de Lara
` (3 subsequent siblings)
7 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-11 0:18 UTC (permalink / raw)
To: dev
Since now rte_hash structure is private, a new function
has been added to let the user iterate through the hash table,
returning next key and data associated on each iteration,
plus the position where they were stored.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
app/test/test_hash.c | 66 ++++++++++++++++++++++++++++++++++++
lib/librte_hash/rte_cuckoo_hash.c | 41 ++++++++++++++++++++++
lib/librte_hash/rte_hash.h | 22 ++++++++++++
lib/librte_hash/rte_hash_version.map | 1 +
4 files changed, 130 insertions(+)
diff --git a/app/test/test_hash.c b/app/test/test_hash.c
index 448586c..7f8c0d3 100644
--- a/app/test/test_hash.c
+++ b/app/test/test_hash.c
@@ -1149,6 +1149,70 @@ static int test_average_table_utilization(void)
return 0;
}
+#define NUM_ENTRIES 1024
+static int test_hash_iteration(void)
+{
+ struct rte_hash *handle;
+ unsigned i;
+ uint8_t keys[NUM_ENTRIES][RTE_HASH_KEY_LENGTH_MAX];
+ const void *next_key;
+ void *next_data;
+ void *data[NUM_ENTRIES];
+ unsigned added_keys;
+ uint32_t iter = 0;
+ int ret = 0;
+
+ ut_params.entries = NUM_ENTRIES;
+ ut_params.name = "test_hash_iteration";
+ ut_params.hash_func = rte_jhash;
+ ut_params.key_len = 16;
+ handle = rte_hash_create(&ut_params);
+ RETURN_IF_ERROR(handle == NULL, "hash creation failed");
+
+ /* Add random entries until key cannot be added */
+ for (added_keys = 0; added_keys < NUM_ENTRIES; added_keys++) {
+ data[added_keys] = (void *) ((uintptr_t) rte_rand());
+ for (i = 0; i < ut_params.key_len; i++)
+ keys[added_keys][i] = rte_rand() % 255;
+ ret = rte_hash_add_key_data(handle, keys[added_keys], data[added_keys]);
+ if (ret < 0)
+ break;
+ }
+
+ /* Iterate through the hash table */
+ while (rte_hash_iterate(handle, &next_key, &next_data, &iter) >= 0) {
+ /* Search for the key in the list of keys added */
+ for (i = 0; i < NUM_ENTRIES; i++) {
+ if (memcmp(next_key, keys[i], ut_params.key_len) == 0) {
+ if (next_data != data[i]) {
+ printf("Data found in the hash table is"
+ "not the data added with the key\n");
+ goto err;
+ }
+ added_keys--;
+ break;
+ }
+ }
+ if (i == NUM_ENTRIES) {
+ printf("Key found in the hash table was not added\n");
+ goto err;
+ }
+ }
+
+ /* Check if all keys have been iterated */
+ if (added_keys != 0) {
+ printf("There were still %u keys to iterate\n", added_keys);
+ goto err;
+ }
+
+ rte_hash_free(handle);
+ return 0;
+
+err:
+ rte_hash_free(handle);
+ return -1;
+}
+
static uint8_t key[16] = {0x00, 0x01, 0x02, 0x03,
0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b,
@@ -1408,6 +1472,8 @@ test_hash(void)
return -1;
if (test_average_table_utilization() < 0)
return -1;
+ if (test_hash_iteration() < 0)
+ return -1;
run_hash_func_tests();
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index cb1ab31..b0c942b 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -1081,6 +1081,47 @@ rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
return __builtin_popcountl(*hit_mask);
}
+int32_t
+rte_hash_iterate(const struct rte_hash *h, const void **key, void **data, uint32_t *next)
+{
+ uint32_t bucket_idx, idx, position;
+ struct rte_hash_key *next_key;
+
+ RETURN_IF_TRUE(((h == NULL) || (next == NULL)), -EINVAL);
+
+ const uint32_t total_entries = h->num_buckets * RTE_HASH_BUCKET_ENTRIES;
+ /* Out of bounds */
+ if (*next >= total_entries)
+ return -ENOENT;
+
+ /* Calculate bucket and index of current iterator */
+ bucket_idx = *next / RTE_HASH_BUCKET_ENTRIES;
+ idx = *next % RTE_HASH_BUCKET_ENTRIES;
+
+ /* If current position is empty, go to the next one */
+ while (h->buckets[bucket_idx].signatures[idx].sig == NULL_SIGNATURE) {
+ (*next)++;
+ /* End of table */
+ if (*next == total_entries)
+ return -ENOENT;
+ bucket_idx = *next / RTE_HASH_BUCKET_ENTRIES;
+ idx = *next % RTE_HASH_BUCKET_ENTRIES;
+ }
+
+ /* Get position of entry in key table */
+ position = h->buckets[bucket_idx].key_idx[idx];
+ next_key = (struct rte_hash_key *) ((char *)h->key_store +
+ position * h->key_entry_size);
+ /* Return key and data */
+ *key = next_key->key;
+ *data = next_key->pdata;
+
+ /* Increment iterator */
+ (*next)++;
+
+ return (position - 1);
+}
+
/* Functions to compare multiple of 16 byte keys (up to 128 bytes) */
static int
rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused)
diff --git a/lib/librte_hash/rte_hash.h b/lib/librte_hash/rte_hash.h
index ff75445..68109d5 100644
--- a/lib/librte_hash/rte_hash.h
+++ b/lib/librte_hash/rte_hash.h
@@ -393,6 +393,28 @@ rte_hash_lookup_bulk_data(const struct rte_hash *h, const void **keys,
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions);
+
+/**
+ * Iterate through the hash table, returning key-value pairs.
+ *
+ * @param h
+ * Hash table to iterate
+ * @param key
+ * Output containing the key where current iterator
+ * was pointing at
+ * @param data
+ * Output containing the data associated with key.
+ * Returns NULL if data was not stored.
+ * @param next
+ * Pointer to iterator. Should be 0 to start iterating the hash table.
+ * Iterator is incremented after each call of this function.
+ * @return
+ * Position where key was stored, if successful.
+ * - -EINVAL if the parameters are invalid.
+ * - -ENOENT if end of the hash table.
+ */
+int32_t
+rte_hash_iterate(const struct rte_hash *h, const void **key, void **data, uint32_t *next);
#ifdef __cplusplus
}
#endif
diff --git a/lib/librte_hash/rte_hash_version.map b/lib/librte_hash/rte_hash_version.map
index a97eac1..5653cb7 100644
--- a/lib/librte_hash/rte_hash_version.map
+++ b/lib/librte_hash/rte_hash_version.map
@@ -25,6 +25,7 @@ DPDK_2.1 {
rte_hash_add_key_data;
rte_hash_add_key_with_hash_data;
rte_hash_create;
+ rte_hash_iterate;
rte_hash_lookup_bulk_data;
rte_hash_lookup_data;
rte_hash_lookup_with_hash_data;
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v7 5/7] MAINTAINERS: claim responsability for hash library
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 0/7] Cuckoo hash - part 3 of Cuckoo hash Pablo de Lara
` (3 preceding siblings ...)
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 4/7] hash: add iterate function Pablo de Lara
@ 2015-07-11 0:18 ` Pablo de Lara
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 6/7] doc: announce ABI change of librte_hash Pablo de Lara
` (2 subsequent siblings)
7 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-11 0:18 UTC (permalink / raw)
To: dev
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
MAINTAINERS | 1 +
1 file changed, 1 insertion(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 5476a73..75a27e9 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -345,6 +345,7 @@ F: doc/guides/sample_app_ug/l3_forward_access_ctrl.rst
Hashes
M: Bruce Richardson <bruce.richardson@intel.com>
+M: Pablo de Lara <pablo.de.lara.guarch@intel.com>
F: lib/librte_hash/
F: doc/guides/prog_guide/hash_lib.rst
F: app/test/test_*hash*
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v7 6/7] doc: announce ABI change of librte_hash
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 0/7] Cuckoo hash - part 3 of Cuckoo hash Pablo de Lara
` (4 preceding siblings ...)
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 5/7] MAINTAINERS: claim responsability for hash library Pablo de Lara
@ 2015-07-11 0:18 ` Pablo de Lara
2015-07-12 22:38 ` Thomas Monjalon
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 7/7] doc: update hash documentation Pablo de Lara
2015-07-12 22:46 ` [dpdk-dev] [PATCH v7 0/7] Cuckoo hash - part 3 of Cuckoo hash Thomas Monjalon
7 siblings, 1 reply; 92+ messages in thread
From: Pablo de Lara @ 2015-07-11 0:18 UTC (permalink / raw)
To: dev
Two of the macros in rte_hash.h are now deprecated, so this patch
adds notice that they will be removed in 2.2.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
doc/guides/rel_notes/abi.rst | 1 +
1 file changed, 1 insertion(+)
diff --git a/doc/guides/rel_notes/abi.rst b/doc/guides/rel_notes/abi.rst
index 9d60a74..b45017c 100644
--- a/doc/guides/rel_notes/abi.rst
+++ b/doc/guides/rel_notes/abi.rst
@@ -21,3 +21,4 @@ Deprecation Notices
1024 queues per port. This change will be in release 2.2.
There is no backward compatibility planned from release 2.2.
All binaries will need to be rebuilt from release 2.2.
+* The Macros #RTE_HASH_BUCKET_ENTRIES_MAX and #RTE_HASH_KEY_LENGTH_MAX are deprecated and will be removed with version 2.2.
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* [dpdk-dev] [PATCH v7 7/7] doc: update hash documentation
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 0/7] Cuckoo hash - part 3 of Cuckoo hash Pablo de Lara
` (5 preceding siblings ...)
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 6/7] doc: announce ABI change of librte_hash Pablo de Lara
@ 2015-07-11 0:18 ` Pablo de Lara
2015-07-12 22:46 ` [dpdk-dev] [PATCH v7 0/7] Cuckoo hash - part 3 of Cuckoo hash Thomas Monjalon
7 siblings, 0 replies; 92+ messages in thread
From: Pablo de Lara @ 2015-07-11 0:18 UTC (permalink / raw)
To: dev
Updates hash library documentation, reflecting
the new implementation changes.
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
doc/guides/prog_guide/hash_lib.rst | 138 +++++++++++++++++++++++++++++++++----
doc/guides/prog_guide/index.rst | 4 ++
2 files changed, 129 insertions(+), 13 deletions(-)
diff --git a/doc/guides/prog_guide/hash_lib.rst b/doc/guides/prog_guide/hash_lib.rst
index 9b83835..466c482 100644
--- a/doc/guides/prog_guide/hash_lib.rst
+++ b/doc/guides/prog_guide/hash_lib.rst
@@ -1,5 +1,5 @@
.. BSD LICENSE
- Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
All rights reserved.
Redistribution and use in source and binary forms, with or without
@@ -50,8 +50,6 @@ The hash also allows the configuration of some low-level implementation related
* Hash function to translate the key into a bucket index
-* Number of entries per bucket
-
The main methods exported by the hash are:
* Add entry with key: The key is provided as input. If a new entry is successfully added to the hash for the specified key,
@@ -65,10 +63,26 @@ The main methods exported by the hash are:
* Lookup for entry with key: The key is provided as input. If an entry with the specified key is found in the hash (lookup hit),
then the position of the entry is returned, otherwise (lookup miss) a negative value is returned.
-The current hash implementation handles the key management only.
-The actual data associated with each key has to be managed by the user using a separate table that
+Apart from these method explained above, the API allows the user three more options:
+
+* Add / lookup / delete with key and precomputed hash: Both the key and its precomputed hash are provided as input. This allows
+ the user to perform these operations faster, as hash is already computed.
+
+* Add / lookup with key and data: A pair of key-value is provided as input. This allows the user to store
+ not only the key, but also data which may be either a 8-byte integer or a pointer to external data (if data size is more than 8 bytes).
+
+* Combination of the two options above: User can provide key, precomputed hash and data.
+
+Also, the API contains a method to allow the user to look up entries in bursts, achieving higher performance
+than looking up individual entries, as the function prefetches next entries at the time it is operating
+with the first ones, which reduces significantly the impact of the necessary memory accesses.
+Notice that this method uses a pipeline of 8 entries (4 stages of 2 entries), so it is highly recommended
+to use at least 8 entries per burst.
+
+The actual data associated with each key can be either managed by the user using a separate table that
mirrors the hash in terms of number of entries and position of each entry,
-as shown in the Flow Classification use case describes in the following sections.
+as shown in the Flow Classification use case describes in the following sections,
+or stored in the hash table itself.
The example hash tables in the L2/L3 Forwarding sample applications defines which port to forward a packet to based on a packet flow identified by the five-tuple lookup.
However, this table could also be used for more sophisticated features and provide many other functions and actions that could be performed on the packets and flows.
@@ -76,17 +90,26 @@ However, this table could also be used for more sophisticated features and provi
Implementation Details
----------------------
-The hash table is implemented as an array of entries which is further divided into buckets,
-with the same number of consecutive array entries in each bucket.
-For any input key, there is always a single bucket where that key can be stored in the hash,
-therefore only the entries within that bucket need to be examined when the key is looked up.
+The hash table has two main tables:
+
+* First table is an array of entries which is further divided into buckets,
+ with the same number of consecutive array entries in each bucket. Each entry contains the computed primary
+ and secondary hashes of a given key (explained below), and an index to the second table.
+
+* The second table is an array of all the keys stored in the hash table and its data associated to each key.
+
+The hash library uses the cuckoo hash method to resolve collisions.
+For any input key, there are two possible buckets (primary and secondary/alternative location)
+where that key can be stored in the hash, therefore only the entries within those bucket need to be examined
+when the key is looked up.
The lookup speed is achieved by reducing the number of entries to be scanned from the total
-number of hash entries down to the number of entries in a hash bucket,
+number of hash entries down to the number of entries in the two hash buckets,
as opposed to the basic method of linearly scanning all the entries in the array.
The hash uses a hash function (configurable) to translate the input key into a 4-byte key signature.
The bucket index is the key signature modulo the number of hash buckets.
-Once the bucket is identified, the scope of the hash add,
-delete and lookup operations is reduced to the entries in that bucket.
+
+Once the buckets are identified, the scope of the hash add,
+delete and lookup operations is reduced to the entries in those buckets (it is very likely that entries are in the primary bucket).
To speed up the search logic within the bucket, each hash entry stores the 4-byte key signature together with the full key for each hash entry.
For large key sizes, comparing the input key against a key from the bucket can take significantly more time than
@@ -95,6 +118,95 @@ Therefore, the signature comparison is done first and the full key comparison do
The full key comparison is still necessary, as two input keys from the same bucket can still potentially have the same 4-byte hash signature,
although this event is relatively rare for hash functions providing good uniform distributions for the set of input keys.
+Example of lookup:
+
+First of all, the primary bucket is identified and entry is likely to be stored there.
+If signature was stored there, we compare its key against the one provided and return the position
+where it was stored and/or the data associated to that key if there is a match.
+If signature is not in the primary bucket, the secondary bucket is looked up, where same procedure
+is carried out. If there is no match there either, key is considered not to be in the table.
+
+Example of addition:
+
+Like lookup, the primary and secondary buckets are indentified. If there is an empty slot in
+the primary bucket, primary and secondary signatures are stored in that slot, key and data (if any) are added to
+the second table and an index to the position in the second table is stored in the slot of the first table.
+If there is no space in the primary bucket, one of the entries on that bucket is pushed to its alternative location,
+and the key to be added is inserted in its position.
+To know where the alternative bucket of the evicted entry is, the secondary signature is looked up and alternative bucket index
+is calculated from doing the modulo, as seen above. If there is room in the alternative bucket, the evicted entry
+is stored in it. If not, same process is repeated (one of the entries gets pushed) until a non full bucket is found.
+Notice that despite all the entry movement in the first table, the second table is not touched, which would impact
+greatly in performance.
+
+In the very unlikely event that table enters in a loop where same entries are being evicted indefinitely,
+key is considered not able to be stored.
+With random keys, this method allows the user to get around 90% of the table utilization, without
+having to drop any stored entry (LRU) or allocate more memory (extended buckets).
+
+Entry distribution in hash table
+--------------------------------
+
+As mentioned above, Cuckoo hash implementation pushes elements out of their bucket,
+if there is a new entry to be added which primary location coincides with their current bucket,
+being pushed to their alternative location.
+Therefore, as user adds more entries to the hash table, distribution of the hash values
+in the buckets will change, being most of them in their primary location and a few in
+their secondary location, which the later will increase, as table gets busier.
+This information is quite useful, as performance may be lower as more entries
+are evicted to their secondary location.
+
+See the tables below showing example entry distribution as table utilization increases.
+
+.. _table_hash_lib_1:
+
+.. table:: Entry distribution measured with an example table with 1024 random entries using jhash algorithm
+
+ +--------------+-----------------------+-------------------------+
+ | % Table used | % In Primary location | % In Secondary location |
+ +==============+=======================+=========================+
+ | 25 | 100 | 0 |
+ +--------------+-----------------------+-------------------------+
+ | 50 | 96.1 | 3.9 |
+ +--------------+-----------------------+-------------------------+
+ | 75 | 88.2 | 11.8 |
+ +--------------+-----------------------+-------------------------+
+ | 80 | 86.3 | 13.7 |
+ +--------------+-----------------------+-------------------------+
+ | 85 | 83.1 | 16.9 |
+ +--------------+-----------------------+-------------------------+
+ | 90 | 77.3 | 22.7 |
+ +--------------+-----------------------+-------------------------+
+ | 95.8 | 64.5 | 35.5 |
+ +--------------+-----------------------+-------------------------+
+
+|
+
+.. _table_hash_lib_2:
+
+.. table:: Entry distribution measured with an example table with 1 million random entries using jhash algorithm
+
+ +--------------+-----------------------+-------------------------+
+ | % Table used | % In Primary location | % In Secondary location |
+ +==============+=======================+=========================+
+ | 50 | 96 | 4 |
+ +--------------+-----------------------+-------------------------+
+ | 75 | 86.9 | 13.1 |
+ +--------------+-----------------------+-------------------------+
+ | 80 | 83.9 | 16.1 |
+ +--------------+-----------------------+-------------------------+
+ | 85 | 80.1 | 19.9 |
+ +--------------+-----------------------+-------------------------+
+ | 90 | 74.8 | 25.2 |
+ +--------------+-----------------------+-------------------------+
+ | 94.5 | 67.4 | 32.6 |
+ +--------------+-----------------------+-------------------------+
+
+.. note::
+
+ Last values on the tables above are the average maximum table
+ utilization with random keys and using Jenkins hash function.
+
Use Case: Flow Classification
-----------------------------
diff --git a/doc/guides/prog_guide/index.rst b/doc/guides/prog_guide/index.rst
index 3295661..036640c 100644
--- a/doc/guides/prog_guide/index.rst
+++ b/doc/guides/prog_guide/index.rst
@@ -241,3 +241,7 @@ Programmer's Guide
:numref:`table_qos_33` :ref:`table_qos_33`
:numref:`table_qos_34` :ref:`table_qos_34`
+
+:numref:`table_hash_lib_1` :ref:`table_hash_lib_1`
+
+:numref:`table_hash_lib_2` :ref:`table_hash_lib_2`
--
2.4.3
^ permalink raw reply [flat|nested] 92+ messages in thread* Re: [dpdk-dev] [PATCH v7 0/7] Cuckoo hash - part 3 of Cuckoo hash
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 0/7] Cuckoo hash - part 3 of Cuckoo hash Pablo de Lara
` (6 preceding siblings ...)
2015-07-11 0:18 ` [dpdk-dev] [PATCH v7 7/7] doc: update hash documentation Pablo de Lara
@ 2015-07-12 22:46 ` Thomas Monjalon
7 siblings, 0 replies; 92+ messages in thread
From: Thomas Monjalon @ 2015-07-12 22:46 UTC (permalink / raw)
To: Pablo de Lara; +Cc: dev
2015-07-11 01:18, Pablo de Lara:
> This patchset is to replace the existing hash library with
> a more efficient and functional approach, using the Cuckoo hash
> method to deal with collisions. This method is based on using
> two different hash functions to have two possible locations
> in the hash table where an entry can be.
> So, if a bucket is full, a new entry can push one of the items
> in that bucket to its alternative location, making space for itself.
>
> Advantages
> ~~
> - Offers the option to store more entries when the target bucket is full
> (unlike the previous implementation)
> - Memory efficient: for storing those entries, it is not necessary to
> request new memory, as the entries will be stored in the same table
> - Constant worst lookup time: in worst case scenario, it always takes
> the same time to look up an entry, as there are only two possible locations
> where an entry can be.
> - Storing data: user can store data in the hash table, unlike the
> previous implementation, but he can still use the old API
>
> This implementation typically offers over 90% utilization.
> Notice that API has been extended, but old API remains.
> Check documentation included to know more about this new implementation
> (including how entries are distributed as table utilization increases).
>
> Changes in v7:
> - Fix inaccurate documentation
>
> Changes in v6:
> - Replace datatype for functions from uintptr_t to void *
>
> Changes in v5:
> - Fix 32-bit compilation issues
>
> Changes in v4:
> - Unit tests enhancements are not part of this patchset anymore.
> - rte_hash structure has been made internal in another patch,
> so it is not part of this patchset anymore.
> - Add function to iterate through the hash table, as rte_hash
> structure has been made private.
> - Added extra_flag parameter in rte_hash_parameter to be able
> to add new parameters in the future without breaking the ABI
> - Remove proposed lookup_bulk_with_hash function, as it is
> not of much use with the existing hash functions
> (there are no vector hash functions).
> - User can store 8-byte integer or pointer as data, instead
> of variable size data, as discussed in the mailing list.
>
> Changes in v3:
>
> - Now user can store variable size data, instead of 32 or 64-bit size data,
> using the new parameter "data_len" in rte_hash_parameters
> - Add lookup_bulk_with_hash function in performance unit tests
> - Add new functions that handle data in performance unit tests
> - Remove duplicates in performance unit tests
> - Fix rte_hash_reset, which was not resetting the last entry
>
> Changes in v2:
>
> - Fixed issue where table could not store maximum number of entries
> - Fixed issue where lookup burst could not be more than 32 (instead of 64)
> - Remove unnecessary macros and add other useful ones
> - Added missing library dependencies
> - Used directly rte_hash_secondary instead of rte_hash_alt
> - Renamed rte_hash.c to rte_cuckoo_hash.c to ease the view of the new library
> - Renamed test_hash_perf.c temporarily to ease the view of the improved unit test
> - Moved rte_hash, rte_bucket and rte_hash_key structures to rte_cuckoo_hash.c to
> make them private
> - Corrected copyright dates
> - Added an optimized function to compare keys that are multiple of 16 bytes
> - Improved the way to use primary/secondary signatures. Now both are stored in
> the bucket, so there is no need to calculate them if required.
> Also, there is no need to use the MSB of a signature to differenciate between
> an empty entry and signature 0, since we are storing both signatures,
> which cannot be both 0.
> - Removed rte_hash_rehash, as it was a very expensive operation.
> Therefore, the add function returns now -ENOSPC if key cannot be added
> because of a loop.
> - Prefetched new slot for new key in add function to improve performance.
> - Made doxygen comments more clear.
> - Removed unnecessary rte_hash_del_key_data and rte_hash_del_key_with_data,
> as we can use the lookup functions if we want to get the data before deleting.
> - Removed some unnecessary includes in rte_hash.h
> - Removed some unnecessary variables in rte_cuckoo_hash.c
> - Removed some unnecessary checks before creating a new hash table
> - Added documentation (in release notes and programmers guide)
> - Added new unit tests and replaced the performance one for hash tables
>
> Series Acked-by: Bruce Richardson <bruce.richardson@intel.com>
Applied, thanks
Some bugs/formatting were fixed in fly.
Some remaining comments may be addressed in further patches.
^ permalink raw reply [flat|nested] 92+ messages in thread