[PATCH] fib: network byte order IPv4 lookup

[Vladimir Medvedkin <vladimir.medvedkin@intel.com>]

Previously when running rte_fib_lookup IPv4 addresses must have been in
host byte order.

This patch adds a new flag RTE_FIB_FLAG_LOOKUP_BE that can be passed on
fib create, which will allow to have IPv4 in network byte order on
lookup.

Signed-off-by: Vladimir Medvedkin <vladimir.medvedkin@intel.com>
---
 app/test/test_fib.c      |  2 +-
 lib/fib/dir24_8.c        | 58 +++++++++++++++++-----------
 lib/fib/dir24_8.h        | 44 ++++++++++++++++++++-
 lib/fib/dir24_8_avx512.c | 82 +++++++++++++++++++++++-----------------
 lib/fib/dir24_8_avx512.h | 15 ++++++++
 lib/fib/rte_fib.c        |  7 +++-
 lib/fib/rte_fib.h        |  4 ++
 7 files changed, 152 insertions(+), 60 deletions(-)

diff --git a/app/test/test_fib.c b/app/test/test_fib.c
index 45dccca1f6..b0e53dbe01 100644
--- a/app/test/test_fib.c
+++ b/app/test/test_fib.c
@@ -319,7 +319,7 @@ int32_t
 test_lookup(void)
 {
 	struct rte_fib *fib = NULL;
-	struct rte_fib_conf config;
+	struct rte_fib_conf config = { 0 };
 	uint64_t def_nh = 100;
 	int ret;
 
diff --git a/lib/fib/dir24_8.c b/lib/fib/dir24_8.c
index c739e92304..2d2a7db697 100644
--- a/lib/fib/dir24_8.c
+++ b/lib/fib/dir24_8.c
@@ -26,41 +26,49 @@
 #define ROUNDUP(x, y)	 RTE_ALIGN_CEIL(x, (1 << (32 - y)))
 
 static inline rte_fib_lookup_fn_t
-get_scalar_fn(enum rte_fib_dir24_8_nh_sz nh_sz)
+get_scalar_fn(enum rte_fib_dir24_8_nh_sz nh_sz, bool be_addr)
 {
 	switch (nh_sz) {
 	case RTE_FIB_DIR24_8_1B:
-		return dir24_8_lookup_bulk_1b;
+		return (be_addr) ? dir24_8_lookup_bulk_1b_be :
+					dir24_8_lookup_bulk_1b;
 	case RTE_FIB_DIR24_8_2B:
-		return dir24_8_lookup_bulk_2b;
+		return (be_addr) ? dir24_8_lookup_bulk_2b_be :
+					dir24_8_lookup_bulk_2b;
 	case RTE_FIB_DIR24_8_4B:
-		return dir24_8_lookup_bulk_4b;
+		return (be_addr) ? dir24_8_lookup_bulk_4b_be :
+					dir24_8_lookup_bulk_4b;
 	case RTE_FIB_DIR24_8_8B:
-		return dir24_8_lookup_bulk_8b;
+		return (be_addr) ? dir24_8_lookup_bulk_8b_be :
+					dir24_8_lookup_bulk_8b;
 	default:
 		return NULL;
 	}
 }
 
 static inline rte_fib_lookup_fn_t
-get_scalar_fn_inlined(enum rte_fib_dir24_8_nh_sz nh_sz)
+get_scalar_fn_inlined(enum rte_fib_dir24_8_nh_sz nh_sz, bool be_addr)
 {
 	switch (nh_sz) {
 	case RTE_FIB_DIR24_8_1B:
-		return dir24_8_lookup_bulk_0;
+		return (be_addr) ? dir24_8_lookup_bulk_0_be :
+					dir24_8_lookup_bulk_0;
 	case RTE_FIB_DIR24_8_2B:
-		return dir24_8_lookup_bulk_1;
+		return (be_addr) ? dir24_8_lookup_bulk_1_be :
+					dir24_8_lookup_bulk_1;
 	case RTE_FIB_DIR24_8_4B:
-		return dir24_8_lookup_bulk_2;
+		return (be_addr) ? dir24_8_lookup_bulk_2_be :
+					dir24_8_lookup_bulk_2;
 	case RTE_FIB_DIR24_8_8B:
-		return dir24_8_lookup_bulk_3;
+		return (be_addr) ? dir24_8_lookup_bulk_3_be :
+					dir24_8_lookup_bulk_3;
 	default:
 		return NULL;
 	}
 }
 
 static inline rte_fib_lookup_fn_t
-get_vector_fn(enum rte_fib_dir24_8_nh_sz nh_sz)
+get_vector_fn(enum rte_fib_dir24_8_nh_sz nh_sz, bool be_addr)
 {
 #ifdef CC_DIR24_8_AVX512_SUPPORT
 	if ((rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) <= 0) ||
@@ -69,13 +77,17 @@ get_vector_fn(enum rte_fib_dir24_8_nh_sz nh_sz)
 
 	switch (nh_sz) {
 	case RTE_FIB_DIR24_8_1B:
-		return rte_dir24_8_vec_lookup_bulk_1b;
+		return (be_addr) ? rte_dir24_8_vec_lookup_bulk_1b_be :
+					rte_dir24_8_vec_lookup_bulk_1b;
 	case RTE_FIB_DIR24_8_2B:
-		return rte_dir24_8_vec_lookup_bulk_2b;
+		return (be_addr) ? rte_dir24_8_vec_lookup_bulk_2b_be :
+					rte_dir24_8_vec_lookup_bulk_2b;
 	case RTE_FIB_DIR24_8_4B:
-		return rte_dir24_8_vec_lookup_bulk_4b;
+		return (be_addr) ? rte_dir24_8_vec_lookup_bulk_4b_be :
+					rte_dir24_8_vec_lookup_bulk_4b;
 	case RTE_FIB_DIR24_8_8B:
-		return rte_dir24_8_vec_lookup_bulk_8b;
+		return (be_addr) ? rte_dir24_8_vec_lookup_bulk_8b_be :
+					rte_dir24_8_vec_lookup_bulk_8b;
 	default:
 		return NULL;
 	}
@@ -86,7 +98,7 @@ get_vector_fn(enum rte_fib_dir24_8_nh_sz nh_sz)
 }
 
 rte_fib_lookup_fn_t
-dir24_8_get_lookup_fn(void *p, enum rte_fib_lookup_type type)
+dir24_8_get_lookup_fn(void *p, enum rte_fib_lookup_type type, bool be_addr)
 {
 	enum rte_fib_dir24_8_nh_sz nh_sz;
 	rte_fib_lookup_fn_t ret_fn;
@@ -99,16 +111,18 @@ dir24_8_get_lookup_fn(void *p, enum rte_fib_lookup_type type)
 
 	switch (type) {
 	case RTE_FIB_LOOKUP_DIR24_8_SCALAR_MACRO:
-		return get_scalar_fn(nh_sz);
+		return get_scalar_fn(nh_sz, be_addr);
 	case RTE_FIB_LOOKUP_DIR24_8_SCALAR_INLINE:
-		return get_scalar_fn_inlined(nh_sz);
+		return get_scalar_fn_inlined(nh_sz, be_addr);
 	case RTE_FIB_LOOKUP_DIR24_8_SCALAR_UNI:
-		return dir24_8_lookup_bulk_uni;
+		return (be_addr) ? dir24_8_lookup_bulk_uni_be :
+						dir24_8_lookup_bulk_uni;
 	case RTE_FIB_LOOKUP_DIR24_8_VECTOR_AVX512:
-		return get_vector_fn(nh_sz);
+		return get_vector_fn(nh_sz, be_addr);
 	case RTE_FIB_LOOKUP_DEFAULT:
-		ret_fn = get_vector_fn(nh_sz);
-		return (ret_fn != NULL) ? ret_fn : get_scalar_fn(nh_sz);
+		ret_fn = get_vector_fn(nh_sz, be_addr);
+		return (ret_fn != NULL) ? ret_fn :
+			get_scalar_fn(nh_sz, be_addr);
 	default:
 		return NULL;
 	}
diff --git a/lib/fib/dir24_8.h b/lib/fib/dir24_8.h
index 7125049f15..2c776e118f 100644
--- a/lib/fib/dir24_8.h
+++ b/lib/fib/dir24_8.h
@@ -7,7 +7,9 @@
 #define _DIR24_8_H_
 
 #include <stdalign.h>
+#include <stdbool.h>
 
+#include <rte_byteorder.h>
 #include <rte_prefetch.h>
 #include <rte_branch_prediction.h>
 
@@ -237,6 +239,46 @@ dir24_8_lookup_bulk_uni(void *p, const uint32_t *ips,
 	}
 }
 
+#define BSWAP_MAX_LENGTH	64
+
+typedef void (*dir24_8_lookup_bulk_be_cb)(void *p, const uint32_t *ips,
+	uint64_t *next_hops, const unsigned int n);
+
+static inline void
+dir24_8_lookup_bulk_be(void *p, const uint32_t *ips,
+	uint64_t *next_hops, const unsigned int n,
+	dir24_8_lookup_bulk_be_cb cb)
+{
+	uint32_t le_ips[BSWAP_MAX_LENGTH];
+	unsigned int i;
+
+	for (i = 0; i < n; i += BSWAP_MAX_LENGTH) {
+		int j;
+		for (j = 0; j < BSWAP_MAX_LENGTH && i + j < n; j++)
+			le_ips[j] = rte_be_to_cpu_32(ips[i + j]);
+
+		cb(p, le_ips, next_hops + i, j);
+	}
+}
+
+#define DECLARE_BE_LOOKUP_FN(name)					\
+static inline void							\
+name##_be(void *p, const uint32_t *ips,					\
+	uint64_t *next_hops, const unsigned int n)			\
+{									\
+	dir24_8_lookup_bulk_be(p, ips, next_hops, n, name);		\
+}
+
+DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_1b)
+DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_2b)
+DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_4b)
+DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_8b)
+DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_0)
+DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_1)
+DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_2)
+DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_3)
+DECLARE_BE_LOOKUP_FN(dir24_8_lookup_bulk_uni)
+
 void *
 dir24_8_create(const char *name, int socket_id, struct rte_fib_conf *conf);
 
@@ -244,7 +286,7 @@ void
 dir24_8_free(void *p);
 
 rte_fib_lookup_fn_t
-dir24_8_get_lookup_fn(void *p, enum rte_fib_lookup_type type);
+dir24_8_get_lookup_fn(void *p, enum rte_fib_lookup_type type, bool be_addr);
 
 int
 dir24_8_modify(struct rte_fib *fib, uint32_t ip, uint8_t depth,
diff --git a/lib/fib/dir24_8_avx512.c b/lib/fib/dir24_8_avx512.c
index 43dba28cfb..edd802abe4 100644
--- a/lib/fib/dir24_8_avx512.c
+++ b/lib/fib/dir24_8_avx512.c
@@ -10,7 +10,7 @@
 
 static __rte_always_inline void
 dir24_8_vec_lookup_x16(void *p, const uint32_t *ips,
-	uint64_t *next_hops, int size)
+	uint64_t *next_hops, int size, bool be_addr)
 {
 	struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
 	__mmask16 msk_ext;
@@ -28,6 +28,16 @@ dir24_8_vec_lookup_x16(void *p, const uint32_t *ips,
 		res_msk = _mm512_set1_epi32(UINT16_MAX);
 
 	ip_vec = _mm512_loadu_si512(ips);
+	if (be_addr) {
+		const __m512i bswap32 = _mm512_set_epi8(
+			12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3,
+			12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3,
+			12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3,
+			12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3
+		);
+		ip_vec = _mm512_shuffle_epi8(ip_vec, bswap32);
+	}
+
 	/* mask 24 most significant bits */
 	idxes = _mm512_srli_epi32(ip_vec, 8);
 
@@ -78,7 +88,7 @@ dir24_8_vec_lookup_x16(void *p, const uint32_t *ips,
 
 static __rte_always_inline void
 dir24_8_vec_lookup_x8_8b(void *p, const uint32_t *ips,
-	uint64_t *next_hops)
+	uint64_t *next_hops, bool be_addr)
 {
 	struct dir24_8_tbl *dp = (struct dir24_8_tbl *)p;
 	const __m512i zero = _mm512_set1_epi32(0);
@@ -89,6 +99,13 @@ dir24_8_vec_lookup_x8_8b(void *p, const uint32_t *ips,
 	__mmask8 msk_ext;
 
 	ip_vec = _mm256_loadu_si256((const void *)ips);
+	if (be_addr) {
+		const __m256i bswap32 = _mm256_set_epi8(
+			12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3,
+			12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3
+		);
+		ip_vec = _mm256_shuffle_epi8(ip_vec, bswap32);
+	}
 	/* mask 24 most significant bits */
 	idxes_256 = _mm256_srli_epi32(ip_vec, 8);
 
@@ -114,52 +131,49 @@ dir24_8_vec_lookup_x8_8b(void *p, const uint32_t *ips,
 	_mm512_storeu_si512(next_hops, res);
 }
 
-void
-rte_dir24_8_vec_lookup_bulk_1b(void *p, const uint32_t *ips,
-	uint64_t *next_hops, const unsigned int n)
-{
-	uint32_t i;
-	for (i = 0; i < (n / 16); i++)
-		dir24_8_vec_lookup_x16(p, ips + i * 16, next_hops + i * 16,
-			sizeof(uint8_t));
-
-	dir24_8_lookup_bulk_1b(p, ips + i * 16, next_hops + i * 16,
-		n - i * 16);
+#define DECLARE_VECTOR_FN(suffix, nh_type, be_addr)			\
+void									\
+rte_dir24_8_vec_lookup_bulk_##suffix(void *p, const uint32_t *ips,	\
+	uint64_t *next_hops, const unsigned int n)			\
+{									\
+	uint32_t i;							\
+									\
+	for (i = 0; i < (n / 16); i++)					\
+		dir24_8_vec_lookup_x16(p, ips + i * 16, next_hops + i * 16, \
+			sizeof(nh_type), be_addr);			\
+									\
+	dir24_8_lookup_bulk_##suffix(p, ips + i * 16, next_hops + i * 16, \
+		n - i * 16);						\
 }
 
-void
-rte_dir24_8_vec_lookup_bulk_2b(void *p, const uint32_t *ips,
-	uint64_t *next_hops, const unsigned int n)
-{
-	uint32_t i;
-	for (i = 0; i < (n / 16); i++)
-		dir24_8_vec_lookup_x16(p, ips + i * 16, next_hops + i * 16,
-			sizeof(uint16_t));
-
-	dir24_8_lookup_bulk_2b(p, ips + i * 16, next_hops + i * 16,
-		n - i * 16);
-}
+DECLARE_VECTOR_FN(1b, uint8_t, false)
+DECLARE_VECTOR_FN(2b, uint16_t, false)
+DECLARE_VECTOR_FN(4b, uint32_t, false)
+DECLARE_VECTOR_FN(1b_be, uint8_t, true)
+DECLARE_VECTOR_FN(2b_be, uint16_t, true)
+DECLARE_VECTOR_FN(4b_be, uint32_t, true)
 
 void
-rte_dir24_8_vec_lookup_bulk_4b(void *p, const uint32_t *ips,
+rte_dir24_8_vec_lookup_bulk_8b(void *p, const uint32_t *ips,
 	uint64_t *next_hops, const unsigned int n)
 {
 	uint32_t i;
-	for (i = 0; i < (n / 16); i++)
-		dir24_8_vec_lookup_x16(p, ips + i * 16, next_hops + i * 16,
-			sizeof(uint32_t));
+	for (i = 0; i < (n / 8); i++)
+		dir24_8_vec_lookup_x8_8b(p, ips + i * 8,
+			next_hops + i * 8, false);
 
-	dir24_8_lookup_bulk_4b(p, ips + i * 16, next_hops + i * 16,
-		n - i * 16);
+	dir24_8_lookup_bulk_8b(p, ips + i * 8, next_hops + i * 8, n - i * 8);
 }
 
 void
-rte_dir24_8_vec_lookup_bulk_8b(void *p, const uint32_t *ips,
+rte_dir24_8_vec_lookup_bulk_8b_be(void *p, const uint32_t *ips,
 	uint64_t *next_hops, const unsigned int n)
 {
 	uint32_t i;
 	for (i = 0; i < (n / 8); i++)
-		dir24_8_vec_lookup_x8_8b(p, ips + i * 8, next_hops + i * 8);
+		dir24_8_vec_lookup_x8_8b(p, ips + i * 8,
+			next_hops + i * 8, true);
 
-	dir24_8_lookup_bulk_8b(p, ips + i * 8, next_hops + i * 8, n - i * 8);
+	dir24_8_lookup_bulk_8b_be(p, ips + i * 8,
+		next_hops + i * 8, n - i * 8);
 }
diff --git a/lib/fib/dir24_8_avx512.h b/lib/fib/dir24_8_avx512.h
index 1d3c2b9317..e9f7b72519 100644
--- a/lib/fib/dir24_8_avx512.h
+++ b/lib/fib/dir24_8_avx512.h
@@ -21,4 +21,19 @@ void
 rte_dir24_8_vec_lookup_bulk_8b(void *p, const uint32_t *ips,
 	uint64_t *next_hops, const unsigned int n);
 
+void
+rte_dir24_8_vec_lookup_bulk_1b_be(void *p, const uint32_t *ips,
+	uint64_t *next_hops, const unsigned int n);
+
+void
+rte_dir24_8_vec_lookup_bulk_2b_be(void *p, const uint32_t *ips,
+	uint64_t *next_hops, const unsigned int n);
+
+void
+rte_dir24_8_vec_lookup_bulk_4b_be(void *p, const uint32_t *ips,
+	uint64_t *next_hops, const unsigned int n);
+
+void
+rte_dir24_8_vec_lookup_bulk_8b_be(void *p, const uint32_t *ips,
+	uint64_t *next_hops, const unsigned int n);
 #endif /* _DIR248_AVX512_H_ */
diff --git a/lib/fib/rte_fib.c b/lib/fib/rte_fib.c
index 4f9fba5a4f..991e48b5ea 100644
--- a/lib/fib/rte_fib.c
+++ b/lib/fib/rte_fib.c
@@ -42,6 +42,7 @@ EAL_REGISTER_TAILQ(rte_fib_tailq)
 struct rte_fib {
 	char			name[RTE_FIB_NAMESIZE];
 	enum rte_fib_type	type;	/**< Type of FIB struct */
+	int flags;					/**< Flags */
 	struct rte_rib		*rib;	/**< RIB helper datastructure */
 	void			*dp;	/**< pointer to the dataplane struct*/
 	rte_fib_lookup_fn_t	lookup;	/**< FIB lookup function */
@@ -110,7 +111,7 @@ init_dataplane(struct rte_fib *fib, __rte_unused int socket_id,
 		if (fib->dp == NULL)
 			return -rte_errno;
 		fib->lookup = dir24_8_get_lookup_fn(fib->dp,
-			RTE_FIB_LOOKUP_DEFAULT);
+			RTE_FIB_LOOKUP_DEFAULT, !!(fib->flags & RTE_FIB_FLAG_LOOKUP_BE));
 		fib->modify = dir24_8_modify;
 		return 0;
 	default:
@@ -214,6 +215,7 @@ rte_fib_create(const char *name, int socket_id, struct rte_fib_conf *conf)
 	rte_strlcpy(fib->name, name, sizeof(fib->name));
 	fib->rib = rib;
 	fib->type = conf->type;
+	fib->flags = conf->flags;
 	fib->def_nh = conf->default_nh;
 	ret = init_dataplane(fib, socket_id, conf);
 	if (ret < 0) {
@@ -329,7 +331,8 @@ rte_fib_select_lookup(struct rte_fib *fib,
 
 	switch (fib->type) {
 	case RTE_FIB_DIR24_8:
-		fn = dir24_8_get_lookup_fn(fib->dp, type);
+		fn = dir24_8_get_lookup_fn(fib->dp, type,
+			!!(fib->flags & RTE_FIB_FLAG_LOOKUP_BE));
 		if (fn == NULL)
 			return -EINVAL;
 		fib->lookup = fn;
diff --git a/lib/fib/rte_fib.h b/lib/fib/rte_fib.h
index d7a5aafe53..1617235e85 100644
--- a/lib/fib/rte_fib.h
+++ b/lib/fib/rte_fib.h
@@ -28,6 +28,9 @@ struct rte_rib;
 /** Maximum depth value possible for IPv4 FIB. */
 #define RTE_FIB_MAXDEPTH	32
 
+/** If set fib lookup is expecting ipv4 in network byte order */
+#define RTE_FIB_FLAG_LOOKUP_BE	1
+
 /** Type of FIB struct */
 enum rte_fib_type {
 	RTE_FIB_DUMMY,		/**< RIB tree based FIB */
@@ -76,6 +79,7 @@ enum rte_fib_lookup_type {
 /** FIB configuration structure */
 struct rte_fib_conf {
 	enum rte_fib_type type; /**< Type of FIB struct */
+	unsigned int flags;
 	/** Default value returned on lookup if there is no route */
 	uint64_t default_nh;
 	int	max_routes;
-- 
2.34.1