[dpdk-dev] [PATCH v1 1/2] net/mlx5: load libmlx5 and libibverbs in run-time

[dpdk-dev] [PATCH v1 1/2] net/mlx5: load libmlx5 and libibverbs in run-time

[Shachar Beiser]

MLX5 PMD loads libraries: libibverbs and libmlx5.
MLX5 PMD is not linked to external libraries.

Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
---
 config/common_base               |   1 +
 drivers/net/mlx5/Makefile        |  27 +-
 drivers/net/mlx5/lib/mlx5_dll.c  | 758 +++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/lib/mlx5_dll.h  |  97 +++++
 drivers/net/mlx5/mlx5.c          |  17 +-
 drivers/net/mlx5/mlx5.h          |   4 +
 drivers/net/mlx5/mlx5_flow.c     |   4 +
 drivers/net/mlx5/mlx5_mac.c      |   4 +
 drivers/net/mlx5/mlx5_mr.c       |   4 +
 drivers/net/mlx5/mlx5_rss.c      |   4 +
 drivers/net/mlx5/mlx5_rxmode.c   |   4 +
 drivers/net/mlx5/mlx5_rxq.c      |   4 +
 drivers/net/mlx5/mlx5_rxtx.c     |   4 +
 drivers/net/mlx5/mlx5_rxtx.h     |   6 +-
 drivers/net/mlx5/mlx5_rxtx_vec.c |   4 +
 drivers/net/mlx5/mlx5_txq.c      |   4 +
 mk/rte.app.mk                    |   8 +-
 17 files changed, 941 insertions(+), 13 deletions(-)
 create mode 100644 drivers/net/mlx5/lib/mlx5_dll.c
 create mode 100644 drivers/net/mlx5/lib/mlx5_dll.h

diff --git a/config/common_base b/config/common_base
index e74febe..3708de4 100644
--- a/config/common_base
+++ b/config/common_base
@@ -237,6 +237,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
 # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
 #
 CONFIG_RTE_LIBRTE_MLX5_PMD=n
+CONFIG_RTE_LIBRTE_MLX5_DLL=y
 CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
 CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
 
diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index a3984eb..1dc0a05 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -53,7 +53,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c
+endif
 # Basic CFLAGS.
 CFLAGS += -O3
 CFLAGS += -std=c11 -Wall -Wextra
@@ -64,7 +66,11 @@ CFLAGS += -D_DEFAULT_SOURCE
 CFLAGS += -D_XOPEN_SOURCE=600
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -Wno-strict-prototypes
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+LDLIBS += -ldl
+else
 LDLIBS += -libverbs -lmlx5
+endif
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_pci
@@ -91,7 +97,11 @@ endif
 ifdef CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE
 CFLAGS += -DMLX5_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE)
 endif
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+CFLAGS += -DMLX5_PMD_DLL
+else
+CFLAGS += -UMLX5_PMD_DLL
+endif
 include $(RTE_SDK)/mk/rte.lib.mk
 
 # Generate and clean-up mlx5_autoconf.h.
@@ -105,26 +115,28 @@ endif
 
 mlx5_autoconf.h.new: FORCE
 
+VERBS_H := infiniband/verbs.h
+MLX5DV_H := infiniband/mlx5dv.h
 mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 	$Q $(RM) -f -- '$@'
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_DEVICE_VXLAN_SUPPORT \
-		infiniband/verbs.h \
+		$(VERBS_H) \
 		enum IBV_DEVICE_VXLAN_SUPPORT \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_WQ_FLAG_RX_END_PADDING \
-		infiniband/verbs.h \
+		$(VERBS_H) \
 		enum IBV_WQ_FLAG_RX_END_PADDING \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_MLX5_MOD_MPW \
-		infiniband/mlx5dv.h \
+		$(MLX5DV_H) \
 		enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
-		infiniband/mlx5dv.h \
+		$(MLX5DV_H) \
 		enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
@@ -144,10 +156,9 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
-		infiniband/verbs.h \
+		$(VERBS_H)  \
 		enum IBV_FLOW_SPEC_ACTION_COUNT \
 		$(AUTOCONF_OUTPUT)
-
 # Create mlx5_autoconf.h or update it in case it differs from the new one.
 
 mlx5_autoconf.h: mlx5_autoconf.h.new
diff --git a/drivers/net/mlx5/lib/mlx5_dll.c b/drivers/net/mlx5/lib/mlx5_dll.c
new file mode 100644
index 0000000..2e679ea
--- /dev/null
+++ b/drivers/net/mlx5/lib/mlx5_dll.c
@@ -0,0 +1,758 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <dlfcn.h>
+#include <rte_log.h>
+#include <mlx5_utils.h>
+#include "mlx5_dll.h"
+
+#define VERBS_LIB_DIR "/usr/lib64/libibverbs"
+#define MLX5_LIB_DIR "/usr/lib64/libmlx5"
+#define DIR_LENGTH 25
+/**
+ * Load a libibverbs and libmlx5 symbols table.
+ *
+ * @return
+ *   0 on success.
+ */
+int mlx5_load_libs(void)
+{
+    void *dlhandle;
+    int ret;
+
+	dlhandle = mlx5_lib_load("ibverbs.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load ibverbs.so");
+		return -1;
+	}
+	ret = mlx5_lverbs_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in libverbs.so ");
+		return ret;
+	}
+	dlhandle = mlx5_lib_load("mlx5.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load mlx5.so");
+		return -1;
+	}
+	ret = mlx5_lmlx5_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in lmlx5.so  ");
+		return ret;
+	}
+	return 0;
+}
+/**
+ * Load a libibverbs or libmlx5 symbols table.
+ *
+ * @param name[in]
+ *   The library name.
+ * @return
+ *   dlhandle on success.
+ */
+void *mlx5_lib_load(const char *name)
+{
+	char *so_name;
+	void *dlhandle;
+	char lib[DIR_LENGTH];
+
+	if (strcmp(name, "ibverbs.so"))
+		strcpy(lib, VERBS_LIB_DIR);
+	else
+		strcpy(lib, MLX5_LIB_DIR);
+	/* If the name is an absolute path then open that path after appending
+	 * the trailer suffix
+	 */
+	if (name[0] == '/') {
+		if (asprintf(&so_name, "%s", name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		if (!dlhandle)
+			goto out_dlopen;
+		free(so_name);
+		return dlhandle;
+	}
+	/* If configured with a provider plugin path then try that next */
+	if (strlen(lib) > 1) {
+		if (asprintf(&so_name, "%s/lib%s", lib, name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		free(so_name);
+		if (dlhandle)
+			return dlhandle;
+	}
+	/* Otherwise use the system libary search path. This is the historical
+	 * behavior of libibverbs
+	 */
+	if (asprintf(&so_name, "lib%s", name) < 0)
+		goto out_asprintf;
+	dlhandle = dlopen(so_name, RTLD_LAZY);
+	if (!dlhandle)
+		goto out_dlopen;
+	free(so_name);
+	return dlhandle;
+out_asprintf:
+	ERROR("Warning: couldn't load driver '%s'.\n", name);
+	return NULL;
+out_dlopen:
+	ERROR("Warning: couldn't load driver '%s': %s\n", so_name, dlerror());
+	free(so_name);
+	return NULL;
+}
+
+/**
+ * Register libibverbs functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx5_lverbs_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx5_libverbs.ibv_destroy_qp) = dlsym(handle,
+													  "ibv_destroy_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_destroy_cq) = dlsym(handle,
+													  "ibv_destroy_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_dealloc_pd) = dlsym(handle,
+													  "ibv_dealloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_device_list) =
+									dlsym(handle, "ibv_get_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_open_device) = dlsym(handle,
+													   "ibv_open_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_query_device) = dlsym(handle,
+														"ibv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_close_device) = dlsym(handle,
+														"ibv_close_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_free_device_list) =
+										dlsym(handle, "ibv_free_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_alloc_pd) = dlsym(handle, "ibv_alloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_device_name) =
+									dlsym(handle, "ibv_get_device_name");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_fork_init) = dlsym(handle,
+													 "ibv_fork_init");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_destroy_comp_channel) =
+									dlsym(handle, "ibv_destroy_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_dereg_mr) = dlsym(handle, "ibv_dereg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_create_comp_channel) =
+									dlsym(handle, "ibv_create_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_cq_event) = dlsym(handle,
+														"ibv_get_cq_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_create_qp) = dlsym(handle,
+													 "ibv_create_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_ack_async_event) =
+									dlsym(handle, "ibv_ack_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_async_event) =
+									dlsym(handle, "ibv_get_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_reg_mr) = dlsym(handle, "ibv_reg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_create_cq) = dlsym(handle,
+													 "ibv_create_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_modify_qp) = dlsym(handle,
+													 "ibv_modify_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_query_port) = dlsym(handle,
+													  "ibv_query_port");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+
+/**
+ * Register libmlx5 functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx5_lmlx5_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx5_lmlx5.mlx5dv_create_cq) = dlsym(handle,
+													 "mlx5dv_create_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_lmlx5.mlx5dv_init_obj) = dlsym(handle,
+													"mlx5dv_init_obj");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_lmlx5.mlx5dv_query_device) = dlsym(handle,
+														"mlx5dv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_lmlx5.mlx5dv_set_context_attr) =
+								 dlsym(handle, "mlx5dv_set_context_attr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param context[in]
+ *   The ibv context.
+ * @param cq_attr[in]
+ *   The completion queue attribute.
+ * @param mlx5_cq_attr[in]
+ *   The mlx5 completion queue attribute.
+ * @return
+ *   a pointer on success.
+ */
+struct ibv_cq_ex *mlx5dv_create_cq(struct ibv_context *context,
+								   struct ibv_cq_init_attr_ex *cq_attr,
+								   struct mlx5dv_cq_init_attr *mlx5_cq_attr)
+{
+	return mlx5_lmlx5.mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr);
+}
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param mlx5dv_obj[in]
+ *   The mlx5 object.
+ * @param obj_type[in]
+ *   The object type.
+ * @return
+ *   a pointer on success.
+ */
+int mlx5dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type)
+{
+	return mlx5_lmlx5.mlx5dv_init_obj(obj, obj_type);
+}
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param ctx_in[in]
+ *   The ibverbs context.
+ * @param attrs_out[out]
+ *   The mlx5 context.
+ * @return
+ *  0 on success.
+ */
+int mlx5dv_query_device(struct ibv_context *ctx_in,
+						struct mlx5dv_context *attrs_out)
+{
+	return mlx5_lmlx5.mlx5dv_query_device(ctx_in, attrs_out);
+}
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param type[in]
+ *   The mlx5 context attribute type.
+ * @param attr[in]
+ *   The mlx5 context attribute.
+ * @return
+ *  0 on success.
+ */
+int mlx5dv_set_context_attr(struct ibv_context *context,
+							enum mlx5dv_set_ctx_attr_type type,
+							void *attr)
+{
+	return mlx5_lmlx5.mlx5dv_set_context_attr(context, type, attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_qp(struct ibv_qp *qp)
+{
+	return mlx5_libverbs.ibv_destroy_qp(qp);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   Completion queue pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_cq(struct ibv_cq *cq)
+{
+	return mlx5_libverbs.ibv_destroy_cq(cq);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   Protection domain.
+ * @return
+ *  0 on success.
+ */
+int ibv_dealloc_pd(struct ibv_pd *pd)
+{
+	return mlx5_libverbs.ibv_dealloc_pd(pd);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param num_devices[in]
+ *   The number of devices.
+ * @return
+ *  0 on success.
+ */
+struct ibv_device **ibv_get_device_list(int *num_devices)
+{
+	return mlx5_libverbs.ibv_get_device_list(num_devices);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param device_attr[in]
+ *   The device attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_device(struct ibv_context *context,
+					 struct ibv_device_attr *device_attr)
+{
+	return mlx5_libverbs.ibv_query_device(context, device_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @return
+ *  0 on success.
+ */
+int ibv_close_device(struct ibv_context *context)
+{
+	return mlx5_libverbs.ibv_close_device(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param list[in]
+ *   The list of devices.
+ */
+void ibv_free_device_list(struct ibv_device **list)
+{
+	return mlx5_libverbs.ibv_free_device_list(list);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   A device pointer.
+ * @return
+ *  The context on success.
+ */
+struct ibv_context *ibv_open_device(struct ibv_device *device)
+{
+	return mlx5_libverbs.ibv_open_device(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @return
+ *  protection domain pointer on success.
+ */
+struct ibv_pd *ibv_alloc_pd(struct ibv_context *context)
+{
+	return mlx5_libverbs.ibv_alloc_pd(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   An ibverbs device pointer.
+ * @return
+ *  device name on success.
+ */
+const char *ibv_get_device_name(struct ibv_device *device)
+{
+	return mlx5_libverbs.ibv_get_device_name(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @return
+ *  0 on success.
+ */
+int ibv_fork_init(void)
+{
+	return mlx5_libverbs.ibv_fork_init();
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A completion channel.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_comp_channel(struct ibv_comp_channel *channel)
+{
+	return mlx5_libverbs.ibv_destroy_comp_channel(channel);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param mr[in]
+ *   A pointer to memory region.
+ * @return
+ *  0 on success.
+ */
+int ibv_dereg_mr(struct ibv_mr *mr)
+{
+	return mlx5_libverbs.ibv_dereg_mr(mr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A pointer to ibverbs context.
+ * @return
+ *  0 on success.
+ */
+struct ibv_comp_channel *ibv_create_comp_channel(
+											  struct ibv_context *context)
+{
+	return mlx5_libverbs.ibv_create_comp_channel(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A pointer to completion channel.
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param cq_context[in]
+ *   A pointer to completion queue context.
+ * @return
+ *  0 on success.
+ */
+int ibv_get_cq_event(struct ibv_comp_channel *channel,
+					 struct ibv_cq **cq, void **cq_context)
+{
+	return mlx5_libverbs.ibv_get_cq_event(channel, cq, cq_context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param nevents[in]
+ *   The number of events.
+ * @return
+ *  0 on success.
+ */
+void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
+{
+	return mlx5_libverbs.ibv_ack_cq_events(cq, nevents);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param qp_init_attr[in]
+ *   The queue pair attributes.
+ * @return
+ *  A pointer to queue pair on success.
+ */
+struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
+							 struct ibv_qp_init_attr *qp_init_attr)
+{
+	return mlx5_libverbs.ibv_create_qp(pd, qp_init_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+void ibv_ack_async_event(struct ibv_async_event *event)
+{
+	return mlx5_libverbs.ibv_ack_async_event(event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A ibverbs context.
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+int ibv_get_async_event(struct ibv_context *context,
+						struct ibv_async_event *event)
+{
+	return mlx5_libverbs.ibv_get_async_event(context, event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param addr[in]
+ *   The address.
+ * @param length[in]
+ *   The length.
+ * @param access[in]
+ *   The access type.
+ * @return
+ *  A pointer to memory region on success.
+ */
+struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr,
+						  size_t length, int access)
+{
+	return mlx5_libverbs.ibv_reg_mr(pd, addr, length, access);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   Ibverbs context.
+ * @param cqe[in]
+ *   A completion queue entry.
+ * @param cq_context[in]
+ *   Completion queue context.
+ * @param channel[in]
+ *   The completion channel.
+ * @param comp_vector[in]
+ *   Completion vector.
+ * @return
+ *  0 on success.
+ */
+struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe,
+							 void *cq_context,
+							 struct ibv_comp_channel *channel,
+							 int comp_vector)
+{
+	return mlx5_libverbs.ibv_create_cq(context, cqe, cq_context, channel,
+									   comp_vector);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair.
+ * @param attr[in]
+ *   The queue pair attribute.
+ * @param attr_mask[in]
+ *   Attribute mask.
+ * @return
+ *  0 on success.
+ */
+int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+				  int attr_mask)
+{
+	return mlx5_libverbs.ibv_modify_qp(qp, attr, attr_mask);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @param port_num[in]
+ *   The port number.
+ * @param port_attr[in]
+ *   Port attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_port(struct ibv_context *context, uint8_t port_num,
+				   struct ibv_port_attr *port_attr)
+{
+	port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
+	port_attr->reserved   = 0;
+	return mlx5_libverbs.ibv_query_port(context, port_num, port_attr);
+}
diff --git a/drivers/net/mlx5/lib/mlx5_dll.h b/drivers/net/mlx5/lib/mlx5_dll.h
new file mode 100644
index 0000000..900d800
--- /dev/null
+++ b/drivers/net/mlx5/lib/mlx5_dll.h
@@ -0,0 +1,97 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef RTE_PMD_MLX5_IBVERBS_H_
+#define RTE_PMD_MLX5_IBVERBS_H_
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
+
+void *mlx5_lib_load(const char *name);
+int mlx5_lverbs_function_register(void *handle);
+int mlx5_lmlx5_function_register(void *handle);
+int mlx5_load_libs(void);
+#undef ibv_query_port
+
+struct {
+	int size;
+	int version;
+	int (*ibv_destroy_qp)(struct ibv_qp *qp);
+	int (*ibv_destroy_cq)(struct ibv_cq *cq);
+	int (*ibv_dealloc_pd)(struct ibv_pd *pd);
+	struct ibv_device **(*ibv_get_device_list)(int *num_devices);
+	int (*ibv_query_device)(struct ibv_context *context,
+							struct ibv_device_attr *device_attr);
+	int (*ibv_close_device)(struct ibv_context *context);
+	void (*ibv_free_device_list)(struct ibv_device **list);
+	struct ibv_context *(*ibv_open_device)(struct ibv_device *device);
+	struct ibv_pd *(*ibv_alloc_pd)(struct ibv_context *context);
+	const char *(*ibv_get_device_name)(struct ibv_device *device);
+	int (*ibv_fork_init)(void);
+	int (*ibv_destroy_comp_channel)(struct ibv_comp_channel *channel);
+	int (*ibv_dereg_mr)(struct ibv_mr *mr);
+	struct ibv_comp_channel *(*ibv_create_comp_channel)(
+							  struct ibv_context *context);
+	int(*ibv_get_cq_event)(struct ibv_comp_channel *channel,
+						   struct ibv_cq **cq, void **cq_context);
+	void (*ibv_ack_cq_events)(struct ibv_cq *cq, unsigned int nevents);
+	struct ibv_qp *(*ibv_create_qp)(struct ibv_pd *pd,
+									struct ibv_qp_init_attr *qp_init_attr);
+	void (*ibv_ack_async_event)(struct ibv_async_event *event);
+	int (*ibv_get_async_event)(struct ibv_context *context,
+							   struct ibv_async_event *event);
+	struct ibv_mr *(*ibv_reg_mr)(struct ibv_pd *pd, void *addr,
+								 size_t length,
+								 int access);
+	struct ibv_cq *(*ibv_create_cq)(struct ibv_context *context,
+									int cqe, void *cq_context,
+									struct ibv_comp_channel *channel,
+									int comp_vector);
+	int (*ibv_modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+						 int attr_mask);
+	int (*ibv_query_port)(struct ibv_context *context, uint8_t port_num,
+						struct ibv_port_attr *port_attr);
+}mlx5_libverbs;
+
+struct {
+	int size;
+	int version;
+	struct ibv_cq_ex *(*mlx5dv_create_cq)(struct ibv_context *context,
+								struct ibv_cq_init_attr_ex *cq_attr,
+								struct mlx5dv_cq_init_attr *mlx5_cq_attr);
+	int (*mlx5dv_init_obj)(struct mlx5dv_obj *obj, uint64_t obj_type);
+	int (*mlx5dv_query_device)(struct ibv_context *ctx_in,
+							   struct mlx5dv_context *attrs_out);
+	int (*mlx5dv_set_context_attr)(struct ibv_context *context,
+								enum mlx5dv_set_ctx_attr_type type,
+								void *attr);
+}mlx5_lmlx5;
+#endif /* RTE_PMD_MLX5_IBVERBS_H_ */
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 0548d17..9420d97 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -30,7 +30,8 @@
  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-
+#define _GNU_SOURCE
+#include <stdio.h>
 #include <stddef.h>
 #include <unistd.h>
 #include <string.h>
@@ -39,13 +40,17 @@
 #include <stdlib.h>
 #include <errno.h>
 #include <net/if.h>
-
+#include <dlfcn.h>
 /* Verbs header. */
 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
@@ -1029,6 +1034,9 @@ struct mlx5_args {
 rte_mlx5_pmd_init(void)
 {
 	/* Build the static table for ptype conversion. */
+#ifdef MLX5_PMD_DLL
+	int ret;
+#endif
 	mlx5_set_ptype_table();
 	/*
 	 * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
@@ -1040,6 +1048,11 @@ struct mlx5_args {
 	/* Match the size of Rx completion entry to the size of a cacheline. */
 	if (RTE_CACHE_LINE_SIZE == 128)
 		setenv("MLX5_CQE_SIZE", "128", 0);
+#ifdef MLX5_PMD_DLL
+	ret = mlx5_load_libs();
+	if ( ret != 0 )
+		return;
+#endif
 	ibv_fork_init();
 	rte_pci_register(&mlx5_driver);
 }
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index e6a69b8..1dde985 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -46,7 +46,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index f32dfdd..ffd419c 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -39,7 +39,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c
index d17b991..9e8902b 100644
--- a/drivers/net/mlx5/mlx5_mac.c
+++ b/drivers/net/mlx5/mlx5_mac.c
@@ -46,7 +46,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 6b29eed..c91fcdf 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -36,7 +36,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c
index f3de46d..9f58147 100644
--- a/drivers/net/mlx5/mlx5_rss.c
+++ b/drivers/net/mlx5/mlx5_rss.c
@@ -42,7 +42,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c
index 0ef2cdf..f9f7d6f 100644
--- a/drivers/net/mlx5/mlx5_rxmode.c
+++ b/drivers/net/mlx5/mlx5_rxmode.c
@@ -40,7 +40,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 85399ef..dfd6400 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -44,8 +44,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 2d30c50..e2215aa 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -41,8 +41,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index d34f3cc..7419f2c 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -43,12 +43,14 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
-#include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
-
 #include <rte_mbuf.h>
 #include <rte_mempool.h>
 #include <rte_common.h>
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c
index ba6c8ce..fc7f701 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec.c
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.c
@@ -41,8 +41,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 9c5860f..8f52525 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -44,7 +44,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 6a6a745..0542c9a 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -51,7 +51,9 @@ endif
 
 # Link only the libraries used in the application
 LDFLAGS += --as-needed
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+LDFLAGS += -ldl
+endif
 # default path for libs
 _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
 
@@ -142,7 +144,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI)        += -lrte_pmd_kni
 endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD)        += -lrte_pmd_lio
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -libverbs -lmlx4
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5
+else
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -libverbs -lmlx5
+endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD)       += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk
 _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD)        += -lrte_pmd_nfp
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL)       += -lrte_pmd_null
-- 
1.8.3.1

[dpdk-dev] [PATCH v1 2/2] net/mlx4: load libmlx4 and libibverbs in run-time

[Shachar Beiser]

MLX4 PMD dynamically loads mlx4 and ibverbs in
run-time and it is not linked to external libraries.

Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
---
 config/common_base              |   1 +
 drivers/net/mlx4/Makefile       |  14 +-
 drivers/net/mlx4/lib/mlx4_dll.c | 731 ++++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx4/lib/mlx4_dll.h |  94 ++++++
 drivers/net/mlx4/mlx4.c         |  12 +
 drivers/net/mlx4/mlx4.h         |   4 +
 drivers/net/mlx4/mlx4_ethdev.c  |   4 +
 drivers/net/mlx4/mlx4_flow.c    |   4 +
 drivers/net/mlx4/mlx4_mr.c      |   4 +
 drivers/net/mlx4/mlx4_prm.h     |   4 +
 drivers/net/mlx4/mlx4_rxq.c     |   4 +
 drivers/net/mlx4/mlx4_rxtx.c    |   4 +
 drivers/net/mlx4/mlx4_rxtx.h    |   4 +
 drivers/net/mlx4/mlx4_txq.c     |   4 +
 mk/rte.app.mk                   |   7 +
 15 files changed, 893 insertions(+), 2 deletions(-)
 create mode 100644 drivers/net/mlx4/lib/mlx4_dll.c
 create mode 100644 drivers/net/mlx4/lib/mlx4_dll.h

diff --git a/config/common_base b/config/common_base
index 3708de4..8ef6be4 100644
--- a/config/common_base
+++ b/config/common_base
@@ -229,6 +229,7 @@ CONFIG_RTE_LIBRTE_FM10K_INC_VECTOR=y
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
 #
 CONFIG_RTE_LIBRTE_MLX4_PMD=n
+CONFIG_RTE_LIBRTE_MLX4_DLL=y
 CONFIG_RTE_LIBRTE_MLX4_DEBUG=n
 CONFIG_RTE_LIBRTE_MLX4_DEBUG_BROKEN_VERBS=n
 CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile
index f1f47c2..aba1d5f 100644
--- a/drivers/net/mlx4/Makefile
+++ b/drivers/net/mlx4/Makefile
@@ -44,7 +44,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_rxq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_rxtx.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_txq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_utils.c
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += lib/mlx4_dll.c
+endif
 # Basic CFLAGS.
 CFLAGS += -O3
 CFLAGS += -std=c11 -Wall -Wextra
@@ -54,7 +56,11 @@ CFLAGS += -D_BSD_SOURCE
 CFLAGS += -D_DEFAULT_SOURCE
 CFLAGS += -D_XOPEN_SOURCE=600
 CFLAGS += $(WERROR_FLAGS)
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+LDLIBS += -ldl
+else
 LDLIBS += -libverbs -lmlx4
+endif
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_pci
@@ -81,7 +87,11 @@ endif
 ifdef CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE
 CFLAGS += -DMLX4_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE)
 endif
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+CFLAGS += -DMLX4_PMD_DLL
+else
+CFLAGS += -UMLX4_PMD_DLL
+endif
 ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DEBUG_BROKEN_VERBS),y)
 CFLAGS += -DMLX4_PMD_DEBUG_BROKEN_VERBS
 endif
diff --git a/drivers/net/mlx4/lib/mlx4_dll.c b/drivers/net/mlx4/lib/mlx4_dll.c
new file mode 100644
index 0000000..6469869
--- /dev/null
+++ b/drivers/net/mlx4/lib/mlx4_dll.c
@@ -0,0 +1,731 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <dlfcn.h>
+#include <rte_log.h>
+#include "../mlx4_utils.h"
+#include "mlx4_dll.h"
+
+#define VERBS_LIB_DIR "/usr/lib64/libibverbs"
+#define MLX5_LIB_DIR "/usr/lib64/libmlx4"
+#define DIR_LENGTH 25
+/**
+ * Load a libibverbs and libmlx4 symbols table.
+ *
+ * @return
+ *   0 on success.
+ */
+int mlx4_load_libs(void)
+{
+    void *dlhandle;
+    int ret;
+
+	dlhandle = mlx4_lib_load("ibverbs.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load ibverbs.so");
+		return -1;
+	}
+	ret = mlx4_lverbs_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in libverbs.so ");
+		return ret;
+	}
+	dlhandle = mlx4_lib_load("mlx4.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load mlx4.so");
+		return -1;
+	}
+	ret = mlx4_lmlx4_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in lmlx4.so  ");
+		return ret;
+	}
+	return 0;
+}
+/**
+ * Load a libibverbs or libmlx4 symbols table.
+ *
+ * @param name[in]
+ *   The library name.
+ * @return
+ *   dlhandle on success.
+ */
+void *mlx4_lib_load(const char *name)
+{
+	char *so_name;
+	void *dlhandle;
+	char lib[DIR_LENGTH];
+
+	if (strcmp(name, "ibverbs.so"))
+		strcpy(lib, VERBS_LIB_DIR);
+	else
+		strcpy(lib, MLX5_LIB_DIR);
+	/* If the name is an absolute path then open that path after appending
+	 * the trailer suffix
+	 */
+	if (name[0] == '/') {
+		if (asprintf(&so_name, "%s", name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		if (!dlhandle)
+			goto out_dlopen;
+		free(so_name);
+		return dlhandle;
+	}
+	/* If configured with a provider plugin path then try that next */
+	if (strlen(lib) > 1) {
+		if (asprintf(&so_name, "%s/lib%s", lib, name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		free(so_name);
+		if (dlhandle)
+			return dlhandle;
+	}
+	/* Otherwise use the system libary search path. This is the historical
+	 * behavior of libibverbs
+	 */
+	if (asprintf(&so_name, "lib%s", name) < 0)
+		goto out_asprintf;
+	dlhandle = dlopen(so_name, RTLD_LAZY);
+	if (!dlhandle)
+		goto out_dlopen;
+	free(so_name);
+	return dlhandle;
+out_asprintf:
+	ERROR("Warning: couldn't load driver '%s'.\n", name);
+	return NULL;
+out_dlopen:
+	ERROR("Warning: couldn't load driver '%s': %s\n", so_name, dlerror());
+	free(so_name);
+	return NULL;
+}
+
+/**
+ * Register libibverbs functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx4_lverbs_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx4_libverbs.ibv_destroy_qp) = dlsym(handle,
+													  "ibv_destroy_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_destroy_cq) = dlsym(handle,
+													  "ibv_destroy_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_dealloc_pd) = dlsym(handle,
+													  "ibv_dealloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_device_list) =
+									dlsym(handle, "ibv_get_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_open_device) = dlsym(handle,
+													   "ibv_open_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_query_device) = dlsym(handle,
+														"ibv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_close_device) = dlsym(handle,
+														"ibv_close_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_free_device_list) =
+										dlsym(handle, "ibv_free_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_alloc_pd) = dlsym(handle, "ibv_alloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_device_name) =
+									dlsym(handle, "ibv_get_device_name");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_fork_init) = dlsym(handle,
+													 "ibv_fork_init");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_destroy_comp_channel) =
+									dlsym(handle, "ibv_destroy_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_dereg_mr) = dlsym(handle, "ibv_dereg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_create_comp_channel) =
+									dlsym(handle, "ibv_create_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_cq_event) = dlsym(handle,
+														"ibv_get_cq_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_create_qp) = dlsym(handle,
+													 "ibv_create_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_ack_async_event) =
+									dlsym(handle, "ibv_ack_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_async_event) =
+									dlsym(handle, "ibv_get_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_reg_mr) = dlsym(handle, "ibv_reg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_create_cq) = dlsym(handle,
+													 "ibv_create_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_modify_qp) = dlsym(handle,
+													 "ibv_modify_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_query_port) = dlsym(handle,
+													  "ibv_query_port");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+
+/**
+ * Register libmlx4 functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx4_lmlx4_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx4_lmlx4.mlx4dv_init_obj) = dlsym(handle,
+													"mlx4dv_init_obj");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_lmlx4.mlx4dv_query_device) = dlsym(handle,
+														"mlx4dv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_lmlx4.mlx4dv_set_context_attr) =
+								 dlsym(handle, "mlx4dv_set_context_attr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+/**
+ * Function pointer calls mlx4 API .
+ *
+ * @param mlx4dv_obj[in]
+ *   The mlx4 object.
+ * @param obj_type[in]
+ *   The object type.
+ * @return
+ *   a pointer on success.
+ */
+int mlx4dv_init_obj(struct mlx4dv_obj *obj, uint64_t obj_type)
+{
+	return mlx4_lmlx4.mlx4dv_init_obj(obj, obj_type);
+}
+/**
+ * Function pointer calls mlx4 API .
+ *
+ * @param ctx_in[in]
+ *   The ibverbs context.
+ * @param attrs_out[out]
+ *   The mlx4 context.
+ * @return
+ *  0 on success.
+ */
+int mlx4dv_query_device(struct ibv_context *ctx_in,
+						struct mlx4dv_context *attrs_out)
+{
+	return mlx4_lmlx4.mlx4dv_query_device(ctx_in, attrs_out);
+}
+/**
+ * Function pointer calls mlx4 API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param type[in]
+ *   The mlx4 context attribute type.
+ * @param attr[in]
+ *   The mlx4 context attribute.
+ * @return
+ *  0 on success.
+ */
+int mlx4dv_set_context_attr(struct ibv_context *context,
+							enum mlx4dv_set_ctx_attr_type type,
+							void *attr)
+{
+	return mlx4_lmlx4.mlx4dv_set_context_attr(context, type, attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_qp(struct ibv_qp *qp)
+{
+	return mlx4_libverbs.ibv_destroy_qp(qp);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   Completion queue pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_cq(struct ibv_cq *cq)
+{
+	return mlx4_libverbs.ibv_destroy_cq(cq);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   Protection domain.
+ * @return
+ *  0 on success.
+ */
+int ibv_dealloc_pd(struct ibv_pd *pd)
+{
+	return mlx4_libverbs.ibv_dealloc_pd(pd);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param num_devices[in]
+ *   The number of devices.
+ * @return
+ *  0 on success.
+ */
+struct ibv_device **ibv_get_device_list(int *num_devices)
+{
+	return mlx4_libverbs.ibv_get_device_list(num_devices);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param device_attr[in]
+ *   The device attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_device(struct ibv_context *context,
+					 struct ibv_device_attr *device_attr)
+{
+	return mlx4_libverbs.ibv_query_device(context, device_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @return
+ *  0 on success.
+ */
+int ibv_close_device(struct ibv_context *context)
+{
+	return mlx4_libverbs.ibv_close_device(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param list[in]
+ *   The list of devices.
+ */
+void ibv_free_device_list(struct ibv_device **list)
+{
+	return mlx4_libverbs.ibv_free_device_list(list);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   A device pointer.
+ * @return
+ *  The context on success.
+ */
+struct ibv_context *ibv_open_device(struct ibv_device *device)
+{
+	return mlx4_libverbs.ibv_open_device(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @return
+ *  protection domain pointer on success.
+ */
+struct ibv_pd *ibv_alloc_pd(struct ibv_context *context)
+{
+	return mlx4_libverbs.ibv_alloc_pd(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   An ibverbs device pointer.
+ * @return
+ *  device name on success.
+ */
+const char *ibv_get_device_name(struct ibv_device *device)
+{
+	return mlx4_libverbs.ibv_get_device_name(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @return
+ *  0 on success.
+ */
+int ibv_fork_init(void)
+{
+	return mlx4_libverbs.ibv_fork_init();
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A completion channel.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_comp_channel(struct ibv_comp_channel *channel)
+{
+	return mlx4_libverbs.ibv_destroy_comp_channel(channel);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param mr[in]
+ *   A pointer to memory region.
+ * @return
+ *  0 on success.
+ */
+int ibv_dereg_mr(struct ibv_mr *mr)
+{
+	return mlx4_libverbs.ibv_dereg_mr(mr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A pointer to ibverbs context.
+ * @return
+ *  0 on success.
+ */
+struct ibv_comp_channel *ibv_create_comp_channel(
+											  struct ibv_context *context)
+{
+	return mlx4_libverbs.ibv_create_comp_channel(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A pointer to completion channel.
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param cq_context[in]
+ *   A pointer to completion queue context.
+ * @return
+ *  0 on success.
+ */
+int ibv_get_cq_event(struct ibv_comp_channel *channel,
+					 struct ibv_cq **cq, void **cq_context)
+{
+	return mlx4_libverbs.ibv_get_cq_event(channel, cq, cq_context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param nevents[in]
+ *   The number of events.
+ * @return
+ *  0 on success.
+ */
+void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
+{
+	return mlx4_libverbs.ibv_ack_cq_events(cq, nevents);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param qp_init_attr[in]
+ *   The queue pair attributes.
+ * @return
+ *  A pointer to queue pair on success.
+ */
+struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
+							 struct ibv_qp_init_attr *qp_init_attr)
+{
+	return mlx4_libverbs.ibv_create_qp(pd, qp_init_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+void ibv_ack_async_event(struct ibv_async_event *event)
+{
+	return mlx4_libverbs.ibv_ack_async_event(event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A ibverbs context.
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+int ibv_get_async_event(struct ibv_context *context,
+						struct ibv_async_event *event)
+{
+	return mlx4_libverbs.ibv_get_async_event(context, event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param addr[in]
+ *   The address.
+ * @param length[in]
+ *   The length.
+ * @param access[in]
+ *   The access type.
+ * @return
+ *  A pointer to memory region on success.
+ */
+struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr,
+						  size_t length, int access)
+{
+	return mlx4_libverbs.ibv_reg_mr(pd, addr, length, access);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   Ibverbs context.
+ * @param cqe[in]
+ *   A completion queue entry.
+ * @param cq_context[in]
+ *   Completion queue context.
+ * @param channel[in]
+ *   The completion channel.
+ * @param comp_vector[in]
+ *   Completion vector.
+ * @return
+ *  0 on success.
+ */
+struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe,
+							 void *cq_context,
+							 struct ibv_comp_channel *channel,
+							 int comp_vector)
+{
+	return mlx4_libverbs.ibv_create_cq(context, cqe, cq_context, channel,
+									   comp_vector);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair.
+ * @param attr[in]
+ *   The queue pair attribute.
+ * @param attr_mask[in]
+ *   Attribute mask.
+ * @return
+ *  0 on success.
+ */
+int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+				  int attr_mask)
+{
+	return mlx4_libverbs.ibv_modify_qp(qp, attr, attr_mask);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @param port_num[in]
+ *   The port number.
+ * @param port_attr[in]
+ *   Port attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_port(struct ibv_context *context, uint8_t port_num,
+				   struct ibv_port_attr *port_attr)
+{
+	port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
+	port_attr->reserved   = 0;
+	return mlx4_libverbs.ibv_query_port(context, port_num, port_attr);
+}
diff --git a/drivers/net/mlx4/lib/mlx4_dll.h b/drivers/net/mlx4/lib/mlx4_dll.h
new file mode 100644
index 0000000..e95b78e
--- /dev/null
+++ b/drivers/net/mlx4/lib/mlx4_dll.h
@@ -0,0 +1,94 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef RTE_PMD_MLX4_IBVERBS_H_
+#define RTE_PMD_MLX4_IBVERBS_H_
+#include <infiniband/verbs.h>
+#include <infiniband/mlx4dv.h>
+
+void *mlx4_lib_load(const char *name);
+int mlx4_lverbs_function_register(void *handle);
+int mlx4_lmlx4_function_register(void *handle);
+int mlx4_load_libs(void);
+#undef ibv_query_port
+
+struct {
+	int size;
+	int version;
+	int (*ibv_destroy_qp)(struct ibv_qp *qp);
+	int (*ibv_destroy_cq)(struct ibv_cq *cq);
+	int (*ibv_dealloc_pd)(struct ibv_pd *pd);
+	struct ibv_device **(*ibv_get_device_list)(int *num_devices);
+	int (*ibv_query_device)(struct ibv_context *context,
+							struct ibv_device_attr *device_attr);
+	int (*ibv_close_device)(struct ibv_context *context);
+	void (*ibv_free_device_list)(struct ibv_device **list);
+	struct ibv_context *(*ibv_open_device)(struct ibv_device *device);
+	struct ibv_pd *(*ibv_alloc_pd)(struct ibv_context *context);
+	const char *(*ibv_get_device_name)(struct ibv_device *device);
+	int (*ibv_fork_init)(void);
+	int (*ibv_destroy_comp_channel)(struct ibv_comp_channel *channel);
+	int (*ibv_dereg_mr)(struct ibv_mr *mr);
+	struct ibv_comp_channel *(*ibv_create_comp_channel)(
+							  struct ibv_context *context);
+	int(*ibv_get_cq_event)(struct ibv_comp_channel *channel,
+						   struct ibv_cq **cq, void **cq_context);
+	void (*ibv_ack_cq_events)(struct ibv_cq *cq, unsigned int nevents);
+	struct ibv_qp *(*ibv_create_qp)(struct ibv_pd *pd,
+									struct ibv_qp_init_attr *qp_init_attr);
+	void (*ibv_ack_async_event)(struct ibv_async_event *event);
+	int (*ibv_get_async_event)(struct ibv_context *context,
+							   struct ibv_async_event *event);
+	struct ibv_mr *(*ibv_reg_mr)(struct ibv_pd *pd, void *addr,
+								 size_t length,
+								 int access);
+	struct ibv_cq *(*ibv_create_cq)(struct ibv_context *context,
+									int cqe, void *cq_context,
+									struct ibv_comp_channel *channel,
+									int comp_vector);
+	int (*ibv_modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+						 int attr_mask);
+	int (*ibv_query_port)(struct ibv_context *context, uint8_t port_num,
+						struct ibv_port_attr *port_attr);
+}mlx4_libverbs;
+
+struct {
+	int size;
+	int version;
+	int (*mlx4dv_init_obj)(struct mlx4dv_obj *obj, uint64_t obj_type);
+	int (*mlx4dv_query_device)(struct ibv_context *ctx_in,
+							   struct mlx4dv_context *attrs_out);
+	int (*mlx4dv_set_context_attr)(struct ibv_context *context,
+								enum mlx4dv_set_ctx_attr_type type,
+								void *attr);
+}mlx4_lmlx4;
+#endif /* RTE_PMD_MLX4_IBVERBS_H_ */
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index f9e4f9d..8fee391 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -49,7 +49,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
@@ -713,7 +717,15 @@ struct mlx4_conf {
 	 * applications to use fork() safely for purposes other than
 	 * using this PMD, which is not supported in forked processes.
 	 */
+#ifdef MLX4_PMD_DLL
+	int ret;
+#endif
 	setenv("RDMAV_HUGEPAGES_SAFE", "1", 1);
+#ifdef MLX4_PMD_DLL
+	ret = mlx4_load_libs();
+	if ( ret != 0 )
+		return;
+#endif
 	ibv_fork_init();
 	rte_pci_register(&mlx4_driver);
 }
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 3aeef87..6e4565b 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -42,7 +42,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_ethdev.c b/drivers/net/mlx4/mlx4_ethdev.c
index 2f69e7d..98134ab 100644
--- a/drivers/net/mlx4/mlx4_ethdev.c
+++ b/drivers/net/mlx4/mlx4_ethdev.c
@@ -56,7 +56,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_flow.c b/drivers/net/mlx4/mlx4_flow.c
index 8b87b29..89d6d17 100644
--- a/drivers/net/mlx4/mlx4_flow.c
+++ b/drivers/net/mlx4/mlx4_flow.c
@@ -49,7 +49,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_mr.c b/drivers/net/mlx4/mlx4_mr.c
index 2a3e269..b494e95 100644
--- a/drivers/net/mlx4/mlx4_mr.c
+++ b/drivers/net/mlx4/mlx4_mr.c
@@ -47,7 +47,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_prm.h b/drivers/net/mlx4/mlx4_prm.h
index fcc7c12..b8854e1 100644
--- a/drivers/net/mlx4/mlx4_prm.h
+++ b/drivers/net/mlx4/mlx4_prm.h
@@ -42,8 +42,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PRD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c
index 53313c5..5680c89 100644
--- a/drivers/net/mlx4/mlx4_rxq.c
+++ b/drivers/net/mlx4/mlx4_rxq.c
@@ -46,8 +46,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_rxtx.c b/drivers/net/mlx4/mlx4_rxtx.c
index 2bfa8b1..a876896 100644
--- a/drivers/net/mlx4/mlx4_rxtx.c
+++ b/drivers/net/mlx4/mlx4_rxtx.c
@@ -44,7 +44,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h
index 463df2b..c115032 100644
--- a/drivers/net/mlx4/mlx4_rxtx.h
+++ b/drivers/net/mlx4/mlx4_rxtx.h
@@ -41,8 +41,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c
index 7882a4d..8ae9eb9 100644
--- a/drivers/net/mlx4/mlx4_txq.c
+++ b/drivers/net/mlx4/mlx4_txq.c
@@ -46,7 +46,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 0542c9a..5589482 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -54,6 +54,9 @@ LDFLAGS += --as-needed
 ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
 LDFLAGS += -ldl
 endif
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+LDFLAGS += -ldl
+endif
 # default path for libs
 _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
 
@@ -143,7 +146,11 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI)        += -lrte_pmd_kni
 endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD)        += -lrte_pmd_lio
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4
+else
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -libverbs -lmlx4
+endif
 ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5
 else
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 1/2] net/mlx5: load libmlx5 and libibverbs in run-time

[Shachar Beiser]

MLX5 PMD loads libraries: libibverbs and libmlx5.
MLX5 PMD is not linked to external libraries.

Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
---
History: 
     V1: 
       http://dpdk.org/patch/31555
V2:
* Fixing checkpatch coding style issues 

---
 config/common_base               |   1 +
 drivers/net/mlx5/Makefile        |  27 +-
 drivers/net/mlx5/lib/mlx5_dll.c  | 758 +++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/lib/mlx5_dll.h  |  97 +++++
 drivers/net/mlx5/mlx5.c          |  17 +-
 drivers/net/mlx5/mlx5.h          |   4 +
 drivers/net/mlx5/mlx5_flow.c     |   4 +
 drivers/net/mlx5/mlx5_mac.c      |   4 +
 drivers/net/mlx5/mlx5_mr.c       |   4 +
 drivers/net/mlx5/mlx5_rss.c      |   4 +
 drivers/net/mlx5/mlx5_rxmode.c   |   4 +
 drivers/net/mlx5/mlx5_rxq.c      |   4 +
 drivers/net/mlx5/mlx5_rxtx.c     |   4 +
 drivers/net/mlx5/mlx5_rxtx.h     |   6 +-
 drivers/net/mlx5/mlx5_rxtx_vec.c |   4 +
 drivers/net/mlx5/mlx5_txq.c      |   4 +
 mk/rte.app.mk                    |   8 +-
 17 files changed, 941 insertions(+), 13 deletions(-)
 create mode 100644 drivers/net/mlx5/lib/mlx5_dll.c
 create mode 100644 drivers/net/mlx5/lib/mlx5_dll.h

diff --git a/config/common_base b/config/common_base
index e74febe..3708de4 100644
--- a/config/common_base
+++ b/config/common_base
@@ -237,6 +237,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
 # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
 #
 CONFIG_RTE_LIBRTE_MLX5_PMD=n
+CONFIG_RTE_LIBRTE_MLX5_DLL=y
 CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
 CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
 
diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index a3984eb..1dc0a05 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -53,7 +53,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c
+endif
 # Basic CFLAGS.
 CFLAGS += -O3
 CFLAGS += -std=c11 -Wall -Wextra
@@ -64,7 +66,11 @@ CFLAGS += -D_DEFAULT_SOURCE
 CFLAGS += -D_XOPEN_SOURCE=600
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -Wno-strict-prototypes
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+LDLIBS += -ldl
+else
 LDLIBS += -libverbs -lmlx5
+endif
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_pci
@@ -91,7 +97,11 @@ endif
 ifdef CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE
 CFLAGS += -DMLX5_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE)
 endif
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+CFLAGS += -DMLX5_PMD_DLL
+else
+CFLAGS += -UMLX5_PMD_DLL
+endif
 include $(RTE_SDK)/mk/rte.lib.mk
 
 # Generate and clean-up mlx5_autoconf.h.
@@ -105,26 +115,28 @@ endif
 
 mlx5_autoconf.h.new: FORCE
 
+VERBS_H := infiniband/verbs.h
+MLX5DV_H := infiniband/mlx5dv.h
 mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 	$Q $(RM) -f -- '$@'
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_DEVICE_VXLAN_SUPPORT \
-		infiniband/verbs.h \
+		$(VERBS_H) \
 		enum IBV_DEVICE_VXLAN_SUPPORT \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_WQ_FLAG_RX_END_PADDING \
-		infiniband/verbs.h \
+		$(VERBS_H) \
 		enum IBV_WQ_FLAG_RX_END_PADDING \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_MLX5_MOD_MPW \
-		infiniband/mlx5dv.h \
+		$(MLX5DV_H) \
 		enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
-		infiniband/mlx5dv.h \
+		$(MLX5DV_H) \
 		enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
@@ -144,10 +156,9 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
-		infiniband/verbs.h \
+		$(VERBS_H)  \
 		enum IBV_FLOW_SPEC_ACTION_COUNT \
 		$(AUTOCONF_OUTPUT)
-
 # Create mlx5_autoconf.h or update it in case it differs from the new one.
 
 mlx5_autoconf.h: mlx5_autoconf.h.new
diff --git a/drivers/net/mlx5/lib/mlx5_dll.c b/drivers/net/mlx5/lib/mlx5_dll.c
new file mode 100644
index 0000000..2228ad1
--- /dev/null
+++ b/drivers/net/mlx5/lib/mlx5_dll.c
@@ -0,0 +1,758 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <dlfcn.h>
+#include <rte_log.h>
+#include <mlx5_utils.h>
+#include "mlx5_dll.h"
+
+#define VERBS_LIB_DIR "/usr/lib64/libibverbs"
+#define MLX5_LIB_DIR "/usr/lib64/libmlx5"
+#define DIR_LENGTH 25
+/**
+ * Load a libibverbs and libmlx5 symbols table.
+ *
+ * @return
+ *   0 on success.
+ */
+int mlx5_load_libs(void)
+{
+	void *dlhandle;
+	int ret;
+
+	dlhandle = mlx5_lib_load("ibverbs.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load ibverbs.so");
+		return -1;
+	}
+	ret = mlx5_lverbs_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in libverbs.so ");
+		return ret;
+	}
+	dlhandle = mlx5_lib_load("mlx5.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load mlx5.so");
+		return -1;
+	}
+	ret = mlx5_lmlx5_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in lmlx5.so  ");
+		return ret;
+	}
+	return 0;
+}
+/**
+ * Load a libibverbs or libmlx5 symbols table.
+ *
+ * @param name[in]
+ *   The library name.
+ * @return
+ *   dlhandle on success.
+ */
+void *mlx5_lib_load(const char *name)
+{
+	char *so_name;
+	void *dlhandle;
+	char lib[DIR_LENGTH];
+
+	if (strcmp(name, "ibverbs.so"))
+		strcpy(lib, VERBS_LIB_DIR);
+	else
+		strcpy(lib, MLX5_LIB_DIR);
+	/* If the name is an absolute path then open that path after appending
+	 * the trailer suffix
+	 */
+	if (name[0] == '/') {
+		if (asprintf(&so_name, "%s", name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		if (!dlhandle)
+			goto out_dlopen;
+		free(so_name);
+		return dlhandle;
+	}
+	/* If configured with a provider plugin path then try that next */
+	if (strlen(lib) > 1) {
+		if (asprintf(&so_name, "%s/lib%s", lib, name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		free(so_name);
+		if (dlhandle)
+			return dlhandle;
+	}
+	/* Otherwise use the system libary search path. This is the historical
+	 * behavior of libibverbs
+	 */
+	if (asprintf(&so_name, "lib%s", name) < 0)
+		goto out_asprintf;
+	dlhandle = dlopen(so_name, RTLD_LAZY);
+	if (!dlhandle)
+		goto out_dlopen;
+	free(so_name);
+	return dlhandle;
+out_asprintf:
+	ERROR("Warning: couldn't load driver '%s'.\n", name);
+	return NULL;
+out_dlopen:
+	ERROR("Warning: couldn't load driver '%s': %s\n", so_name, dlerror());
+	free(so_name);
+	return NULL;
+}
+
+/**
+ * Register libibverbs functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx5_lverbs_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx5_libverbs.ibv_destroy_qp) = dlsym(handle,
+													  "ibv_destroy_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_destroy_cq) = dlsym(handle,
+													  "ibv_destroy_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_dealloc_pd) = dlsym(handle,
+													  "ibv_dealloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_device_list) =
+									dlsym(handle, "ibv_get_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_open_device) = dlsym(handle,
+													   "ibv_open_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_query_device) = dlsym(handle,
+														"ibv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_close_device) = dlsym(handle,
+														"ibv_close_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_free_device_list) =
+										dlsym(handle, "ibv_free_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_alloc_pd) = dlsym(handle, "ibv_alloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_device_name) =
+									dlsym(handle, "ibv_get_device_name");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_fork_init) = dlsym(handle,
+													 "ibv_fork_init");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_destroy_comp_channel) =
+									dlsym(handle, "ibv_destroy_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_dereg_mr) = dlsym(handle, "ibv_dereg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_create_comp_channel) =
+									dlsym(handle, "ibv_create_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_cq_event) = dlsym(handle,
+														"ibv_get_cq_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_create_qp) = dlsym(handle,
+													 "ibv_create_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_ack_async_event) =
+									dlsym(handle, "ibv_ack_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_async_event) =
+									dlsym(handle, "ibv_get_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_reg_mr) = dlsym(handle, "ibv_reg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_create_cq) = dlsym(handle,
+													 "ibv_create_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_modify_qp) = dlsym(handle,
+													 "ibv_modify_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_query_port) = dlsym(handle,
+													  "ibv_query_port");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+
+/**
+ * Register libmlx5 functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx5_lmlx5_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx5_lmlx5.mlx5dv_create_cq) = dlsym(handle,
+													 "mlx5dv_create_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_lmlx5.mlx5dv_init_obj) = dlsym(handle,
+													"mlx5dv_init_obj");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_lmlx5.mlx5dv_query_device) = dlsym(handle,
+														"mlx5dv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_lmlx5.mlx5dv_set_context_attr) =
+								 dlsym(handle, "mlx5dv_set_context_attr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param context[in]
+ *   The ibv context.
+ * @param cq_attr[in]
+ *   The completion queue attribute.
+ * @param mlx5_cq_attr[in]
+ *   The mlx5 completion queue attribute.
+ * @return
+ *   a pointer on success.
+ */
+struct ibv_cq_ex *mlx5dv_create_cq(struct ibv_context *context,
+								   struct ibv_cq_init_attr_ex *cq_attr,
+								   struct mlx5dv_cq_init_attr *mlx5_cq_attr)
+{
+	return mlx5_lmlx5.mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr);
+}
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param mlx5dv_obj[in]
+ *   The mlx5 object.
+ * @param obj_type[in]
+ *   The object type.
+ * @return
+ *   a pointer on success.
+ */
+int mlx5dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type)
+{
+	return mlx5_lmlx5.mlx5dv_init_obj(obj, obj_type);
+}
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param ctx_in[in]
+ *   The ibverbs context.
+ * @param attrs_out[out]
+ *   The mlx5 context.
+ * @return
+ *  0 on success.
+ */
+int mlx5dv_query_device(struct ibv_context *ctx_in,
+						struct mlx5dv_context *attrs_out)
+{
+	return mlx5_lmlx5.mlx5dv_query_device(ctx_in, attrs_out);
+}
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param type[in]
+ *   The mlx5 context attribute type.
+ * @param attr[in]
+ *   The mlx5 context attribute.
+ * @return
+ *  0 on success.
+ */
+int mlx5dv_set_context_attr(struct ibv_context *context,
+							enum mlx5dv_set_ctx_attr_type type,
+							void *attr)
+{
+	return mlx5_lmlx5.mlx5dv_set_context_attr(context, type, attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_qp(struct ibv_qp *qp)
+{
+	return mlx5_libverbs.ibv_destroy_qp(qp);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   Completion queue pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_cq(struct ibv_cq *cq)
+{
+	return mlx5_libverbs.ibv_destroy_cq(cq);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   Protection domain.
+ * @return
+ *  0 on success.
+ */
+int ibv_dealloc_pd(struct ibv_pd *pd)
+{
+	return mlx5_libverbs.ibv_dealloc_pd(pd);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param num_devices[in]
+ *   The number of devices.
+ * @return
+ *  0 on success.
+ */
+struct ibv_device **ibv_get_device_list(int *num_devices)
+{
+	return mlx5_libverbs.ibv_get_device_list(num_devices);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param device_attr[in]
+ *   The device attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_device(struct ibv_context *context,
+					 struct ibv_device_attr *device_attr)
+{
+	return mlx5_libverbs.ibv_query_device(context, device_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @return
+ *  0 on success.
+ */
+int ibv_close_device(struct ibv_context *context)
+{
+	return mlx5_libverbs.ibv_close_device(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param list[in]
+ *   The list of devices.
+ */
+void ibv_free_device_list(struct ibv_device **list)
+{
+	return mlx5_libverbs.ibv_free_device_list(list);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   A device pointer.
+ * @return
+ *  The context on success.
+ */
+struct ibv_context *ibv_open_device(struct ibv_device *device)
+{
+	return mlx5_libverbs.ibv_open_device(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @return
+ *  protection domain pointer on success.
+ */
+struct ibv_pd *ibv_alloc_pd(struct ibv_context *context)
+{
+	return mlx5_libverbs.ibv_alloc_pd(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   An ibverbs device pointer.
+ * @return
+ *  device name on success.
+ */
+const char *ibv_get_device_name(struct ibv_device *device)
+{
+	return mlx5_libverbs.ibv_get_device_name(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @return
+ *  0 on success.
+ */
+int ibv_fork_init(void)
+{
+	return mlx5_libverbs.ibv_fork_init();
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A completion channel.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_comp_channel(struct ibv_comp_channel *channel)
+{
+	return mlx5_libverbs.ibv_destroy_comp_channel(channel);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param mr[in]
+ *   A pointer to memory region.
+ * @return
+ *  0 on success.
+ */
+int ibv_dereg_mr(struct ibv_mr *mr)
+{
+	return mlx5_libverbs.ibv_dereg_mr(mr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A pointer to ibverbs context.
+ * @return
+ *  0 on success.
+ */
+struct ibv_comp_channel *ibv_create_comp_channel(
+											  struct ibv_context *context)
+{
+	return mlx5_libverbs.ibv_create_comp_channel(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A pointer to completion channel.
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param cq_context[in]
+ *   A pointer to completion queue context.
+ * @return
+ *  0 on success.
+ */
+int ibv_get_cq_event(struct ibv_comp_channel *channel,
+					 struct ibv_cq **cq, void **cq_context)
+{
+	return mlx5_libverbs.ibv_get_cq_event(channel, cq, cq_context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param nevents[in]
+ *   The number of events.
+ * @return
+ *  0 on success.
+ */
+void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
+{
+	return mlx5_libverbs.ibv_ack_cq_events(cq, nevents);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param qp_init_attr[in]
+ *   The queue pair attributes.
+ * @return
+ *  A pointer to queue pair on success.
+ */
+struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
+							 struct ibv_qp_init_attr *qp_init_attr)
+{
+	return mlx5_libverbs.ibv_create_qp(pd, qp_init_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+void ibv_ack_async_event(struct ibv_async_event *event)
+{
+	return mlx5_libverbs.ibv_ack_async_event(event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A ibverbs context.
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+int ibv_get_async_event(struct ibv_context *context,
+						struct ibv_async_event *event)
+{
+	return mlx5_libverbs.ibv_get_async_event(context, event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param addr[in]
+ *   The address.
+ * @param length[in]
+ *   The length.
+ * @param access[in]
+ *   The access type.
+ * @return
+ *  A pointer to memory region on success.
+ */
+struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr,
+						  size_t length, int access)
+{
+	return mlx5_libverbs.ibv_reg_mr(pd, addr, length, access);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   Ibverbs context.
+ * @param cqe[in]
+ *   A completion queue entry.
+ * @param cq_context[in]
+ *   Completion queue context.
+ * @param channel[in]
+ *   The completion channel.
+ * @param comp_vector[in]
+ *   Completion vector.
+ * @return
+ *  0 on success.
+ */
+struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe,
+							 void *cq_context,
+							 struct ibv_comp_channel *channel,
+							 int comp_vector)
+{
+	return mlx5_libverbs.ibv_create_cq(context, cqe, cq_context, channel,
+									   comp_vector);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair.
+ * @param attr[in]
+ *   The queue pair attribute.
+ * @param attr_mask[in]
+ *   Attribute mask.
+ * @return
+ *  0 on success.
+ */
+int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+				  int attr_mask)
+{
+	return mlx5_libverbs.ibv_modify_qp(qp, attr, attr_mask);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @param port_num[in]
+ *   The port number.
+ * @param port_attr[in]
+ *   Port attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_port(struct ibv_context *context, uint8_t port_num,
+				   struct ibv_port_attr *port_attr)
+{
+	port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
+	port_attr->reserved   = 0;
+	return mlx5_libverbs.ibv_query_port(context, port_num, port_attr);
+}
diff --git a/drivers/net/mlx5/lib/mlx5_dll.h b/drivers/net/mlx5/lib/mlx5_dll.h
new file mode 100644
index 0000000..3add2e6
--- /dev/null
+++ b/drivers/net/mlx5/lib/mlx5_dll.h
@@ -0,0 +1,97 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef RTE_PMD_MLX5_IBVERBS_H_
+#define RTE_PMD_MLX5_IBVERBS_H_
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
+
+void *mlx5_lib_load(const char *name);
+int mlx5_lverbs_function_register(void *handle);
+int mlx5_lmlx5_function_register(void *handle);
+int mlx5_load_libs(void);
+#undef ibv_query_port
+
+struct {
+	int size;
+	int version;
+	int (*ibv_destroy_qp)(struct ibv_qp *qp);
+	int (*ibv_destroy_cq)(struct ibv_cq *cq);
+	int (*ibv_dealloc_pd)(struct ibv_pd *pd);
+	struct ibv_device **(*ibv_get_device_list)(int *num_devices);
+	int (*ibv_query_device)(struct ibv_context *context,
+							struct ibv_device_attr *device_attr);
+	int (*ibv_close_device)(struct ibv_context *context);
+	void (*ibv_free_device_list)(struct ibv_device **list);
+	struct ibv_context *(*ibv_open_device)(struct ibv_device *device);
+	struct ibv_pd *(*ibv_alloc_pd)(struct ibv_context *context);
+	const char *(*ibv_get_device_name)(struct ibv_device *device);
+	int (*ibv_fork_init)(void);
+	int (*ibv_destroy_comp_channel)(struct ibv_comp_channel *channel);
+	int (*ibv_dereg_mr)(struct ibv_mr *mr);
+	struct ibv_comp_channel *(*ibv_create_comp_channel)(
+							  struct ibv_context *context);
+	int(*ibv_get_cq_event)(struct ibv_comp_channel *channel,
+						   struct ibv_cq **cq, void **cq_context);
+	void (*ibv_ack_cq_events)(struct ibv_cq *cq, unsigned int nevents);
+	struct ibv_qp *(*ibv_create_qp)(struct ibv_pd *pd,
+									struct ibv_qp_init_attr *qp_init_attr);
+	void (*ibv_ack_async_event)(struct ibv_async_event *event);
+	int (*ibv_get_async_event)(struct ibv_context *context,
+							   struct ibv_async_event *event);
+	struct ibv_mr *(*ibv_reg_mr)(struct ibv_pd *pd, void *addr,
+								 size_t length,
+								 int access);
+	struct ibv_cq *(*ibv_create_cq)(struct ibv_context *context,
+									int cqe, void *cq_context,
+									struct ibv_comp_channel *channel,
+									int comp_vector);
+	int (*ibv_modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+						 int attr_mask);
+	int (*ibv_query_port)(struct ibv_context *context, uint8_t port_num,
+						struct ibv_port_attr *port_attr);
+} mlx5_libverbs;
+
+struct {
+	int size;
+	int version;
+	struct ibv_cq_ex *(*mlx5dv_create_cq)(struct ibv_context *context,
+								struct ibv_cq_init_attr_ex *cq_attr,
+								struct mlx5dv_cq_init_attr *mlx5_cq_attr);
+	int (*mlx5dv_init_obj)(struct mlx5dv_obj *obj, uint64_t obj_type);
+	int (*mlx5dv_query_device)(struct ibv_context *ctx_in,
+							   struct mlx5dv_context *attrs_out);
+	int (*mlx5dv_set_context_attr)(struct ibv_context *context,
+								   enum mlx5dv_set_ctx_attr_type type,
+								   void *attr);
+} mlx5_lmlx5;
+#endif /* RTE_PMD_MLX5_IBVERBS_H_ */
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 0548d17..323ced7 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -30,7 +30,8 @@
  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-
+#define _GNU_SOURCE
+#include <stdio.h>
 #include <stddef.h>
 #include <unistd.h>
 #include <string.h>
@@ -39,13 +40,17 @@
 #include <stdlib.h>
 #include <errno.h>
 #include <net/if.h>
-
+#include <dlfcn.h>
 /* Verbs header. */
 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
@@ -1029,6 +1034,9 @@ struct mlx5_args {
 rte_mlx5_pmd_init(void)
 {
 	/* Build the static table for ptype conversion. */
+#ifdef MLX5_PMD_DLL
+	int ret;
+#endif
 	mlx5_set_ptype_table();
 	/*
 	 * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
@@ -1040,6 +1048,11 @@ struct mlx5_args {
 	/* Match the size of Rx completion entry to the size of a cacheline. */
 	if (RTE_CACHE_LINE_SIZE == 128)
 		setenv("MLX5_CQE_SIZE", "128", 0);
+#ifdef MLX5_PMD_DLL
+	ret = mlx5_load_libs();
+	if (ret != 0)
+		return;
+#endif
 	ibv_fork_init();
 	rte_pci_register(&mlx5_driver);
 }
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index e6a69b8..1dde985 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -46,7 +46,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index f32dfdd..ffd419c 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -39,7 +39,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c
index d17b991..9e8902b 100644
--- a/drivers/net/mlx5/mlx5_mac.c
+++ b/drivers/net/mlx5/mlx5_mac.c
@@ -46,7 +46,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 6b29eed..c91fcdf 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -36,7 +36,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c
index f3de46d..9f58147 100644
--- a/drivers/net/mlx5/mlx5_rss.c
+++ b/drivers/net/mlx5/mlx5_rss.c
@@ -42,7 +42,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c
index 0ef2cdf..f9f7d6f 100644
--- a/drivers/net/mlx5/mlx5_rxmode.c
+++ b/drivers/net/mlx5/mlx5_rxmode.c
@@ -40,7 +40,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 85399ef..dfd6400 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -44,8 +44,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 2d30c50..e2215aa 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -41,8 +41,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index d34f3cc..7419f2c 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -43,12 +43,14 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
-#include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
-
 #include <rte_mbuf.h>
 #include <rte_mempool.h>
 #include <rte_common.h>
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c
index ba6c8ce..fc7f701 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec.c
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.c
@@ -41,8 +41,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 9c5860f..8f52525 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -44,7 +44,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 6a6a745..0542c9a 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -51,7 +51,9 @@ endif
 
 # Link only the libraries used in the application
 LDFLAGS += --as-needed
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+LDFLAGS += -ldl
+endif
 # default path for libs
 _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
 
@@ -142,7 +144,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI)        += -lrte_pmd_kni
 endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD)        += -lrte_pmd_lio
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -libverbs -lmlx4
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5
+else
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -libverbs -lmlx5
+endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD)       += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk
 _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD)        += -lrte_pmd_nfp
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL)       += -lrte_pmd_null
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 2/2] net/mlx4: load libmlx4 and libibverbs in run-time

[Shachar Beiser]

MLX4 PMD dynamically loads mlx4 and ibverbs in
run-time and it is not linked to external libraries.

Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
---
    History:
        V1: http://dpdk.org/dev/patchwork/patch/31555/
    V2:
        Fixing checkpatch code styling warnings
---
 config/common_base              |   1 +
 drivers/net/mlx4/Makefile       |  14 +-
 drivers/net/mlx4/lib/mlx4_dll.c | 731 ++++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx4/lib/mlx4_dll.h |  94 ++++++
 drivers/net/mlx4/mlx4.c         |  12 +
 drivers/net/mlx4/mlx4.h         |   4 +
 drivers/net/mlx4/mlx4_ethdev.c  |   4 +
 drivers/net/mlx4/mlx4_flow.c    |   4 +
 drivers/net/mlx4/mlx4_mr.c      |   4 +
 drivers/net/mlx4/mlx4_prm.h     |   4 +
 drivers/net/mlx4/mlx4_rxq.c     |   4 +
 drivers/net/mlx4/mlx4_rxtx.c    |   4 +
 drivers/net/mlx4/mlx4_rxtx.h    |   4 +
 drivers/net/mlx4/mlx4_txq.c     |   4 +
 drivers/net/mlx5/lib/mlx5_dll.c |   4 +-
 mk/rte.app.mk                   |   7 +
 16 files changed, 895 insertions(+), 4 deletions(-)
 create mode 100644 drivers/net/mlx4/lib/mlx4_dll.c
 create mode 100644 drivers/net/mlx4/lib/mlx4_dll.h

diff --git a/config/common_base b/config/common_base
index 3708de4..8ef6be4 100644
--- a/config/common_base
+++ b/config/common_base
@@ -229,6 +229,7 @@ CONFIG_RTE_LIBRTE_FM10K_INC_VECTOR=y
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
 #
 CONFIG_RTE_LIBRTE_MLX4_PMD=n
+CONFIG_RTE_LIBRTE_MLX4_DLL=y
 CONFIG_RTE_LIBRTE_MLX4_DEBUG=n
 CONFIG_RTE_LIBRTE_MLX4_DEBUG_BROKEN_VERBS=n
 CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile
index f1f47c2..aba1d5f 100644
--- a/drivers/net/mlx4/Makefile
+++ b/drivers/net/mlx4/Makefile
@@ -44,7 +44,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_rxq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_rxtx.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_txq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_utils.c
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += lib/mlx4_dll.c
+endif
 # Basic CFLAGS.
 CFLAGS += -O3
 CFLAGS += -std=c11 -Wall -Wextra
@@ -54,7 +56,11 @@ CFLAGS += -D_BSD_SOURCE
 CFLAGS += -D_DEFAULT_SOURCE
 CFLAGS += -D_XOPEN_SOURCE=600
 CFLAGS += $(WERROR_FLAGS)
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+LDLIBS += -ldl
+else
 LDLIBS += -libverbs -lmlx4
+endif
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_pci
@@ -81,7 +87,11 @@ endif
 ifdef CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE
 CFLAGS += -DMLX4_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE)
 endif
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+CFLAGS += -DMLX4_PMD_DLL
+else
+CFLAGS += -UMLX4_PMD_DLL
+endif
 ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DEBUG_BROKEN_VERBS),y)
 CFLAGS += -DMLX4_PMD_DEBUG_BROKEN_VERBS
 endif
diff --git a/drivers/net/mlx4/lib/mlx4_dll.c b/drivers/net/mlx4/lib/mlx4_dll.c
new file mode 100644
index 0000000..0d8e7a9
--- /dev/null
+++ b/drivers/net/mlx4/lib/mlx4_dll.c
@@ -0,0 +1,731 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <dlfcn.h>
+#include <rte_log.h>
+#include "../mlx4_utils.h"
+#include "mlx4_dll.h"
+
+#define VERBS_LIB_DIR "/usr/lib64/libibverbs"
+#define MLX5_LIB_DIR "/usr/lib64/libmlx4"
+#define DIR_LENGTH 25
+/**
+ * Load a libibverbs and libmlx4 symbols table.
+ *
+ * @return
+ *   0 on success.
+ */
+int mlx4_load_libs(void)
+{
+	void *dlhandle;
+	int ret;
+
+	dlhandle = mlx4_lib_load("ibverbs.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load ibverbs.so");
+		return -1;
+	}
+	ret = mlx4_lverbs_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in libverbs.so ");
+		return ret;
+	}
+	dlhandle = mlx4_lib_load("mlx4.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load mlx4.so");
+		return -1;
+	}
+	ret = mlx4_lmlx4_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in lmlx4.so  ");
+		return ret;
+	}
+	return 0;
+}
+/**
+ * Load a libibverbs or libmlx4 symbols table.
+ *
+ * @param name[in]
+ *   The library name.
+ * @return
+ *   dlhandle on success.
+ */
+void *mlx4_lib_load(const char *name)
+{
+	char *so_name;
+	void *dlhandle;
+	char lib[DIR_LENGTH];
+
+	if (strcmp(name, "ibverbs.so"))
+		strcpy(lib, VERBS_LIB_DIR);
+	else
+		strcpy(lib, MLX5_LIB_DIR);
+	/* If the name is an absolute path then open that path after appending
+	 * the trailer suffix
+	 */
+	if (name[0] == '/') {
+		if (asprintf(&so_name, "%s", name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		if (!dlhandle)
+			goto out_dlopen;
+		free(so_name);
+		return dlhandle;
+	}
+	/* If configured with a provider plugin path then try that next */
+	if (strlen(lib) > 1) {
+		if (asprintf(&so_name, "%s/lib%s", lib, name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		free(so_name);
+		if (dlhandle)
+			return dlhandle;
+	}
+	/* Otherwise use the system libary search path. This is the historical
+	 * behavior of libibverbs
+	 */
+	if (asprintf(&so_name, "lib%s", name) < 0)
+		goto out_asprintf;
+	dlhandle = dlopen(so_name, RTLD_LAZY);
+	if (!dlhandle)
+		goto out_dlopen;
+	free(so_name);
+	return dlhandle;
+out_asprintf:
+	ERROR("Warning: couldn't load driver '%s'.\n", name);
+	return NULL;
+out_dlopen:
+	ERROR("Warning: couldn't load driver '%s': %s\n", so_name, dlerror());
+	free(so_name);
+	return NULL;
+}
+
+/**
+ * Register libibverbs functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx4_lverbs_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx4_libverbs.ibv_destroy_qp) = dlsym(handle,
+													  "ibv_destroy_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_destroy_cq) = dlsym(handle,
+													  "ibv_destroy_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_dealloc_pd) = dlsym(handle,
+													  "ibv_dealloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_device_list) =
+									dlsym(handle, "ibv_get_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_open_device) = dlsym(handle,
+													   "ibv_open_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_query_device) = dlsym(handle,
+														"ibv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_close_device) = dlsym(handle,
+														"ibv_close_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_free_device_list) =
+										dlsym(handle, "ibv_free_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_alloc_pd) = dlsym(handle, "ibv_alloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_device_name) =
+									dlsym(handle, "ibv_get_device_name");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_fork_init) = dlsym(handle,
+													 "ibv_fork_init");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_destroy_comp_channel) =
+									dlsym(handle, "ibv_destroy_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_dereg_mr) = dlsym(handle, "ibv_dereg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_create_comp_channel) =
+									dlsym(handle, "ibv_create_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_cq_event) = dlsym(handle,
+														"ibv_get_cq_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_create_qp) = dlsym(handle,
+													 "ibv_create_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_ack_async_event) =
+									dlsym(handle, "ibv_ack_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_async_event) =
+									dlsym(handle, "ibv_get_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_reg_mr) = dlsym(handle, "ibv_reg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_create_cq) = dlsym(handle,
+													 "ibv_create_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_modify_qp) = dlsym(handle,
+													 "ibv_modify_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_query_port) = dlsym(handle,
+													  "ibv_query_port");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+
+/**
+ * Register libmlx4 functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx4_lmlx4_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx4_lmlx4.mlx4dv_init_obj) = dlsym(handle,
+													"mlx4dv_init_obj");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_lmlx4.mlx4dv_query_device) = dlsym(handle,
+														"mlx4dv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_lmlx4.mlx4dv_set_context_attr) =
+								 dlsym(handle, "mlx4dv_set_context_attr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+/**
+ * Function pointer calls mlx4 API .
+ *
+ * @param mlx4dv_obj[in]
+ *   The mlx4 object.
+ * @param obj_type[in]
+ *   The object type.
+ * @return
+ *   a pointer on success.
+ */
+int mlx4dv_init_obj(struct mlx4dv_obj *obj, uint64_t obj_type)
+{
+	return mlx4_lmlx4.mlx4dv_init_obj(obj, obj_type);
+}
+/**
+ * Function pointer calls mlx4 API .
+ *
+ * @param ctx_in[in]
+ *   The ibverbs context.
+ * @param attrs_out[out]
+ *   The mlx4 context.
+ * @return
+ *  0 on success.
+ */
+int mlx4dv_query_device(struct ibv_context *ctx_in,
+						struct mlx4dv_context *attrs_out)
+{
+	return mlx4_lmlx4.mlx4dv_query_device(ctx_in, attrs_out);
+}
+/**
+ * Function pointer calls mlx4 API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param type[in]
+ *   The mlx4 context attribute type.
+ * @param attr[in]
+ *   The mlx4 context attribute.
+ * @return
+ *  0 on success.
+ */
+int mlx4dv_set_context_attr(struct ibv_context *context,
+							enum mlx4dv_set_ctx_attr_type type,
+							void *attr)
+{
+	return mlx4_lmlx4.mlx4dv_set_context_attr(context, type, attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_qp(struct ibv_qp *qp)
+{
+	return mlx4_libverbs.ibv_destroy_qp(qp);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   Completion queue pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_cq(struct ibv_cq *cq)
+{
+	return mlx4_libverbs.ibv_destroy_cq(cq);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   Protection domain.
+ * @return
+ *  0 on success.
+ */
+int ibv_dealloc_pd(struct ibv_pd *pd)
+{
+	return mlx4_libverbs.ibv_dealloc_pd(pd);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param num_devices[in]
+ *   The number of devices.
+ * @return
+ *  0 on success.
+ */
+struct ibv_device **ibv_get_device_list(int *num_devices)
+{
+	return mlx4_libverbs.ibv_get_device_list(num_devices);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param device_attr[in]
+ *   The device attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_device(struct ibv_context *context,
+					 struct ibv_device_attr *device_attr)
+{
+	return mlx4_libverbs.ibv_query_device(context, device_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @return
+ *  0 on success.
+ */
+int ibv_close_device(struct ibv_context *context)
+{
+	return mlx4_libverbs.ibv_close_device(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param list[in]
+ *   The list of devices.
+ */
+void ibv_free_device_list(struct ibv_device **list)
+{
+	return mlx4_libverbs.ibv_free_device_list(list);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   A device pointer.
+ * @return
+ *  The context on success.
+ */
+struct ibv_context *ibv_open_device(struct ibv_device *device)
+{
+	return mlx4_libverbs.ibv_open_device(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @return
+ *  protection domain pointer on success.
+ */
+struct ibv_pd *ibv_alloc_pd(struct ibv_context *context)
+{
+	return mlx4_libverbs.ibv_alloc_pd(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   An ibverbs device pointer.
+ * @return
+ *  device name on success.
+ */
+const char *ibv_get_device_name(struct ibv_device *device)
+{
+	return mlx4_libverbs.ibv_get_device_name(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @return
+ *  0 on success.
+ */
+int ibv_fork_init(void)
+{
+	return mlx4_libverbs.ibv_fork_init();
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A completion channel.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_comp_channel(struct ibv_comp_channel *channel)
+{
+	return mlx4_libverbs.ibv_destroy_comp_channel(channel);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param mr[in]
+ *   A pointer to memory region.
+ * @return
+ *  0 on success.
+ */
+int ibv_dereg_mr(struct ibv_mr *mr)
+{
+	return mlx4_libverbs.ibv_dereg_mr(mr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A pointer to ibverbs context.
+ * @return
+ *  0 on success.
+ */
+struct ibv_comp_channel *ibv_create_comp_channel(
+											  struct ibv_context *context)
+{
+	return mlx4_libverbs.ibv_create_comp_channel(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A pointer to completion channel.
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param cq_context[in]
+ *   A pointer to completion queue context.
+ * @return
+ *  0 on success.
+ */
+int ibv_get_cq_event(struct ibv_comp_channel *channel,
+					 struct ibv_cq **cq, void **cq_context)
+{
+	return mlx4_libverbs.ibv_get_cq_event(channel, cq, cq_context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param nevents[in]
+ *   The number of events.
+ * @return
+ *  0 on success.
+ */
+void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
+{
+	return mlx4_libverbs.ibv_ack_cq_events(cq, nevents);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param qp_init_attr[in]
+ *   The queue pair attributes.
+ * @return
+ *  A pointer to queue pair on success.
+ */
+struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
+							 struct ibv_qp_init_attr *qp_init_attr)
+{
+	return mlx4_libverbs.ibv_create_qp(pd, qp_init_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+void ibv_ack_async_event(struct ibv_async_event *event)
+{
+	return mlx4_libverbs.ibv_ack_async_event(event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A ibverbs context.
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+int ibv_get_async_event(struct ibv_context *context,
+						struct ibv_async_event *event)
+{
+	return mlx4_libverbs.ibv_get_async_event(context, event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param addr[in]
+ *   The address.
+ * @param length[in]
+ *   The length.
+ * @param access[in]
+ *   The access type.
+ * @return
+ *  A pointer to memory region on success.
+ */
+struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr,
+						  size_t length, int access)
+{
+	return mlx4_libverbs.ibv_reg_mr(pd, addr, length, access);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   Ibverbs context.
+ * @param cqe[in]
+ *   A completion queue entry.
+ * @param cq_context[in]
+ *   Completion queue context.
+ * @param channel[in]
+ *   The completion channel.
+ * @param comp_vector[in]
+ *   Completion vector.
+ * @return
+ *  0 on success.
+ */
+struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe,
+							 void *cq_context,
+							 struct ibv_comp_channel *channel,
+							 int comp_vector)
+{
+	return mlx4_libverbs.ibv_create_cq(context, cqe, cq_context, channel,
+									   comp_vector);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair.
+ * @param attr[in]
+ *   The queue pair attribute.
+ * @param attr_mask[in]
+ *   Attribute mask.
+ * @return
+ *  0 on success.
+ */
+int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+				  int attr_mask)
+{
+	return mlx4_libverbs.ibv_modify_qp(qp, attr, attr_mask);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @param port_num[in]
+ *   The port number.
+ * @param port_attr[in]
+ *   Port attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_port(struct ibv_context *context, uint8_t port_num,
+				   struct ibv_port_attr *port_attr)
+{
+	port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
+	port_attr->reserved   = 0;
+	return mlx4_libverbs.ibv_query_port(context, port_num, port_attr);
+}
diff --git a/drivers/net/mlx4/lib/mlx4_dll.h b/drivers/net/mlx4/lib/mlx4_dll.h
new file mode 100644
index 0000000..e95b78e
--- /dev/null
+++ b/drivers/net/mlx4/lib/mlx4_dll.h
@@ -0,0 +1,94 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef RTE_PMD_MLX4_IBVERBS_H_
+#define RTE_PMD_MLX4_IBVERBS_H_
+#include <infiniband/verbs.h>
+#include <infiniband/mlx4dv.h>
+
+void *mlx4_lib_load(const char *name);
+int mlx4_lverbs_function_register(void *handle);
+int mlx4_lmlx4_function_register(void *handle);
+int mlx4_load_libs(void);
+#undef ibv_query_port
+
+struct {
+	int size;
+	int version;
+	int (*ibv_destroy_qp)(struct ibv_qp *qp);
+	int (*ibv_destroy_cq)(struct ibv_cq *cq);
+	int (*ibv_dealloc_pd)(struct ibv_pd *pd);
+	struct ibv_device **(*ibv_get_device_list)(int *num_devices);
+	int (*ibv_query_device)(struct ibv_context *context,
+							struct ibv_device_attr *device_attr);
+	int (*ibv_close_device)(struct ibv_context *context);
+	void (*ibv_free_device_list)(struct ibv_device **list);
+	struct ibv_context *(*ibv_open_device)(struct ibv_device *device);
+	struct ibv_pd *(*ibv_alloc_pd)(struct ibv_context *context);
+	const char *(*ibv_get_device_name)(struct ibv_device *device);
+	int (*ibv_fork_init)(void);
+	int (*ibv_destroy_comp_channel)(struct ibv_comp_channel *channel);
+	int (*ibv_dereg_mr)(struct ibv_mr *mr);
+	struct ibv_comp_channel *(*ibv_create_comp_channel)(
+							  struct ibv_context *context);
+	int(*ibv_get_cq_event)(struct ibv_comp_channel *channel,
+						   struct ibv_cq **cq, void **cq_context);
+	void (*ibv_ack_cq_events)(struct ibv_cq *cq, unsigned int nevents);
+	struct ibv_qp *(*ibv_create_qp)(struct ibv_pd *pd,
+									struct ibv_qp_init_attr *qp_init_attr);
+	void (*ibv_ack_async_event)(struct ibv_async_event *event);
+	int (*ibv_get_async_event)(struct ibv_context *context,
+							   struct ibv_async_event *event);
+	struct ibv_mr *(*ibv_reg_mr)(struct ibv_pd *pd, void *addr,
+								 size_t length,
+								 int access);
+	struct ibv_cq *(*ibv_create_cq)(struct ibv_context *context,
+									int cqe, void *cq_context,
+									struct ibv_comp_channel *channel,
+									int comp_vector);
+	int (*ibv_modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+						 int attr_mask);
+	int (*ibv_query_port)(struct ibv_context *context, uint8_t port_num,
+						struct ibv_port_attr *port_attr);
+}mlx4_libverbs;
+
+struct {
+	int size;
+	int version;
+	int (*mlx4dv_init_obj)(struct mlx4dv_obj *obj, uint64_t obj_type);
+	int (*mlx4dv_query_device)(struct ibv_context *ctx_in,
+							   struct mlx4dv_context *attrs_out);
+	int (*mlx4dv_set_context_attr)(struct ibv_context *context,
+								enum mlx4dv_set_ctx_attr_type type,
+								void *attr);
+}mlx4_lmlx4;
+#endif /* RTE_PMD_MLX4_IBVERBS_H_ */
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index f9e4f9d..8fee391 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -49,7 +49,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
@@ -713,7 +717,15 @@ struct mlx4_conf {
 	 * applications to use fork() safely for purposes other than
 	 * using this PMD, which is not supported in forked processes.
 	 */
+#ifdef MLX4_PMD_DLL
+	int ret;
+#endif
 	setenv("RDMAV_HUGEPAGES_SAFE", "1", 1);
+#ifdef MLX4_PMD_DLL
+	ret = mlx4_load_libs();
+	if ( ret != 0 )
+		return;
+#endif
 	ibv_fork_init();
 	rte_pci_register(&mlx4_driver);
 }
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 3aeef87..6e4565b 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -42,7 +42,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_ethdev.c b/drivers/net/mlx4/mlx4_ethdev.c
index 2f69e7d..98134ab 100644
--- a/drivers/net/mlx4/mlx4_ethdev.c
+++ b/drivers/net/mlx4/mlx4_ethdev.c
@@ -56,7 +56,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_flow.c b/drivers/net/mlx4/mlx4_flow.c
index 8b87b29..89d6d17 100644
--- a/drivers/net/mlx4/mlx4_flow.c
+++ b/drivers/net/mlx4/mlx4_flow.c
@@ -49,7 +49,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_mr.c b/drivers/net/mlx4/mlx4_mr.c
index 2a3e269..b494e95 100644
--- a/drivers/net/mlx4/mlx4_mr.c
+++ b/drivers/net/mlx4/mlx4_mr.c
@@ -47,7 +47,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_prm.h b/drivers/net/mlx4/mlx4_prm.h
index fcc7c12..b8854e1 100644
--- a/drivers/net/mlx4/mlx4_prm.h
+++ b/drivers/net/mlx4/mlx4_prm.h
@@ -42,8 +42,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PRD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c
index 53313c5..5680c89 100644
--- a/drivers/net/mlx4/mlx4_rxq.c
+++ b/drivers/net/mlx4/mlx4_rxq.c
@@ -46,8 +46,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_rxtx.c b/drivers/net/mlx4/mlx4_rxtx.c
index 2bfa8b1..a876896 100644
--- a/drivers/net/mlx4/mlx4_rxtx.c
+++ b/drivers/net/mlx4/mlx4_rxtx.c
@@ -44,7 +44,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h
index 463df2b..c115032 100644
--- a/drivers/net/mlx4/mlx4_rxtx.h
+++ b/drivers/net/mlx4/mlx4_rxtx.h
@@ -41,8 +41,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c
index 7882a4d..8ae9eb9 100644
--- a/drivers/net/mlx4/mlx4_txq.c
+++ b/drivers/net/mlx4/mlx4_txq.c
@@ -46,7 +46,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/lib/mlx5_dll.c b/drivers/net/mlx5/lib/mlx5_dll.c
index 2228ad1..d0ff97f 100644
--- a/drivers/net/mlx5/lib/mlx5_dll.c
+++ b/drivers/net/mlx5/lib/mlx5_dll.c
@@ -115,7 +115,7 @@ void *mlx5_lib_load(const char *name)
 		if (dlhandle)
 			return dlhandle;
 	}
-	/* Otherwise use the system libary search path. This is the historical
+	/* Otherwise use the system library search path. This is the historical
 	 * behavior of libibverbs
 	 */
 	if (asprintf(&so_name, "lib%s", name) < 0)
@@ -204,7 +204,7 @@ int mlx5_lverbs_function_register(void *handle)
 		goto exit;
 	}
 	*(void **)(&mlx5_libverbs.ibv_free_device_list) =
-										dlsym(handle, "ibv_free_device_list");
+									dlsym(handle, "ibv_free_device_list");
 	error = dlerror();
 	if ((error) != NULL) {
 		ERROR("%s\n", error);
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 0542c9a..5589482 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -54,6 +54,9 @@ LDFLAGS += --as-needed
 ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
 LDFLAGS += -ldl
 endif
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+LDFLAGS += -ldl
+endif
 # default path for libs
 _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
 
@@ -143,7 +146,11 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI)        += -lrte_pmd_kni
 endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD)        += -lrte_pmd_lio
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4
+else
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -libverbs -lmlx4
+endif
 ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5
 else
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 1/2] net/mlx5: load libmlx5 and libibverbs in run-time

[Shachar Beiser]

MLX5 PMD loads libraries: libibverbs and libmlx5.
MLX5 PMD is not linked to external libraries.

Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
---
History: 
     V1: 
       http://dpdk.org/patch/31555
V2:
* Fixing checkpatch coding style issues 

---
 config/common_base               |   1 +
 drivers/net/mlx5/Makefile        |  27 +-
 drivers/net/mlx5/lib/mlx5_dll.c  | 758 +++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/lib/mlx5_dll.h  |  97 +++++
 drivers/net/mlx5/mlx5.c          |  17 +-
 drivers/net/mlx5/mlx5.h          |   4 +
 drivers/net/mlx5/mlx5_flow.c     |   4 +
 drivers/net/mlx5/mlx5_mac.c      |   4 +
 drivers/net/mlx5/mlx5_mr.c       |   4 +
 drivers/net/mlx5/mlx5_rss.c      |   4 +
 drivers/net/mlx5/mlx5_rxmode.c   |   4 +
 drivers/net/mlx5/mlx5_rxq.c      |   4 +
 drivers/net/mlx5/mlx5_rxtx.c     |   4 +
 drivers/net/mlx5/mlx5_rxtx.h     |   6 +-
 drivers/net/mlx5/mlx5_rxtx_vec.c |   4 +
 drivers/net/mlx5/mlx5_txq.c      |   4 +
 mk/rte.app.mk                    |   8 +-
 17 files changed, 941 insertions(+), 13 deletions(-)
 create mode 100644 drivers/net/mlx5/lib/mlx5_dll.c
 create mode 100644 drivers/net/mlx5/lib/mlx5_dll.h

diff --git a/config/common_base b/config/common_base
index e74febe..3708de4 100644
--- a/config/common_base
+++ b/config/common_base
@@ -237,6 +237,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
 # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
 #
 CONFIG_RTE_LIBRTE_MLX5_PMD=n
+CONFIG_RTE_LIBRTE_MLX5_DLL=y
 CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
 CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
 
diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index a3984eb..1dc0a05 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -53,7 +53,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c
+endif
 # Basic CFLAGS.
 CFLAGS += -O3
 CFLAGS += -std=c11 -Wall -Wextra
@@ -64,7 +66,11 @@ CFLAGS += -D_DEFAULT_SOURCE
 CFLAGS += -D_XOPEN_SOURCE=600
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -Wno-strict-prototypes
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+LDLIBS += -ldl
+else
 LDLIBS += -libverbs -lmlx5
+endif
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_pci
@@ -91,7 +97,11 @@ endif
 ifdef CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE
 CFLAGS += -DMLX5_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE)
 endif
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+CFLAGS += -DMLX5_PMD_DLL
+else
+CFLAGS += -UMLX5_PMD_DLL
+endif
 include $(RTE_SDK)/mk/rte.lib.mk
 
 # Generate and clean-up mlx5_autoconf.h.
@@ -105,26 +115,28 @@ endif
 
 mlx5_autoconf.h.new: FORCE
 
+VERBS_H := infiniband/verbs.h
+MLX5DV_H := infiniband/mlx5dv.h
 mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 	$Q $(RM) -f -- '$@'
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_DEVICE_VXLAN_SUPPORT \
-		infiniband/verbs.h \
+		$(VERBS_H) \
 		enum IBV_DEVICE_VXLAN_SUPPORT \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_WQ_FLAG_RX_END_PADDING \
-		infiniband/verbs.h \
+		$(VERBS_H) \
 		enum IBV_WQ_FLAG_RX_END_PADDING \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_MLX5_MOD_MPW \
-		infiniband/mlx5dv.h \
+		$(MLX5DV_H) \
 		enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
-		infiniband/mlx5dv.h \
+		$(MLX5DV_H) \
 		enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
@@ -144,10 +156,9 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
-		infiniband/verbs.h \
+		$(VERBS_H)  \
 		enum IBV_FLOW_SPEC_ACTION_COUNT \
 		$(AUTOCONF_OUTPUT)
-
 # Create mlx5_autoconf.h or update it in case it differs from the new one.
 
 mlx5_autoconf.h: mlx5_autoconf.h.new
diff --git a/drivers/net/mlx5/lib/mlx5_dll.c b/drivers/net/mlx5/lib/mlx5_dll.c
new file mode 100644
index 0000000..2228ad1
--- /dev/null
+++ b/drivers/net/mlx5/lib/mlx5_dll.c
@@ -0,0 +1,758 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <dlfcn.h>
+#include <rte_log.h>
+#include <mlx5_utils.h>
+#include "mlx5_dll.h"
+
+#define VERBS_LIB_DIR "/usr/lib64/libibverbs"
+#define MLX5_LIB_DIR "/usr/lib64/libmlx5"
+#define DIR_LENGTH 25
+/**
+ * Load a libibverbs and libmlx5 symbols table.
+ *
+ * @return
+ *   0 on success.
+ */
+int mlx5_load_libs(void)
+{
+	void *dlhandle;
+	int ret;
+
+	dlhandle = mlx5_lib_load("ibverbs.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load ibverbs.so");
+		return -1;
+	}
+	ret = mlx5_lverbs_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in libverbs.so ");
+		return ret;
+	}
+	dlhandle = mlx5_lib_load("mlx5.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load mlx5.so");
+		return -1;
+	}
+	ret = mlx5_lmlx5_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in lmlx5.so  ");
+		return ret;
+	}
+	return 0;
+}
+/**
+ * Load a libibverbs or libmlx5 symbols table.
+ *
+ * @param name[in]
+ *   The library name.
+ * @return
+ *   dlhandle on success.
+ */
+void *mlx5_lib_load(const char *name)
+{
+	char *so_name;
+	void *dlhandle;
+	char lib[DIR_LENGTH];
+
+	if (strcmp(name, "ibverbs.so"))
+		strcpy(lib, VERBS_LIB_DIR);
+	else
+		strcpy(lib, MLX5_LIB_DIR);
+	/* If the name is an absolute path then open that path after appending
+	 * the trailer suffix
+	 */
+	if (name[0] == '/') {
+		if (asprintf(&so_name, "%s", name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		if (!dlhandle)
+			goto out_dlopen;
+		free(so_name);
+		return dlhandle;
+	}
+	/* If configured with a provider plugin path then try that next */
+	if (strlen(lib) > 1) {
+		if (asprintf(&so_name, "%s/lib%s", lib, name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		free(so_name);
+		if (dlhandle)
+			return dlhandle;
+	}
+	/* Otherwise use the system libary search path. This is the historical
+	 * behavior of libibverbs
+	 */
+	if (asprintf(&so_name, "lib%s", name) < 0)
+		goto out_asprintf;
+	dlhandle = dlopen(so_name, RTLD_LAZY);
+	if (!dlhandle)
+		goto out_dlopen;
+	free(so_name);
+	return dlhandle;
+out_asprintf:
+	ERROR("Warning: couldn't load driver '%s'.\n", name);
+	return NULL;
+out_dlopen:
+	ERROR("Warning: couldn't load driver '%s': %s\n", so_name, dlerror());
+	free(so_name);
+	return NULL;
+}
+
+/**
+ * Register libibverbs functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx5_lverbs_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx5_libverbs.ibv_destroy_qp) = dlsym(handle,
+													  "ibv_destroy_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_destroy_cq) = dlsym(handle,
+													  "ibv_destroy_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_dealloc_pd) = dlsym(handle,
+													  "ibv_dealloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_device_list) =
+									dlsym(handle, "ibv_get_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_open_device) = dlsym(handle,
+													   "ibv_open_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_query_device) = dlsym(handle,
+														"ibv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_close_device) = dlsym(handle,
+														"ibv_close_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_free_device_list) =
+										dlsym(handle, "ibv_free_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_alloc_pd) = dlsym(handle, "ibv_alloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_device_name) =
+									dlsym(handle, "ibv_get_device_name");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_fork_init) = dlsym(handle,
+													 "ibv_fork_init");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_destroy_comp_channel) =
+									dlsym(handle, "ibv_destroy_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_dereg_mr) = dlsym(handle, "ibv_dereg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_create_comp_channel) =
+									dlsym(handle, "ibv_create_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_cq_event) = dlsym(handle,
+														"ibv_get_cq_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_create_qp) = dlsym(handle,
+													 "ibv_create_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_ack_async_event) =
+									dlsym(handle, "ibv_ack_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_async_event) =
+									dlsym(handle, "ibv_get_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_reg_mr) = dlsym(handle, "ibv_reg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_create_cq) = dlsym(handle,
+													 "ibv_create_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_modify_qp) = dlsym(handle,
+													 "ibv_modify_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_query_port) = dlsym(handle,
+													  "ibv_query_port");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+
+/**
+ * Register libmlx5 functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx5_lmlx5_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx5_lmlx5.mlx5dv_create_cq) = dlsym(handle,
+													 "mlx5dv_create_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_lmlx5.mlx5dv_init_obj) = dlsym(handle,
+													"mlx5dv_init_obj");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_lmlx5.mlx5dv_query_device) = dlsym(handle,
+														"mlx5dv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_lmlx5.mlx5dv_set_context_attr) =
+								 dlsym(handle, "mlx5dv_set_context_attr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param context[in]
+ *   The ibv context.
+ * @param cq_attr[in]
+ *   The completion queue attribute.
+ * @param mlx5_cq_attr[in]
+ *   The mlx5 completion queue attribute.
+ * @return
+ *   a pointer on success.
+ */
+struct ibv_cq_ex *mlx5dv_create_cq(struct ibv_context *context,
+								   struct ibv_cq_init_attr_ex *cq_attr,
+								   struct mlx5dv_cq_init_attr *mlx5_cq_attr)
+{
+	return mlx5_lmlx5.mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr);
+}
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param mlx5dv_obj[in]
+ *   The mlx5 object.
+ * @param obj_type[in]
+ *   The object type.
+ * @return
+ *   a pointer on success.
+ */
+int mlx5dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type)
+{
+	return mlx5_lmlx5.mlx5dv_init_obj(obj, obj_type);
+}
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param ctx_in[in]
+ *   The ibverbs context.
+ * @param attrs_out[out]
+ *   The mlx5 context.
+ * @return
+ *  0 on success.
+ */
+int mlx5dv_query_device(struct ibv_context *ctx_in,
+						struct mlx5dv_context *attrs_out)
+{
+	return mlx5_lmlx5.mlx5dv_query_device(ctx_in, attrs_out);
+}
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param type[in]
+ *   The mlx5 context attribute type.
+ * @param attr[in]
+ *   The mlx5 context attribute.
+ * @return
+ *  0 on success.
+ */
+int mlx5dv_set_context_attr(struct ibv_context *context,
+							enum mlx5dv_set_ctx_attr_type type,
+							void *attr)
+{
+	return mlx5_lmlx5.mlx5dv_set_context_attr(context, type, attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_qp(struct ibv_qp *qp)
+{
+	return mlx5_libverbs.ibv_destroy_qp(qp);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   Completion queue pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_cq(struct ibv_cq *cq)
+{
+	return mlx5_libverbs.ibv_destroy_cq(cq);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   Protection domain.
+ * @return
+ *  0 on success.
+ */
+int ibv_dealloc_pd(struct ibv_pd *pd)
+{
+	return mlx5_libverbs.ibv_dealloc_pd(pd);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param num_devices[in]
+ *   The number of devices.
+ * @return
+ *  0 on success.
+ */
+struct ibv_device **ibv_get_device_list(int *num_devices)
+{
+	return mlx5_libverbs.ibv_get_device_list(num_devices);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param device_attr[in]
+ *   The device attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_device(struct ibv_context *context,
+					 struct ibv_device_attr *device_attr)
+{
+	return mlx5_libverbs.ibv_query_device(context, device_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @return
+ *  0 on success.
+ */
+int ibv_close_device(struct ibv_context *context)
+{
+	return mlx5_libverbs.ibv_close_device(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param list[in]
+ *   The list of devices.
+ */
+void ibv_free_device_list(struct ibv_device **list)
+{
+	return mlx5_libverbs.ibv_free_device_list(list);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   A device pointer.
+ * @return
+ *  The context on success.
+ */
+struct ibv_context *ibv_open_device(struct ibv_device *device)
+{
+	return mlx5_libverbs.ibv_open_device(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @return
+ *  protection domain pointer on success.
+ */
+struct ibv_pd *ibv_alloc_pd(struct ibv_context *context)
+{
+	return mlx5_libverbs.ibv_alloc_pd(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   An ibverbs device pointer.
+ * @return
+ *  device name on success.
+ */
+const char *ibv_get_device_name(struct ibv_device *device)
+{
+	return mlx5_libverbs.ibv_get_device_name(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @return
+ *  0 on success.
+ */
+int ibv_fork_init(void)
+{
+	return mlx5_libverbs.ibv_fork_init();
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A completion channel.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_comp_channel(struct ibv_comp_channel *channel)
+{
+	return mlx5_libverbs.ibv_destroy_comp_channel(channel);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param mr[in]
+ *   A pointer to memory region.
+ * @return
+ *  0 on success.
+ */
+int ibv_dereg_mr(struct ibv_mr *mr)
+{
+	return mlx5_libverbs.ibv_dereg_mr(mr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A pointer to ibverbs context.
+ * @return
+ *  0 on success.
+ */
+struct ibv_comp_channel *ibv_create_comp_channel(
+											  struct ibv_context *context)
+{
+	return mlx5_libverbs.ibv_create_comp_channel(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A pointer to completion channel.
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param cq_context[in]
+ *   A pointer to completion queue context.
+ * @return
+ *  0 on success.
+ */
+int ibv_get_cq_event(struct ibv_comp_channel *channel,
+					 struct ibv_cq **cq, void **cq_context)
+{
+	return mlx5_libverbs.ibv_get_cq_event(channel, cq, cq_context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param nevents[in]
+ *   The number of events.
+ * @return
+ *  0 on success.
+ */
+void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
+{
+	return mlx5_libverbs.ibv_ack_cq_events(cq, nevents);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param qp_init_attr[in]
+ *   The queue pair attributes.
+ * @return
+ *  A pointer to queue pair on success.
+ */
+struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
+							 struct ibv_qp_init_attr *qp_init_attr)
+{
+	return mlx5_libverbs.ibv_create_qp(pd, qp_init_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+void ibv_ack_async_event(struct ibv_async_event *event)
+{
+	return mlx5_libverbs.ibv_ack_async_event(event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A ibverbs context.
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+int ibv_get_async_event(struct ibv_context *context,
+						struct ibv_async_event *event)
+{
+	return mlx5_libverbs.ibv_get_async_event(context, event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param addr[in]
+ *   The address.
+ * @param length[in]
+ *   The length.
+ * @param access[in]
+ *   The access type.
+ * @return
+ *  A pointer to memory region on success.
+ */
+struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr,
+						  size_t length, int access)
+{
+	return mlx5_libverbs.ibv_reg_mr(pd, addr, length, access);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   Ibverbs context.
+ * @param cqe[in]
+ *   A completion queue entry.
+ * @param cq_context[in]
+ *   Completion queue context.
+ * @param channel[in]
+ *   The completion channel.
+ * @param comp_vector[in]
+ *   Completion vector.
+ * @return
+ *  0 on success.
+ */
+struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe,
+							 void *cq_context,
+							 struct ibv_comp_channel *channel,
+							 int comp_vector)
+{
+	return mlx5_libverbs.ibv_create_cq(context, cqe, cq_context, channel,
+									   comp_vector);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair.
+ * @param attr[in]
+ *   The queue pair attribute.
+ * @param attr_mask[in]
+ *   Attribute mask.
+ * @return
+ *  0 on success.
+ */
+int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+				  int attr_mask)
+{
+	return mlx5_libverbs.ibv_modify_qp(qp, attr, attr_mask);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @param port_num[in]
+ *   The port number.
+ * @param port_attr[in]
+ *   Port attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_port(struct ibv_context *context, uint8_t port_num,
+				   struct ibv_port_attr *port_attr)
+{
+	port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
+	port_attr->reserved   = 0;
+	return mlx5_libverbs.ibv_query_port(context, port_num, port_attr);
+}
diff --git a/drivers/net/mlx5/lib/mlx5_dll.h b/drivers/net/mlx5/lib/mlx5_dll.h
new file mode 100644
index 0000000..3add2e6
--- /dev/null
+++ b/drivers/net/mlx5/lib/mlx5_dll.h
@@ -0,0 +1,97 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef RTE_PMD_MLX5_IBVERBS_H_
+#define RTE_PMD_MLX5_IBVERBS_H_
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
+
+void *mlx5_lib_load(const char *name);
+int mlx5_lverbs_function_register(void *handle);
+int mlx5_lmlx5_function_register(void *handle);
+int mlx5_load_libs(void);
+#undef ibv_query_port
+
+struct {
+	int size;
+	int version;
+	int (*ibv_destroy_qp)(struct ibv_qp *qp);
+	int (*ibv_destroy_cq)(struct ibv_cq *cq);
+	int (*ibv_dealloc_pd)(struct ibv_pd *pd);
+	struct ibv_device **(*ibv_get_device_list)(int *num_devices);
+	int (*ibv_query_device)(struct ibv_context *context,
+							struct ibv_device_attr *device_attr);
+	int (*ibv_close_device)(struct ibv_context *context);
+	void (*ibv_free_device_list)(struct ibv_device **list);
+	struct ibv_context *(*ibv_open_device)(struct ibv_device *device);
+	struct ibv_pd *(*ibv_alloc_pd)(struct ibv_context *context);
+	const char *(*ibv_get_device_name)(struct ibv_device *device);
+	int (*ibv_fork_init)(void);
+	int (*ibv_destroy_comp_channel)(struct ibv_comp_channel *channel);
+	int (*ibv_dereg_mr)(struct ibv_mr *mr);
+	struct ibv_comp_channel *(*ibv_create_comp_channel)(
+							  struct ibv_context *context);
+	int(*ibv_get_cq_event)(struct ibv_comp_channel *channel,
+						   struct ibv_cq **cq, void **cq_context);
+	void (*ibv_ack_cq_events)(struct ibv_cq *cq, unsigned int nevents);
+	struct ibv_qp *(*ibv_create_qp)(struct ibv_pd *pd,
+									struct ibv_qp_init_attr *qp_init_attr);
+	void (*ibv_ack_async_event)(struct ibv_async_event *event);
+	int (*ibv_get_async_event)(struct ibv_context *context,
+							   struct ibv_async_event *event);
+	struct ibv_mr *(*ibv_reg_mr)(struct ibv_pd *pd, void *addr,
+								 size_t length,
+								 int access);
+	struct ibv_cq *(*ibv_create_cq)(struct ibv_context *context,
+									int cqe, void *cq_context,
+									struct ibv_comp_channel *channel,
+									int comp_vector);
+	int (*ibv_modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+						 int attr_mask);
+	int (*ibv_query_port)(struct ibv_context *context, uint8_t port_num,
+						struct ibv_port_attr *port_attr);
+} mlx5_libverbs;
+
+struct {
+	int size;
+	int version;
+	struct ibv_cq_ex *(*mlx5dv_create_cq)(struct ibv_context *context,
+								struct ibv_cq_init_attr_ex *cq_attr,
+								struct mlx5dv_cq_init_attr *mlx5_cq_attr);
+	int (*mlx5dv_init_obj)(struct mlx5dv_obj *obj, uint64_t obj_type);
+	int (*mlx5dv_query_device)(struct ibv_context *ctx_in,
+							   struct mlx5dv_context *attrs_out);
+	int (*mlx5dv_set_context_attr)(struct ibv_context *context,
+								   enum mlx5dv_set_ctx_attr_type type,
+								   void *attr);
+} mlx5_lmlx5;
+#endif /* RTE_PMD_MLX5_IBVERBS_H_ */
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 0548d17..323ced7 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -30,7 +30,8 @@
  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-
+#define _GNU_SOURCE
+#include <stdio.h>
 #include <stddef.h>
 #include <unistd.h>
 #include <string.h>
@@ -39,13 +40,17 @@
 #include <stdlib.h>
 #include <errno.h>
 #include <net/if.h>
-
+#include <dlfcn.h>
 /* Verbs header. */
 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
@@ -1029,6 +1034,9 @@ struct mlx5_args {
 rte_mlx5_pmd_init(void)
 {
 	/* Build the static table for ptype conversion. */
+#ifdef MLX5_PMD_DLL
+	int ret;
+#endif
 	mlx5_set_ptype_table();
 	/*
 	 * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
@@ -1040,6 +1048,11 @@ struct mlx5_args {
 	/* Match the size of Rx completion entry to the size of a cacheline. */
 	if (RTE_CACHE_LINE_SIZE == 128)
 		setenv("MLX5_CQE_SIZE", "128", 0);
+#ifdef MLX5_PMD_DLL
+	ret = mlx5_load_libs();
+	if (ret != 0)
+		return;
+#endif
 	ibv_fork_init();
 	rte_pci_register(&mlx5_driver);
 }
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index e6a69b8..1dde985 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -46,7 +46,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index f32dfdd..ffd419c 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -39,7 +39,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c
index d17b991..9e8902b 100644
--- a/drivers/net/mlx5/mlx5_mac.c
+++ b/drivers/net/mlx5/mlx5_mac.c
@@ -46,7 +46,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 6b29eed..c91fcdf 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -36,7 +36,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c
index f3de46d..9f58147 100644
--- a/drivers/net/mlx5/mlx5_rss.c
+++ b/drivers/net/mlx5/mlx5_rss.c
@@ -42,7 +42,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c
index 0ef2cdf..f9f7d6f 100644
--- a/drivers/net/mlx5/mlx5_rxmode.c
+++ b/drivers/net/mlx5/mlx5_rxmode.c
@@ -40,7 +40,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 85399ef..dfd6400 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -44,8 +44,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 2d30c50..e2215aa 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -41,8 +41,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index d34f3cc..7419f2c 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -43,12 +43,14 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
-#include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
-
 #include <rte_mbuf.h>
 #include <rte_mempool.h>
 #include <rte_common.h>
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c
index ba6c8ce..fc7f701 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec.c
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.c
@@ -41,8 +41,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 9c5860f..8f52525 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -44,7 +44,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 6a6a745..0542c9a 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -51,7 +51,9 @@ endif
 
 # Link only the libraries used in the application
 LDFLAGS += --as-needed
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+LDFLAGS += -ldl
+endif
 # default path for libs
 _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
 
@@ -142,7 +144,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI)        += -lrte_pmd_kni
 endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD)        += -lrte_pmd_lio
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -libverbs -lmlx4
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5
+else
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -libverbs -lmlx5
+endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD)       += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk
 _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD)        += -lrte_pmd_nfp
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL)       += -lrte_pmd_null
-- 
1.8.3.1

[dpdk-dev] [PATCH v2 2/2] net/mlx4: load libmlx4 and libibverbs in run-time

[Shachar Beiser]

MLX4 PMD dynamically loads mlx4 and ibverbs in
run-time and it is not linked to external libraries.

Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
---
    History:
        V1: http://dpdk.org/dev/patchwork/patch/31555/
    V2:
        Fixing checkpatch code styling warnings
---
 config/common_base              |   1 +
 drivers/net/mlx4/Makefile       |  14 +-
 drivers/net/mlx4/lib/mlx4_dll.c | 731 ++++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx4/lib/mlx4_dll.h |  94 ++++++
 drivers/net/mlx4/mlx4.c         |  12 +
 drivers/net/mlx4/mlx4.h         |   4 +
 drivers/net/mlx4/mlx4_ethdev.c  |   4 +
 drivers/net/mlx4/mlx4_flow.c    |   4 +
 drivers/net/mlx4/mlx4_mr.c      |   4 +
 drivers/net/mlx4/mlx4_prm.h     |   4 +
 drivers/net/mlx4/mlx4_rxq.c     |   4 +
 drivers/net/mlx4/mlx4_rxtx.c    |   4 +
 drivers/net/mlx4/mlx4_rxtx.h    |   4 +
 drivers/net/mlx4/mlx4_txq.c     |   4 +
 drivers/net/mlx5/lib/mlx5_dll.c |   4 +-
 mk/rte.app.mk                   |   7 +
 16 files changed, 895 insertions(+), 4 deletions(-)
 create mode 100644 drivers/net/mlx4/lib/mlx4_dll.c
 create mode 100644 drivers/net/mlx4/lib/mlx4_dll.h

diff --git a/config/common_base b/config/common_base
index 3708de4..8ef6be4 100644
--- a/config/common_base
+++ b/config/common_base
@@ -229,6 +229,7 @@ CONFIG_RTE_LIBRTE_FM10K_INC_VECTOR=y
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
 #
 CONFIG_RTE_LIBRTE_MLX4_PMD=n
+CONFIG_RTE_LIBRTE_MLX4_DLL=y
 CONFIG_RTE_LIBRTE_MLX4_DEBUG=n
 CONFIG_RTE_LIBRTE_MLX4_DEBUG_BROKEN_VERBS=n
 CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile
index f1f47c2..aba1d5f 100644
--- a/drivers/net/mlx4/Makefile
+++ b/drivers/net/mlx4/Makefile
@@ -44,7 +44,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_rxq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_rxtx.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_txq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_utils.c
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += lib/mlx4_dll.c
+endif
 # Basic CFLAGS.
 CFLAGS += -O3
 CFLAGS += -std=c11 -Wall -Wextra
@@ -54,7 +56,11 @@ CFLAGS += -D_BSD_SOURCE
 CFLAGS += -D_DEFAULT_SOURCE
 CFLAGS += -D_XOPEN_SOURCE=600
 CFLAGS += $(WERROR_FLAGS)
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+LDLIBS += -ldl
+else
 LDLIBS += -libverbs -lmlx4
+endif
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_pci
@@ -81,7 +87,11 @@ endif
 ifdef CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE
 CFLAGS += -DMLX4_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE)
 endif
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+CFLAGS += -DMLX4_PMD_DLL
+else
+CFLAGS += -UMLX4_PMD_DLL
+endif
 ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DEBUG_BROKEN_VERBS),y)
 CFLAGS += -DMLX4_PMD_DEBUG_BROKEN_VERBS
 endif
diff --git a/drivers/net/mlx4/lib/mlx4_dll.c b/drivers/net/mlx4/lib/mlx4_dll.c
new file mode 100644
index 0000000..0d8e7a9
--- /dev/null
+++ b/drivers/net/mlx4/lib/mlx4_dll.c
@@ -0,0 +1,731 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <dlfcn.h>
+#include <rte_log.h>
+#include "../mlx4_utils.h"
+#include "mlx4_dll.h"
+
+#define VERBS_LIB_DIR "/usr/lib64/libibverbs"
+#define MLX5_LIB_DIR "/usr/lib64/libmlx4"
+#define DIR_LENGTH 25
+/**
+ * Load a libibverbs and libmlx4 symbols table.
+ *
+ * @return
+ *   0 on success.
+ */
+int mlx4_load_libs(void)
+{
+	void *dlhandle;
+	int ret;
+
+	dlhandle = mlx4_lib_load("ibverbs.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load ibverbs.so");
+		return -1;
+	}
+	ret = mlx4_lverbs_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in libverbs.so ");
+		return ret;
+	}
+	dlhandle = mlx4_lib_load("mlx4.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load mlx4.so");
+		return -1;
+	}
+	ret = mlx4_lmlx4_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in lmlx4.so  ");
+		return ret;
+	}
+	return 0;
+}
+/**
+ * Load a libibverbs or libmlx4 symbols table.
+ *
+ * @param name[in]
+ *   The library name.
+ * @return
+ *   dlhandle on success.
+ */
+void *mlx4_lib_load(const char *name)
+{
+	char *so_name;
+	void *dlhandle;
+	char lib[DIR_LENGTH];
+
+	if (strcmp(name, "ibverbs.so"))
+		strcpy(lib, VERBS_LIB_DIR);
+	else
+		strcpy(lib, MLX5_LIB_DIR);
+	/* If the name is an absolute path then open that path after appending
+	 * the trailer suffix
+	 */
+	if (name[0] == '/') {
+		if (asprintf(&so_name, "%s", name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		if (!dlhandle)
+			goto out_dlopen;
+		free(so_name);
+		return dlhandle;
+	}
+	/* If configured with a provider plugin path then try that next */
+	if (strlen(lib) > 1) {
+		if (asprintf(&so_name, "%s/lib%s", lib, name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		free(so_name);
+		if (dlhandle)
+			return dlhandle;
+	}
+	/* Otherwise use the system libary search path. This is the historical
+	 * behavior of libibverbs
+	 */
+	if (asprintf(&so_name, "lib%s", name) < 0)
+		goto out_asprintf;
+	dlhandle = dlopen(so_name, RTLD_LAZY);
+	if (!dlhandle)
+		goto out_dlopen;
+	free(so_name);
+	return dlhandle;
+out_asprintf:
+	ERROR("Warning: couldn't load driver '%s'.\n", name);
+	return NULL;
+out_dlopen:
+	ERROR("Warning: couldn't load driver '%s': %s\n", so_name, dlerror());
+	free(so_name);
+	return NULL;
+}
+
+/**
+ * Register libibverbs functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx4_lverbs_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx4_libverbs.ibv_destroy_qp) = dlsym(handle,
+													  "ibv_destroy_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_destroy_cq) = dlsym(handle,
+													  "ibv_destroy_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_dealloc_pd) = dlsym(handle,
+													  "ibv_dealloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_device_list) =
+									dlsym(handle, "ibv_get_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_open_device) = dlsym(handle,
+													   "ibv_open_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_query_device) = dlsym(handle,
+														"ibv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_close_device) = dlsym(handle,
+														"ibv_close_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_free_device_list) =
+										dlsym(handle, "ibv_free_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_alloc_pd) = dlsym(handle, "ibv_alloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_device_name) =
+									dlsym(handle, "ibv_get_device_name");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_fork_init) = dlsym(handle,
+													 "ibv_fork_init");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_destroy_comp_channel) =
+									dlsym(handle, "ibv_destroy_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_dereg_mr) = dlsym(handle, "ibv_dereg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_create_comp_channel) =
+									dlsym(handle, "ibv_create_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_cq_event) = dlsym(handle,
+														"ibv_get_cq_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_create_qp) = dlsym(handle,
+													 "ibv_create_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_ack_async_event) =
+									dlsym(handle, "ibv_ack_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_async_event) =
+									dlsym(handle, "ibv_get_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_reg_mr) = dlsym(handle, "ibv_reg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_create_cq) = dlsym(handle,
+													 "ibv_create_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_modify_qp) = dlsym(handle,
+													 "ibv_modify_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_query_port) = dlsym(handle,
+													  "ibv_query_port");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+
+/**
+ * Register libmlx4 functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx4_lmlx4_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx4_lmlx4.mlx4dv_init_obj) = dlsym(handle,
+													"mlx4dv_init_obj");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_lmlx4.mlx4dv_query_device) = dlsym(handle,
+														"mlx4dv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_lmlx4.mlx4dv_set_context_attr) =
+								 dlsym(handle, "mlx4dv_set_context_attr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+/**
+ * Function pointer calls mlx4 API .
+ *
+ * @param mlx4dv_obj[in]
+ *   The mlx4 object.
+ * @param obj_type[in]
+ *   The object type.
+ * @return
+ *   a pointer on success.
+ */
+int mlx4dv_init_obj(struct mlx4dv_obj *obj, uint64_t obj_type)
+{
+	return mlx4_lmlx4.mlx4dv_init_obj(obj, obj_type);
+}
+/**
+ * Function pointer calls mlx4 API .
+ *
+ * @param ctx_in[in]
+ *   The ibverbs context.
+ * @param attrs_out[out]
+ *   The mlx4 context.
+ * @return
+ *  0 on success.
+ */
+int mlx4dv_query_device(struct ibv_context *ctx_in,
+						struct mlx4dv_context *attrs_out)
+{
+	return mlx4_lmlx4.mlx4dv_query_device(ctx_in, attrs_out);
+}
+/**
+ * Function pointer calls mlx4 API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param type[in]
+ *   The mlx4 context attribute type.
+ * @param attr[in]
+ *   The mlx4 context attribute.
+ * @return
+ *  0 on success.
+ */
+int mlx4dv_set_context_attr(struct ibv_context *context,
+							enum mlx4dv_set_ctx_attr_type type,
+							void *attr)
+{
+	return mlx4_lmlx4.mlx4dv_set_context_attr(context, type, attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_qp(struct ibv_qp *qp)
+{
+	return mlx4_libverbs.ibv_destroy_qp(qp);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   Completion queue pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_cq(struct ibv_cq *cq)
+{
+	return mlx4_libverbs.ibv_destroy_cq(cq);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   Protection domain.
+ * @return
+ *  0 on success.
+ */
+int ibv_dealloc_pd(struct ibv_pd *pd)
+{
+	return mlx4_libverbs.ibv_dealloc_pd(pd);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param num_devices[in]
+ *   The number of devices.
+ * @return
+ *  0 on success.
+ */
+struct ibv_device **ibv_get_device_list(int *num_devices)
+{
+	return mlx4_libverbs.ibv_get_device_list(num_devices);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param device_attr[in]
+ *   The device attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_device(struct ibv_context *context,
+					 struct ibv_device_attr *device_attr)
+{
+	return mlx4_libverbs.ibv_query_device(context, device_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @return
+ *  0 on success.
+ */
+int ibv_close_device(struct ibv_context *context)
+{
+	return mlx4_libverbs.ibv_close_device(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param list[in]
+ *   The list of devices.
+ */
+void ibv_free_device_list(struct ibv_device **list)
+{
+	return mlx4_libverbs.ibv_free_device_list(list);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   A device pointer.
+ * @return
+ *  The context on success.
+ */
+struct ibv_context *ibv_open_device(struct ibv_device *device)
+{
+	return mlx4_libverbs.ibv_open_device(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @return
+ *  protection domain pointer on success.
+ */
+struct ibv_pd *ibv_alloc_pd(struct ibv_context *context)
+{
+	return mlx4_libverbs.ibv_alloc_pd(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   An ibverbs device pointer.
+ * @return
+ *  device name on success.
+ */
+const char *ibv_get_device_name(struct ibv_device *device)
+{
+	return mlx4_libverbs.ibv_get_device_name(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @return
+ *  0 on success.
+ */
+int ibv_fork_init(void)
+{
+	return mlx4_libverbs.ibv_fork_init();
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A completion channel.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_comp_channel(struct ibv_comp_channel *channel)
+{
+	return mlx4_libverbs.ibv_destroy_comp_channel(channel);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param mr[in]
+ *   A pointer to memory region.
+ * @return
+ *  0 on success.
+ */
+int ibv_dereg_mr(struct ibv_mr *mr)
+{
+	return mlx4_libverbs.ibv_dereg_mr(mr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A pointer to ibverbs context.
+ * @return
+ *  0 on success.
+ */
+struct ibv_comp_channel *ibv_create_comp_channel(
+											  struct ibv_context *context)
+{
+	return mlx4_libverbs.ibv_create_comp_channel(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A pointer to completion channel.
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param cq_context[in]
+ *   A pointer to completion queue context.
+ * @return
+ *  0 on success.
+ */
+int ibv_get_cq_event(struct ibv_comp_channel *channel,
+					 struct ibv_cq **cq, void **cq_context)
+{
+	return mlx4_libverbs.ibv_get_cq_event(channel, cq, cq_context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param nevents[in]
+ *   The number of events.
+ * @return
+ *  0 on success.
+ */
+void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
+{
+	return mlx4_libverbs.ibv_ack_cq_events(cq, nevents);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param qp_init_attr[in]
+ *   The queue pair attributes.
+ * @return
+ *  A pointer to queue pair on success.
+ */
+struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
+							 struct ibv_qp_init_attr *qp_init_attr)
+{
+	return mlx4_libverbs.ibv_create_qp(pd, qp_init_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+void ibv_ack_async_event(struct ibv_async_event *event)
+{
+	return mlx4_libverbs.ibv_ack_async_event(event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A ibverbs context.
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+int ibv_get_async_event(struct ibv_context *context,
+						struct ibv_async_event *event)
+{
+	return mlx4_libverbs.ibv_get_async_event(context, event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param addr[in]
+ *   The address.
+ * @param length[in]
+ *   The length.
+ * @param access[in]
+ *   The access type.
+ * @return
+ *  A pointer to memory region on success.
+ */
+struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr,
+						  size_t length, int access)
+{
+	return mlx4_libverbs.ibv_reg_mr(pd, addr, length, access);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   Ibverbs context.
+ * @param cqe[in]
+ *   A completion queue entry.
+ * @param cq_context[in]
+ *   Completion queue context.
+ * @param channel[in]
+ *   The completion channel.
+ * @param comp_vector[in]
+ *   Completion vector.
+ * @return
+ *  0 on success.
+ */
+struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe,
+							 void *cq_context,
+							 struct ibv_comp_channel *channel,
+							 int comp_vector)
+{
+	return mlx4_libverbs.ibv_create_cq(context, cqe, cq_context, channel,
+									   comp_vector);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair.
+ * @param attr[in]
+ *   The queue pair attribute.
+ * @param attr_mask[in]
+ *   Attribute mask.
+ * @return
+ *  0 on success.
+ */
+int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+				  int attr_mask)
+{
+	return mlx4_libverbs.ibv_modify_qp(qp, attr, attr_mask);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @param port_num[in]
+ *   The port number.
+ * @param port_attr[in]
+ *   Port attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_port(struct ibv_context *context, uint8_t port_num,
+				   struct ibv_port_attr *port_attr)
+{
+	port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
+	port_attr->reserved   = 0;
+	return mlx4_libverbs.ibv_query_port(context, port_num, port_attr);
+}
diff --git a/drivers/net/mlx4/lib/mlx4_dll.h b/drivers/net/mlx4/lib/mlx4_dll.h
new file mode 100644
index 0000000..e95b78e
--- /dev/null
+++ b/drivers/net/mlx4/lib/mlx4_dll.h
@@ -0,0 +1,94 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef RTE_PMD_MLX4_IBVERBS_H_
+#define RTE_PMD_MLX4_IBVERBS_H_
+#include <infiniband/verbs.h>
+#include <infiniband/mlx4dv.h>
+
+void *mlx4_lib_load(const char *name);
+int mlx4_lverbs_function_register(void *handle);
+int mlx4_lmlx4_function_register(void *handle);
+int mlx4_load_libs(void);
+#undef ibv_query_port
+
+struct {
+	int size;
+	int version;
+	int (*ibv_destroy_qp)(struct ibv_qp *qp);
+	int (*ibv_destroy_cq)(struct ibv_cq *cq);
+	int (*ibv_dealloc_pd)(struct ibv_pd *pd);
+	struct ibv_device **(*ibv_get_device_list)(int *num_devices);
+	int (*ibv_query_device)(struct ibv_context *context,
+							struct ibv_device_attr *device_attr);
+	int (*ibv_close_device)(struct ibv_context *context);
+	void (*ibv_free_device_list)(struct ibv_device **list);
+	struct ibv_context *(*ibv_open_device)(struct ibv_device *device);
+	struct ibv_pd *(*ibv_alloc_pd)(struct ibv_context *context);
+	const char *(*ibv_get_device_name)(struct ibv_device *device);
+	int (*ibv_fork_init)(void);
+	int (*ibv_destroy_comp_channel)(struct ibv_comp_channel *channel);
+	int (*ibv_dereg_mr)(struct ibv_mr *mr);
+	struct ibv_comp_channel *(*ibv_create_comp_channel)(
+							  struct ibv_context *context);
+	int(*ibv_get_cq_event)(struct ibv_comp_channel *channel,
+						   struct ibv_cq **cq, void **cq_context);
+	void (*ibv_ack_cq_events)(struct ibv_cq *cq, unsigned int nevents);
+	struct ibv_qp *(*ibv_create_qp)(struct ibv_pd *pd,
+									struct ibv_qp_init_attr *qp_init_attr);
+	void (*ibv_ack_async_event)(struct ibv_async_event *event);
+	int (*ibv_get_async_event)(struct ibv_context *context,
+							   struct ibv_async_event *event);
+	struct ibv_mr *(*ibv_reg_mr)(struct ibv_pd *pd, void *addr,
+								 size_t length,
+								 int access);
+	struct ibv_cq *(*ibv_create_cq)(struct ibv_context *context,
+									int cqe, void *cq_context,
+									struct ibv_comp_channel *channel,
+									int comp_vector);
+	int (*ibv_modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+						 int attr_mask);
+	int (*ibv_query_port)(struct ibv_context *context, uint8_t port_num,
+						struct ibv_port_attr *port_attr);
+}mlx4_libverbs;
+
+struct {
+	int size;
+	int version;
+	int (*mlx4dv_init_obj)(struct mlx4dv_obj *obj, uint64_t obj_type);
+	int (*mlx4dv_query_device)(struct ibv_context *ctx_in,
+							   struct mlx4dv_context *attrs_out);
+	int (*mlx4dv_set_context_attr)(struct ibv_context *context,
+								enum mlx4dv_set_ctx_attr_type type,
+								void *attr);
+}mlx4_lmlx4;
+#endif /* RTE_PMD_MLX4_IBVERBS_H_ */
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index f9e4f9d..8fee391 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -49,7 +49,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
@@ -713,7 +717,15 @@ struct mlx4_conf {
 	 * applications to use fork() safely for purposes other than
 	 * using this PMD, which is not supported in forked processes.
 	 */
+#ifdef MLX4_PMD_DLL
+	int ret;
+#endif
 	setenv("RDMAV_HUGEPAGES_SAFE", "1", 1);
+#ifdef MLX4_PMD_DLL
+	ret = mlx4_load_libs();
+	if ( ret != 0 )
+		return;
+#endif
 	ibv_fork_init();
 	rte_pci_register(&mlx4_driver);
 }
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 3aeef87..6e4565b 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -42,7 +42,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_ethdev.c b/drivers/net/mlx4/mlx4_ethdev.c
index 2f69e7d..98134ab 100644
--- a/drivers/net/mlx4/mlx4_ethdev.c
+++ b/drivers/net/mlx4/mlx4_ethdev.c
@@ -56,7 +56,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_flow.c b/drivers/net/mlx4/mlx4_flow.c
index 8b87b29..89d6d17 100644
--- a/drivers/net/mlx4/mlx4_flow.c
+++ b/drivers/net/mlx4/mlx4_flow.c
@@ -49,7 +49,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_mr.c b/drivers/net/mlx4/mlx4_mr.c
index 2a3e269..b494e95 100644
--- a/drivers/net/mlx4/mlx4_mr.c
+++ b/drivers/net/mlx4/mlx4_mr.c
@@ -47,7 +47,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_prm.h b/drivers/net/mlx4/mlx4_prm.h
index fcc7c12..b8854e1 100644
--- a/drivers/net/mlx4/mlx4_prm.h
+++ b/drivers/net/mlx4/mlx4_prm.h
@@ -42,8 +42,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PRD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c
index 53313c5..5680c89 100644
--- a/drivers/net/mlx4/mlx4_rxq.c
+++ b/drivers/net/mlx4/mlx4_rxq.c
@@ -46,8 +46,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_rxtx.c b/drivers/net/mlx4/mlx4_rxtx.c
index 2bfa8b1..a876896 100644
--- a/drivers/net/mlx4/mlx4_rxtx.c
+++ b/drivers/net/mlx4/mlx4_rxtx.c
@@ -44,7 +44,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h
index 463df2b..c115032 100644
--- a/drivers/net/mlx4/mlx4_rxtx.h
+++ b/drivers/net/mlx4/mlx4_rxtx.h
@@ -41,8 +41,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c
index 7882a4d..8ae9eb9 100644
--- a/drivers/net/mlx4/mlx4_txq.c
+++ b/drivers/net/mlx4/mlx4_txq.c
@@ -46,7 +46,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/lib/mlx5_dll.c b/drivers/net/mlx5/lib/mlx5_dll.c
index 2228ad1..d0ff97f 100644
--- a/drivers/net/mlx5/lib/mlx5_dll.c
+++ b/drivers/net/mlx5/lib/mlx5_dll.c
@@ -115,7 +115,7 @@ void *mlx5_lib_load(const char *name)
 		if (dlhandle)
 			return dlhandle;
 	}
-	/* Otherwise use the system libary search path. This is the historical
+	/* Otherwise use the system library search path. This is the historical
 	 * behavior of libibverbs
 	 */
 	if (asprintf(&so_name, "lib%s", name) < 0)
@@ -204,7 +204,7 @@ int mlx5_lverbs_function_register(void *handle)
 		goto exit;
 	}
 	*(void **)(&mlx5_libverbs.ibv_free_device_list) =
-										dlsym(handle, "ibv_free_device_list");
+									dlsym(handle, "ibv_free_device_list");
 	error = dlerror();
 	if ((error) != NULL) {
 		ERROR("%s\n", error);
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 0542c9a..5589482 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -54,6 +54,9 @@ LDFLAGS += --as-needed
 ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
 LDFLAGS += -ldl
 endif
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+LDFLAGS += -ldl
+endif
 # default path for libs
 _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
 
@@ -143,7 +146,11 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI)        += -lrte_pmd_kni
 endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD)        += -lrte_pmd_lio
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4
+else
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -libverbs -lmlx4
+endif
 ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5
 else
-- 
1.8.3.1

[dpdk-dev] [PATCH v3 1/2] net/mlx5: load libmlx5 and libibverbs in run-time

[Shachar Beiser]

MLX5 PMD loads libraries: libibverbs and libmlx5.
MLX5 PMD is not linked to external libraries.

Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
---
    v1:
        load external libraries in run-time
    v2
    * fix checkpatch warnings 
    v3:
    * fix checkpatch warnings 
---
 config/common_base               |   1 +
 drivers/net/mlx5/Makefile        |  27 +-
 drivers/net/mlx5/lib/mlx5_dll.c  | 756 +++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/lib/mlx5_dll.h  |  94 +++++
 drivers/net/mlx5/mlx5.c          |  17 +-
 drivers/net/mlx5/mlx5.h          |   4 +
 drivers/net/mlx5/mlx5_flow.c     |   4 +
 drivers/net/mlx5/mlx5_mac.c      |   4 +
 drivers/net/mlx5/mlx5_mr.c       |   4 +
 drivers/net/mlx5/mlx5_rss.c      |   4 +
 drivers/net/mlx5/mlx5_rxmode.c   |   4 +
 drivers/net/mlx5/mlx5_rxq.c      |   4 +
 drivers/net/mlx5/mlx5_rxtx.c     |   4 +
 drivers/net/mlx5/mlx5_rxtx.h     |   6 +-
 drivers/net/mlx5/mlx5_rxtx_vec.c |   4 +
 drivers/net/mlx5/mlx5_txq.c      |   4 +
 mk/rte.app.mk                    |   8 +-
 17 files changed, 936 insertions(+), 13 deletions(-)
 create mode 100644 drivers/net/mlx5/lib/mlx5_dll.c
 create mode 100644 drivers/net/mlx5/lib/mlx5_dll.h

diff --git a/config/common_base b/config/common_base
index e74febe..3708de4 100644
--- a/config/common_base
+++ b/config/common_base
@@ -237,6 +237,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
 # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
 #
 CONFIG_RTE_LIBRTE_MLX5_PMD=n
+CONFIG_RTE_LIBRTE_MLX5_DLL=y
 CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
 CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
 
diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index a3984eb..1dc0a05 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -53,7 +53,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c
+endif
 # Basic CFLAGS.
 CFLAGS += -O3
 CFLAGS += -std=c11 -Wall -Wextra
@@ -64,7 +66,11 @@ CFLAGS += -D_DEFAULT_SOURCE
 CFLAGS += -D_XOPEN_SOURCE=600
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -Wno-strict-prototypes
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+LDLIBS += -ldl
+else
 LDLIBS += -libverbs -lmlx5
+endif
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_pci
@@ -91,7 +97,11 @@ endif
 ifdef CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE
 CFLAGS += -DMLX5_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE)
 endif
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+CFLAGS += -DMLX5_PMD_DLL
+else
+CFLAGS += -UMLX5_PMD_DLL
+endif
 include $(RTE_SDK)/mk/rte.lib.mk
 
 # Generate and clean-up mlx5_autoconf.h.
@@ -105,26 +115,28 @@ endif
 
 mlx5_autoconf.h.new: FORCE
 
+VERBS_H := infiniband/verbs.h
+MLX5DV_H := infiniband/mlx5dv.h
 mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 	$Q $(RM) -f -- '$@'
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_DEVICE_VXLAN_SUPPORT \
-		infiniband/verbs.h \
+		$(VERBS_H) \
 		enum IBV_DEVICE_VXLAN_SUPPORT \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_WQ_FLAG_RX_END_PADDING \
-		infiniband/verbs.h \
+		$(VERBS_H) \
 		enum IBV_WQ_FLAG_RX_END_PADDING \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_MLX5_MOD_MPW \
-		infiniband/mlx5dv.h \
+		$(MLX5DV_H) \
 		enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
-		infiniband/mlx5dv.h \
+		$(MLX5DV_H) \
 		enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
@@ -144,10 +156,9 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
-		infiniband/verbs.h \
+		$(VERBS_H)  \
 		enum IBV_FLOW_SPEC_ACTION_COUNT \
 		$(AUTOCONF_OUTPUT)
-
 # Create mlx5_autoconf.h or update it in case it differs from the new one.
 
 mlx5_autoconf.h: mlx5_autoconf.h.new
diff --git a/drivers/net/mlx5/lib/mlx5_dll.c b/drivers/net/mlx5/lib/mlx5_dll.c
new file mode 100644
index 0000000..2c4ac74
--- /dev/null
+++ b/drivers/net/mlx5/lib/mlx5_dll.c
@@ -0,0 +1,756 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <dlfcn.h>
+#include <rte_log.h>
+#include <mlx5_utils.h>
+#include "mlx5_dll.h"
+
+#define VERBS_LIB_DIR "/usr/lib64/libibverbs"
+#define MLX5_LIB_DIR "/usr/lib64/libmlx5"
+#define DIR_LENGTH 25
+/**
+ * Load a libibverbs and libmlx5 symbols table.
+ *
+ * @return
+ *   0 on success.
+ */
+int mlx5_load_libs(void)
+{
+	void *dlhandle;
+	int ret;
+
+	dlhandle = mlx5_lib_load("ibverbs.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load ibverbs.so");
+		return -1;
+	}
+	ret = mlx5_lverbs_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in libverbs.so ");
+		return ret;
+	}
+	dlhandle = mlx5_lib_load("mlx5.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load mlx5.so");
+		return -1;
+	}
+	ret = mlx5_lmlx5_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in lmlx5.so  ");
+		return ret;
+	}
+	return 0;
+}
+/**
+ * Load a libibverbs or libmlx5 symbols table.
+ *
+ * @param name[in]
+ *   The library name.
+ * @return
+ *   dlhandle on success.
+ */
+void *mlx5_lib_load(const char *name)
+{
+	char *so_name;
+	void *dlhandle;
+	char lib[DIR_LENGTH];
+
+	if (strcmp(name, "ibverbs.so"))
+		strcpy(lib, VERBS_LIB_DIR);
+	else
+		strcpy(lib, MLX5_LIB_DIR);
+	/* If the name is an absolute path then open that path after appending
+	 * the trailer suffix
+	 */
+	if (name[0] == '/') {
+		if (asprintf(&so_name, "%s", name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		if (!dlhandle)
+			goto out_dlopen;
+		free(so_name);
+		return dlhandle;
+	}
+	/* If configured with a provider plugin path then try that next */
+	if (strlen(lib) > 1) {
+		if (asprintf(&so_name, "%s/lib%s", lib, name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		free(so_name);
+		if (dlhandle)
+			return dlhandle;
+	}
+	/* Otherwise use the system library search path. This is the historical
+	 * behavior of libibverbs
+	 */
+	if (asprintf(&so_name, "lib%s", name) < 0)
+		goto out_asprintf;
+	dlhandle = dlopen(so_name, RTLD_LAZY);
+	if (!dlhandle)
+		goto out_dlopen;
+	free(so_name);
+	return dlhandle;
+out_asprintf:
+	ERROR("Warning: couldn't load driver '%s'.\n", name);
+	return NULL;
+out_dlopen:
+	ERROR("Warning: couldn't load driver '%s': %s\n", so_name, dlerror());
+	free(so_name);
+	return NULL;
+}
+
+/**
+ * Register libibverbs functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx5_lverbs_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx5_libverbs.ibv_destroy_qp) =
+			dlsym(handle, "ibv_destroy_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_destroy_cq) =
+			dlsym(handle, "ibv_destroy_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_dealloc_pd) =
+			dlsym(handle, "ibv_dealloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_device_list) =
+			dlsym(handle, "ibv_get_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_open_device) =
+			dlsym(handle, "ibv_open_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_query_device) =
+			dlsym(handle, "ibv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_close_device) =
+			dlsym(handle, "ibv_close_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_free_device_list) =
+			dlsym(handle, "ibv_free_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_alloc_pd) =
+			dlsym(handle, "ibv_alloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_device_name) =
+			dlsym(handle, "ibv_get_device_name");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_fork_init) =
+			dlsym(handle, "ibv_fork_init");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_destroy_comp_channel) =
+			dlsym(handle, "ibv_destroy_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_dereg_mr) =
+			dlsym(handle, "ibv_dereg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_create_comp_channel) =
+			dlsym(handle, "ibv_create_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_cq_event) =
+			dlsym(handle, "ibv_get_cq_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_create_qp) =
+			dlsym(handle, "ibv_create_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_ack_async_event) =
+			dlsym(handle, "ibv_ack_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_get_async_event) =
+			dlsym(handle, "ibv_get_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_reg_mr) =
+			dlsym(handle, "ibv_reg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_create_cq) =
+			dlsym(handle, "ibv_create_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_modify_qp) =
+			dlsym(handle, "ibv_modify_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_libverbs.ibv_query_port) =
+			dlsym(handle, "ibv_query_port");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+
+/**
+ * Register libmlx5 functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx5_lmlx5_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx5_lmlx5.mlx5dv_create_cq) =
+			dlsym(handle, "mlx5dv_create_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_lmlx5.mlx5dv_init_obj) =
+			dlsym(handle, "mlx5dv_init_obj");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_lmlx5.mlx5dv_query_device) =
+			dlsym(handle, "mlx5dv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx5_lmlx5.mlx5dv_set_context_attr) =
+			dlsym(handle, "mlx5dv_set_context_attr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param context[in]
+ *   The ibv context.
+ * @param cq_attr[in]
+ *   The completion queue attribute.
+ * @param mlx5_cq_attr[in]
+ *   The mlx5 completion queue attribute.
+ * @return
+ *   a pointer on success.
+ */
+struct ibv_cq_ex *mlx5dv_create_cq(struct ibv_context *context,
+		struct ibv_cq_init_attr_ex *cq_attr,
+		struct mlx5dv_cq_init_attr *mlx5_cq_attr)
+{
+	return mlx5_lmlx5.mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr);
+}
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param mlx5dv_obj[in]
+ *   The mlx5 object.
+ * @param obj_type[in]
+ *   The object type.
+ * @return
+ *   a pointer on success.
+ */
+int mlx5dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type)
+{
+	return mlx5_lmlx5.mlx5dv_init_obj(obj, obj_type);
+}
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param ctx_in[in]
+ *   The ibverbs context.
+ * @param attrs_out[out]
+ *   The mlx5 context.
+ * @return
+ *  0 on success.
+ */
+int mlx5dv_query_device(struct ibv_context *ctx_in,
+		struct mlx5dv_context *attrs_out)
+{
+	return mlx5_lmlx5.mlx5dv_query_device(ctx_in, attrs_out);
+}
+/**
+ * Function pointer calls mlx5 API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param type[in]
+ *   The mlx5 context attribute type.
+ * @param attr[in]
+ *   The mlx5 context attribute.
+ * @return
+ *  0 on success.
+ */
+int mlx5dv_set_context_attr(struct ibv_context *context,
+		enum mlx5dv_set_ctx_attr_type type, void *attr)
+{
+	return mlx5_lmlx5.mlx5dv_set_context_attr(context, type, attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_qp(struct ibv_qp *qp)
+{
+	return mlx5_libverbs.ibv_destroy_qp(qp);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   Completion queue pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_cq(struct ibv_cq *cq)
+{
+	return mlx5_libverbs.ibv_destroy_cq(cq);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   Protection domain.
+ * @return
+ *  0 on success.
+ */
+int ibv_dealloc_pd(struct ibv_pd *pd)
+{
+	return mlx5_libverbs.ibv_dealloc_pd(pd);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param num_devices[in]
+ *   The number of devices.
+ * @return
+ *  0 on success.
+ */
+struct ibv_device **ibv_get_device_list(int *num_devices)
+{
+	return mlx5_libverbs.ibv_get_device_list(num_devices);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param device_attr[in]
+ *   The device attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_device(struct ibv_context *context,
+		struct ibv_device_attr *device_attr)
+{
+	return mlx5_libverbs.ibv_query_device(context, device_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @return
+ *  0 on success.
+ */
+int ibv_close_device(struct ibv_context *context)
+{
+	return mlx5_libverbs.ibv_close_device(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param list[in]
+ *   The list of devices.
+ */
+void ibv_free_device_list(struct ibv_device **list)
+{
+	return mlx5_libverbs.ibv_free_device_list(list);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   A device pointer.
+ * @return
+ *  The context on success.
+ */
+struct ibv_context *ibv_open_device(struct ibv_device *device)
+{
+	return mlx5_libverbs.ibv_open_device(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @return
+ *  protection domain pointer on success.
+ */
+struct ibv_pd *ibv_alloc_pd(struct ibv_context *context)
+{
+	return mlx5_libverbs.ibv_alloc_pd(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   An ibverbs device pointer.
+ * @return
+ *  device name on success.
+ */
+const char *ibv_get_device_name(struct ibv_device *device)
+{
+	return mlx5_libverbs.ibv_get_device_name(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @return
+ *  0 on success.
+ */
+int ibv_fork_init(void)
+{
+	return mlx5_libverbs.ibv_fork_init();
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A completion channel.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_comp_channel(struct ibv_comp_channel *channel)
+{
+	return mlx5_libverbs.ibv_destroy_comp_channel(channel);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param mr[in]
+ *   A pointer to memory region.
+ * @return
+ *  0 on success.
+ */
+int ibv_dereg_mr(struct ibv_mr *mr)
+{
+	return mlx5_libverbs.ibv_dereg_mr(mr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A pointer to ibverbs context.
+ * @return
+ *  0 on success.
+ */
+struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context)
+{
+	return mlx5_libverbs.ibv_create_comp_channel(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A pointer to completion channel.
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param cq_context[in]
+ *   A pointer to completion queue context.
+ * @return
+ *  0 on success.
+ */
+int ibv_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq,
+		void **cq_context)
+{
+	return mlx5_libverbs.ibv_get_cq_event(channel, cq, cq_context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param nevents[in]
+ *   The number of events.
+ * @return
+ *  0 on success.
+ */
+void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
+{
+	return mlx5_libverbs.ibv_ack_cq_events(cq, nevents);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param qp_init_attr[in]
+ *   The queue pair attributes.
+ * @return
+ *  A pointer to queue pair on success.
+ */
+struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
+		struct ibv_qp_init_attr *qp_init_attr)
+{
+	return mlx5_libverbs.ibv_create_qp(pd, qp_init_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+void ibv_ack_async_event(struct ibv_async_event *event)
+{
+	return mlx5_libverbs.ibv_ack_async_event(event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A ibverbs context.
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+int ibv_get_async_event(struct ibv_context *context,
+		struct ibv_async_event *event)
+{
+	return mlx5_libverbs.ibv_get_async_event(context, event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param addr[in]
+ *   The address.
+ * @param length[in]
+ *   The length.
+ * @param access[in]
+ *   The access type.
+ * @return
+ *  A pointer to memory region on success.
+ */
+struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
+		int access)
+{
+	return mlx5_libverbs.ibv_reg_mr(pd, addr, length, access);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   Ibverbs context.
+ * @param cqe[in]
+ *   A completion queue entry.
+ * @param cq_context[in]
+ *   Completion queue context.
+ * @param channel[in]
+ *   The completion channel.
+ * @param comp_vector[in]
+ *   Completion vector.
+ * @return
+ *  0 on success.
+ */
+struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe,
+	void *cq_context, struct ibv_comp_channel *channel, int comp_vector)
+{
+	return mlx5_libverbs.ibv_create_cq(context, cqe, cq_context, channel,
+			comp_vector);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair.
+ * @param attr[in]
+ *   The queue pair attribute.
+ * @param attr_mask[in]
+ *   Attribute mask.
+ * @return
+ *  0 on success.
+ */
+int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
+{
+	return mlx5_libverbs.ibv_modify_qp(qp, attr, attr_mask);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @param port_num[in]
+ *   The port number.
+ * @param port_attr[in]
+ *   Port attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_port(struct ibv_context *context, uint8_t port_num,
+		struct ibv_port_attr *port_attr)
+{
+	port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
+	port_attr->reserved = 0;
+	return mlx5_libverbs.ibv_query_port(context, port_num, port_attr);
+}
diff --git a/drivers/net/mlx5/lib/mlx5_dll.h b/drivers/net/mlx5/lib/mlx5_dll.h
new file mode 100644
index 0000000..120083d
--- /dev/null
+++ b/drivers/net/mlx5/lib/mlx5_dll.h
@@ -0,0 +1,94 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef RTE_PMD_MLX5_IBVERBS_H_
+#define RTE_PMD_MLX5_IBVERBS_H_
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
+
+void *mlx5_lib_load(const char *name);
+int mlx5_lverbs_function_register(void *handle);
+int mlx5_lmlx5_function_register(void *handle);
+int mlx5_load_libs(void);
+#undef ibv_query_port
+
+struct {
+	int size;
+	int version;
+	int (*ibv_destroy_qp)(struct ibv_qp *qp);
+	int (*ibv_destroy_cq)(struct ibv_cq *cq);
+	int (*ibv_dealloc_pd)(struct ibv_pd *pd);
+	struct ibv_device **(*ibv_get_device_list)(int *num_devices);
+	int (*ibv_query_device)(struct ibv_context *context,
+			struct ibv_device_attr *device_attr);
+	int (*ibv_close_device)(struct ibv_context *context);
+	void (*ibv_free_device_list)(struct ibv_device **list);
+	struct ibv_context *(*ibv_open_device)(struct ibv_device *device);
+	struct ibv_pd *(*ibv_alloc_pd)(struct ibv_context *context);
+	const char *(*ibv_get_device_name)(struct ibv_device *device);
+	int (*ibv_fork_init)(void);
+	int (*ibv_destroy_comp_channel)(struct ibv_comp_channel *channel);
+	int (*ibv_dereg_mr)(struct ibv_mr *mr);
+	struct ibv_comp_channel *(*ibv_create_comp_channel)(
+			struct ibv_context *context);
+	int (*ibv_get_cq_event)(struct ibv_comp_channel *channel,
+			struct ibv_cq **cq, void **cq_context);
+	void (*ibv_ack_cq_events)(struct ibv_cq *cq, unsigned int nevents);
+	struct ibv_qp *(*ibv_create_qp)(struct ibv_pd *pd,
+			struct ibv_qp_init_attr *qp_init_attr);
+	void (*ibv_ack_async_event)(struct ibv_async_event *event);
+	int (*ibv_get_async_event)(struct ibv_context *context,
+			struct ibv_async_event *event);
+	struct ibv_mr *(*ibv_reg_mr)(struct ibv_pd *pd, void *addr,
+			size_t length, int access);
+	struct ibv_cq *(*ibv_create_cq)(struct ibv_context *context, int cqe,
+			void *cq_context, struct ibv_comp_channel *channel,
+			int comp_vector);
+	int (*ibv_modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+			int attr_mask);
+	int (*ibv_query_port)(struct ibv_context *context, uint8_t port_num,
+			struct ibv_port_attr *port_attr);
+} mlx5_libverbs;
+
+struct {
+	int size;
+	int version;
+	struct ibv_cq_ex *(*mlx5dv_create_cq)(struct ibv_context *context,
+			struct ibv_cq_init_attr_ex *cq_attr,
+			struct mlx5dv_cq_init_attr *mlx5_cq_attr);
+	int (*mlx5dv_init_obj)(struct mlx5dv_obj *obj, uint64_t obj_type);
+	int (*mlx5dv_query_device)(struct ibv_context *ctx_in,
+			struct mlx5dv_context *attrs_out);
+	int (*mlx5dv_set_context_attr)(struct ibv_context *context,
+			enum mlx5dv_set_ctx_attr_type type, void *attr);
+} mlx5_lmlx5;
+#endif /* RTE_PMD_MLX5_IBVERBS_H_ */
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 0548d17..323ced7 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -30,7 +30,8 @@
  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-
+#define _GNU_SOURCE
+#include <stdio.h>
 #include <stddef.h>
 #include <unistd.h>
 #include <string.h>
@@ -39,13 +40,17 @@
 #include <stdlib.h>
 #include <errno.h>
 #include <net/if.h>
-
+#include <dlfcn.h>
 /* Verbs header. */
 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
@@ -1029,6 +1034,9 @@ struct mlx5_args {
 rte_mlx5_pmd_init(void)
 {
 	/* Build the static table for ptype conversion. */
+#ifdef MLX5_PMD_DLL
+	int ret;
+#endif
 	mlx5_set_ptype_table();
 	/*
 	 * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
@@ -1040,6 +1048,11 @@ struct mlx5_args {
 	/* Match the size of Rx completion entry to the size of a cacheline. */
 	if (RTE_CACHE_LINE_SIZE == 128)
 		setenv("MLX5_CQE_SIZE", "128", 0);
+#ifdef MLX5_PMD_DLL
+	ret = mlx5_load_libs();
+	if (ret != 0)
+		return;
+#endif
 	ibv_fork_init();
 	rte_pci_register(&mlx5_driver);
 }
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index e6a69b8..1dde985 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -46,7 +46,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index f32dfdd..ffd419c 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -39,7 +39,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c
index d17b991..9e8902b 100644
--- a/drivers/net/mlx5/mlx5_mac.c
+++ b/drivers/net/mlx5/mlx5_mac.c
@@ -46,7 +46,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 6b29eed..c91fcdf 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -36,7 +36,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c
index f3de46d..9f58147 100644
--- a/drivers/net/mlx5/mlx5_rss.c
+++ b/drivers/net/mlx5/mlx5_rss.c
@@ -42,7 +42,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c
index 0ef2cdf..f9f7d6f 100644
--- a/drivers/net/mlx5/mlx5_rxmode.c
+++ b/drivers/net/mlx5/mlx5_rxmode.c
@@ -40,7 +40,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 85399ef..dfd6400 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -44,8 +44,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 2d30c50..e2215aa 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -41,8 +41,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index d34f3cc..7419f2c 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -43,12 +43,14 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
-#include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
-
 #include <rte_mbuf.h>
 #include <rte_mempool.h>
 #include <rte_common.h>
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c
index ba6c8ce..fc7f701 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec.c
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.c
@@ -41,8 +41,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 9c5860f..8f52525 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -44,7 +44,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX5_PMD_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 6a6a745..0542c9a 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -51,7 +51,9 @@ endif
 
 # Link only the libraries used in the application
 LDFLAGS += --as-needed
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+LDFLAGS += -ldl
+endif
 # default path for libs
 _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
 
@@ -142,7 +144,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI)        += -lrte_pmd_kni
 endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD)        += -lrte_pmd_lio
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -libverbs -lmlx4
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5
+else
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -libverbs -lmlx5
+endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD)       += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk
 _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD)        += -lrte_pmd_nfp
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL)       += -lrte_pmd_null
-- 
1.8.3.1

[dpdk-dev] [PATCH v3 2/2] net/mlx4: load libmlx4 and libibverbs in run-time

[Shachar Beiser]

MLX4 PMD dynamically loads mlx4 and ibverbs in
run-time and it is not linked to external libraries.

Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
---
    v1:
        load external libraries in run-time
    v2: 
        fix checkpatch warnings
    v3:
        fix checkpatch warnings
---
 config/common_base              |   1 +
 drivers/net/mlx4/Makefile       |  14 +-
 drivers/net/mlx4/lib/mlx4_dll.c | 729 ++++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx4/lib/mlx4_dll.h |  91 +++++
 drivers/net/mlx4/mlx4.c         |  12 +
 drivers/net/mlx4/mlx4.h         |   4 +
 drivers/net/mlx4/mlx4_ethdev.c  |   4 +
 drivers/net/mlx4/mlx4_flow.c    |   4 +
 drivers/net/mlx4/mlx4_mr.c      |   4 +
 drivers/net/mlx4/mlx4_prm.h     |   4 +
 drivers/net/mlx4/mlx4_rxq.c     |   4 +
 drivers/net/mlx4/mlx4_rxtx.c    |   4 +
 drivers/net/mlx4/mlx4_rxtx.h    |   4 +
 drivers/net/mlx4/mlx4_txq.c     |   4 +
 mk/rte.app.mk                   |   7 +
 15 files changed, 888 insertions(+), 2 deletions(-)
 create mode 100644 drivers/net/mlx4/lib/mlx4_dll.c
 create mode 100644 drivers/net/mlx4/lib/mlx4_dll.h

diff --git a/config/common_base b/config/common_base
index 3708de4..8ef6be4 100644
--- a/config/common_base
+++ b/config/common_base
@@ -229,6 +229,7 @@ CONFIG_RTE_LIBRTE_FM10K_INC_VECTOR=y
 # Compile burst-oriented Mellanox ConnectX-3 (MLX4) PMD
 #
 CONFIG_RTE_LIBRTE_MLX4_PMD=n
+CONFIG_RTE_LIBRTE_MLX4_DLL=y
 CONFIG_RTE_LIBRTE_MLX4_DEBUG=n
 CONFIG_RTE_LIBRTE_MLX4_DEBUG_BROKEN_VERBS=n
 CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile
index f1f47c2..aba1d5f 100644
--- a/drivers/net/mlx4/Makefile
+++ b/drivers/net/mlx4/Makefile
@@ -44,7 +44,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_rxq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_rxtx.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_txq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_utils.c
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += lib/mlx4_dll.c
+endif
 # Basic CFLAGS.
 CFLAGS += -O3
 CFLAGS += -std=c11 -Wall -Wextra
@@ -54,7 +56,11 @@ CFLAGS += -D_BSD_SOURCE
 CFLAGS += -D_DEFAULT_SOURCE
 CFLAGS += -D_XOPEN_SOURCE=600
 CFLAGS += $(WERROR_FLAGS)
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+LDLIBS += -ldl
+else
 LDLIBS += -libverbs -lmlx4
+endif
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_pci
@@ -81,7 +87,11 @@ endif
 ifdef CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE
 CFLAGS += -DMLX4_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE)
 endif
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+CFLAGS += -DMLX4_PMD_DLL
+else
+CFLAGS += -UMLX4_PMD_DLL
+endif
 ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DEBUG_BROKEN_VERBS),y)
 CFLAGS += -DMLX4_PMD_DEBUG_BROKEN_VERBS
 endif
diff --git a/drivers/net/mlx4/lib/mlx4_dll.c b/drivers/net/mlx4/lib/mlx4_dll.c
new file mode 100644
index 0000000..8c7cede
--- /dev/null
+++ b/drivers/net/mlx4/lib/mlx4_dll.c
@@ -0,0 +1,729 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <dlfcn.h>
+#include <rte_log.h>
+#include "../mlx4_utils.h"
+#include "mlx4_dll.h"
+
+#define VERBS_LIB_DIR "/usr/lib64/libibverbs"
+#define MLX5_LIB_DIR "/usr/lib64/libmlx4"
+#define DIR_LENGTH 25
+/**
+ * Load a libibverbs and libmlx4 symbols table.
+ *
+ * @return
+ *   0 on success.
+ */
+int mlx4_load_libs(void)
+{
+	void *dlhandle;
+	int ret;
+
+	dlhandle = mlx4_lib_load("ibverbs.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load ibverbs.so");
+		return -1;
+	}
+	ret = mlx4_lverbs_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in libverbs.so ");
+		return ret;
+	}
+	dlhandle = mlx4_lib_load("mlx4.so");
+	if (dlhandle == NULL) {
+		ERROR("cannot load mlx4.so");
+		return -1;
+	}
+	ret = mlx4_lmlx4_function_register(dlhandle);
+	if (ret == -1) {
+		ERROR("cannot register a function in lmlx4.so  ");
+		return ret;
+	}
+	return 0;
+}
+/**
+ * Load a libibverbs or libmlx4 symbols table.
+ *
+ * @param name[in]
+ *   The library name.
+ * @return
+ *   dlhandle on success.
+ */
+void *mlx4_lib_load(const char *name)
+{
+	char *so_name;
+	void *dlhandle;
+	char lib[DIR_LENGTH];
+
+	if (strcmp(name, "ibverbs.so"))
+		strcpy(lib, VERBS_LIB_DIR);
+	else
+		strcpy(lib, MLX5_LIB_DIR);
+	/* If the name is an absolute path then open that path after appending
+	 * the trailer suffix
+	 */
+	if (name[0] == '/') {
+		if (asprintf(&so_name, "%s", name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		if (!dlhandle)
+			goto out_dlopen;
+		free(so_name);
+		return dlhandle;
+	}
+	/* If configured with a provider plugin path then try that next */
+	if (strlen(lib) > 1) {
+		if (asprintf(&so_name, "%s/lib%s", lib, name) < 0)
+			goto out_asprintf;
+		dlhandle = dlopen(so_name, RTLD_LAZY);
+		free(so_name);
+		if (dlhandle)
+			return dlhandle;
+	}
+	/* Otherwise use the system library search path. This is the historical
+	 * behavior of libibverbs
+	 */
+	if (asprintf(&so_name, "lib%s", name) < 0)
+		goto out_asprintf;
+	dlhandle = dlopen(so_name, RTLD_LAZY);
+	if (!dlhandle)
+		goto out_dlopen;
+	free(so_name);
+	return dlhandle;
+out_asprintf:
+	ERROR("Warning: couldn't load driver '%s'.\n", name);
+	return NULL;
+out_dlopen:
+	ERROR("Warning: couldn't load driver '%s': %s\n", so_name, dlerror());
+	free(so_name);
+	return NULL;
+}
+
+/**
+ * Register libibverbs functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx4_lverbs_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx4_libverbs.ibv_destroy_qp) =
+			dlsym(handle, "ibv_destroy_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_destroy_cq) =
+			dlsym(handle, "ibv_destroy_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_dealloc_pd) =
+			dlsym(handle, "ibv_dealloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_device_list) =
+			dlsym(handle, "ibv_get_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_open_device) =
+			dlsym(handle, "ibv_open_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_query_device) =
+			dlsym(handle, "ibv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_close_device) =
+			dlsym(handle, "ibv_close_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_free_device_list) =
+			dlsym(handle, "ibv_free_device_list");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_alloc_pd) =
+			dlsym(handle, "ibv_alloc_pd");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_device_name) =
+			dlsym(handle, "ibv_get_device_name");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_fork_init) =
+			dlsym(handle, "ibv_fork_init");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_destroy_comp_channel) =
+			dlsym(handle, "ibv_destroy_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_dereg_mr) =
+			dlsym(handle, "ibv_dereg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_create_comp_channel) =
+			dlsym(handle, "ibv_create_comp_channel");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_cq_event) =
+			dlsym(handle, "ibv_get_cq_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_create_qp) =
+			dlsym(handle, "ibv_create_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_ack_async_event) =
+			dlsym(handle, "ibv_ack_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_get_async_event) =
+			dlsym(handle, "ibv_get_async_event");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_reg_mr) =
+			dlsym(handle, "ibv_reg_mr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_create_cq) =
+			dlsym(handle, "ibv_create_cq");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_modify_qp) =
+			dlsym(handle, "ibv_modify_qp");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_libverbs.ibv_query_port) =
+			dlsym(handle, "ibv_query_port");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+
+/**
+ * Register libmlx4 functions apis .
+ *
+ * @param handle[in]
+ *   The library handle.
+ * @return
+ *   0 on success.
+ */
+int mlx4_lmlx4_function_register(void *handle)
+{
+	char *error;
+	int ret = 0;
+
+	*(void **)(&mlx4_lmlx4.mlx4dv_init_obj) =
+			dlsym(handle, "mlx4dv_init_obj");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_lmlx4.mlx4dv_query_device) =
+			dlsym(handle, "mlx4dv_query_device");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	*(void **)(&mlx4_lmlx4.mlx4dv_set_context_attr) =
+			dlsym(handle, "mlx4dv_set_context_attr");
+	error = dlerror();
+	if ((error) != NULL) {
+		ERROR("%s\n", error);
+		ret = EINVAL;
+		goto exit;
+	}
+	return 0;
+exit:
+	return -ret;
+}
+/**
+ * Function pointer calls mlx4 API .
+ *
+ * @param mlx4dv_obj[in]
+ *   The mlx4 object.
+ * @param obj_type[in]
+ *   The object type.
+ * @return
+ *   a pointer on success.
+ */
+int mlx4dv_init_obj(struct mlx4dv_obj *obj, uint64_t obj_type)
+{
+	return mlx4_lmlx4.mlx4dv_init_obj(obj, obj_type);
+}
+/**
+ * Function pointer calls mlx4 API .
+ *
+ * @param ctx_in[in]
+ *   The ibverbs context.
+ * @param attrs_out[out]
+ *   The mlx4 context.
+ * @return
+ *  0 on success.
+ */
+int mlx4dv_query_device(struct ibv_context *ctx_in,
+		struct mlx4dv_context *attrs_out)
+{
+	return mlx4_lmlx4.mlx4dv_query_device(ctx_in, attrs_out);
+}
+/**
+ * Function pointer calls mlx4 API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param type[in]
+ *   The mlx4 context attribute type.
+ * @param attr[in]
+ *   The mlx4 context attribute.
+ * @return
+ *  0 on success.
+ */
+int mlx4dv_set_context_attr(struct ibv_context *context,
+		enum mlx4dv_set_ctx_attr_type type, void *attr)
+{
+	return mlx4_lmlx4.mlx4dv_set_context_attr(context, type, attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_qp(struct ibv_qp *qp)
+{
+	return mlx4_libverbs.ibv_destroy_qp(qp);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   Completion queue pointer.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_cq(struct ibv_cq *cq)
+{
+	return mlx4_libverbs.ibv_destroy_cq(cq);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   Protection domain.
+ * @return
+ *  0 on success.
+ */
+int ibv_dealloc_pd(struct ibv_pd *pd)
+{
+	return mlx4_libverbs.ibv_dealloc_pd(pd);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param num_devices[in]
+ *   The number of devices.
+ * @return
+ *  0 on success.
+ */
+struct ibv_device **ibv_get_device_list(int *num_devices)
+{
+	return mlx4_libverbs.ibv_get_device_list(num_devices);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @param device_attr[in]
+ *   The device attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_device(struct ibv_context *context,
+		struct ibv_device_attr *device_attr)
+{
+	return mlx4_libverbs.ibv_query_device(context, device_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   The ibverbs context.
+ * @return
+ *  0 on success.
+ */
+int ibv_close_device(struct ibv_context *context)
+{
+	return mlx4_libverbs.ibv_close_device(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param list[in]
+ *   The list of devices.
+ */
+void ibv_free_device_list(struct ibv_device **list)
+{
+	return mlx4_libverbs.ibv_free_device_list(list);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   A device pointer.
+ * @return
+ *  The context on success.
+ */
+struct ibv_context *ibv_open_device(struct ibv_device *device)
+{
+	return mlx4_libverbs.ibv_open_device(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @return
+ *  protection domain pointer on success.
+ */
+struct ibv_pd *ibv_alloc_pd(struct ibv_context *context)
+{
+	return mlx4_libverbs.ibv_alloc_pd(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param device[in]
+ *   An ibverbs device pointer.
+ * @return
+ *  device name on success.
+ */
+const char *ibv_get_device_name(struct ibv_device *device)
+{
+	return mlx4_libverbs.ibv_get_device_name(device);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @return
+ *  0 on success.
+ */
+int ibv_fork_init(void)
+{
+	return mlx4_libverbs.ibv_fork_init();
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A completion channel.
+ * @return
+ *  0 on success.
+ */
+int ibv_destroy_comp_channel(struct ibv_comp_channel *channel)
+{
+	return mlx4_libverbs.ibv_destroy_comp_channel(channel);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param mr[in]
+ *   A pointer to memory region.
+ * @return
+ *  0 on success.
+ */
+int ibv_dereg_mr(struct ibv_mr *mr)
+{
+	return mlx4_libverbs.ibv_dereg_mr(mr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A pointer to ibverbs context.
+ * @return
+ *  0 on success.
+ */
+struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context)
+{
+	return mlx4_libverbs.ibv_create_comp_channel(context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param channel[in]
+ *   A pointer to completion channel.
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param cq_context[in]
+ *   A pointer to completion queue context.
+ * @return
+ *  0 on success.
+ */
+int ibv_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq,
+		void **cq_context)
+{
+	return mlx4_libverbs.ibv_get_cq_event(channel, cq, cq_context);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param cq[in]
+ *   A pointer to completion queue.
+ * @param nevents[in]
+ *   The number of events.
+ * @return
+ *  0 on success.
+ */
+void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
+{
+	return mlx4_libverbs.ibv_ack_cq_events(cq, nevents);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param qp_init_attr[in]
+ *   The queue pair attributes.
+ * @return
+ *  A pointer to queue pair on success.
+ */
+struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
+		struct ibv_qp_init_attr *qp_init_attr)
+{
+	return mlx4_libverbs.ibv_create_qp(pd, qp_init_attr);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+void ibv_ack_async_event(struct ibv_async_event *event)
+{
+	return mlx4_libverbs.ibv_ack_async_event(event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   A ibverbs context.
+ * @param event[in]
+ *   A pointer to a-synchronic event.
+ */
+int ibv_get_async_event(struct ibv_context *context,
+		struct ibv_async_event *event)
+{
+	return mlx4_libverbs.ibv_get_async_event(context, event);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param pd[in]
+ *   The protection domain pointer.
+ * @param addr[in]
+ *   The address.
+ * @param length[in]
+ *   The length.
+ * @param access[in]
+ *   The access type.
+ * @return
+ *  A pointer to memory region on success.
+ */
+struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
+		int access)
+{
+	return mlx4_libverbs.ibv_reg_mr(pd, addr, length, access);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   Ibverbs context.
+ * @param cqe[in]
+ *   A completion queue entry.
+ * @param cq_context[in]
+ *   Completion queue context.
+ * @param channel[in]
+ *   The completion channel.
+ * @param comp_vector[in]
+ *   Completion vector.
+ * @return
+ *  0 on success.
+ */
+struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe,
+	void *cq_context, struct ibv_comp_channel *channel, int comp_vector)
+{
+	return mlx4_libverbs.ibv_create_cq(context, cqe, cq_context, channel,
+			comp_vector);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param qp[in]
+ *   Queue pair.
+ * @param attr[in]
+ *   The queue pair attribute.
+ * @param attr_mask[in]
+ *   Attribute mask.
+ * @return
+ *  0 on success.
+ */
+int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
+{
+	return mlx4_libverbs.ibv_modify_qp(qp, attr, attr_mask);
+}
+/**
+ * Function pointer calls libibverbs API .
+ *
+ * @param context[in]
+ *   An ibverbs context.
+ * @param port_num[in]
+ *   The port number.
+ * @param port_attr[in]
+ *   Port attribute.
+ * @return
+ *  0 on success.
+ */
+int ibv_query_port(struct ibv_context *context, uint8_t port_num,
+		struct ibv_port_attr *port_attr)
+{
+	port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
+	port_attr->reserved = 0;
+	return mlx4_libverbs.ibv_query_port(context, port_num, port_attr);
+}
diff --git a/drivers/net/mlx4/lib/mlx4_dll.h b/drivers/net/mlx4/lib/mlx4_dll.h
new file mode 100644
index 0000000..9048040
--- /dev/null
+++ b/drivers/net/mlx4/lib/mlx4_dll.h
@@ -0,0 +1,91 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2015 6WIND S.A.
+ *   Copyright 2015 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef RTE_PMD_MLX4_IBVERBS_H_
+#define RTE_PMD_MLX4_IBVERBS_H_
+#include <infiniband/verbs.h>
+#include <infiniband/mlx4dv.h>
+
+void *mlx4_lib_load(const char *name);
+int mlx4_lverbs_function_register(void *handle);
+int mlx4_lmlx4_function_register(void *handle);
+int mlx4_load_libs(void);
+#undef ibv_query_port
+
+struct {
+	int size;
+	int version;
+	int (*ibv_destroy_qp)(struct ibv_qp *qp);
+	int (*ibv_destroy_cq)(struct ibv_cq *cq);
+	int (*ibv_dealloc_pd)(struct ibv_pd *pd);
+	struct ibv_device **(*ibv_get_device_list)(int *num_devices);
+	int (*ibv_query_device)(struct ibv_context *context,
+			struct ibv_device_attr *device_attr);
+	int (*ibv_close_device)(struct ibv_context *context);
+	void (*ibv_free_device_list)(struct ibv_device **list);
+	struct ibv_context *(*ibv_open_device)(struct ibv_device *device);
+	struct ibv_pd *(*ibv_alloc_pd)(struct ibv_context *context);
+	const char *(*ibv_get_device_name)(struct ibv_device *device);
+	int (*ibv_fork_init)(void);
+	int (*ibv_destroy_comp_channel)(struct ibv_comp_channel *channel);
+	int (*ibv_dereg_mr)(struct ibv_mr *mr);
+	struct ibv_comp_channel *(*ibv_create_comp_channel)(
+			struct ibv_context *context);
+	int (*ibv_get_cq_event)(struct ibv_comp_channel *channel,
+			struct ibv_cq **cq, void **cq_context);
+	void (*ibv_ack_cq_events)(struct ibv_cq *cq, unsigned int nevents);
+	struct ibv_qp *(*ibv_create_qp)(struct ibv_pd *pd,
+			struct ibv_qp_init_attr *qp_init_attr);
+	void (*ibv_ack_async_event)(struct ibv_async_event *event);
+	int (*ibv_get_async_event)(struct ibv_context *context,
+			struct ibv_async_event *event);
+	struct ibv_mr *(*ibv_reg_mr)(struct ibv_pd *pd, void *addr,
+			size_t length, int access);
+	struct ibv_cq *(*ibv_create_cq)(struct ibv_context *context, int cqe,
+			void *cq_context, struct ibv_comp_channel *channel,
+			int comp_vector);
+	int (*ibv_modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+			int attr_mask);
+	int (*ibv_query_port)(struct ibv_context *context, uint8_t port_num,
+			struct ibv_port_attr *port_attr);
+} mlx4_libverbs;
+
+struct {
+	int size;
+	int version;
+	int (*mlx4dv_init_obj)(struct mlx4dv_obj *obj, uint64_t obj_type);
+	int (*mlx4dv_query_device)(struct ibv_context *ctx_in,
+			struct mlx4dv_context *attrs_out);
+	int (*mlx4dv_set_context_attr)(struct ibv_context *context,
+			enum mlx4dv_set_ctx_attr_type type, void *attr);
+} mlx4_lmlx4;
+#endif /* RTE_PMD_MLX4_IBVERBS_H_ */
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index f9e4f9d..90f9ed3 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -49,7 +49,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
@@ -713,7 +717,15 @@ struct mlx4_conf {
 	 * applications to use fork() safely for purposes other than
 	 * using this PMD, which is not supported in forked processes.
 	 */
+#ifdef MLX4_PMD_DLL
+	int ret;
+#endif
 	setenv("RDMAV_HUGEPAGES_SAFE", "1", 1);
+#ifdef MLX4_PMD_DLL
+	ret = mlx4_load_libs();
+	if (ret != 0)
+		return;
+#endif
 	ibv_fork_init();
 	rte_pci_register(&mlx4_driver);
 }
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 3aeef87..6e4565b 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -42,7 +42,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_ethdev.c b/drivers/net/mlx4/mlx4_ethdev.c
index 2f69e7d..98134ab 100644
--- a/drivers/net/mlx4/mlx4_ethdev.c
+++ b/drivers/net/mlx4/mlx4_ethdev.c
@@ -56,7 +56,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_flow.c b/drivers/net/mlx4/mlx4_flow.c
index 8b87b29..89d6d17 100644
--- a/drivers/net/mlx4/mlx4_flow.c
+++ b/drivers/net/mlx4/mlx4_flow.c
@@ -49,7 +49,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_mr.c b/drivers/net/mlx4/mlx4_mr.c
index 2a3e269..b494e95 100644
--- a/drivers/net/mlx4/mlx4_mr.c
+++ b/drivers/net/mlx4/mlx4_mr.c
@@ -47,7 +47,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_prm.h b/drivers/net/mlx4/mlx4_prm.h
index fcc7c12..b8854e1 100644
--- a/drivers/net/mlx4/mlx4_prm.h
+++ b/drivers/net/mlx4/mlx4_prm.h
@@ -42,8 +42,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PRD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c
index 53313c5..5680c89 100644
--- a/drivers/net/mlx4/mlx4_rxq.c
+++ b/drivers/net/mlx4/mlx4_rxq.c
@@ -46,8 +46,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_rxtx.c b/drivers/net/mlx4/mlx4_rxtx.c
index 2bfa8b1..a876896 100644
--- a/drivers/net/mlx4/mlx4_rxtx.c
+++ b/drivers/net/mlx4/mlx4_rxtx.c
@@ -44,7 +44,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h
index 463df2b..c115032 100644
--- a/drivers/net/mlx4/mlx4_rxtx.h
+++ b/drivers/net/mlx4/mlx4_rxtx.h
@@ -41,8 +41,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c
index 7882a4d..8ae9eb9 100644
--- a/drivers/net/mlx4/mlx4_txq.c
+++ b/drivers/net/mlx4/mlx4_txq.c
@@ -46,7 +46,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef MLX4_PMD_DLL
+#include "lib/mlx4_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 0542c9a..5589482 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -54,6 +54,9 @@ LDFLAGS += --as-needed
 ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
 LDFLAGS += -ldl
 endif
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+LDFLAGS += -ldl
+endif
 # default path for libs
 _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
 
@@ -143,7 +146,11 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI)        += -lrte_pmd_kni
 endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD)        += -lrte_pmd_lio
+ifeq ($(CONFIG_RTE_LIBRTE_MLX4_DLL),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4
+else
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -libverbs -lmlx4
+endif
 ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5
 else
-- 
1.8.3.1

[dpdk-dev] [PATCH v4] net/mlx5: load libmlx5 and libibverbs in run-time

[Shachar Beiser]

MLX5 PMD loads libraries: libibverbs and libmlx5.
MLX5 PMD is not linked to external libraries.

Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
---
v1: 
    load external libraries in run-time
v2:
    * fix checkpatch warnings 
v3:
    * fix checkpatch warnings
v4:
    New  MACROs in order to reuse code  
---
 config/common_base               |   1 +
 drivers/net/mlx5/Makefile        |  22 ++-
 drivers/net/mlx5/lib/mlx5_dll.c  | 294 +++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/lib/mlx5_dll.h  | 103 ++++++++++++++
 drivers/net/mlx5/mlx5.c          |  17 ++-
 drivers/net/mlx5/mlx5.h          |   4 +
 drivers/net/mlx5/mlx5_flow.c     |   4 +
 drivers/net/mlx5/mlx5_mac.c      |   4 +
 drivers/net/mlx5/mlx5_mr.c       |   4 +
 drivers/net/mlx5/mlx5_rss.c      |   4 +
 drivers/net/mlx5/mlx5_rxmode.c   |   4 +
 drivers/net/mlx5/mlx5_rxq.c      |   4 +
 drivers/net/mlx5/mlx5_rxtx.c     |   4 +
 drivers/net/mlx5/mlx5_rxtx.h     |   6 +-
 drivers/net/mlx5/mlx5_rxtx_vec.c |   4 +
 drivers/net/mlx5/mlx5_txq.c      |   4 +
 mk/rte.app.mk                    |   8 +-
 17 files changed, 479 insertions(+), 12 deletions(-)
 create mode 100644 drivers/net/mlx5/lib/mlx5_dll.c
 create mode 100644 drivers/net/mlx5/lib/mlx5_dll.h

diff --git a/config/common_base b/config/common_base
index b8ee8f9..30c8fcf 100644
--- a/config/common_base
+++ b/config/common_base
@@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
 # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
 #
 CONFIG_RTE_LIBRTE_MLX5_PMD=n
+CONFIG_RTE_LIBRTE_MLX5_DLL=y
 CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
 CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
 
diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index a3984eb..24fa127 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -53,18 +53,25 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c
+endif
 # Basic CFLAGS.
 CFLAGS += -O3
 CFLAGS += -std=c11 -Wall -Wextra
 CFLAGS += -g
 CFLAGS += -I.
+CFLAGS += -I$(SRCDIR)
 CFLAGS += -D_BSD_SOURCE
 CFLAGS += -D_DEFAULT_SOURCE
 CFLAGS += -D_XOPEN_SOURCE=600
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -Wno-strict-prototypes
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+LDLIBS += -ldl
+else
 LDLIBS += -libverbs -lmlx5
+endif
 LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
 LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
 LDLIBS += -lrte_bus_pci
@@ -105,26 +112,28 @@ endif
 
 mlx5_autoconf.h.new: FORCE
 
+VERBS_H := infiniband/verbs.h
+MLX5DV_H := infiniband/mlx5dv.h
 mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 	$Q $(RM) -f -- '$@'
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_DEVICE_VXLAN_SUPPORT \
-		infiniband/verbs.h \
+		$(VERBS_H) \
 		enum IBV_DEVICE_VXLAN_SUPPORT \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_WQ_FLAG_RX_END_PADDING \
-		infiniband/verbs.h \
+		$(VERBS_H) \
 		enum IBV_WQ_FLAG_RX_END_PADDING \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_MLX5_MOD_MPW \
-		infiniband/mlx5dv.h \
+		$(MLX5DV_H) \
 		enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
-		infiniband/mlx5dv.h \
+		$(MLX5DV_H) \
 		enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
@@ -144,10 +153,9 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
-		infiniband/verbs.h \
+		$(VERBS_H)  \
 		enum IBV_FLOW_SPEC_ACTION_COUNT \
 		$(AUTOCONF_OUTPUT)
-
 # Create mlx5_autoconf.h or update it in case it differs from the new one.
 
 mlx5_autoconf.h: mlx5_autoconf.h.new
diff --git a/drivers/net/mlx5/lib/mlx5_dll.c b/drivers/net/mlx5/lib/mlx5_dll.c
new file mode 100644
index 0000000..cafc78a
--- /dev/null
+++ b/drivers/net/mlx5/lib/mlx5_dll.c
@@ -0,0 +1,294 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2017 6WIND S.A.
+ *   Copyright 2017 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <dlfcn.h>
+
+#include <rte_log.h>
+
+#include "mlx5_utils.h"
+#include "mlx5_dll.h"
+
+MLX5_MLX5DV_FUNC(create_cq, struct ibv_cq_ex *,
+		 struct ibv_context *context,
+		 struct ibv_cq_init_attr_ex *cq_attr,
+		 struct mlx5dv_cq_init_attr *mlx5_cq_attr)
+{
+	return __MLX5_MLX5DV_FUNC(create_cq, context, cq_attr, mlx5_cq_attr);
+}
+
+MLX5_MLX5DV_FUNC(init_obj, int, struct mlx5dv_obj *obj, uint64_t obj_type)
+{
+	return __MLX5_MLX5DV_FUNC(init_obj, obj, obj_type);
+}
+
+MLX5_MLX5DV_FUNC(query_device, int, struct ibv_context *ctx_in,
+		 struct mlx5dv_context *attrs_out)
+{
+	return __MLX5_MLX5DV_FUNC(query_device, ctx_in, attrs_out);
+}
+
+MLX5_MLX5DV_FUNC(set_context_attr, int, struct ibv_context *context,
+		 enum mlx5dv_set_ctx_attr_type type, void *attr)
+{
+	return __MLX5_MLX5DV_FUNC(set_context_attr, context, type, attr);
+}
+
+MLX5_IBVERBS_FUNC(destroy_qp, int, struct ibv_qp *qp)
+{
+	return __MLX5_IBVERBS_FUNC(destroy_qp, qp);
+}
+
+MLX5_IBVERBS_FUNC(destroy_cq, int, struct ibv_cq *cq)
+{
+	return __MLX5_IBVERBS_FUNC(destroy_cq, cq);
+}
+
+MLX5_IBVERBS_FUNC(dealloc_pd, int, struct ibv_pd *pd)
+{
+	return __MLX5_IBVERBS_FUNC(dealloc_pd, pd);
+}
+
+MLX5_IBVERBS_FUNC(get_device_list, struct ibv_device **, int *num_devices)
+{
+	return __MLX5_IBVERBS_FUNC(get_device_list, num_devices);
+}
+
+MLX5_IBVERBS_FUNC(query_device, int, struct ibv_context *context,
+		  struct ibv_device_attr *device_attr)
+{
+	return __MLX5_IBVERBS_FUNC(query_device, context, device_attr);
+}
+
+MLX5_IBVERBS_FUNC(close_device, int, struct ibv_context *context)
+{
+	return __MLX5_IBVERBS_FUNC(close_device, context);
+}
+
+MLX5_IBVERBS_FUNC(free_device_list, void, struct ibv_device **list)
+{
+	__MLX5_IBVERBS_FUNC(free_device_list, list);
+}
+
+MLX5_IBVERBS_FUNC(open_device, struct ibv_context *, struct ibv_device *device)
+{
+	return __MLX5_IBVERBS_FUNC(open_device, device);
+}
+
+MLX5_IBVERBS_FUNC(alloc_pd, struct ibv_pd *, struct ibv_context *context)
+{
+	return __MLX5_IBVERBS_FUNC(alloc_pd, context);
+}
+
+MLX5_IBVERBS_FUNC(get_device_name, const char *, struct ibv_device *device)
+{
+	return __MLX5_IBVERBS_FUNC(get_device_name, device);
+}
+
+MLX5_IBVERBS_FUNC(fork_init, int, void)
+{
+	return __MLX5_IBVERBS_FUNC(fork_init,);
+}
+
+MLX5_IBVERBS_FUNC(destroy_comp_channel, int, struct ibv_comp_channel *channel)
+{
+	return __MLX5_IBVERBS_FUNC(destroy_comp_channel, channel);
+}
+
+MLX5_IBVERBS_FUNC(dereg_mr, int, struct ibv_mr *mr)
+{
+	return __MLX5_IBVERBS_FUNC(dereg_mr, mr);
+}
+
+MLX5_IBVERBS_FUNC(create_comp_channel, struct ibv_comp_channel *,
+		  struct ibv_context *context)
+{
+	return __MLX5_IBVERBS_FUNC(create_comp_channel, context);
+}
+
+MLX5_IBVERBS_FUNC(get_cq_event, int, struct ibv_comp_channel *channel,
+		  struct ibv_cq **cq, void **cq_context)
+{
+	return __MLX5_IBVERBS_FUNC(get_cq_event, channel, cq, cq_context);
+}
+
+MLX5_IBVERBS_FUNC(ack_cq_events, void, struct ibv_cq *cq, unsigned int nevents)
+{
+	__MLX5_IBVERBS_FUNC(ack_cq_events, cq, nevents);
+}
+
+MLX5_IBVERBS_FUNC(create_qp, struct ibv_qp *, struct ibv_pd *pd,
+		  struct ibv_qp_init_attr *qp_init_attr)
+{
+	return __MLX5_IBVERBS_FUNC(create_qp, pd, qp_init_attr);
+}
+
+MLX5_IBVERBS_FUNC(ack_async_event, void, struct ibv_async_event *event)
+{
+	__MLX5_IBVERBS_FUNC(ack_async_event, event);
+}
+
+MLX5_IBVERBS_FUNC(get_async_event, int, struct ibv_context *context,
+		  struct ibv_async_event *event)
+{
+	return __MLX5_IBVERBS_FUNC(get_async_event, context, event);
+}
+
+MLX5_IBVERBS_FUNC(reg_mr, struct ibv_mr *, struct ibv_pd *pd, void *addr,
+		  size_t length, int access)
+{
+	return __MLX5_IBVERBS_FUNC(reg_mr, pd, addr, length, access);
+}
+
+MLX5_IBVERBS_FUNC(create_cq, struct ibv_cq*, struct ibv_context *context,
+		  int cqe, void *cq_context, struct ibv_comp_channel *channel,
+		  int comp_vector)
+{
+	return __MLX5_IBVERBS_FUNC(create_cq, context, cqe, cq_context, channel,
+				   comp_vector);
+}
+
+MLX5_IBVERBS_FUNC(modify_qp, int, struct ibv_qp *qp, struct ibv_qp_attr *attr,
+		  int attr_mask)
+{
+	return __MLX5_IBVERBS_FUNC(modify_qp, qp, attr, attr_mask);
+}
+
+MLX5_IBVERBS_FUNC(query_port, int, struct ibv_context *context,
+		  uint8_t port_num, struct ibv_port_attr *port_attr)
+{
+	port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
+	port_attr->reserved = 0;
+	return __MLX5_IBVERBS_FUNC(query_port, context, port_num, port_attr);
+}
+
+MLX5_IBVERBS_FUNC(port_state_str, const char *, enum ibv_port_state port_state)
+{
+	return __MLX5_IBVERBS_FUNC(port_state_str, port_state);
+}
+
+/**
+ * Register libibverbs APIs.
+ *
+ * @param[in] handle
+ *   Library handle.
+ *
+ * @return
+ *   0 on success, -1 on failure.
+ */
+int mlx5_libverbs_function_register(void *handle)
+{
+	MLX5_DLSYM_IBVERBS_FUNC(handle, modify_qp);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_qp);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_cq);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, dealloc_pd);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, alloc_pd);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, get_device_list);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, open_device);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, query_device);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, close_device);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, free_device_list);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, get_device_name);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, fork_init);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_comp_channel);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, create_comp_channel);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, dereg_mr);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, get_cq_event);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, create_qp);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, ack_async_event);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, get_async_event);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, reg_mr);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, create_cq);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, modify_qp);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, query_port);
+	MLX5_DLSYM_IBVERBS_FUNC(handle, port_state_str);
+	return 0;
+}
+
+/**
+ * Register libmlx5 APIs.
+ *
+ * @param[in] handle
+ *   Library handle.
+ *
+ * @return
+ *   0 on success, -1 on failure.
+ */
+int mlx5_lmlx5_function_register(void *handle)
+{
+	MLX5_DLSYM_MLX5DV_FUNC(handle, create_cq);
+	MLX5_DLSYM_MLX5DV_FUNC(handle, init_obj);
+	MLX5_DLSYM_MLX5DV_FUNC(handle, query_device);
+	MLX5_DLSYM_MLX5DV_FUNC(handle, set_context_attr);
+	return 0;
+}
+
+/**
+ * Load symbols of libibverbs and libmlx5.
+ *
+ * @return
+ *   0 on success, -1 on failure.
+ */
+int mlx5_load_libs(void)
+{
+	void *dlhandle;
+	int ret;
+
+	dlhandle = dlopen(MLX5_LIB_IBVERBS_FILENAME, RTLD_LAZY);
+	if (dlhandle == NULL) {
+		ERROR("failed to open %s: %s\n",
+		      MLX5_LIB_IBVERBS_FILENAME, dlerror());
+		return -1;
+	}
+	ret = mlx5_libverbs_function_register(dlhandle);
+	if (ret) {
+		ERROR("failed to register a function in %s",
+		      MLX5_LIB_IBVERBS_FILENAME);
+		return -1;
+	}
+	dlhandle = dlopen(MLX5_LIB_MLX5_FILENAME, RTLD_LAZY);
+	if (dlhandle == NULL) {
+		ERROR("failed to open %s: %s\n",
+		      MLX5_LIB_MLX5_FILENAME, dlerror());
+		return -1;
+	}
+	ret = mlx5_lmlx5_function_register(dlhandle);
+	if (ret) {
+		ERROR("failed to register a function in %s",
+		      MLX5_LIB_MLX5_FILENAME);
+		return -1;
+	}
+	return 0;
+}
diff --git a/drivers/net/mlx5/lib/mlx5_dll.h b/drivers/net/mlx5/lib/mlx5_dll.h
new file mode 100644
index 0000000..9c3d604
--- /dev/null
+++ b/drivers/net/mlx5/lib/mlx5_dll.h
@@ -0,0 +1,103 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2017 6WIND S.A.
+ *   Copyright 2017 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef MLX5_DLL_H_
+#define MLX5_DLL_H_
+
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
+
+#define MLX5_LIB_IBVERBS_FILENAME "libibverbs.so"
+#define MLX5_LIB_MLX5_FILENAME "libmlx5.so"
+
+#ifndef RTE_LIBRTE_MLX4_PMD
+
+#define MLX5_IBVERBS_FUNC(name, ret, ...) \
+	ret(*__mlx5_pmd_ibv_##name)(__VA_ARGS__); \
+	ret(ibv_##name)(__VA_ARGS__)
+
+#define __MLX5_IBVERBS_FUNC(name, ...) \
+	(*__mlx5_pmd_ibv_##name)(__VA_ARGS__)
+
+#define MLX5_DLSYM_IBVERBS_FUNC(handle, name) \
+	do { \
+		char *err; \
+		*(void **)(&__mlx5_pmd_ibv_##name) = \
+			dlsym(handle, "ibv_"#name); \
+		err = dlerror(); \
+		if (err != NULL) { \
+			ERROR("failed to link a symbol: %s", err); \
+			return -1; \
+		} \
+	} while (0)
+
+#else /* RTE_LIBRTE_MLX4_PMD */
+
+#define MLX5_IBVERBS_FUNC(name, ret, ...) \
+	ret(___mlx5_pmd_ibv_##name)(__VA_ARGS__); \
+	ret(___mlx5_pmd_ibv_##name)(__VA_ARGS__)
+
+#define __MLX5_IBVERBS_FUNC(name, ...) \
+	(ibv_##name)(__VA_ARGS__)
+
+#define MLX5_DLSYM_IBVERBS_FUNC(handle, name) do { (void)handle; } while (0)
+
+#endif /* RTE_LIBRTE_MLX4_PMD */
+
+#define MLX5_MLX5DV_FUNC(name, ret, ...) \
+	ret(*__mlx5_pmd_mlx5dv_##name)(__VA_ARGS__); \
+	ret(mlx5dv_##name)(__VA_ARGS__)
+
+#define __MLX5_MLX5DV_FUNC(name, ...) \
+	(*__mlx5_pmd_mlx5dv_##name)(__VA_ARGS__)
+
+#define MLX5_DLSYM_MLX5DV_FUNC(handle, name) \
+	do { \
+		char *err; \
+		*(void **)(&__mlx5_pmd_mlx5dv_##name) = \
+			dlsym(handle, "mlx5dv_"#name); \
+		err = dlerror(); \
+		if (err != NULL) { \
+			ERROR("failed to link a symbol: %s", err); \
+			return -1; \
+		} \
+	} while (0)
+
+#undef ibv_query_port
+
+void *mlx5_lib_load(const char *name);
+int mlx5_libverbs_function_register(void *handle);
+int mlx5_lmlx5_function_register(void *handle);
+int mlx5_load_libs(void);
+
+#endif /* RTE_PMD_MLX5_DLL_H_ */
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cd66fe1..eeef782 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -30,7 +30,8 @@
  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-
+#define _GNU_SOURCE
+#include <stdio.h>
 #include <stddef.h>
 #include <unistd.h>
 #include <string.h>
@@ -39,13 +40,17 @@
 #include <stdlib.h>
 #include <errno.h>
 #include <net/if.h>
-
+#include <dlfcn.h>
 /* Verbs header. */
 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
@@ -1027,6 +1032,9 @@ struct mlx5_args {
 rte_mlx5_pmd_init(void)
 {
 	/* Build the static table for ptype conversion. */
+#ifdef RTE_LIBRTE_MLX5_DLL
+	int ret;
+#endif
 	mlx5_set_ptype_table();
 	/*
 	 * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
@@ -1038,6 +1046,11 @@ struct mlx5_args {
 	/* Match the size of Rx completion entry to the size of a cacheline. */
 	if (RTE_CACHE_LINE_SIZE == 128)
 		setenv("MLX5_CQE_SIZE", "128", 0);
+#ifdef RTE_LIBRTE_MLX5_DLL
+	ret = mlx5_load_libs();
+	if (ret != 0)
+		return;
+#endif
 	ibv_fork_init();
 	rte_pci_register(&mlx5_driver);
 }
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index e6a69b8..9db7024 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -46,7 +46,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 6605cfd..49f8972 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -39,7 +39,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c
index 9fb5ba5..c9f4bfd 100644
--- a/drivers/net/mlx5/mlx5_mac.c
+++ b/drivers/net/mlx5/mlx5_mac.c
@@ -46,7 +46,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 6b29eed..f27a428 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -36,7 +36,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c
index f491135..63cbfa0 100644
--- a/drivers/net/mlx5/mlx5_rss.c
+++ b/drivers/net/mlx5/mlx5_rss.c
@@ -42,7 +42,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c
index 6fb245b..1ddad70 100644
--- a/drivers/net/mlx5/mlx5_rxmode.c
+++ b/drivers/net/mlx5/mlx5_rxmode.c
@@ -40,7 +40,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index a4cdd37..aaf4ce1 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -44,8 +44,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 67e3db1..4faf302 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -41,8 +41,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index e70d523..e637555 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -43,12 +43,14 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
-#include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
-
 #include <rte_mbuf.h>
 #include <rte_mempool.h>
 #include <rte_common.h>
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c
index 8d23dae..679bee7 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec.c
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.c
@@ -41,8 +41,12 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
 #include <infiniband/mlx5dv.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 89b16fd..8b0cdf4 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -44,7 +44,11 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#ifdef RTE_LIBRTE_MLX5_DLL
+#include "lib/mlx5_dll.h"
+#else
 #include <infiniband/verbs.h>
+#endif
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 6a6a745..2dd2f6b 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -51,7 +51,9 @@ endif
 
 # Link only the libraries used in the application
 LDFLAGS += --as-needed
-
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+LDFLAGS += -ldl
+endif
 # default path for libs
 _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
 
@@ -142,7 +144,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI)        += -lrte_pmd_kni
 endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD)        += -lrte_pmd_lio
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -libverbs -lmlx4
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -ldl
+else
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -libverbs -lmlx5
+endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD)       += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk
 _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD)        += -lrte_pmd_nfp
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL)       += -lrte_pmd_null
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v4] net/mlx5: load libmlx5 and libibverbs in run-time

[Nelio Laranjeiro]

Hi Shachar,

Please see small comment bellow,

On Sun, Dec 31, 2017 at 07:52:51AM +0000, Shachar Beiser wrote:
> MLX5 PMD loads libraries: libibverbs and libmlx5.
> MLX5 PMD is not linked to external libraries.
> 
> Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
> ---
> v1: 
>     load external libraries in run-time
> v2:
>     * fix checkpatch warnings 
> v3:
>     * fix checkpatch warnings
> v4:
>     New  MACROs in order to reuse code  
> ---
>  config/common_base               |   1 +
>  drivers/net/mlx5/Makefile        |  22 ++-
>  drivers/net/mlx5/lib/mlx5_dll.c  | 294 +++++++++++++++++++++++++++++++++++++++
>  drivers/net/mlx5/lib/mlx5_dll.h  | 103 ++++++++++++++
>  drivers/net/mlx5/mlx5.c          |  17 ++-
>  drivers/net/mlx5/mlx5.h          |   4 +
>  drivers/net/mlx5/mlx5_flow.c     |   4 +
>  drivers/net/mlx5/mlx5_mac.c      |   4 +
>  drivers/net/mlx5/mlx5_mr.c       |   4 +
>  drivers/net/mlx5/mlx5_rss.c      |   4 +
>  drivers/net/mlx5/mlx5_rxmode.c   |   4 +
>  drivers/net/mlx5/mlx5_rxq.c      |   4 +
>  drivers/net/mlx5/mlx5_rxtx.c     |   4 +
>  drivers/net/mlx5/mlx5_rxtx.h     |   6 +-
>  drivers/net/mlx5/mlx5_rxtx_vec.c |   4 +
>  drivers/net/mlx5/mlx5_txq.c      |   4 +
>  mk/rte.app.mk                    |   8 +-
>  17 files changed, 479 insertions(+), 12 deletions(-)
>  create mode 100644 drivers/net/mlx5/lib/mlx5_dll.c
>  create mode 100644 drivers/net/mlx5/lib/mlx5_dll.h
> 
> diff --git a/config/common_base b/config/common_base
> index b8ee8f9..30c8fcf 100644
> --- a/config/common_base
> +++ b/config/common_base
> @@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
>  # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
>  #
>  CONFIG_RTE_LIBRTE_MLX5_PMD=n
> +CONFIG_RTE_LIBRTE_MLX5_DLL=y
>  CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
>  CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8

Not sure a new configuration item is allowed.  If it is, the
documentation of such variable is missing.

> diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> index a3984eb..24fa127 100644
> --- a/drivers/net/mlx5/Makefile
> +++ b/drivers/net/mlx5/Makefile
> @@ -53,18 +53,25 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c
>  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
>  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
>  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
> -
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c
> +endif
>  # Basic CFLAGS.
>  CFLAGS += -O3
>  CFLAGS += -std=c11 -Wall -Wextra
>  CFLAGS += -g
>  CFLAGS += -I.
> +CFLAGS += -I$(SRCDIR)
>  CFLAGS += -D_BSD_SOURCE
>  CFLAGS += -D_DEFAULT_SOURCE
>  CFLAGS += -D_XOPEN_SOURCE=600
>  CFLAGS += $(WERROR_FLAGS)
>  CFLAGS += -Wno-strict-prototypes
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +LDLIBS += -ldl
> +else
>  LDLIBS += -libverbs -lmlx5
> +endif
>  LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
>  LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
>  LDLIBS += -lrte_bus_pci
> @@ -105,26 +112,28 @@ endif
>  
>  mlx5_autoconf.h.new: FORCE
>  
> +VERBS_H := infiniband/verbs.h
> +MLX5DV_H := infiniband/mlx5dv.h
>  mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
>  	$Q $(RM) -f -- '$@'
>  	$Q sh -- '$<' '$@' \
>  		HAVE_IBV_DEVICE_VXLAN_SUPPORT \
> -		infiniband/verbs.h \
> +		$(VERBS_H) \
>  		enum IBV_DEVICE_VXLAN_SUPPORT \
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
>  		HAVE_IBV_WQ_FLAG_RX_END_PADDING \
> -		infiniband/verbs.h \
> +		$(VERBS_H) \
>  		enum IBV_WQ_FLAG_RX_END_PADDING \
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
>  		HAVE_IBV_MLX5_MOD_MPW \
> -		infiniband/mlx5dv.h \
> +		$(MLX5DV_H) \
>  		enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
>  		HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
> -		infiniband/mlx5dv.h \
> +		$(MLX5DV_H) \
>  		enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
> @@ -144,10 +153,9 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
>  		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> -		infiniband/verbs.h \
> +		$(VERBS_H)  \
>  		enum IBV_FLOW_SPEC_ACTION_COUNT \
>  		$(AUTOCONF_OUTPUT)

This modification should be inside its own patch, it is not directly
related to the this patch itself.

> -
>  # Create mlx5_autoconf.h or update it in case it differs from the new one.
>  
>  mlx5_autoconf.h: mlx5_autoconf.h.new
<snip/>
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
> index cd66fe1..eeef782 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -30,7 +30,8 @@
>   *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>   *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
>   */
> -
> +#define _GNU_SOURCE
> +#include <stdio.h>
>  #include <stddef.h>
>  #include <unistd.h>
>  #include <string.h>
> @@ -39,13 +40,17 @@
>  #include <stdlib.h>
>  #include <errno.h>
>  #include <net/if.h>
> -
> +#include <dlfcn.h>

The empty line should remain.

>  /* Verbs header. */
>  /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic ignored "-Wpedantic"
>  #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
>  #include <infiniband/verbs.h>
> +#endif

This could be done by the mlx5_dll.h file which could include the
correct header according to the configuration.

<snip/>
> diff --git a/mk/rte.app.mk b/mk/rte.app.mk
> index 6a6a745..2dd2f6b 100644
> --- a/mk/rte.app.mk
> +++ b/mk/rte.app.mk
> @@ -51,7 +51,9 @@ endif
>  
>  # Link only the libraries used in the application
>  LDFLAGS += --as-needed
> -
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +LDFLAGS += -ldl
> +endif
>  # default path for libs
>  _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
>  
> @@ -142,7 +144,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI)        += -lrte_pmd_kni
>  endif
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD)        += -lrte_pmd_lio
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -libverbs -lmlx4
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -ldl
> +else
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -libverbs -lmlx5
> +endif
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD)       += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD)        += -lrte_pmd_nfp
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL)       += -lrte_pmd_null
> -- 
> 1.8.3.1
> 

Thanks,

-- 
Nélio Laranjeiro
6WIND

Re: [dpdk-dev] [PATCH v4] net/mlx5: load libmlx5 and libibverbs in run-time

[Shachar Beiser]

> -----Original Message-----
> From: Nelio Laranjeiro [mailto:nelio.laranjeiro@6wind.com]
> Sent: Tuesday, January 2, 2018 4:07 PM
> To: Shachar Beiser <shacharbe@mellanox.com>
> Cc: dev@dpdk.org; Adrien Mazarguil <adrien.mazarguil@6wind.com>
> Subject: Re: [PATCH v4] net/mlx5: load libmlx5 and libibverbs in run-time
> 
> Hi Shachar,
> 
> Please see small comment bellow,
> 
> On Sun, Dec 31, 2017 at 07:52:51AM +0000, Shachar Beiser wrote:
> > MLX5 PMD loads libraries: libibverbs and libmlx5.
> > MLX5 PMD is not linked to external libraries.
> >
> > Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
> > ---
> > v1:
> >     load external libraries in run-time
> > v2:
> >     * fix checkpatch warnings
> > v3:
> >     * fix checkpatch warnings
> > v4:
> >     New  MACROs in order to reuse code
> > ---
> >  config/common_base               |   1 +
> >  drivers/net/mlx5/Makefile        |  22 ++-
> >  drivers/net/mlx5/lib/mlx5_dll.c  | 294
> > +++++++++++++++++++++++++++++++++++++++
> >  drivers/net/mlx5/lib/mlx5_dll.h  | 103 ++++++++++++++
> >  drivers/net/mlx5/mlx5.c          |  17 ++-
> >  drivers/net/mlx5/mlx5.h          |   4 +
> >  drivers/net/mlx5/mlx5_flow.c     |   4 +
> >  drivers/net/mlx5/mlx5_mac.c      |   4 +
> >  drivers/net/mlx5/mlx5_mr.c       |   4 +
> >  drivers/net/mlx5/mlx5_rss.c      |   4 +
> >  drivers/net/mlx5/mlx5_rxmode.c   |   4 +
> >  drivers/net/mlx5/mlx5_rxq.c      |   4 +
> >  drivers/net/mlx5/mlx5_rxtx.c     |   4 +
> >  drivers/net/mlx5/mlx5_rxtx.h     |   6 +-
> >  drivers/net/mlx5/mlx5_rxtx_vec.c |   4 +
> >  drivers/net/mlx5/mlx5_txq.c      |   4 +
> >  mk/rte.app.mk                    |   8 +-
> >  17 files changed, 479 insertions(+), 12 deletions(-)  create mode
> > 100644 drivers/net/mlx5/lib/mlx5_dll.c  create mode 100644
> > drivers/net/mlx5/lib/mlx5_dll.h
> >
> > diff --git a/config/common_base b/config/common_base index
> > b8ee8f9..30c8fcf 100644
> > --- a/config/common_base
> > +++ b/config/common_base
> > @@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
> >  # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
> > #  CONFIG_RTE_LIBRTE_MLX5_PMD=n
> > +CONFIG_RTE_LIBRTE_MLX5_DLL=y
> >  CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
> >  CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
> 
> Not sure a new configuration item is allowed.  If it is, the documentation of
> such variable is missing.

[S.B] The patch is based on this CONFIG_RTE_LIBRTE_MLX5_DLL , it was required by Adrian in the design phase to enable/disable this linkage mode.
          I will update the documentation .
> 
> > diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> > index a3984eb..24fa127 100644
> > --- a/drivers/net/mlx5/Makefile
> > +++ b/drivers/net/mlx5/Makefile
> > @@ -53,18 +53,25 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) +=
> mlx5_rss.c
> >  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
> >  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
> >  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
> > -
> > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> > +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c endif
> >  # Basic CFLAGS.
> >  CFLAGS += -O3
> >  CFLAGS += -std=c11 -Wall -Wextra
> >  CFLAGS += -g
> >  CFLAGS += -I.
> > +CFLAGS += -I$(SRCDIR)
> >  CFLAGS += -D_BSD_SOURCE
> >  CFLAGS += -D_DEFAULT_SOURCE
> >  CFLAGS += -D_XOPEN_SOURCE=600
> >  CFLAGS += $(WERROR_FLAGS)
> >  CFLAGS += -Wno-strict-prototypes
> > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) LDLIBS += -ldl else
> >  LDLIBS += -libverbs -lmlx5
> > +endif
> >  LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring  LDLIBS +=
> > -lrte_ethdev -lrte_net -lrte_kvargs  LDLIBS += -lrte_bus_pci @@
> > -105,26 +112,28 @@ endif
> >
> >  mlx5_autoconf.h.new: FORCE
> >
> > +VERBS_H := infiniband/verbs.h
> > +MLX5DV_H := infiniband/mlx5dv.h
> >  mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
> >  	$Q $(RM) -f -- '$@'
> >  	$Q sh -- '$<' '$@' \
> >  		HAVE_IBV_DEVICE_VXLAN_SUPPORT \
> > -		infiniband/verbs.h \
> > +		$(VERBS_H) \
> >  		enum IBV_DEVICE_VXLAN_SUPPORT \
> >  		$(AUTOCONF_OUTPUT)
> >  	$Q sh -- '$<' '$@' \
> >  		HAVE_IBV_WQ_FLAG_RX_END_PADDING \
> > -		infiniband/verbs.h \
> > +		$(VERBS_H) \
> >  		enum IBV_WQ_FLAG_RX_END_PADDING \
> >  		$(AUTOCONF_OUTPUT)
> >  	$Q sh -- '$<' '$@' \
> >  		HAVE_IBV_MLX5_MOD_MPW \
> > -		infiniband/mlx5dv.h \
> > +		$(MLX5DV_H) \
> >  		enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
> >  		$(AUTOCONF_OUTPUT)
> >  	$Q sh -- '$<' '$@' \
> >  		HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
> > -		infiniband/mlx5dv.h \
> > +		$(MLX5DV_H) \
> >  		enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
> >  		$(AUTOCONF_OUTPUT)
> >  	$Q sh -- '$<' '$@' \
> > @@ -144,10 +153,9 @@ mlx5_autoconf.h.new:
> $(RTE_SDK)/buildtools/auto-config-h.sh
> >  		$(AUTOCONF_OUTPUT)
> >  	$Q sh -- '$<' '$@' \
> >  		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> > -		infiniband/verbs.h \
> > +		$(VERBS_H)  \
> >  		enum IBV_FLOW_SPEC_ACTION_COUNT \
> >  		$(AUTOCONF_OUTPUT)
> 
> This modification should be inside its own patch, it is not directly related to
> the this patch itself.
[S.B] I can revert the VERBS_H ,  MLX5DV_H  and not change it all . 
           There is no need to change in a different patch. 

> 
> > -
> >  # Create mlx5_autoconf.h or update it in case it differs from the new one.
> >
> >  mlx5_autoconf.h: mlx5_autoconf.h.new
> <snip/>
> > diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> > cd66fe1..eeef782 100644
> > --- a/drivers/net/mlx5/mlx5.c
> > +++ b/drivers/net/mlx5/mlx5.c
> > @@ -30,7 +30,8 @@
> >   *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
> OF THE USE
> >   *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
> DAMAGE.
> >   */
> > -
> > +#define _GNU_SOURCE
> > +#include <stdio.h>
> >  #include <stddef.h>
> >  #include <unistd.h>
> >  #include <string.h>
> > @@ -39,13 +40,17 @@
> >  #include <stdlib.h>
> >  #include <errno.h>
> >  #include <net/if.h>
> > -
> > +#include <dlfcn.h>
> 
> The empty line should remain.
[S.B] OK.
> 
> >  /* Verbs header. */
> >  /* ISO C doesn't support unnamed structs/unions, disabling -pedantic.
> > */  #ifdef PEDANTIC  #pragma GCC diagnostic ignored "-Wpedantic"
> >  #endif
> > +#ifdef RTE_LIBRTE_MLX5_DLL
> > +#include "lib/mlx5_dll.h"
> > +#else
> >  #include <infiniband/verbs.h>
> > +#endif
> 
> This could be done by the mlx5_dll.h file which could include the correct
> header according to the configuration.
[S.B] I guess you refer to all *.c files that includes the verbs.h .
           I will change them all ? 
> 
> <snip/>
> > diff --git a/mk/rte.app.mk b/mk/rte.app.mk index 6a6a745..2dd2f6b
> > 100644
> > --- a/mk/rte.app.mk
> > +++ b/mk/rte.app.mk
> > @@ -51,7 +51,9 @@ endif
> >
> >  # Link only the libraries used in the application  LDFLAGS +=
> > --as-needed
> > -
> > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) LDFLAGS += -ldl endif
> >  # default path for libs
> >  _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
> >
> > @@ -142,7 +144,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI)        +=
> -lrte_pmd_kni
> >  endif
> >  _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD)        += -lrte_pmd_lio
> >  _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -
> libverbs -lmlx4
> > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> > +_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -ldl
> > +else
> >  _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -
> libverbs -lmlx5
> > +endif
> >  _LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD)       += -lrte_pmd_mrvl -
> L$(LIBMUSDK_PATH)/lib -lmusdk
> >  _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD)        += -lrte_pmd_nfp
> >  _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL)       += -lrte_pmd_null
> > --
> > 1.8.3.1
> >
> 
> Thanks,
> 
> --
> Nélio Laranjeiro
> 6WIND

Re: [dpdk-dev] [PATCH v4] net/mlx5: load libmlx5 and libibverbs in run-time

[Nélio Laranjeiro]

Hi Shachar,

On Wed, Jan 03, 2018 at 03:00:46PM +0000, Shachar Beiser wrote:
<snip/>
> > > --- a/config/common_base
> > > +++ b/config/common_base
> > > @@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
> > >  # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
> > > #  CONFIG_RTE_LIBRTE_MLX5_PMD=n
> > > +CONFIG_RTE_LIBRTE_MLX5_DLL=y
> > >  CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
> > >  CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
> > 
> > Not sure a new configuration item is allowed.  If it is, the documentation of
> > such variable is missing.
> 
> [S.B] The patch is based on this CONFIG_RTE_LIBRTE_MLX5_DLL , it was
> required by Adrian in the design phase to enable/disable this linkage
> mode.
>           I will update the documentation .

Before updating the documentation you should speak with Thomas or
Ferruh, not sure new items are allowed anymore.

> > > diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> > > index a3984eb..24fa127 100644
> > > --- a/drivers/net/mlx5/Makefile
> > > +++ b/drivers/net/mlx5/Makefile
> > > @@ -53,18 +53,25 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) +=
> > mlx5_rss.c
> > >  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
> > >  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
> > >  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
> > > -
> > > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> > > +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c endif
> > >  # Basic CFLAGS.
> > >  CFLAGS += -O3
> > >  CFLAGS += -std=c11 -Wall -Wextra
> > >  CFLAGS += -g
> > >  CFLAGS += -I.
> > > +CFLAGS += -I$(SRCDIR)
> > >  CFLAGS += -D_BSD_SOURCE
> > >  CFLAGS += -D_DEFAULT_SOURCE
> > >  CFLAGS += -D_XOPEN_SOURCE=600
> > >  CFLAGS += $(WERROR_FLAGS)
> > >  CFLAGS += -Wno-strict-prototypes
> > > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) LDLIBS += -ldl else
> > >  LDLIBS += -libverbs -lmlx5
> > > +endif
> > >  LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring  LDLIBS +=
> > > -lrte_ethdev -lrte_net -lrte_kvargs  LDLIBS += -lrte_bus_pci @@
> > > -105,26 +112,28 @@ endif
> > >
> > >  mlx5_autoconf.h.new: FORCE
> > >
> > > +VERBS_H := infiniband/verbs.h
> > > +MLX5DV_H := infiniband/mlx5dv.h
> > >  mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
> > >  	$Q $(RM) -f -- '$@'
> > >  	$Q sh -- '$<' '$@' \
> > >  		HAVE_IBV_DEVICE_VXLAN_SUPPORT \
> > > -		infiniband/verbs.h \
> > > +		$(VERBS_H) \
> > >  		enum IBV_DEVICE_VXLAN_SUPPORT \
> > >  		$(AUTOCONF_OUTPUT)
> > >  	$Q sh -- '$<' '$@' \
> > >  		HAVE_IBV_WQ_FLAG_RX_END_PADDING \
> > > -		infiniband/verbs.h \
> > > +		$(VERBS_H) \
> > >  		enum IBV_WQ_FLAG_RX_END_PADDING \
> > >  		$(AUTOCONF_OUTPUT)
> > >  	$Q sh -- '$<' '$@' \
> > >  		HAVE_IBV_MLX5_MOD_MPW \
> > > -		infiniband/mlx5dv.h \
> > > +		$(MLX5DV_H) \
> > >  		enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
> > >  		$(AUTOCONF_OUTPUT)
> > >  	$Q sh -- '$<' '$@' \
> > >  		HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
> > > -		infiniband/mlx5dv.h \
> > > +		$(MLX5DV_H) \
> > >  		enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
> > >  		$(AUTOCONF_OUTPUT)
> > >  	$Q sh -- '$<' '$@' \
> > > @@ -144,10 +153,9 @@ mlx5_autoconf.h.new:
> > $(RTE_SDK)/buildtools/auto-config-h.sh
> > >  		$(AUTOCONF_OUTPUT)
> > >  	$Q sh -- '$<' '$@' \
> > >  		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> > > -		infiniband/verbs.h \
> > > +		$(VERBS_H)  \
> > >  		enum IBV_FLOW_SPEC_ACTION_COUNT \
> > >  		$(AUTOCONF_OUTPUT)
> > 
> > This modification should be inside its own patch, it is not directly related to
> > the this patch itself.
> [S.B] I can revert the VERBS_H ,  MLX5DV_H  and not change it all . 
>            There is no need to change in a different patch. 

As you wish.

> > > -
> > >  # Create mlx5_autoconf.h or update it in case it differs from the new one.
> > >
> > >  mlx5_autoconf.h: mlx5_autoconf.h.new
> > <snip/>
> > > diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> > > cd66fe1..eeef782 100644
> > > --- a/drivers/net/mlx5/mlx5.c
> > > +++ b/drivers/net/mlx5/mlx5.c
> > > @@ -30,7 +30,8 @@
> > >   *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
> > OF THE USE
> > >   *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
> > DAMAGE.
> > >   */
> > > -
> > > +#define _GNU_SOURCE
> > > +#include <stdio.h>
> > >  #include <stddef.h>
> > >  #include <unistd.h>
> > >  #include <string.h>
> > > @@ -39,13 +40,17 @@
> > >  #include <stdlib.h>
> > >  #include <errno.h>
> > >  #include <net/if.h>
> > > -
> > > +#include <dlfcn.h>
> > 
> > The empty line should remain.
> [S.B] OK.

To be sure, the empty line should be between the system include and
verbs ones.

> > 
> > >  /* Verbs header. */
> > >  /* ISO C doesn't support unnamed structs/unions, disabling -pedantic.
> > > */  #ifdef PEDANTIC  #pragma GCC diagnostic ignored "-Wpedantic"
> > >  #endif
> > > +#ifdef RTE_LIBRTE_MLX5_DLL
> > > +#include "lib/mlx5_dll.h"
> > > +#else
> > >  #include <infiniband/verbs.h>
> > > +#endif
> > 
> > This could be done by the mlx5_dll.h file which could include the correct
> > header according to the configuration.
> [S.B] I guess you refer to all *.c files that includes the verbs.h .
>            I will change them all ? 

Yes

<snip/>
 
Thanks,

-- 
Nélio Laranjeiro
6WIND

Re: [dpdk-dev] [PATCH v4] net/mlx5: load libmlx5 and libibverbs in run-time

[Thomas Monjalon]

Hi,

31/12/2017 08:52, Shachar Beiser:
> MLX5 PMD loads libraries: libibverbs and libmlx5.
> MLX5 PMD is not linked to external libraries.
> 
> Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>

Please better describe the change, even if it is obvious:
how it is before, after and why making this change.

> --- /dev/null
> +++ b/drivers/net/mlx5/lib/mlx5_dll.c

What means "dll"? Isn't it a Windows wording?
Maybe this file is better described as "verbs glue"?

> @@ -0,0 +1,294 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright 2017 6WIND S.A.
> + *   Copyright 2017 Mellanox.

I think it should be
	Copyright 2017 Mellanox Technologies, Ltd.

> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of 6WIND S.A. nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */

Please replace the license text by SPDX identifier.
Example: https://dpdk.org/patch/32809/

Re: [dpdk-dev] [PATCH v4] net/mlx5: load libmlx5 and libibverbs in run-time

[Thomas Monjalon]

04/01/2018 08:36, Nélio Laranjeiro:
> Hi Shachar,
> 
> On Wed, Jan 03, 2018 at 03:00:46PM +0000, Shachar Beiser wrote:
> <snip/>
> > > > --- a/config/common_base
> > > > +++ b/config/common_base
> > > > @@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
> > > >  # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
> > > > #  CONFIG_RTE_LIBRTE_MLX5_PMD=n
> > > > +CONFIG_RTE_LIBRTE_MLX5_DLL=y
> > > >  CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
> > > >  CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
> > > 
> > > Not sure a new configuration item is allowed.  If it is, the documentation of
> > > such variable is missing.
> > 
> > [S.B] The patch is based on this CONFIG_RTE_LIBRTE_MLX5_DLL , it was
> > required by Adrian in the design phase to enable/disable this linkage
> > mode.
> >           I will update the documentation .
> 
> Before updating the documentation you should speak with Thomas or
> Ferruh, not sure new items are allowed anymore.

It is OK to add a compilation option to configure the compilation.

Re: [dpdk-dev] [PATCH v4] net/mlx5: load libmlx5 and libibverbs in run-time

[Marcelo Ricardo Leitner]

Hi,

On Sun, Dec 31, 2017 at 07:52:51AM +0000, Shachar Beiser wrote:
> MLX5 PMD loads libraries: libibverbs and libmlx5.
> MLX5 PMD is not linked to external libraries.
> 
> Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
> ---
> v1: 
>     load external libraries in run-time
> v2:
>     * fix checkpatch warnings 
> v3:
>     * fix checkpatch warnings
> v4:
>     New  MACROs in order to reuse code  
> ---
>  config/common_base               |   1 +
>  drivers/net/mlx5/Makefile        |  22 ++-
>  drivers/net/mlx5/lib/mlx5_dll.c  | 294 +++++++++++++++++++++++++++++++++++++++
>  drivers/net/mlx5/lib/mlx5_dll.h  | 103 ++++++++++++++
>  drivers/net/mlx5/mlx5.c          |  17 ++-
>  drivers/net/mlx5/mlx5.h          |   4 +
>  drivers/net/mlx5/mlx5_flow.c     |   4 +
>  drivers/net/mlx5/mlx5_mac.c      |   4 +
>  drivers/net/mlx5/mlx5_mr.c       |   4 +
>  drivers/net/mlx5/mlx5_rss.c      |   4 +
>  drivers/net/mlx5/mlx5_rxmode.c   |   4 +
>  drivers/net/mlx5/mlx5_rxq.c      |   4 +
>  drivers/net/mlx5/mlx5_rxtx.c     |   4 +
>  drivers/net/mlx5/mlx5_rxtx.h     |   6 +-
>  drivers/net/mlx5/mlx5_rxtx_vec.c |   4 +
>  drivers/net/mlx5/mlx5_txq.c      |   4 +
>  mk/rte.app.mk                    |   8 +-
>  17 files changed, 479 insertions(+), 12 deletions(-)
>  create mode 100644 drivers/net/mlx5/lib/mlx5_dll.c
>  create mode 100644 drivers/net/mlx5/lib/mlx5_dll.h
> 
> diff --git a/config/common_base b/config/common_base
> index b8ee8f9..30c8fcf 100644
> --- a/config/common_base
> +++ b/config/common_base
> @@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
>  # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
>  #
>  CONFIG_RTE_LIBRTE_MLX5_PMD=n
> +CONFIG_RTE_LIBRTE_MLX5_DLL=y
>  CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
>  CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
>  
> diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> index a3984eb..24fa127 100644
> --- a/drivers/net/mlx5/Makefile
> +++ b/drivers/net/mlx5/Makefile
> @@ -53,18 +53,25 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c
>  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
>  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
>  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
> -
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c
> +endif
>  # Basic CFLAGS.
>  CFLAGS += -O3
>  CFLAGS += -std=c11 -Wall -Wextra
>  CFLAGS += -g
>  CFLAGS += -I.
> +CFLAGS += -I$(SRCDIR)
>  CFLAGS += -D_BSD_SOURCE
>  CFLAGS += -D_DEFAULT_SOURCE
>  CFLAGS += -D_XOPEN_SOURCE=600
>  CFLAGS += $(WERROR_FLAGS)
>  CFLAGS += -Wno-strict-prototypes
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +LDLIBS += -ldl
> +else
>  LDLIBS += -libverbs -lmlx5
> +endif
>  LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
>  LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
>  LDLIBS += -lrte_bus_pci
> @@ -105,26 +112,28 @@ endif
>  
>  mlx5_autoconf.h.new: FORCE
>  
> +VERBS_H := infiniband/verbs.h
> +MLX5DV_H := infiniband/mlx5dv.h
>  mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
>  	$Q $(RM) -f -- '$@'
>  	$Q sh -- '$<' '$@' \
>  		HAVE_IBV_DEVICE_VXLAN_SUPPORT \
> -		infiniband/verbs.h \
> +		$(VERBS_H) \
>  		enum IBV_DEVICE_VXLAN_SUPPORT \
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
>  		HAVE_IBV_WQ_FLAG_RX_END_PADDING \
> -		infiniband/verbs.h \
> +		$(VERBS_H) \
>  		enum IBV_WQ_FLAG_RX_END_PADDING \
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
>  		HAVE_IBV_MLX5_MOD_MPW \
> -		infiniband/mlx5dv.h \
> +		$(MLX5DV_H) \
>  		enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
>  		HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
> -		infiniband/mlx5dv.h \
> +		$(MLX5DV_H) \
>  		enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
> @@ -144,10 +153,9 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
>  		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> -		infiniband/verbs.h \
> +		$(VERBS_H)  \
>  		enum IBV_FLOW_SPEC_ACTION_COUNT \
>  		$(AUTOCONF_OUTPUT)
> -
>  # Create mlx5_autoconf.h or update it in case it differs from the new one.
>  
>  mlx5_autoconf.h: mlx5_autoconf.h.new
> diff --git a/drivers/net/mlx5/lib/mlx5_dll.c b/drivers/net/mlx5/lib/mlx5_dll.c
> new file mode 100644
> index 0000000..cafc78a
> --- /dev/null
> +++ b/drivers/net/mlx5/lib/mlx5_dll.c
> @@ -0,0 +1,294 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright 2017 6WIND S.A.
> + *   Copyright 2017 Mellanox.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of 6WIND S.A. nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#define _GNU_SOURCE
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <dlfcn.h>
> +
> +#include <rte_log.h>
> +
> +#include "mlx5_utils.h"
> +#include "mlx5_dll.h"
> +
> +MLX5_MLX5DV_FUNC(create_cq, struct ibv_cq_ex *,
> +		 struct ibv_context *context,
> +		 struct ibv_cq_init_attr_ex *cq_attr,
> +		 struct mlx5dv_cq_init_attr *mlx5_cq_attr)
> +{
> +	return __MLX5_MLX5DV_FUNC(create_cq, context, cq_attr, mlx5_cq_attr);
> +}
> +
> +MLX5_MLX5DV_FUNC(init_obj, int, struct mlx5dv_obj *obj, uint64_t obj_type)
> +{
> +	return __MLX5_MLX5DV_FUNC(init_obj, obj, obj_type);
> +}
> +
> +MLX5_MLX5DV_FUNC(query_device, int, struct ibv_context *ctx_in,
> +		 struct mlx5dv_context *attrs_out)
> +{
> +	return __MLX5_MLX5DV_FUNC(query_device, ctx_in, attrs_out);
> +}
> +
> +MLX5_MLX5DV_FUNC(set_context_attr, int, struct ibv_context *context,
> +		 enum mlx5dv_set_ctx_attr_type type, void *attr)
> +{
> +	return __MLX5_MLX5DV_FUNC(set_context_attr, context, type, attr);
> +}
> +
> +MLX5_IBVERBS_FUNC(destroy_qp, int, struct ibv_qp *qp)
> +{
> +	return __MLX5_IBVERBS_FUNC(destroy_qp, qp);
> +}
> +
> +MLX5_IBVERBS_FUNC(destroy_cq, int, struct ibv_cq *cq)
> +{
> +	return __MLX5_IBVERBS_FUNC(destroy_cq, cq);
> +}
> +
> +MLX5_IBVERBS_FUNC(dealloc_pd, int, struct ibv_pd *pd)
> +{
> +	return __MLX5_IBVERBS_FUNC(dealloc_pd, pd);
> +}
> +
> +MLX5_IBVERBS_FUNC(get_device_list, struct ibv_device **, int *num_devices)
> +{
> +	return __MLX5_IBVERBS_FUNC(get_device_list, num_devices);
> +}
> +
> +MLX5_IBVERBS_FUNC(query_device, int, struct ibv_context *context,
> +		  struct ibv_device_attr *device_attr)
> +{
> +	return __MLX5_IBVERBS_FUNC(query_device, context, device_attr);
> +}
> +
> +MLX5_IBVERBS_FUNC(close_device, int, struct ibv_context *context)
> +{
> +	return __MLX5_IBVERBS_FUNC(close_device, context);
> +}
> +
> +MLX5_IBVERBS_FUNC(free_device_list, void, struct ibv_device **list)
> +{
> +	__MLX5_IBVERBS_FUNC(free_device_list, list);
> +}
> +
> +MLX5_IBVERBS_FUNC(open_device, struct ibv_context *, struct ibv_device *device)
> +{
> +	return __MLX5_IBVERBS_FUNC(open_device, device);
> +}
> +
> +MLX5_IBVERBS_FUNC(alloc_pd, struct ibv_pd *, struct ibv_context *context)
> +{
> +	return __MLX5_IBVERBS_FUNC(alloc_pd, context);
> +}
> +
> +MLX5_IBVERBS_FUNC(get_device_name, const char *, struct ibv_device *device)
> +{
> +	return __MLX5_IBVERBS_FUNC(get_device_name, device);
> +}
> +
> +MLX5_IBVERBS_FUNC(fork_init, int, void)
> +{
> +	return __MLX5_IBVERBS_FUNC(fork_init,);
> +}
> +
> +MLX5_IBVERBS_FUNC(destroy_comp_channel, int, struct ibv_comp_channel *channel)
> +{
> +	return __MLX5_IBVERBS_FUNC(destroy_comp_channel, channel);
> +}
> +
> +MLX5_IBVERBS_FUNC(dereg_mr, int, struct ibv_mr *mr)
> +{
> +	return __MLX5_IBVERBS_FUNC(dereg_mr, mr);
> +}
> +
> +MLX5_IBVERBS_FUNC(create_comp_channel, struct ibv_comp_channel *,
> +		  struct ibv_context *context)
> +{
> +	return __MLX5_IBVERBS_FUNC(create_comp_channel, context);
> +}
> +
> +MLX5_IBVERBS_FUNC(get_cq_event, int, struct ibv_comp_channel *channel,
> +		  struct ibv_cq **cq, void **cq_context)
> +{
> +	return __MLX5_IBVERBS_FUNC(get_cq_event, channel, cq, cq_context);
> +}
> +
> +MLX5_IBVERBS_FUNC(ack_cq_events, void, struct ibv_cq *cq, unsigned int nevents)
> +{
> +	__MLX5_IBVERBS_FUNC(ack_cq_events, cq, nevents);
> +}
> +
> +MLX5_IBVERBS_FUNC(create_qp, struct ibv_qp *, struct ibv_pd *pd,
> +		  struct ibv_qp_init_attr *qp_init_attr)
> +{
> +	return __MLX5_IBVERBS_FUNC(create_qp, pd, qp_init_attr);
> +}
> +
> +MLX5_IBVERBS_FUNC(ack_async_event, void, struct ibv_async_event *event)
> +{
> +	__MLX5_IBVERBS_FUNC(ack_async_event, event);
> +}
> +
> +MLX5_IBVERBS_FUNC(get_async_event, int, struct ibv_context *context,
> +		  struct ibv_async_event *event)
> +{
> +	return __MLX5_IBVERBS_FUNC(get_async_event, context, event);
> +}
> +
> +MLX5_IBVERBS_FUNC(reg_mr, struct ibv_mr *, struct ibv_pd *pd, void *addr,
> +		  size_t length, int access)
> +{
> +	return __MLX5_IBVERBS_FUNC(reg_mr, pd, addr, length, access);
> +}
> +
> +MLX5_IBVERBS_FUNC(create_cq, struct ibv_cq*, struct ibv_context *context,
> +		  int cqe, void *cq_context, struct ibv_comp_channel *channel,
> +		  int comp_vector)
> +{
> +	return __MLX5_IBVERBS_FUNC(create_cq, context, cqe, cq_context, channel,
> +				   comp_vector);
> +}
> +
> +MLX5_IBVERBS_FUNC(modify_qp, int, struct ibv_qp *qp, struct ibv_qp_attr *attr,
> +		  int attr_mask)
> +{
> +	return __MLX5_IBVERBS_FUNC(modify_qp, qp, attr, attr_mask);
> +}
> +
> +MLX5_IBVERBS_FUNC(query_port, int, struct ibv_context *context,
> +		  uint8_t port_num, struct ibv_port_attr *port_attr)
> +{
> +	port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
> +	port_attr->reserved = 0;

This was supposed to be just a wrapper around the function pointer.
Why is this extra code needed? Isn't it also needed when using the
function directly?

> +	return __MLX5_IBVERBS_FUNC(query_port, context, port_num, port_attr);
> +}
> +
> +MLX5_IBVERBS_FUNC(port_state_str, const char *, enum ibv_port_state port_state)
> +{
> +	return __MLX5_IBVERBS_FUNC(port_state_str, port_state);
> +}
> +
> +/**
> + * Register libibverbs APIs.
> + *
> + * @param[in] handle
> + *   Library handle.
> + *
> + * @return
> + *   0 on success, -1 on failure.
> + */
> +int mlx5_libverbs_function_register(void *handle)
> +{
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, modify_qp);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_qp);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_cq);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, dealloc_pd);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, alloc_pd);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, get_device_list);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, open_device);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, query_device);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, close_device);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, free_device_list);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, get_device_name);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, fork_init);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_comp_channel);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, create_comp_channel);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, dereg_mr);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, get_cq_event);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, create_qp);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, ack_async_event);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, get_async_event);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, reg_mr);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, create_cq);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, modify_qp);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, query_port);
> +	MLX5_DLSYM_IBVERBS_FUNC(handle, port_state_str);
> +	return 0;
> +}
> +
> +/**
> + * Register libmlx5 APIs.
> + *
> + * @param[in] handle
> + *   Library handle.
> + *
> + * @return
> + *   0 on success, -1 on failure.
> + */
> +int mlx5_lmlx5_function_register(void *handle)
> +{
> +	MLX5_DLSYM_MLX5DV_FUNC(handle, create_cq);
> +	MLX5_DLSYM_MLX5DV_FUNC(handle, init_obj);
> +	MLX5_DLSYM_MLX5DV_FUNC(handle, query_device);
> +	MLX5_DLSYM_MLX5DV_FUNC(handle, set_context_attr);
> +	return 0;
> +}
> +
> +/**
> + * Load symbols of libibverbs and libmlx5.
> + *
> + * @return
> + *   0 on success, -1 on failure.
> + */
> +int mlx5_load_libs(void)
> +{
> +	void *dlhandle;
> +	int ret;
> +
> +	dlhandle = dlopen(MLX5_LIB_IBVERBS_FILENAME, RTLD_LAZY);
> +	if (dlhandle == NULL) {
> +		ERROR("failed to open %s: %s\n",
> +		      MLX5_LIB_IBVERBS_FILENAME, dlerror());
> +		return -1;
> +	}
> +	ret = mlx5_libverbs_function_register(dlhandle);
> +	if (ret) {
> +		ERROR("failed to register a function in %s",
> +		      MLX5_LIB_IBVERBS_FILENAME);
> +		return -1;
> +	}
> +	dlhandle = dlopen(MLX5_LIB_MLX5_FILENAME, RTLD_LAZY);
> +	if (dlhandle == NULL) {
> +		ERROR("failed to open %s: %s\n",
> +		      MLX5_LIB_MLX5_FILENAME, dlerror());
> +		return -1;
> +	}
> +	ret = mlx5_lmlx5_function_register(dlhandle);
> +	if (ret) {
> +		ERROR("failed to register a function in %s",
> +		      MLX5_LIB_MLX5_FILENAME);
> +		return -1;
> +	}
> +	return 0;
> +}
> diff --git a/drivers/net/mlx5/lib/mlx5_dll.h b/drivers/net/mlx5/lib/mlx5_dll.h
> new file mode 100644
> index 0000000..9c3d604
> --- /dev/null
> +++ b/drivers/net/mlx5/lib/mlx5_dll.h
> @@ -0,0 +1,103 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright 2017 6WIND S.A.
> + *   Copyright 2017 Mellanox.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of 6WIND S.A. nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#ifndef MLX5_DLL_H_
> +#define MLX5_DLL_H_
> +
> +#include <infiniband/verbs.h>
> +#include <infiniband/mlx5dv.h>
> +
> +#define MLX5_LIB_IBVERBS_FILENAME "libibverbs.so"
> +#define MLX5_LIB_MLX5_FILENAME "libmlx5.so"

These are only packaged in -devel packages. You probably want to
specify the major version here. As in, libmlx5.so.1.

> +
> +#ifndef RTE_LIBRTE_MLX4_PMD

This may be project specific, thus I'm sorry if I'm violating some
code style policy but most of the macros below don't need/shouldn't be
here in the header as they are only used by the dll wrapper and
shouldn't be used outside of it.

Perhaps the only thing really needed here is the 
+int mlx5_load_libs(void);
declaration and the includes above.

> +
> +#define MLX5_IBVERBS_FUNC(name, ret, ...) \
> +	ret(*__mlx5_pmd_ibv_##name)(__VA_ARGS__); \
> +	ret(ibv_##name)(__VA_ARGS__)
> +
> +#define __MLX5_IBVERBS_FUNC(name, ...) \
> +	(*__mlx5_pmd_ibv_##name)(__VA_ARGS__)
> +
> +#define MLX5_DLSYM_IBVERBS_FUNC(handle, name) \
> +	do { \
> +		char *err; \
> +		*(void **)(&__mlx5_pmd_ibv_##name) = \
> +			dlsym(handle, "ibv_"#name); \
> +		err = dlerror(); \
> +		if (err != NULL) { \
> +			ERROR("failed to link a symbol: %s", err); \
> +			return -1; \
> +		} \
> +	} while (0)
> +
> +#else /* RTE_LIBRTE_MLX4_PMD */
> +
> +#define MLX5_IBVERBS_FUNC(name, ret, ...) \
> +	ret(___mlx5_pmd_ibv_##name)(__VA_ARGS__); \
> +	ret(___mlx5_pmd_ibv_##name)(__VA_ARGS__)
> +
> +#define __MLX5_IBVERBS_FUNC(name, ...) \
> +	(ibv_##name)(__VA_ARGS__)
> +
> +#define MLX5_DLSYM_IBVERBS_FUNC(handle, name) do { (void)handle; } while (0)
> +
> +#endif /* RTE_LIBRTE_MLX4_PMD */
> +
> +#define MLX5_MLX5DV_FUNC(name, ret, ...) \
> +	ret(*__mlx5_pmd_mlx5dv_##name)(__VA_ARGS__); \
> +	ret(mlx5dv_##name)(__VA_ARGS__)
> +
> +#define __MLX5_MLX5DV_FUNC(name, ...) \
> +	(*__mlx5_pmd_mlx5dv_##name)(__VA_ARGS__)
> +
> +#define MLX5_DLSYM_MLX5DV_FUNC(handle, name) \
> +	do { \
> +		char *err; \
> +		*(void **)(&__mlx5_pmd_mlx5dv_##name) = \
> +			dlsym(handle, "mlx5dv_"#name); \
> +		err = dlerror(); \
> +		if (err != NULL) { \
> +			ERROR("failed to link a symbol: %s", err); \
> +			return -1; \
> +		} \
> +	} while (0)
> +
> +#undef ibv_query_port
> +
> +void *mlx5_lib_load(const char *name);
> +int mlx5_libverbs_function_register(void *handle);
> +int mlx5_lmlx5_function_register(void *handle);
> +int mlx5_load_libs(void);
> +
> +#endif /* RTE_PMD_MLX5_DLL_H_ */
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
> index cd66fe1..eeef782 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -30,7 +30,8 @@
>   *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>   *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
>   */
> -
> +#define _GNU_SOURCE
> +#include <stdio.h>

this and

>  #include <stddef.h>
>  #include <unistd.h>
>  #include <string.h>
> @@ -39,13 +40,17 @@
>  #include <stdlib.h>
>  #include <errno.h>
>  #include <net/if.h>
> -
> +#include <dlfcn.h>

this change shouldn't be needed in this file, as all it does is call
an initialization function below.

>  /* Verbs header. */
>  /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic ignored "-Wpedantic"
>  #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
>  #include <infiniband/verbs.h>
> +#endif
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic error "-Wpedantic"
>  #endif
> @@ -1027,6 +1032,9 @@ struct mlx5_args {
>  rte_mlx5_pmd_init(void)
>  {
>  	/* Build the static table for ptype conversion. */
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +	int ret;
> +#endif
>  	mlx5_set_ptype_table();
>  	/*
>  	 * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
> @@ -1038,6 +1046,11 @@ struct mlx5_args {
>  	/* Match the size of Rx completion entry to the size of a cacheline. */
>  	if (RTE_CACHE_LINE_SIZE == 128)
>  		setenv("MLX5_CQE_SIZE", "128", 0);
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +	ret = mlx5_load_libs();
> +	if (ret != 0)
> +		return;
> +#endif

Also, depends on project code style, but defining an empty
mlx5_load_libs() in case RTE_LIBRTE_MLX5_DLL is NOT defined would
avoid the two ifdef above.

  Marcelo

>  	ibv_fork_init();
>  	rte_pci_register(&mlx5_driver);
>  }
> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
> index e6a69b8..9db7024 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -46,7 +46,11 @@
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic ignored "-Wpedantic"
>  #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
>  #include <infiniband/verbs.h>
> +#endif
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic error "-Wpedantic"
>  #endif
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index 6605cfd..49f8972 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -39,7 +39,11 @@
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic ignored "-Wpedantic"
>  #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
>  #include <infiniband/verbs.h>
> +#endif
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic error "-Wpedantic"
>  #endif
> diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c
> index 9fb5ba5..c9f4bfd 100644
> --- a/drivers/net/mlx5/mlx5_mac.c
> +++ b/drivers/net/mlx5/mlx5_mac.c
> @@ -46,7 +46,11 @@
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic ignored "-Wpedantic"
>  #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
>  #include <infiniband/verbs.h>
> +#endif
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic error "-Wpedantic"
>  #endif
> diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
> index 6b29eed..f27a428 100644
> --- a/drivers/net/mlx5/mlx5_mr.c
> +++ b/drivers/net/mlx5/mlx5_mr.c
> @@ -36,7 +36,11 @@
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic ignored "-Wpedantic"
>  #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
>  #include <infiniband/verbs.h>
> +#endif
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic error "-Wpedantic"
>  #endif
> diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c
> index f491135..63cbfa0 100644
> --- a/drivers/net/mlx5/mlx5_rss.c
> +++ b/drivers/net/mlx5/mlx5_rss.c
> @@ -42,7 +42,11 @@
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic ignored "-Wpedantic"
>  #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
>  #include <infiniband/verbs.h>
> +#endif
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic error "-Wpedantic"
>  #endif
> diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c
> index 6fb245b..1ddad70 100644
> --- a/drivers/net/mlx5/mlx5_rxmode.c
> +++ b/drivers/net/mlx5/mlx5_rxmode.c
> @@ -40,7 +40,11 @@
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic ignored "-Wpedantic"
>  #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
>  #include <infiniband/verbs.h>
> +#endif
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic error "-Wpedantic"
>  #endif
> diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
> index a4cdd37..aaf4ce1 100644
> --- a/drivers/net/mlx5/mlx5_rxq.c
> +++ b/drivers/net/mlx5/mlx5_rxq.c
> @@ -44,8 +44,12 @@
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic ignored "-Wpedantic"
>  #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
>  #include <infiniband/verbs.h>
>  #include <infiniband/mlx5dv.h>
> +#endif
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic error "-Wpedantic"
>  #endif
> diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
> index 67e3db1..4faf302 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.c
> +++ b/drivers/net/mlx5/mlx5_rxtx.c
> @@ -41,8 +41,12 @@
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic ignored "-Wpedantic"
>  #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
>  #include <infiniband/verbs.h>
>  #include <infiniband/mlx5dv.h>
> +#endif
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic error "-Wpedantic"
>  #endif
> diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> index e70d523..e637555 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.h
> +++ b/drivers/net/mlx5/mlx5_rxtx.h
> @@ -43,12 +43,14 @@
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic ignored "-Wpedantic"
>  #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
>  #include <infiniband/verbs.h>
> -#include <infiniband/mlx5dv.h>
> +#endif
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic error "-Wpedantic"
>  #endif
> -
>  #include <rte_mbuf.h>
>  #include <rte_mempool.h>
>  #include <rte_common.h>
> diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c
> index 8d23dae..679bee7 100644
> --- a/drivers/net/mlx5/mlx5_rxtx_vec.c
> +++ b/drivers/net/mlx5/mlx5_rxtx_vec.c
> @@ -41,8 +41,12 @@
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic ignored "-Wpedantic"
>  #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
>  #include <infiniband/verbs.h>
>  #include <infiniband/mlx5dv.h>
> +#endif
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic error "-Wpedantic"
>  #endif
> diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
> index 89b16fd..8b0cdf4 100644
> --- a/drivers/net/mlx5/mlx5_txq.c
> +++ b/drivers/net/mlx5/mlx5_txq.c
> @@ -44,7 +44,11 @@
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic ignored "-Wpedantic"
>  #endif
> +#ifdef RTE_LIBRTE_MLX5_DLL
> +#include "lib/mlx5_dll.h"
> +#else
>  #include <infiniband/verbs.h>
> +#endif
>  #ifdef PEDANTIC
>  #pragma GCC diagnostic error "-Wpedantic"
>  #endif
> diff --git a/mk/rte.app.mk b/mk/rte.app.mk
> index 6a6a745..2dd2f6b 100644
> --- a/mk/rte.app.mk
> +++ b/mk/rte.app.mk
> @@ -51,7 +51,9 @@ endif
>  
>  # Link only the libraries used in the application
>  LDFLAGS += --as-needed
> -
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +LDFLAGS += -ldl
> +endif
>  # default path for libs
>  _LDLIBS-y += -L$(RTE_SDK_BIN)/lib
>  
> @@ -142,7 +144,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI)        += -lrte_pmd_kni
>  endif
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD)        += -lrte_pmd_lio
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -libverbs -lmlx4
> +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -ldl
> +else
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -libverbs -lmlx5
> +endif
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD)       += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD)        += -lrte_pmd_nfp
>  _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL)       += -lrte_pmd_null
> -- 
> 1.8.3.1
> 

Re: [dpdk-dev] [PATCH v4] net/mlx5: load libmlx5 and libibverbs in run-time

[Marcelo Ricardo Leitner]

Hello,

On Thu, Jan 04, 2018 at 08:36:08AM +0100, Nélio Laranjeiro wrote:
> Hi Shachar,
> 
> On Wed, Jan 03, 2018 at 03:00:46PM +0000, Shachar Beiser wrote:
> <snip/>
> > > > --- a/config/common_base
> > > > +++ b/config/common_base
> > > > @@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
> > > >  # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
> > > > #  CONFIG_RTE_LIBRTE_MLX5_PMD=n
> > > > +CONFIG_RTE_LIBRTE_MLX5_DLL=y
> > > >  CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
> > > >  CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
> > > 
> > > Not sure a new configuration item is allowed.  If it is, the documentation of
> > > such variable is missing.
> > 
> > [S.B] The patch is based on this CONFIG_RTE_LIBRTE_MLX5_DLL , it was
> > required by Adrian in the design phase to enable/disable this linkage
> > mode.
> >           I will update the documentation .
> 
> Before updating the documentation you should speak with Thomas or
> Ferruh, not sure new items are allowed anymore.
> 
> > > > diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> > > > index a3984eb..24fa127 100644
> > > > --- a/drivers/net/mlx5/Makefile
> > > > +++ b/drivers/net/mlx5/Makefile
> > > > @@ -53,18 +53,25 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) +=
> > > mlx5_rss.c
> > > >  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
> > > >  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
> > > >  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
> > > > -
> > > > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y)
> > > > +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c endif
> > > >  # Basic CFLAGS.
> > > >  CFLAGS += -O3
> > > >  CFLAGS += -std=c11 -Wall -Wextra
> > > >  CFLAGS += -g
> > > >  CFLAGS += -I.
> > > > +CFLAGS += -I$(SRCDIR)
> > > >  CFLAGS += -D_BSD_SOURCE
> > > >  CFLAGS += -D_DEFAULT_SOURCE
> > > >  CFLAGS += -D_XOPEN_SOURCE=600
> > > >  CFLAGS += $(WERROR_FLAGS)
> > > >  CFLAGS += -Wno-strict-prototypes
> > > > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) LDLIBS += -ldl else
> > > >  LDLIBS += -libverbs -lmlx5
> > > > +endif
> > > >  LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring  LDLIBS +=
> > > > -lrte_ethdev -lrte_net -lrte_kvargs  LDLIBS += -lrte_bus_pci @@
> > > > -105,26 +112,28 @@ endif
> > > >
> > > >  mlx5_autoconf.h.new: FORCE
> > > >
> > > > +VERBS_H := infiniband/verbs.h
> > > > +MLX5DV_H := infiniband/mlx5dv.h
> > > >  mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
> > > >  	$Q $(RM) -f -- '$@'
> > > >  	$Q sh -- '$<' '$@' \
> > > >  		HAVE_IBV_DEVICE_VXLAN_SUPPORT \
> > > > -		infiniband/verbs.h \
> > > > +		$(VERBS_H) \
> > > >  		enum IBV_DEVICE_VXLAN_SUPPORT \
> > > >  		$(AUTOCONF_OUTPUT)
> > > >  	$Q sh -- '$<' '$@' \
> > > >  		HAVE_IBV_WQ_FLAG_RX_END_PADDING \
> > > > -		infiniband/verbs.h \
> > > > +		$(VERBS_H) \
> > > >  		enum IBV_WQ_FLAG_RX_END_PADDING \
> > > >  		$(AUTOCONF_OUTPUT)
> > > >  	$Q sh -- '$<' '$@' \
> > > >  		HAVE_IBV_MLX5_MOD_MPW \
> > > > -		infiniband/mlx5dv.h \
> > > > +		$(MLX5DV_H) \
> > > >  		enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
> > > >  		$(AUTOCONF_OUTPUT)
> > > >  	$Q sh -- '$<' '$@' \
> > > >  		HAVE_IBV_MLX5_MOD_CQE_128B_COMP \
> > > > -		infiniband/mlx5dv.h \
> > > > +		$(MLX5DV_H) \
> > > >  		enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \
> > > >  		$(AUTOCONF_OUTPUT)
> > > >  	$Q sh -- '$<' '$@' \
> > > > @@ -144,10 +153,9 @@ mlx5_autoconf.h.new:
> > > $(RTE_SDK)/buildtools/auto-config-h.sh
> > > >  		$(AUTOCONF_OUTPUT)
> > > >  	$Q sh -- '$<' '$@' \
> > > >  		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> > > > -		infiniband/verbs.h \
> > > > +		$(VERBS_H)  \
> > > >  		enum IBV_FLOW_SPEC_ACTION_COUNT \
> > > >  		$(AUTOCONF_OUTPUT)
> > > 
> > > This modification should be inside its own patch, it is not directly related to
> > > the this patch itself.
> > [S.B] I can revert the VERBS_H ,  MLX5DV_H  and not change it all . 
> >            There is no need to change in a different patch. 
> 
> As you wish.
> 
> > > > -
> > > >  # Create mlx5_autoconf.h or update it in case it differs from the new one.
> > > >
> > > >  mlx5_autoconf.h: mlx5_autoconf.h.new
> > > <snip/>
> > > > diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> > > > cd66fe1..eeef782 100644
> > > > --- a/drivers/net/mlx5/mlx5.c
> > > > +++ b/drivers/net/mlx5/mlx5.c
> > > > @@ -30,7 +30,8 @@
> > > >   *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
> > > OF THE USE
> > > >   *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
> > > DAMAGE.
> > > >   */
> > > > -
> > > > +#define _GNU_SOURCE
> > > > +#include <stdio.h>
> > > >  #include <stddef.h>
> > > >  #include <unistd.h>
> > > >  #include <string.h>
> > > > @@ -39,13 +40,17 @@
> > > >  #include <stdlib.h>
> > > >  #include <errno.h>
> > > >  #include <net/if.h>
> > > > -
> > > > +#include <dlfcn.h>
> > > 
> > > The empty line should remain.
> > [S.B] OK.
> 
> To be sure, the empty line should be between the system include and
> verbs ones.
> 
> > > 
> > > >  /* Verbs header. */
> > > >  /* ISO C doesn't support unnamed structs/unions, disabling -pedantic.
> > > > */  #ifdef PEDANTIC  #pragma GCC diagnostic ignored "-Wpedantic"
> > > >  #endif
> > > > +#ifdef RTE_LIBRTE_MLX5_DLL
> > > > +#include "lib/mlx5_dll.h"
> > > > +#else
> > > >  #include <infiniband/verbs.h>
> > > > +#endif
> > > 
> > > This could be done by the mlx5_dll.h file which could include the correct
> > > header according to the configuration.
> > [S.B] I guess you refer to all *.c files that includes the verbs.h .
> >            I will change them all ? 
> 
> Yes

Actually, why this change? It shouldn't be necessary. As the patch is
defining functions with the very same prototype, it shouldn't matter
if their declarations are coming from one header or another.

  Marcelo