From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by dpdk.org (Postfix) with ESMTP id 67CF23195 for ; Fri, 19 Jan 2018 19:48:32 +0100 (CET) Received: from smtp.corp.redhat.com (int-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.11]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id AF7E95F179; Fri, 19 Jan 2018 18:48:31 +0000 (UTC) Received: from localhost.localdomain (ovpn-116-5.gru2.redhat.com [10.97.116.5]) by smtp.corp.redhat.com (Postfix) with SMTP id 186A5600D3; Fri, 19 Jan 2018 18:48:29 +0000 (UTC) Date: Fri, 19 Jan 2018 16:48:29 -0200 From: Marcelo Ricardo Leitner To: Shachar Beiser Cc: dev@dpdk.org, Adrien Mazarguil , Nelio Laranjeiro Message-ID: <20180119184829.GA6615@localhost.localdomain> References: <37757d5cfe32610ef1c7a17d72ea6bc1466023de.1511450393.git.shacharbe@mellanox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline In-Reply-To: User-Agent: Mutt/1.9.1 (2017-09-22) X-Scanned-By: MIMEDefang 2.79 on 10.5.11.11 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.25]); Fri, 19 Jan 2018 18:48:31 +0000 (UTC) Subject: Re: [dpdk-dev] [PATCH v4] net/mlx5: load libmlx5 and libibverbs in run-time X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 19 Jan 2018 18:48:33 -0000 Hi, On Sun, Dec 31, 2017 at 07:52:51AM +0000, Shachar Beiser wrote: > MLX5 PMD loads libraries: libibverbs and libmlx5. > MLX5 PMD is not linked to external libraries. > > Signed-off-by: Shachar Beiser > --- > v1: > load external libraries in run-time > v2: > * fix checkpatch warnings > v3: > * fix checkpatch warnings > v4: > New MACROs in order to reuse code > --- > config/common_base | 1 + > drivers/net/mlx5/Makefile | 22 ++- > drivers/net/mlx5/lib/mlx5_dll.c | 294 +++++++++++++++++++++++++++++++++++++++ > drivers/net/mlx5/lib/mlx5_dll.h | 103 ++++++++++++++ > drivers/net/mlx5/mlx5.c | 17 ++- > drivers/net/mlx5/mlx5.h | 4 + > drivers/net/mlx5/mlx5_flow.c | 4 + > drivers/net/mlx5/mlx5_mac.c | 4 + > drivers/net/mlx5/mlx5_mr.c | 4 + > drivers/net/mlx5/mlx5_rss.c | 4 + > drivers/net/mlx5/mlx5_rxmode.c | 4 + > drivers/net/mlx5/mlx5_rxq.c | 4 + > drivers/net/mlx5/mlx5_rxtx.c | 4 + > drivers/net/mlx5/mlx5_rxtx.h | 6 +- > drivers/net/mlx5/mlx5_rxtx_vec.c | 4 + > drivers/net/mlx5/mlx5_txq.c | 4 + > mk/rte.app.mk | 8 +- > 17 files changed, 479 insertions(+), 12 deletions(-) > create mode 100644 drivers/net/mlx5/lib/mlx5_dll.c > create mode 100644 drivers/net/mlx5/lib/mlx5_dll.h > > diff --git a/config/common_base b/config/common_base > index b8ee8f9..30c8fcf 100644 > --- a/config/common_base > +++ b/config/common_base > @@ -236,6 +236,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8 > # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD > # > CONFIG_RTE_LIBRTE_MLX5_PMD=n > +CONFIG_RTE_LIBRTE_MLX5_DLL=y > CONFIG_RTE_LIBRTE_MLX5_DEBUG=n > CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8 > > diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile > index a3984eb..24fa127 100644 > --- a/drivers/net/mlx5/Makefile > +++ b/drivers/net/mlx5/Makefile > @@ -53,18 +53,25 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c > SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c > SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c > SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c > - > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) > +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c > +endif > # Basic CFLAGS. > CFLAGS += -O3 > CFLAGS += -std=c11 -Wall -Wextra > CFLAGS += -g > CFLAGS += -I. > +CFLAGS += -I$(SRCDIR) > CFLAGS += -D_BSD_SOURCE > CFLAGS += -D_DEFAULT_SOURCE > CFLAGS += -D_XOPEN_SOURCE=600 > CFLAGS += $(WERROR_FLAGS) > CFLAGS += -Wno-strict-prototypes > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) > +LDLIBS += -ldl > +else > LDLIBS += -libverbs -lmlx5 > +endif > LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring > LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs > LDLIBS += -lrte_bus_pci > @@ -105,26 +112,28 @@ endif > > mlx5_autoconf.h.new: FORCE > > +VERBS_H := infiniband/verbs.h > +MLX5DV_H := infiniband/mlx5dv.h > mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh > $Q $(RM) -f -- '$@' > $Q sh -- '$<' '$@' \ > HAVE_IBV_DEVICE_VXLAN_SUPPORT \ > - infiniband/verbs.h \ > + $(VERBS_H) \ > enum IBV_DEVICE_VXLAN_SUPPORT \ > $(AUTOCONF_OUTPUT) > $Q sh -- '$<' '$@' \ > HAVE_IBV_WQ_FLAG_RX_END_PADDING \ > - infiniband/verbs.h \ > + $(VERBS_H) \ > enum IBV_WQ_FLAG_RX_END_PADDING \ > $(AUTOCONF_OUTPUT) > $Q sh -- '$<' '$@' \ > HAVE_IBV_MLX5_MOD_MPW \ > - infiniband/mlx5dv.h \ > + $(MLX5DV_H) \ > enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \ > $(AUTOCONF_OUTPUT) > $Q sh -- '$<' '$@' \ > HAVE_IBV_MLX5_MOD_CQE_128B_COMP \ > - infiniband/mlx5dv.h \ > + $(MLX5DV_H) \ > enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \ > $(AUTOCONF_OUTPUT) > $Q sh -- '$<' '$@' \ > @@ -144,10 +153,9 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh > $(AUTOCONF_OUTPUT) > $Q sh -- '$<' '$@' \ > HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \ > - infiniband/verbs.h \ > + $(VERBS_H) \ > enum IBV_FLOW_SPEC_ACTION_COUNT \ > $(AUTOCONF_OUTPUT) > - > # Create mlx5_autoconf.h or update it in case it differs from the new one. > > mlx5_autoconf.h: mlx5_autoconf.h.new > diff --git a/drivers/net/mlx5/lib/mlx5_dll.c b/drivers/net/mlx5/lib/mlx5_dll.c > new file mode 100644 > index 0000000..cafc78a > --- /dev/null > +++ b/drivers/net/mlx5/lib/mlx5_dll.c > @@ -0,0 +1,294 @@ > +/*- > + * BSD LICENSE > + * > + * Copyright 2017 6WIND S.A. > + * Copyright 2017 Mellanox. > + * > + * Redistribution and use in source and binary forms, with or without > + * modification, are permitted provided that the following conditions > + * are met: > + * > + * * Redistributions of source code must retain the above copyright > + * notice, this list of conditions and the following disclaimer. > + * * Redistributions in binary form must reproduce the above copyright > + * notice, this list of conditions and the following disclaimer in > + * the documentation and/or other materials provided with the > + * distribution. > + * * Neither the name of 6WIND S.A. nor the names of its > + * contributors may be used to endorse or promote products derived > + * from this software without specific prior written permission. > + * > + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > + */ > + > +#define _GNU_SOURCE > +#include > +#include > +#include > + > +#include > + > +#include "mlx5_utils.h" > +#include "mlx5_dll.h" > + > +MLX5_MLX5DV_FUNC(create_cq, struct ibv_cq_ex *, > + struct ibv_context *context, > + struct ibv_cq_init_attr_ex *cq_attr, > + struct mlx5dv_cq_init_attr *mlx5_cq_attr) > +{ > + return __MLX5_MLX5DV_FUNC(create_cq, context, cq_attr, mlx5_cq_attr); > +} > + > +MLX5_MLX5DV_FUNC(init_obj, int, struct mlx5dv_obj *obj, uint64_t obj_type) > +{ > + return __MLX5_MLX5DV_FUNC(init_obj, obj, obj_type); > +} > + > +MLX5_MLX5DV_FUNC(query_device, int, struct ibv_context *ctx_in, > + struct mlx5dv_context *attrs_out) > +{ > + return __MLX5_MLX5DV_FUNC(query_device, ctx_in, attrs_out); > +} > + > +MLX5_MLX5DV_FUNC(set_context_attr, int, struct ibv_context *context, > + enum mlx5dv_set_ctx_attr_type type, void *attr) > +{ > + return __MLX5_MLX5DV_FUNC(set_context_attr, context, type, attr); > +} > + > +MLX5_IBVERBS_FUNC(destroy_qp, int, struct ibv_qp *qp) > +{ > + return __MLX5_IBVERBS_FUNC(destroy_qp, qp); > +} > + > +MLX5_IBVERBS_FUNC(destroy_cq, int, struct ibv_cq *cq) > +{ > + return __MLX5_IBVERBS_FUNC(destroy_cq, cq); > +} > + > +MLX5_IBVERBS_FUNC(dealloc_pd, int, struct ibv_pd *pd) > +{ > + return __MLX5_IBVERBS_FUNC(dealloc_pd, pd); > +} > + > +MLX5_IBVERBS_FUNC(get_device_list, struct ibv_device **, int *num_devices) > +{ > + return __MLX5_IBVERBS_FUNC(get_device_list, num_devices); > +} > + > +MLX5_IBVERBS_FUNC(query_device, int, struct ibv_context *context, > + struct ibv_device_attr *device_attr) > +{ > + return __MLX5_IBVERBS_FUNC(query_device, context, device_attr); > +} > + > +MLX5_IBVERBS_FUNC(close_device, int, struct ibv_context *context) > +{ > + return __MLX5_IBVERBS_FUNC(close_device, context); > +} > + > +MLX5_IBVERBS_FUNC(free_device_list, void, struct ibv_device **list) > +{ > + __MLX5_IBVERBS_FUNC(free_device_list, list); > +} > + > +MLX5_IBVERBS_FUNC(open_device, struct ibv_context *, struct ibv_device *device) > +{ > + return __MLX5_IBVERBS_FUNC(open_device, device); > +} > + > +MLX5_IBVERBS_FUNC(alloc_pd, struct ibv_pd *, struct ibv_context *context) > +{ > + return __MLX5_IBVERBS_FUNC(alloc_pd, context); > +} > + > +MLX5_IBVERBS_FUNC(get_device_name, const char *, struct ibv_device *device) > +{ > + return __MLX5_IBVERBS_FUNC(get_device_name, device); > +} > + > +MLX5_IBVERBS_FUNC(fork_init, int, void) > +{ > + return __MLX5_IBVERBS_FUNC(fork_init,); > +} > + > +MLX5_IBVERBS_FUNC(destroy_comp_channel, int, struct ibv_comp_channel *channel) > +{ > + return __MLX5_IBVERBS_FUNC(destroy_comp_channel, channel); > +} > + > +MLX5_IBVERBS_FUNC(dereg_mr, int, struct ibv_mr *mr) > +{ > + return __MLX5_IBVERBS_FUNC(dereg_mr, mr); > +} > + > +MLX5_IBVERBS_FUNC(create_comp_channel, struct ibv_comp_channel *, > + struct ibv_context *context) > +{ > + return __MLX5_IBVERBS_FUNC(create_comp_channel, context); > +} > + > +MLX5_IBVERBS_FUNC(get_cq_event, int, struct ibv_comp_channel *channel, > + struct ibv_cq **cq, void **cq_context) > +{ > + return __MLX5_IBVERBS_FUNC(get_cq_event, channel, cq, cq_context); > +} > + > +MLX5_IBVERBS_FUNC(ack_cq_events, void, struct ibv_cq *cq, unsigned int nevents) > +{ > + __MLX5_IBVERBS_FUNC(ack_cq_events, cq, nevents); > +} > + > +MLX5_IBVERBS_FUNC(create_qp, struct ibv_qp *, struct ibv_pd *pd, > + struct ibv_qp_init_attr *qp_init_attr) > +{ > + return __MLX5_IBVERBS_FUNC(create_qp, pd, qp_init_attr); > +} > + > +MLX5_IBVERBS_FUNC(ack_async_event, void, struct ibv_async_event *event) > +{ > + __MLX5_IBVERBS_FUNC(ack_async_event, event); > +} > + > +MLX5_IBVERBS_FUNC(get_async_event, int, struct ibv_context *context, > + struct ibv_async_event *event) > +{ > + return __MLX5_IBVERBS_FUNC(get_async_event, context, event); > +} > + > +MLX5_IBVERBS_FUNC(reg_mr, struct ibv_mr *, struct ibv_pd *pd, void *addr, > + size_t length, int access) > +{ > + return __MLX5_IBVERBS_FUNC(reg_mr, pd, addr, length, access); > +} > + > +MLX5_IBVERBS_FUNC(create_cq, struct ibv_cq*, struct ibv_context *context, > + int cqe, void *cq_context, struct ibv_comp_channel *channel, > + int comp_vector) > +{ > + return __MLX5_IBVERBS_FUNC(create_cq, context, cqe, cq_context, channel, > + comp_vector); > +} > + > +MLX5_IBVERBS_FUNC(modify_qp, int, struct ibv_qp *qp, struct ibv_qp_attr *attr, > + int attr_mask) > +{ > + return __MLX5_IBVERBS_FUNC(modify_qp, qp, attr, attr_mask); > +} > + > +MLX5_IBVERBS_FUNC(query_port, int, struct ibv_context *context, > + uint8_t port_num, struct ibv_port_attr *port_attr) > +{ > + port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED; > + port_attr->reserved = 0; This was supposed to be just a wrapper around the function pointer. Why is this extra code needed? Isn't it also needed when using the function directly? > + return __MLX5_IBVERBS_FUNC(query_port, context, port_num, port_attr); > +} > + > +MLX5_IBVERBS_FUNC(port_state_str, const char *, enum ibv_port_state port_state) > +{ > + return __MLX5_IBVERBS_FUNC(port_state_str, port_state); > +} > + > +/** > + * Register libibverbs APIs. > + * > + * @param[in] handle > + * Library handle. > + * > + * @return > + * 0 on success, -1 on failure. > + */ > +int mlx5_libverbs_function_register(void *handle) > +{ > + MLX5_DLSYM_IBVERBS_FUNC(handle, modify_qp); > + MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_qp); > + MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_cq); > + MLX5_DLSYM_IBVERBS_FUNC(handle, dealloc_pd); > + MLX5_DLSYM_IBVERBS_FUNC(handle, alloc_pd); > + MLX5_DLSYM_IBVERBS_FUNC(handle, get_device_list); > + MLX5_DLSYM_IBVERBS_FUNC(handle, open_device); > + MLX5_DLSYM_IBVERBS_FUNC(handle, query_device); > + MLX5_DLSYM_IBVERBS_FUNC(handle, close_device); > + MLX5_DLSYM_IBVERBS_FUNC(handle, free_device_list); > + MLX5_DLSYM_IBVERBS_FUNC(handle, get_device_name); > + MLX5_DLSYM_IBVERBS_FUNC(handle, fork_init); > + MLX5_DLSYM_IBVERBS_FUNC(handle, destroy_comp_channel); > + MLX5_DLSYM_IBVERBS_FUNC(handle, create_comp_channel); > + MLX5_DLSYM_IBVERBS_FUNC(handle, dereg_mr); > + MLX5_DLSYM_IBVERBS_FUNC(handle, get_cq_event); > + MLX5_DLSYM_IBVERBS_FUNC(handle, create_qp); > + MLX5_DLSYM_IBVERBS_FUNC(handle, ack_async_event); > + MLX5_DLSYM_IBVERBS_FUNC(handle, get_async_event); > + MLX5_DLSYM_IBVERBS_FUNC(handle, reg_mr); > + MLX5_DLSYM_IBVERBS_FUNC(handle, create_cq); > + MLX5_DLSYM_IBVERBS_FUNC(handle, modify_qp); > + MLX5_DLSYM_IBVERBS_FUNC(handle, query_port); > + MLX5_DLSYM_IBVERBS_FUNC(handle, port_state_str); > + return 0; > +} > + > +/** > + * Register libmlx5 APIs. > + * > + * @param[in] handle > + * Library handle. > + * > + * @return > + * 0 on success, -1 on failure. > + */ > +int mlx5_lmlx5_function_register(void *handle) > +{ > + MLX5_DLSYM_MLX5DV_FUNC(handle, create_cq); > + MLX5_DLSYM_MLX5DV_FUNC(handle, init_obj); > + MLX5_DLSYM_MLX5DV_FUNC(handle, query_device); > + MLX5_DLSYM_MLX5DV_FUNC(handle, set_context_attr); > + return 0; > +} > + > +/** > + * Load symbols of libibverbs and libmlx5. > + * > + * @return > + * 0 on success, -1 on failure. > + */ > +int mlx5_load_libs(void) > +{ > + void *dlhandle; > + int ret; > + > + dlhandle = dlopen(MLX5_LIB_IBVERBS_FILENAME, RTLD_LAZY); > + if (dlhandle == NULL) { > + ERROR("failed to open %s: %s\n", > + MLX5_LIB_IBVERBS_FILENAME, dlerror()); > + return -1; > + } > + ret = mlx5_libverbs_function_register(dlhandle); > + if (ret) { > + ERROR("failed to register a function in %s", > + MLX5_LIB_IBVERBS_FILENAME); > + return -1; > + } > + dlhandle = dlopen(MLX5_LIB_MLX5_FILENAME, RTLD_LAZY); > + if (dlhandle == NULL) { > + ERROR("failed to open %s: %s\n", > + MLX5_LIB_MLX5_FILENAME, dlerror()); > + return -1; > + } > + ret = mlx5_lmlx5_function_register(dlhandle); > + if (ret) { > + ERROR("failed to register a function in %s", > + MLX5_LIB_MLX5_FILENAME); > + return -1; > + } > + return 0; > +} > diff --git a/drivers/net/mlx5/lib/mlx5_dll.h b/drivers/net/mlx5/lib/mlx5_dll.h > new file mode 100644 > index 0000000..9c3d604 > --- /dev/null > +++ b/drivers/net/mlx5/lib/mlx5_dll.h > @@ -0,0 +1,103 @@ > +/*- > + * BSD LICENSE > + * > + * Copyright 2017 6WIND S.A. > + * Copyright 2017 Mellanox. > + * > + * Redistribution and use in source and binary forms, with or without > + * modification, are permitted provided that the following conditions > + * are met: > + * > + * * Redistributions of source code must retain the above copyright > + * notice, this list of conditions and the following disclaimer. > + * * Redistributions in binary form must reproduce the above copyright > + * notice, this list of conditions and the following disclaimer in > + * the documentation and/or other materials provided with the > + * distribution. > + * * Neither the name of 6WIND S.A. nor the names of its > + * contributors may be used to endorse or promote products derived > + * from this software without specific prior written permission. > + * > + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > + */ > + > +#ifndef MLX5_DLL_H_ > +#define MLX5_DLL_H_ > + > +#include > +#include > + > +#define MLX5_LIB_IBVERBS_FILENAME "libibverbs.so" > +#define MLX5_LIB_MLX5_FILENAME "libmlx5.so" These are only packaged in -devel packages. You probably want to specify the major version here. As in, libmlx5.so.1. > + > +#ifndef RTE_LIBRTE_MLX4_PMD This may be project specific, thus I'm sorry if I'm violating some code style policy but most of the macros below don't need/shouldn't be here in the header as they are only used by the dll wrapper and shouldn't be used outside of it. Perhaps the only thing really needed here is the +int mlx5_load_libs(void); declaration and the includes above. > + > +#define MLX5_IBVERBS_FUNC(name, ret, ...) \ > + ret(*__mlx5_pmd_ibv_##name)(__VA_ARGS__); \ > + ret(ibv_##name)(__VA_ARGS__) > + > +#define __MLX5_IBVERBS_FUNC(name, ...) \ > + (*__mlx5_pmd_ibv_##name)(__VA_ARGS__) > + > +#define MLX5_DLSYM_IBVERBS_FUNC(handle, name) \ > + do { \ > + char *err; \ > + *(void **)(&__mlx5_pmd_ibv_##name) = \ > + dlsym(handle, "ibv_"#name); \ > + err = dlerror(); \ > + if (err != NULL) { \ > + ERROR("failed to link a symbol: %s", err); \ > + return -1; \ > + } \ > + } while (0) > + > +#else /* RTE_LIBRTE_MLX4_PMD */ > + > +#define MLX5_IBVERBS_FUNC(name, ret, ...) \ > + ret(___mlx5_pmd_ibv_##name)(__VA_ARGS__); \ > + ret(___mlx5_pmd_ibv_##name)(__VA_ARGS__) > + > +#define __MLX5_IBVERBS_FUNC(name, ...) \ > + (ibv_##name)(__VA_ARGS__) > + > +#define MLX5_DLSYM_IBVERBS_FUNC(handle, name) do { (void)handle; } while (0) > + > +#endif /* RTE_LIBRTE_MLX4_PMD */ > + > +#define MLX5_MLX5DV_FUNC(name, ret, ...) \ > + ret(*__mlx5_pmd_mlx5dv_##name)(__VA_ARGS__); \ > + ret(mlx5dv_##name)(__VA_ARGS__) > + > +#define __MLX5_MLX5DV_FUNC(name, ...) \ > + (*__mlx5_pmd_mlx5dv_##name)(__VA_ARGS__) > + > +#define MLX5_DLSYM_MLX5DV_FUNC(handle, name) \ > + do { \ > + char *err; \ > + *(void **)(&__mlx5_pmd_mlx5dv_##name) = \ > + dlsym(handle, "mlx5dv_"#name); \ > + err = dlerror(); \ > + if (err != NULL) { \ > + ERROR("failed to link a symbol: %s", err); \ > + return -1; \ > + } \ > + } while (0) > + > +#undef ibv_query_port > + > +void *mlx5_lib_load(const char *name); > +int mlx5_libverbs_function_register(void *handle); > +int mlx5_lmlx5_function_register(void *handle); > +int mlx5_load_libs(void); > + > +#endif /* RTE_PMD_MLX5_DLL_H_ */ > diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c > index cd66fe1..eeef782 100644 > --- a/drivers/net/mlx5/mlx5.c > +++ b/drivers/net/mlx5/mlx5.c > @@ -30,7 +30,8 @@ > * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > */ > - > +#define _GNU_SOURCE > +#include this and > #include > #include > #include > @@ -39,13 +40,17 @@ > #include > #include > #include > - > +#include this change shouldn't be needed in this file, as all it does is call an initialization function below. > /* Verbs header. */ > /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ > #ifdef PEDANTIC > #pragma GCC diagnostic ignored "-Wpedantic" > #endif > +#ifdef RTE_LIBRTE_MLX5_DLL > +#include "lib/mlx5_dll.h" > +#else > #include > +#endif > #ifdef PEDANTIC > #pragma GCC diagnostic error "-Wpedantic" > #endif > @@ -1027,6 +1032,9 @@ struct mlx5_args { > rte_mlx5_pmd_init(void) > { > /* Build the static table for ptype conversion. */ > +#ifdef RTE_LIBRTE_MLX5_DLL > + int ret; > +#endif > mlx5_set_ptype_table(); > /* > * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use > @@ -1038,6 +1046,11 @@ struct mlx5_args { > /* Match the size of Rx completion entry to the size of a cacheline. */ > if (RTE_CACHE_LINE_SIZE == 128) > setenv("MLX5_CQE_SIZE", "128", 0); > +#ifdef RTE_LIBRTE_MLX5_DLL > + ret = mlx5_load_libs(); > + if (ret != 0) > + return; > +#endif Also, depends on project code style, but defining an empty mlx5_load_libs() in case RTE_LIBRTE_MLX5_DLL is NOT defined would avoid the two ifdef above. Marcelo > ibv_fork_init(); > rte_pci_register(&mlx5_driver); > } > diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h > index e6a69b8..9db7024 100644 > --- a/drivers/net/mlx5/mlx5.h > +++ b/drivers/net/mlx5/mlx5.h > @@ -46,7 +46,11 @@ > #ifdef PEDANTIC > #pragma GCC diagnostic ignored "-Wpedantic" > #endif > +#ifdef RTE_LIBRTE_MLX5_DLL > +#include "lib/mlx5_dll.h" > +#else > #include > +#endif > #ifdef PEDANTIC > #pragma GCC diagnostic error "-Wpedantic" > #endif > diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c > index 6605cfd..49f8972 100644 > --- a/drivers/net/mlx5/mlx5_flow.c > +++ b/drivers/net/mlx5/mlx5_flow.c > @@ -39,7 +39,11 @@ > #ifdef PEDANTIC > #pragma GCC diagnostic ignored "-Wpedantic" > #endif > +#ifdef RTE_LIBRTE_MLX5_DLL > +#include "lib/mlx5_dll.h" > +#else > #include > +#endif > #ifdef PEDANTIC > #pragma GCC diagnostic error "-Wpedantic" > #endif > diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c > index 9fb5ba5..c9f4bfd 100644 > --- a/drivers/net/mlx5/mlx5_mac.c > +++ b/drivers/net/mlx5/mlx5_mac.c > @@ -46,7 +46,11 @@ > #ifdef PEDANTIC > #pragma GCC diagnostic ignored "-Wpedantic" > #endif > +#ifdef RTE_LIBRTE_MLX5_DLL > +#include "lib/mlx5_dll.h" > +#else > #include > +#endif > #ifdef PEDANTIC > #pragma GCC diagnostic error "-Wpedantic" > #endif > diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c > index 6b29eed..f27a428 100644 > --- a/drivers/net/mlx5/mlx5_mr.c > +++ b/drivers/net/mlx5/mlx5_mr.c > @@ -36,7 +36,11 @@ > #ifdef PEDANTIC > #pragma GCC diagnostic ignored "-Wpedantic" > #endif > +#ifdef RTE_LIBRTE_MLX5_DLL > +#include "lib/mlx5_dll.h" > +#else > #include > +#endif > #ifdef PEDANTIC > #pragma GCC diagnostic error "-Wpedantic" > #endif > diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c > index f491135..63cbfa0 100644 > --- a/drivers/net/mlx5/mlx5_rss.c > +++ b/drivers/net/mlx5/mlx5_rss.c > @@ -42,7 +42,11 @@ > #ifdef PEDANTIC > #pragma GCC diagnostic ignored "-Wpedantic" > #endif > +#ifdef RTE_LIBRTE_MLX5_DLL > +#include "lib/mlx5_dll.h" > +#else > #include > +#endif > #ifdef PEDANTIC > #pragma GCC diagnostic error "-Wpedantic" > #endif > diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c > index 6fb245b..1ddad70 100644 > --- a/drivers/net/mlx5/mlx5_rxmode.c > +++ b/drivers/net/mlx5/mlx5_rxmode.c > @@ -40,7 +40,11 @@ > #ifdef PEDANTIC > #pragma GCC diagnostic ignored "-Wpedantic" > #endif > +#ifdef RTE_LIBRTE_MLX5_DLL > +#include "lib/mlx5_dll.h" > +#else > #include > +#endif > #ifdef PEDANTIC > #pragma GCC diagnostic error "-Wpedantic" > #endif > diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c > index a4cdd37..aaf4ce1 100644 > --- a/drivers/net/mlx5/mlx5_rxq.c > +++ b/drivers/net/mlx5/mlx5_rxq.c > @@ -44,8 +44,12 @@ > #ifdef PEDANTIC > #pragma GCC diagnostic ignored "-Wpedantic" > #endif > +#ifdef RTE_LIBRTE_MLX5_DLL > +#include "lib/mlx5_dll.h" > +#else > #include > #include > +#endif > #ifdef PEDANTIC > #pragma GCC diagnostic error "-Wpedantic" > #endif > diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c > index 67e3db1..4faf302 100644 > --- a/drivers/net/mlx5/mlx5_rxtx.c > +++ b/drivers/net/mlx5/mlx5_rxtx.c > @@ -41,8 +41,12 @@ > #ifdef PEDANTIC > #pragma GCC diagnostic ignored "-Wpedantic" > #endif > +#ifdef RTE_LIBRTE_MLX5_DLL > +#include "lib/mlx5_dll.h" > +#else > #include > #include > +#endif > #ifdef PEDANTIC > #pragma GCC diagnostic error "-Wpedantic" > #endif > diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h > index e70d523..e637555 100644 > --- a/drivers/net/mlx5/mlx5_rxtx.h > +++ b/drivers/net/mlx5/mlx5_rxtx.h > @@ -43,12 +43,14 @@ > #ifdef PEDANTIC > #pragma GCC diagnostic ignored "-Wpedantic" > #endif > +#ifdef RTE_LIBRTE_MLX5_DLL > +#include "lib/mlx5_dll.h" > +#else > #include > -#include > +#endif > #ifdef PEDANTIC > #pragma GCC diagnostic error "-Wpedantic" > #endif > - > #include > #include > #include > diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c > index 8d23dae..679bee7 100644 > --- a/drivers/net/mlx5/mlx5_rxtx_vec.c > +++ b/drivers/net/mlx5/mlx5_rxtx_vec.c > @@ -41,8 +41,12 @@ > #ifdef PEDANTIC > #pragma GCC diagnostic ignored "-Wpedantic" > #endif > +#ifdef RTE_LIBRTE_MLX5_DLL > +#include "lib/mlx5_dll.h" > +#else > #include > #include > +#endif > #ifdef PEDANTIC > #pragma GCC diagnostic error "-Wpedantic" > #endif > diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c > index 89b16fd..8b0cdf4 100644 > --- a/drivers/net/mlx5/mlx5_txq.c > +++ b/drivers/net/mlx5/mlx5_txq.c > @@ -44,7 +44,11 @@ > #ifdef PEDANTIC > #pragma GCC diagnostic ignored "-Wpedantic" > #endif > +#ifdef RTE_LIBRTE_MLX5_DLL > +#include "lib/mlx5_dll.h" > +#else > #include > +#endif > #ifdef PEDANTIC > #pragma GCC diagnostic error "-Wpedantic" > #endif > diff --git a/mk/rte.app.mk b/mk/rte.app.mk > index 6a6a745..2dd2f6b 100644 > --- a/mk/rte.app.mk > +++ b/mk/rte.app.mk > @@ -51,7 +51,9 @@ endif > > # Link only the libraries used in the application > LDFLAGS += --as-needed > - > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) > +LDFLAGS += -ldl > +endif > # default path for libs > _LDLIBS-y += -L$(RTE_SDK_BIN)/lib > > @@ -142,7 +144,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI) += -lrte_pmd_kni > endif > _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += -lrte_pmd_lio > _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4 -libverbs -lmlx4 > +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) > +_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -ldl > +else > _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -libverbs -lmlx5 > +endif > _LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD) += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk > _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += -lrte_pmd_nfp > _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += -lrte_pmd_null > -- > 1.8.3.1 >