From: Adrien Mazarguil <adrien.mazarguil@6wind.com>
To: Shahaf Shuler <shahafs@mellanox.com>
Cc: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>,
dev@dpdk.org, Marcelo Ricardo Leitner <mleitner@redhat.com>
Subject: [dpdk-dev] [PATCH v4 4/4] net/mlx5: spawn rdma-core dependency plug-in
Date: Tue, 30 Jan 2018 16:34:58 +0100 [thread overview]
Message-ID: <20180130144512.21872-5-adrien.mazarguil@6wind.com> (raw)
In-Reply-To: <20180130144512.21872-1-adrien.mazarguil@6wind.com>
When mlx5 is not compiled directly as an independent shared object (e.g.
CONFIG_RTE_BUILD_SHARED_LIB not enabled for performance reasons), DPDK
applications inherit its dependencies on libibverbs and libmlx5 through
rte.app.mk.
This is an issue both when DPDK is delivered as a binary package (Linux
distributions) and for end users because rdma-core then propagates as a
mandatory dependency for everything.
Application writers relying on binary DPDK packages are not necessarily
aware of this fact and may end up delivering packages with broken
dependencies.
This patch therefore introduces an intermediate internal plug-in
hard-linked with rdma-core (to preserve symbol versioning) loaded by the
PMD through dlopen(), so that a missing rdma-core does not cause unresolved
symbols, allowing applications to start normally.
Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
---
config/common_base | 1 +
doc/guides/nics/mlx5.rst | 13 ++++++++++++
drivers/net/mlx5/Makefile | 31 ++++++++++++++++++++++++++-
drivers/net/mlx5/mlx5.c | 48 ++++++++++++++++++++++++++++++++++++++++++
mk/rte.app.mk | 4 ++++
5 files changed, 96 insertions(+), 1 deletion(-)
diff --git a/config/common_base b/config/common_base
index 6df20be0e..b9b421db7 100644
--- a/config/common_base
+++ b/config/common_base
@@ -306,6 +306,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
#
CONFIG_RTE_LIBRTE_MLX5_PMD=n
CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
+CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS=n
CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
#
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 2e6d1e45a..a9e4bf51a 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -159,6 +159,19 @@ These options can be modified in the ``.config`` file.
Toggle compilation of librte_pmd_mlx5 itself.
+- ``CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS`` (default **n**)
+
+ Build PMD with additional code to make it loadable without hard
+ dependencies on **libibverbs** nor **libmlx5**, which may not be installed
+ on the target system.
+
+ In this mode, their presence is still required for it to run properly,
+ however their absence won't prevent a DPDK application from starting (with
+ ``CONFIG_RTE_BUILD_SHARED_LIB`` disabled) and they won't show up as
+ missing with ``ldd(1)``.
+
+ This option has no performance impact.
+
- ``CONFIG_RTE_LIBRTE_MLX5_DEBUG`` (default **n**)
Toggle debugging code and stricter compilation flags. Enabling this option
diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index bdec30692..4b20d718b 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -33,9 +33,13 @@ include $(RTE_SDK)/mk/rte.vars.mk
# Library name.
LIB = librte_pmd_mlx5.a
+LIB_GLUE = librte_pmd_mlx5_glue.so
# Sources.
SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c
+ifneq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_glue.c
+endif
SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxq.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_txq.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx.c
@@ -53,7 +57,10 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c
-SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_glue.c
+
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y)
+INSTALL-$(CONFIG_RTE_LIBRTE_MLX5_PMD)-lib += $(LIB_GLUE)
+endif
# Basic CFLAGS.
CFLAGS += -O3
@@ -65,7 +72,13 @@ CFLAGS += -D_DEFAULT_SOURCE
CFLAGS += -D_XOPEN_SOURCE=600
CFLAGS += $(WERROR_FLAGS)
CFLAGS += -Wno-strict-prototypes
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y)
+CFLAGS += -DMLX5_GLUE='"$(LIB_GLUE)"'
+CFLAGS_mlx5_glue.o += -fPIC
+LDLIBS += -ldl
+else
LDLIBS += -libverbs -lmlx5
+endif
LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
LDLIBS += -lrte_bus_pci
@@ -158,7 +171,23 @@ mlx5_autoconf.h: mlx5_autoconf.h.new
$(SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD):.c=.o): mlx5_autoconf.h
+# Generate dependency plug-in for rdma-core when the PMD must not be linked
+# directly, so that applications do not inherit this dependency.
+
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y)
+
+$(LIB): $(LIB_GLUE)
+
+$(LIB_GLUE): mlx5_glue.o
+ $Q $(LD) $(LDFLAGS) $(EXTRA_LDFLAGS) \
+ -s -shared -o $@ $< -libverbs -lmlx5
+
+mlx5_glue.o: mlx5_autoconf.h
+
+endif
+
clean_mlx5: FORCE
$Q rm -f -- mlx5_autoconf.h mlx5_autoconf.h.new
+ $Q rm -f -- mlx5_glue.o $(LIB_GLUE)
clean: clean_mlx5
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index b911d0ad4..466a8b0a2 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -35,6 +35,7 @@
#include <unistd.h>
#include <string.h>
#include <assert.h>
+#include <dlfcn.h>
#include <stdint.h>
#include <stdlib.h>
#include <errno.h>
@@ -57,6 +58,7 @@
#include <rte_pci.h>
#include <rte_bus_pci.h>
#include <rte_common.h>
+#include <rte_config.h>
#include <rte_eal_memconfig.h>
#include <rte_kvargs.h>
@@ -1075,6 +1077,47 @@ static struct rte_pci_driver mlx5_driver = {
.drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV,
};
+#ifdef RTE_LIBRTE_MLX5_DLOPEN_DEPS
+
+/**
+ * Initialization routine for run-time dependency on rdma-core.
+ */
+static int
+mlx5_glue_init(void)
+{
+ void *handle = NULL;
+ void **sym;
+ const char *dlmsg;
+
+ handle = dlopen(MLX5_GLUE, RTLD_LAZY);
+ if (!handle) {
+ rte_errno = EINVAL;
+ dlmsg = dlerror();
+ if (dlmsg)
+ ERROR("cannot load glue library: %s", dlmsg);
+ goto glue_error;
+ }
+ sym = dlsym(handle, "mlx5_glue");
+ if (!sym || !*sym) {
+ rte_errno = EINVAL;
+ dlmsg = dlerror();
+ if (dlmsg)
+ ERROR("cannot resolve glue symbol: %s", dlmsg);
+ goto glue_error;
+ }
+ mlx5_glue = *sym;
+ return 0;
+glue_error:
+ if (handle)
+ dlclose(handle);
+ ERROR("cannot initialize PMD due to missing run-time"
+ " dependency on rdma-core libraries (libibverbs,"
+ " libmlx5)");
+ return -rte_errno;
+}
+
+#endif
+
/**
* Driver initialization routine.
*/
@@ -1094,6 +1137,11 @@ rte_mlx5_pmd_init(void)
/* Match the size of Rx completion entry to the size of a cacheline. */
if (RTE_CACHE_LINE_SIZE == 128)
setenv("MLX5_CQE_SIZE", "128", 0);
+#ifdef RTE_LIBRTE_MLX5_DLOPEN_DEPS
+ if (mlx5_glue_init())
+ return;
+ assert(mlx5_glue);
+#endif
mlx5_glue->fork_init();
rte_pci_register(&mlx5_driver);
}
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index fdbe36630..88f626049 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -148,7 +148,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4 -ldl
else
_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4 -libverbs -lmlx4
endif
+ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -ldl
+else
_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -libverbs -lmlx5
+endif
_LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD) += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk
_LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += -lrte_pmd_nfp
_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += -lrte_pmd_null
--
2.11.0
next prev parent reply other threads:[~2018-01-30 15:35 UTC|newest]
Thread overview: 31+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-01-24 23:24 [dpdk-dev] [PATCH v1 0/4] net/mlx: make rdma-core optional at run-time Adrien Mazarguil
2018-01-24 23:25 ` [dpdk-dev] [PATCH v1 1/4] net/mlx4: move rdma-core calls to separate file Adrien Mazarguil
2018-01-24 23:58 ` Stephen Hemminger
2018-01-25 11:31 ` Adrien Mazarguil
2018-01-24 23:25 ` [dpdk-dev] [PATCH v1 2/4] net/mlx4: spawn rdma-core dependency plug-in Adrien Mazarguil
2018-01-26 10:06 ` Thomas Monjalon
2018-01-24 23:25 ` [dpdk-dev] [PATCH v1 3/4] net/mlx5: move rdma-core calls to separate file Adrien Mazarguil
2018-01-24 23:25 ` [dpdk-dev] [PATCH v1 4/4] net/mlx5: spawn rdma-core dependency plug-in Adrien Mazarguil
2018-01-26 14:18 ` [dpdk-dev] [PATCH v2 0/4] net/mlx: make rdma-core optional at run-time Adrien Mazarguil
2018-01-26 14:18 ` [dpdk-dev] [PATCH v2 1/4] net/mlx4: move rdma-core calls to separate file Adrien Mazarguil
2018-01-26 14:19 ` [dpdk-dev] [PATCH v2 2/4] net/mlx4: spawn rdma-core dependency plug-in Adrien Mazarguil
2018-01-27 15:03 ` Marcelo Ricardo Leitner
2018-01-28 9:04 ` Shahaf Shuler
2018-01-28 11:17 ` Marcelo Ricardo Leitner
2018-01-28 11:46 ` Shahaf Shuler
2018-01-26 14:19 ` [dpdk-dev] [PATCH v2 3/4] net/mlx5: move rdma-core calls to separate file Adrien Mazarguil
2018-01-26 14:19 ` [dpdk-dev] [PATCH v2 4/4] net/mlx5: spawn rdma-core dependency plug-in Adrien Mazarguil
2018-01-26 15:58 ` [dpdk-dev] [PATCH v2 0/4] net/mlx: make rdma-core optional at run-time Nélio Laranjeiro
2018-01-29 17:19 ` [dpdk-dev] [PATCH v3 " Adrien Mazarguil
2018-01-29 17:19 ` [dpdk-dev] [PATCH v3 1/4] net/mlx4: move rdma-core calls to separate file Adrien Mazarguil
2018-01-29 17:19 ` [dpdk-dev] [PATCH v3 2/4] net/mlx4: spawn rdma-core dependency plug-in Adrien Mazarguil
2018-01-29 17:19 ` [dpdk-dev] [PATCH v3 3/4] net/mlx5: move rdma-core calls to separate file Adrien Mazarguil
2018-01-29 17:19 ` [dpdk-dev] [PATCH v3 4/4] net/mlx5: spawn rdma-core dependency plug-in Adrien Mazarguil
2018-01-30 15:34 ` [dpdk-dev] [PATCH v4 0/4] net/mlx: make rdma-core optional at run-time Adrien Mazarguil
2018-01-30 15:34 ` [dpdk-dev] [PATCH v4 1/4] net/mlx4: move rdma-core calls to separate file Adrien Mazarguil
2018-01-30 15:34 ` [dpdk-dev] [PATCH v4 2/4] net/mlx4: spawn rdma-core dependency plug-in Adrien Mazarguil
2018-01-30 17:54 ` Marcelo Ricardo Leitner
2018-01-30 18:32 ` Adrien Mazarguil
2018-01-30 15:34 ` [dpdk-dev] [PATCH v4 3/4] net/mlx5: move rdma-core calls to separate file Adrien Mazarguil
2018-01-30 15:34 ` Adrien Mazarguil [this message]
2018-01-31 10:13 ` [dpdk-dev] [PATCH v4 0/4] net/mlx: make rdma-core optional at run-time Shahaf Shuler
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20180130144512.21872-5-adrien.mazarguil@6wind.com \
--to=adrien.mazarguil@6wind.com \
--cc=dev@dpdk.org \
--cc=mleitner@redhat.com \
--cc=nelio.laranjeiro@6wind.com \
--cc=shahafs@mellanox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).