[dpdk-dev] [PATCH] net/af_xdp: use bpf link for XDP programs

[dpdk-dev] [PATCH] net/af_xdp: use bpf link for XDP programs

[Ciara Loftus]

Since v0.4.0, if the underlying kernel supports it, libbpf uses 'bpf link'
to manage the programs on the interfaces of the xsks. This has two
repercussions for the PMD.

1. In the case where the PMD asks libbpf to load the default XDP program,
the PMD no longer needs to remove it on teardown. This is because bpf link
handles the unloading under the hood.
2. In the case where the PMD loads a custom program, libbpf expects this
program to be linked via bpf link prior to creating the socket.

This patch introduces probes for the libbpf version and kernel support
for bpf link and orchestrates the loading and unloading of
programs according to the capabilities of the kernel and libbpf. The
libbpf version is checked with meson and pkg-config. The probe for
kernel support mirrors how it is implemented in libbpf. A bpf_link is
created and looked up on loopback device. If successful, bpf_link will be
used for the AF_XDP netdev.

Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
---
 drivers/net/af_xdp/compat.h         | 120 ++++++++++++++++++++++++++++
 drivers/net/af_xdp/meson.build      |   7 ++
 drivers/net/af_xdp/rte_eth_af_xdp.c |  13 +--
 3 files changed, 135 insertions(+), 5 deletions(-)

diff --git a/drivers/net/af_xdp/compat.h b/drivers/net/af_xdp/compat.h
index 3880dc7dd7..1243de436c 100644
--- a/drivers/net/af_xdp/compat.h
+++ b/drivers/net/af_xdp/compat.h
@@ -2,9 +2,11 @@
  * Copyright(c) 2020 Intel Corporation.
  */
 
+#include <bpf/bpf.h>
 #include <bpf/xsk.h>
 #include <linux/version.h>
 #include <poll.h>
+#include <xdp/filter.h>
 
 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE && \
 	defined(RTE_LIBRTE_AF_XDP_PMD_SHARED_UMEM)
@@ -54,3 +56,121 @@ tx_syscall_needed(struct xsk_ring_prod *q __rte_unused)
 	return 1;
 }
 #endif
+
+#ifdef RTE_LIBRTE_AF_XDP_PMD_BPF_LINK
+static int link_lookup(int ifindex, int *link_fd)
+{
+	struct bpf_link_info link_info;
+	__u32 link_len;
+	__u32 id = 0;
+	int err;
+	int fd;
+
+	while (true) {
+		err = bpf_link_get_next_id(id, &id);
+		if (err) {
+			if (errno == ENOENT) {
+				err = 0;
+				break;
+			}
+			break;
+		}
+
+		fd = bpf_link_get_fd_by_id(id);
+		if (fd < 0) {
+			if (errno == ENOENT)
+				continue;
+			err = -errno;
+			break;
+		}
+
+		link_len = sizeof(struct bpf_link_info);
+		memset(&link_info, 0, link_len);
+		err = bpf_obj_get_info_by_fd(fd, &link_info, &link_len);
+		if (err) {
+			close(fd);
+			break;
+		}
+		if (link_info.type == BPF_LINK_TYPE_XDP) {
+			if ((int)link_info.xdp.ifindex == ifindex) {
+				*link_fd = fd;
+				break;
+			}
+		}
+		close(fd);
+	}
+
+	return err;
+}
+
+static bool probe_bpf_link(void)
+{
+	DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
+			    .flags = XDP_FLAGS_SKB_MODE);
+	struct bpf_load_program_attr prog_attr;
+	struct bpf_insn insns[2] = {
+		BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
+		BPF_EXIT_INSN()
+	};
+	int prog_fd, link_fd = -1;
+	int ifindex_lo = 1;
+	bool ret = false;
+	int err;
+
+	err = link_lookup(ifindex_lo, &link_fd);
+	if (err)
+		return ret;
+
+	if (link_fd >= 0)
+		return true;
+
+	memset(&prog_attr, 0, sizeof(prog_attr));
+	prog_attr.prog_type = BPF_PROG_TYPE_XDP;
+	prog_attr.insns = insns;
+	prog_attr.insns_cnt = RTE_DIM(insns);
+	prog_attr.license = "GPL";
+
+	prog_fd = bpf_load_program_xattr(&prog_attr, NULL, 0);
+	if (prog_fd < 0)
+		return ret;
+
+	link_fd = bpf_link_create(prog_fd, ifindex_lo, BPF_XDP, &opts);
+	close(prog_fd);
+
+	if (link_fd >= 0) {
+		ret = true;
+		close(link_fd);
+	}
+
+	return ret;
+}
+
+static int link_xdp_program(int if_index, int prog_fd, bool use_bpf_link)
+{
+	DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts);
+	int link_fd, ret = 0;
+
+	if (!use_bpf_link)
+		return bpf_set_link_xdp_fd(if_index, prog_fd,
+					   XDP_FLAGS_UPDATE_IF_NOEXIST);
+
+	opts.flags = 0;
+	link_fd = bpf_link_create(prog_fd, if_index, BPF_XDP, &opts);
+	if (link_fd < 0)
+		ret = -1;
+
+	return ret;
+}
+#else
+static bool probe_bpf_link(void)
+{
+	return false;
+}
+
+static int link_xdp_program(int if_index, int prog_fd,
+			    bool use_bpf_link __rte_unused)
+{
+	return bpf_set_link_xdp_fd(if_index, prog_fd,
+				   XDP_FLAGS_UPDATE_IF_NOEXIST);
+}
+#endif
diff --git a/drivers/net/af_xdp/meson.build b/drivers/net/af_xdp/meson.build
index 3ed2b29784..42dc6d69ac 100644
--- a/drivers/net/af_xdp/meson.build
+++ b/drivers/net/af_xdp/meson.build
@@ -16,11 +16,18 @@ endif
 
 if bpf_dep.found() and cc.has_header('bpf/xsk.h') and cc.has_header('linux/if_xdp.h')
     ext_deps += bpf_dep
+    # check for libbpf shared umem APIs
     bpf_ver_dep = dependency('libbpf', version : '>=0.2.0',
             required: false, method: 'pkg-config')
     if bpf_ver_dep.found()
         dpdk_conf.set('RTE_LIBRTE_AF_XDP_PMD_SHARED_UMEM', 1)
     endif
+    # check for libbpf bpf link support
+    bpf_link_dep = dependency('libbpf', version : '>=0.4.0',
+            required: false, method: 'pkg-config')
+    if bpf_link_dep.found()
+        dpdk_conf.set('RTE_LIBRTE_AF_XDP_PMD_BPF_LINK', 1)
+    endif
 else
     build = false
     reason = 'missing dependency, "libbpf"'
diff --git a/drivers/net/af_xdp/rte_eth_af_xdp.c b/drivers/net/af_xdp/rte_eth_af_xdp.c
index a619dd218d..ce88e6d5e1 100644
--- a/drivers/net/af_xdp/rte_eth_af_xdp.c
+++ b/drivers/net/af_xdp/rte_eth_af_xdp.c
@@ -138,6 +138,7 @@ struct pmd_internals {
 	bool shared_umem;
 	char prog_path[PATH_MAX];
 	bool custom_prog_configured;
+	bool use_bpf_link;
 
 	struct rte_ether_addr eth_addr;
 
@@ -971,7 +972,8 @@ eth_dev_close(struct rte_eth_dev *dev)
 	 */
 	dev->data->mac_addrs = NULL;
 
-	remove_xdp_program(internals);
+	if (!internals->use_bpf_link)
+		remove_xdp_program(internals);
 
 	if (internals->shared_umem) {
 		struct internal_list *list;
@@ -1146,7 +1148,7 @@ xsk_umem_info *xdp_umem_configure(struct pmd_internals *internals,
 }
 
 static int
-load_custom_xdp_prog(const char *prog_path, int if_index)
+load_custom_xdp_prog(const char *prog_path, int if_index, bool use_bpf_link)
 {
 	int ret, prog_fd = -1;
 	struct bpf_object *obj;
@@ -1170,8 +1172,7 @@ load_custom_xdp_prog(const char *prog_path, int if_index)
 	}
 
 	/* Link the program with the given network device */
-	ret = bpf_set_link_xdp_fd(if_index, prog_fd,
-					XDP_FLAGS_UPDATE_IF_NOEXIST);
+	ret = link_xdp_program(if_index, prog_fd, use_bpf_link);
 	if (ret) {
 		AF_XDP_LOG(ERR, "Failed to set prog fd %d on interface\n",
 				prog_fd);
@@ -1271,7 +1272,8 @@ xsk_configure(struct pmd_internals *internals, struct pkt_rx_queue *rxq,
 	if (strnlen(internals->prog_path, PATH_MAX) &&
 				!internals->custom_prog_configured) {
 		ret = load_custom_xdp_prog(internals->prog_path,
-					   internals->if_index);
+					   internals->if_index,
+					   internals->use_bpf_link);
 		if (ret) {
 			AF_XDP_LOG(ERR, "Failed to load custom XDP program %s\n",
 					internals->prog_path);
@@ -1688,6 +1690,7 @@ init_internals(struct rte_vdev_device *dev, const char *if_name,
 	strlcpy(internals->if_name, if_name, IFNAMSIZ);
 	strlcpy(internals->prog_path, prog_path, PATH_MAX);
 	internals->custom_prog_configured = 0;
+	internals->use_bpf_link = probe_bpf_link();
 
 #ifndef ETH_AF_XDP_SHARED_UMEM
 	if (shared_umem) {
-- 
2.17.1

Re: [dpdk-dev] [PATCH] net/af_xdp: use bpf link for XDP programs

[Ferruh Yigit]

On 10/14/2021 10:50 AM, Ciara Loftus wrote:
> --- a/drivers/net/af_xdp/compat.h
> +++ b/drivers/net/af_xdp/compat.h
> @@ -2,9 +2,11 @@
>    * Copyright(c) 2020 Intel Corporation.
>    */
>   
> +#include <bpf/bpf.h>
>   #include <bpf/xsk.h>
>   #include <linux/version.h>
>   #include <poll.h>
> +#include <xdp/filter.h>

Hi Ciara,

I am getting build error because xdp/filter.h is missing [1], where
that header should be?
And should the meson recognize the missing header/library and behave
according, or is the build error expected?


[1]
In file included from ../drivers/net/af_xdp/rte_eth_af_xdp.c:42:
../drivers/net/af_xdp/compat.h:9:10: fatal error: xdp/filter.h: No such file or directory
     9 | #include <xdp/filter.h>
       |

Re: [dpdk-dev] [PATCH] net/af_xdp: use bpf link for XDP programs

[Loftus, Ciara]

> 
> On 10/14/2021 10:50 AM, Ciara Loftus wrote:
> > --- a/drivers/net/af_xdp/compat.h
> > +++ b/drivers/net/af_xdp/compat.h
> > @@ -2,9 +2,11 @@
> >    * Copyright(c) 2020 Intel Corporation.
> >    */
> >
> > +#include <bpf/bpf.h>
> >   #include <bpf/xsk.h>
> >   #include <linux/version.h>
> >   #include <poll.h>
> > +#include <xdp/filter.h>
> 
> Hi Ciara,
> 
> I am getting build error because xdp/filter.h is missing [1], where
> that header should be?
> And should the meson recognize the missing header/library and behave
> according, or is the build error expected?

My mistake. This header shouldn't be included. Thank you for catching this.
It was to provide the macros BPF_MOV64_IMM and BPF_EXIT_INSN but we can derive these from the bpf/bph.h header and remove this dependency. I will implement this in the v2.

PS I noticed the PMD will not initialize since the "fix max Rx packet length" series. Probably an incorrect value for max_rx_pktlen in the PMD. I will look into it and provide a patch.

Thanks,
Ciara

> 
> 
> [1]
> In file included from ../drivers/net/af_xdp/rte_eth_af_xdp.c:42:
> ../drivers/net/af_xdp/compat.h:9:10: fatal error: xdp/filter.h: No such file or
> directory
>      9 | #include <xdp/filter.h>
>        |

[dpdk-dev] [PATCH v2] net/af_xdp: use bpf link for XDP programs

[Ciara Loftus]

Since v0.4.0, if the underlying kernel supports it, libbpf uses 'bpf link'
to manage the programs on the interfaces of the xsks. This has two
repercussions for the PMD.

1. In the case where the PMD asks libbpf to load the default XDP program,
the PMD no longer needs to remove it on teardown. This is because bpf link
handles the unloading under the hood.
2. In the case where the PMD loads a custom program, libbpf expects this
program to be linked via bpf link prior to creating the socket.

This patch introduces probes for the libbpf version and kernel support
for bpf link and orchestrates the loading and unloading of
programs according to the capabilities of the kernel and libbpf. The
libbpf version is checked with meson and pkg-config. The probe for
kernel support mirrors how it is implemented in libbpf. A bpf_link is
created and looked up on loopback device. If successful, bpf_link will be
used for the AF_XDP netdev.

Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
---
 drivers/net/af_xdp/compat.h         | 124 ++++++++++++++++++++++++++++
 drivers/net/af_xdp/meson.build      |   7 ++
 drivers/net/af_xdp/rte_eth_af_xdp.c |  13 +--
 3 files changed, 139 insertions(+), 5 deletions(-)

diff --git a/drivers/net/af_xdp/compat.h b/drivers/net/af_xdp/compat.h
index 3880dc7dd7..1d66f5e318 100644
--- a/drivers/net/af_xdp/compat.h
+++ b/drivers/net/af_xdp/compat.h
@@ -2,6 +2,7 @@
  * Copyright(c) 2020 Intel Corporation.
  */
 
+#include <bpf/bpf.h>
 #include <bpf/xsk.h>
 #include <linux/version.h>
 #include <poll.h>
@@ -54,3 +55,126 @@ tx_syscall_needed(struct xsk_ring_prod *q __rte_unused)
 	return 1;
 }
 #endif
+
+#ifdef RTE_LIBRTE_AF_XDP_PMD_BPF_LINK
+static int link_lookup(int ifindex, int *link_fd)
+{
+	struct bpf_link_info link_info;
+	__u32 link_len;
+	__u32 id = 0;
+	int err;
+	int fd;
+
+	while (true) {
+		err = bpf_link_get_next_id(id, &id);
+		if (err) {
+			if (errno == ENOENT) {
+				err = 0;
+				break;
+			}
+			break;
+		}
+
+		fd = bpf_link_get_fd_by_id(id);
+		if (fd < 0) {
+			if (errno == ENOENT)
+				continue;
+			err = -errno;
+			break;
+		}
+
+		link_len = sizeof(struct bpf_link_info);
+		memset(&link_info, 0, link_len);
+		err = bpf_obj_get_info_by_fd(fd, &link_info, &link_len);
+		if (err) {
+			close(fd);
+			break;
+		}
+		if (link_info.type == BPF_LINK_TYPE_XDP) {
+			if ((int)link_info.xdp.ifindex == ifindex) {
+				*link_fd = fd;
+				break;
+			}
+		}
+		close(fd);
+	}
+
+	return err;
+}
+
+static bool probe_bpf_link(void)
+{
+	DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
+			    .flags = XDP_FLAGS_SKB_MODE);
+	struct bpf_load_program_attr prog_attr;
+	struct bpf_insn insns[2];
+	int prog_fd, link_fd = -1;
+	int ifindex_lo = 1;
+	bool ret = false;
+	int err;
+
+	err = link_lookup(ifindex_lo, &link_fd);
+	if (err)
+		return ret;
+
+	if (link_fd >= 0)
+		return true;
+
+	/* BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), */
+	insns[0].code = BPF_ALU64 | BPF_MOV | BPF_K;
+	insns[0].dst_reg = BPF_REG_0;
+	insns[0].imm = XDP_PASS;
+
+	/* BPF_EXIT_INSN() */
+	insns[1].code = BPF_JMP | BPF_EXIT;
+
+	memset(&prog_attr, 0, sizeof(prog_attr));
+	prog_attr.prog_type = BPF_PROG_TYPE_XDP;
+	prog_attr.insns = insns;
+	prog_attr.insns_cnt = RTE_DIM(insns);
+	prog_attr.license = "GPL";
+
+	prog_fd = bpf_load_program_xattr(&prog_attr, NULL, 0);
+	if (prog_fd < 0)
+		return ret;
+
+	link_fd = bpf_link_create(prog_fd, ifindex_lo, BPF_XDP, &opts);
+	close(prog_fd);
+
+	if (link_fd >= 0) {
+		ret = true;
+		close(link_fd);
+	}
+
+	return ret;
+}
+
+static int link_xdp_program(int if_index, int prog_fd, bool use_bpf_link)
+{
+	DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts);
+	int link_fd, ret = 0;
+
+	if (!use_bpf_link)
+		return bpf_set_link_xdp_fd(if_index, prog_fd,
+					   XDP_FLAGS_UPDATE_IF_NOEXIST);
+
+	opts.flags = 0;
+	link_fd = bpf_link_create(prog_fd, if_index, BPF_XDP, &opts);
+	if (link_fd < 0)
+		ret = -1;
+
+	return ret;
+}
+#else
+static bool probe_bpf_link(void)
+{
+	return false;
+}
+
+static int link_xdp_program(int if_index, int prog_fd,
+			    bool use_bpf_link __rte_unused)
+{
+	return bpf_set_link_xdp_fd(if_index, prog_fd,
+				   XDP_FLAGS_UPDATE_IF_NOEXIST);
+}
+#endif
diff --git a/drivers/net/af_xdp/meson.build b/drivers/net/af_xdp/meson.build
index 3ed2b29784..42dc6d69ac 100644
--- a/drivers/net/af_xdp/meson.build
+++ b/drivers/net/af_xdp/meson.build
@@ -16,11 +16,18 @@ endif
 
 if bpf_dep.found() and cc.has_header('bpf/xsk.h') and cc.has_header('linux/if_xdp.h')
     ext_deps += bpf_dep
+    # check for libbpf shared umem APIs
     bpf_ver_dep = dependency('libbpf', version : '>=0.2.0',
             required: false, method: 'pkg-config')
     if bpf_ver_dep.found()
         dpdk_conf.set('RTE_LIBRTE_AF_XDP_PMD_SHARED_UMEM', 1)
     endif
+    # check for libbpf bpf link support
+    bpf_link_dep = dependency('libbpf', version : '>=0.4.0',
+            required: false, method: 'pkg-config')
+    if bpf_link_dep.found()
+        dpdk_conf.set('RTE_LIBRTE_AF_XDP_PMD_BPF_LINK', 1)
+    endif
 else
     build = false
     reason = 'missing dependency, "libbpf"'
diff --git a/drivers/net/af_xdp/rte_eth_af_xdp.c b/drivers/net/af_xdp/rte_eth_af_xdp.c
index b362ccdcd3..3d295d9277 100644
--- a/drivers/net/af_xdp/rte_eth_af_xdp.c
+++ b/drivers/net/af_xdp/rte_eth_af_xdp.c
@@ -138,6 +138,7 @@ struct pmd_internals {
 	bool shared_umem;
 	char prog_path[PATH_MAX];
 	bool custom_prog_configured;
+	bool use_bpf_link;
 
 	struct rte_ether_addr eth_addr;
 
@@ -971,7 +972,8 @@ eth_dev_close(struct rte_eth_dev *dev)
 	 */
 	dev->data->mac_addrs = NULL;
 
-	remove_xdp_program(internals);
+	if (!internals->use_bpf_link)
+		remove_xdp_program(internals);
 
 	if (internals->shared_umem) {
 		struct internal_list *list;
@@ -1146,7 +1148,7 @@ xsk_umem_info *xdp_umem_configure(struct pmd_internals *internals,
 }
 
 static int
-load_custom_xdp_prog(const char *prog_path, int if_index)
+load_custom_xdp_prog(const char *prog_path, int if_index, bool use_bpf_link)
 {
 	int ret, prog_fd = -1;
 	struct bpf_object *obj;
@@ -1170,8 +1172,7 @@ load_custom_xdp_prog(const char *prog_path, int if_index)
 	}
 
 	/* Link the program with the given network device */
-	ret = bpf_set_link_xdp_fd(if_index, prog_fd,
-					XDP_FLAGS_UPDATE_IF_NOEXIST);
+	ret = link_xdp_program(if_index, prog_fd, use_bpf_link);
 	if (ret) {
 		AF_XDP_LOG(ERR, "Failed to set prog fd %d on interface\n",
 				prog_fd);
@@ -1271,7 +1272,8 @@ xsk_configure(struct pmd_internals *internals, struct pkt_rx_queue *rxq,
 	if (strnlen(internals->prog_path, PATH_MAX) &&
 				!internals->custom_prog_configured) {
 		ret = load_custom_xdp_prog(internals->prog_path,
-					   internals->if_index);
+					   internals->if_index,
+					   internals->use_bpf_link);
 		if (ret) {
 			AF_XDP_LOG(ERR, "Failed to load custom XDP program %s\n",
 					internals->prog_path);
@@ -1688,6 +1690,7 @@ init_internals(struct rte_vdev_device *dev, const char *if_name,
 	strlcpy(internals->if_name, if_name, IFNAMSIZ);
 	strlcpy(internals->prog_path, prog_path, PATH_MAX);
 	internals->custom_prog_configured = 0;
+	internals->use_bpf_link = probe_bpf_link();
 
 #ifndef ETH_AF_XDP_SHARED_UMEM
 	if (shared_umem) {
-- 
2.17.1

Re: [dpdk-dev] [PATCH v2] net/af_xdp: use bpf link for XDP programs

[Ferruh Yigit]

On 10/22/2021 11:42 AM, Ciara Loftus wrote:
> Since v0.4.0, if the underlying kernel supports it, libbpf uses 'bpf link'
> to manage the programs on the interfaces of the xsks. This has two
> repercussions for the PMD.
> 
> 1. In the case where the PMD asks libbpf to load the default XDP program,
> the PMD no longer needs to remove it on teardown. This is because bpf link
> handles the unloading under the hood.
> 2. In the case where the PMD loads a custom program, libbpf expects this
> program to be linked via bpf link prior to creating the socket.
> 
> This patch introduces probes for the libbpf version and kernel support
> for bpf link and orchestrates the loading and unloading of
> programs according to the capabilities of the kernel and libbpf. The
> libbpf version is checked with meson and pkg-config. The probe for
> kernel support mirrors how it is implemented in libbpf. A bpf_link is
> created and looked up on loopback device. If successful, bpf_link will be
> used for the AF_XDP netdev.
> 
> Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
> Signed-off-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>

Applied to dpdk-next-net/main, thanks.