[dpdk-dev] [PATCH 3/5] bpf: add support for packet data load instructions

[Konstantin Ananyev <konstantin.ananyev@intel.com>]

To fill the gap with linux kernel eBPF implementation,
add support for two non-generic instructions:
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD)
which are used to access packet data.
These instructions can only be used when BPF context is a pointer
to 'struct rte_mbuf' (i.e: RTE_BPF_ARG_PTR_MBUF type).

Signed-off-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
---
 doc/guides/prog_guide/bpf_lib.rst | 30 +++++++++++-
 lib/librte_bpf/bpf_exec.c         | 57 +++++++++++++++++++++++
 lib/librte_bpf/bpf_validate.c     | 77 +++++++++++++++++++++++++++++++
 3 files changed, 163 insertions(+), 1 deletion(-)

diff --git a/doc/guides/prog_guide/bpf_lib.rst b/doc/guides/prog_guide/bpf_lib.rst
index 9c728da7b..1feb7734a 100644
--- a/doc/guides/prog_guide/bpf_lib.rst
+++ b/doc/guides/prog_guide/bpf_lib.rst
@@ -27,6 +27,35 @@ The library API provides the following basic operations:
 
 *   Load BPF program from the ELF file and install callback to execute it on given ethdev port/queue.
 
+Packet data load instructions
+-----------------------------
+
+DPDK supports two non-generic instructions: ``(BPF_ABS | size | BPF_LD)``
+and ``(BPF_IND | size | BPF_LD)`` which are used to access packet data.
+These instructions can only be used when execution context is a pointer to
+``struct rte_mbuf`` and have seven implicit operands.
+Register ``R6`` is an implicit input that must contain pointer to ``rte_mbuf``.
+Register ``R0`` is an implicit output which contains the data fetched from the
+packet. Registers ``R1-R5`` are scratch registers
+and must not be used to store the data across these instructions.
+These instructions have implicit program exit condition as well. When
+eBPF program is trying to access the data beyond the packet boundary,
+the interpreter will abort the execution of the program. JIT compilers
+therefore must preserve this property. ``src_reg`` and ``imm32`` fields are
+explicit inputs to these instructions.
+For example, ``(BPF_IND | BPF_W | BPF_LD)`` means:
+
+.. code-block:: c
+
+    uint32_t tmp;
+    R0 = rte_pktmbuf_read((const struct rte_mbuf *)R6,  src_reg + imm32,
+	sizeof(tmp), &tmp);
+    if (R0 == NULL) return FAILED;
+    R0 = ntohl(*(uint32_t *)R0);
+
+and ``R1-R5`` were scratched.
+
+
 Not currently supported eBPF features
 -------------------------------------
 
@@ -34,5 +63,4 @@ Not currently supported eBPF features
  - cBPF
  - tail-pointer call
  - eBPF MAP
- - skb
  - external function calls for 32-bit platforms
diff --git a/lib/librte_bpf/bpf_exec.c b/lib/librte_bpf/bpf_exec.c
index 1bb226643..b921112fe 100644
--- a/lib/librte_bpf/bpf_exec.c
+++ b/lib/librte_bpf/bpf_exec.c
@@ -74,6 +74,26 @@
 		(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \
 		reg[ins->src_reg]))
 
+/* BPF_LD | BPF_ABS/BPF_IND */
+
+#define	NOP(x)	(x)
+
+#define BPF_LD_ABS(bpf, reg, ins, type, op) do { \
+	const type *p = bpf_ld_mbuf(bpf, reg, ins, (ins)->imm, sizeof(type)); \
+	if (p == NULL)  \
+		return 0; \
+	reg[EBPF_REG_0] = op(p[0]); \
+} while (0)
+
+#define BPF_LD_IND(bpf, reg, ins, type, op) do { \
+	uint32_t ofs = reg[ins->src_reg] + (ins)->imm; \
+	const type *p = bpf_ld_mbuf(bpf, reg, ins, ofs, sizeof(type)); \
+	if (p == NULL)  \
+		return 0; \
+	reg[EBPF_REG_0] = op(p[0]); \
+} while (0)
+
+
 static inline void
 bpf_alu_be(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)
 {
@@ -112,6 +132,23 @@ bpf_alu_le(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)
 	}
 }
 
+static inline const void *
+bpf_ld_mbuf(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM],
+	const struct ebpf_insn *ins, uint32_t off, uint32_t len)
+{
+	const struct rte_mbuf *mb;
+	const void *p;
+
+	mb = (const struct rte_mbuf *)(uintptr_t)reg[EBPF_REG_6];
+	p = rte_pktmbuf_read(mb, off, len, reg + EBPF_REG_0);
+	if (p == NULL)
+		RTE_BPF_LOG(DEBUG, "%s(bpf=%p, mbuf=%p, ofs=%u, len=%u): "
+			"load beyond packet boundary at pc: %#zx;\n",
+			__func__, bpf, mb, off, len,
+			(uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins);
+	return p;
+}
+
 static inline uint64_t
 bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM])
 {
@@ -296,6 +333,26 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM])
 				(uint64_t)(uint32_t)ins[1].imm << 32;
 			ins++;
 			break;
+		/* load absolute instructions */
+		case (BPF_LD | BPF_ABS | BPF_B):
+			BPF_LD_ABS(bpf, reg, ins, uint8_t, NOP);
+			break;
+		case (BPF_LD | BPF_ABS | BPF_H):
+			BPF_LD_ABS(bpf, reg, ins, uint16_t, rte_be_to_cpu_16);
+			break;
+		case (BPF_LD | BPF_ABS | BPF_W):
+			BPF_LD_ABS(bpf, reg, ins, uint32_t, rte_be_to_cpu_32);
+			break;
+		/* load indirect instructions */
+		case (BPF_LD | BPF_IND | BPF_B):
+			BPF_LD_IND(bpf, reg, ins, uint8_t, NOP);
+			break;
+		case (BPF_LD | BPF_IND | BPF_H):
+			BPF_LD_IND(bpf, reg, ins, uint16_t, rte_be_to_cpu_16);
+			break;
+		case (BPF_LD | BPF_IND | BPF_W):
+			BPF_LD_IND(bpf, reg, ins, uint32_t, rte_be_to_cpu_32);
+			break;
 		/* store instructions */
 		case (BPF_STX | BPF_MEM | BPF_B):
 			BPF_ST_REG(reg, ins, uint8_t);
diff --git a/lib/librte_bpf/bpf_validate.c b/lib/librte_bpf/bpf_validate.c
index 80d21fabb..fecdda0e1 100644
--- a/lib/librte_bpf/bpf_validate.c
+++ b/lib/librte_bpf/bpf_validate.c
@@ -102,6 +102,9 @@ struct bpf_ins_check {
 #define	WRT_REGS	RTE_LEN2MASK(EBPF_REG_10, uint16_t)
 #define	ZERO_REG	RTE_LEN2MASK(EBPF_REG_1, uint16_t)
 
+/* For LD_IND R6 is an implicit CTX register. */
+#define	IND_SRC_REGS	(WRT_REGS ^ 1 << EBPF_REG_6)
+
 /*
  * check and evaluate functions for particular instruction types.
  */
@@ -580,6 +583,42 @@ eval_neg(struct bpf_reg_val *rd, size_t opsz, uint64_t msk)
 	rd->s.min = RTE_MIN(sx, sy);
 }
 
+static const char *
+eval_ld_mbuf(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
+{
+	uint32_t i, mode;
+	struct bpf_reg_val *rv, ri, rs;
+
+	mode = BPF_MODE(ins->code);
+
+	/* R6 is an implicit input that must contain pointer to mbuf */
+	if (bvf->evst->rv[EBPF_REG_6].v.type != RTE_BPF_ARG_PTR_MBUF)
+		return "invalid type for implicit ctx register";
+
+	if (mode == BPF_IND) {
+		rs = bvf->evst->rv[ins->src_reg];
+		if (rs.v.type != RTE_BPF_ARG_RAW)
+			return "unexpected type for src register";
+
+		eval_fill_imm(&ri, UINT64_MAX, ins->imm);
+		eval_add(&rs, &ri, UINT64_MAX);
+
+		if (rs.s.max < 0 || rs.u.min > UINT32_MAX)
+			return "mbuf boundary violation";
+	}
+
+	/* R1-R5 scratch registers */
+	for (i = EBPF_REG_1; i != EBPF_REG_6; i++)
+		bvf->evst->rv[i].v.type = RTE_BPF_ARG_UNDEF;
+
+	/* R0 is an implicit output, contains data fetched from the packet */
+	rv = bvf->evst->rv + EBPF_REG_0;
+	rv->v.size = bpf_size(BPF_SIZE(ins->code));
+	eval_fill_max_bound(rv, RTE_LEN2MASK(rv->v.size * CHAR_BIT, uint64_t));
+
+	return NULL;
+}
+
 /*
  * check that destination and source operand are in defined state.
  */
@@ -1425,6 +1464,44 @@ static const struct bpf_ins_check ins_chk[UINT8_MAX + 1] = {
 		.imm = { .min = 0, .max = UINT32_MAX},
 		.eval = eval_ld_imm64,
 	},
+	/* load absolute instructions */
+	[(BPF_LD | BPF_ABS | BPF_B)] = {
+		.mask = {. dreg = ZERO_REG, .sreg = ZERO_REG},
+		.off = { .min = 0, .max = 0},
+		.imm = { .min = 0, .max = INT32_MAX},
+		.eval = eval_ld_mbuf,
+	},
+	[(BPF_LD | BPF_ABS | BPF_H)] = {
+		.mask = {. dreg = ZERO_REG, .sreg = ZERO_REG},
+		.off = { .min = 0, .max = 0},
+		.imm = { .min = 0, .max = INT32_MAX},
+		.eval = eval_ld_mbuf,
+	},
+	[(BPF_LD | BPF_ABS | BPF_W)] = {
+		.mask = {. dreg = ZERO_REG, .sreg = ZERO_REG},
+		.off = { .min = 0, .max = 0},
+		.imm = { .min = 0, .max = INT32_MAX},
+		.eval = eval_ld_mbuf,
+	},
+	/* load indirect instructions */
+	[(BPF_LD | BPF_IND | BPF_B)] = {
+		.mask = {. dreg = ZERO_REG, .sreg = IND_SRC_REGS},
+		.off = { .min = 0, .max = 0},
+		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_ld_mbuf,
+	},
+	[(BPF_LD | BPF_IND | BPF_H)] = {
+		.mask = {. dreg = ZERO_REG, .sreg = IND_SRC_REGS},
+		.off = { .min = 0, .max = 0},
+		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_ld_mbuf,
+	},
+	[(BPF_LD | BPF_IND | BPF_W)] = {
+		.mask = {. dreg = ZERO_REG, .sreg = IND_SRC_REGS},
+		.off = { .min = 0, .max = 0},
+		.imm = { .min = 0, .max = UINT32_MAX},
+		.eval = eval_ld_mbuf,
+	},
 	/* store REG instructions */
 	[(BPF_STX | BPF_MEM | BPF_B)] = {
 		.mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
-- 
2.17.1