DPDK patches and discussions
 help / color / mirror / Atom feed
From: Konstantin Ananyev <konstantin.ananyev@intel.com>
To: dev@dpdk.org
Cc: stephen@networkplumber.org, jerinj@marvell.com,
	Konstantin Ananyev <konstantin.ananyev@intel.com>
Subject: [dpdk-dev] [PATCH 5/5] bpf: x86 JIT support for packet data load instructions
Date: Mon, 18 May 2020 16:52:45 +0100	[thread overview]
Message-ID: <20200518155245.11380-6-konstantin.ananyev@intel.com> (raw)
In-Reply-To: <20200518155245.11380-1-konstantin.ananyev@intel.com>

Make x86 JIT to generate native code for
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD)
instructions.

Signed-off-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
---
 lib/librte_bpf/bpf_jit_x86.c  | 181 ++++++++++++++++++++++++++++++++++
 lib/librte_bpf/bpf_validate.c |  17 +++-
 2 files changed, 197 insertions(+), 1 deletion(-)

diff --git a/lib/librte_bpf/bpf_jit_x86.c b/lib/librte_bpf/bpf_jit_x86.c
index f70cd6be5..8a7ad45b3 100644
--- a/lib/librte_bpf/bpf_jit_x86.c
+++ b/lib/librte_bpf/bpf_jit_x86.c
@@ -87,6 +87,14 @@ enum {
 	REG_TMP1 = R10,
 };
 
+/* LD_ABS/LD_IMM offsets */
+enum {
+	LDMB_FSP_OFS, /* fast-path */
+	LDMB_SLP_OFS, /* slow-path */
+	LDMB_FIN_OFS, /* final part */
+	LDMB_OFS_NUM
+};
+
 /*
  * callee saved registers list.
  * keep RBP as the last one.
@@ -100,6 +108,9 @@ struct bpf_jit_state {
 		uint32_t num;
 		int32_t off;
 	} exit;
+	struct {
+		uint32_t stack_ofs;
+	} ldmb;
 	uint32_t reguse;
 	int32_t *off;
 	uint8_t *ins;
@@ -1024,6 +1035,166 @@ emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
 		emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
 }
 
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates code for 'fast_path':
+ * calculate load offset and check is it inside first packet segment.
+ */
+static void
+emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7],
+	uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm,
+	const int32_t ofs[LDMB_OFS_NUM])
+{
+	/* make R2 contain *off* value */
+
+	if (sreg != rg[EBPF_REG_2]) {
+		emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K,
+			rg[EBPF_REG_2], imm);
+		if (mode == BPF_IND)
+			emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+				sreg, rg[EBPF_REG_2]);
+	} else
+		/* BPF_IND with sreg == R2 */
+		emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K,
+			rg[EBPF_REG_2], imm);
+
+	/* R3 = mbuf->data_len */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
+		rg[EBPF_REG_6], rg[EBPF_REG_3],
+		offsetof(struct rte_mbuf, data_len));
+
+	/* R3 = R3 - R2 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_SUB | BPF_X,
+		rg[EBPF_REG_2], rg[EBPF_REG_3]);
+
+	/* JSLE R3, <sz> <slow_path> */
+	emit_cmp_imm(st, EBPF_ALU64, rg[EBPF_REG_3], sz);
+	emit_abs_jcc(st, BPF_JMP | EBPF_JSLE | BPF_K, ofs[LDMB_SLP_OFS]);
+
+	/* R3 = mbuf->data_off */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
+		rg[EBPF_REG_6], rg[EBPF_REG_3],
+		offsetof(struct rte_mbuf, data_off));
+
+	/* R0 = mbuf->buf_addr */
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
+		rg[EBPF_REG_6], rg[EBPF_REG_0],
+		offsetof(struct rte_mbuf, buf_addr));
+
+	/* R0 = R0 + R3 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+		rg[EBPF_REG_3], rg[EBPF_REG_0]);
+
+	/* R0 = R0 + R2 */
+	emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
+		rg[EBPF_REG_2], rg[EBPF_REG_0]);
+
+	/* JMP <fin_part> */
+	emit_abs_jmp(st, ofs[LDMB_FIN_OFS]);
+}
+
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates code for 'slow_path':
+ * call __rte_pktmbuf_read() and check return value.
+ */
+static void
+emit_ldmb_slow_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7],
+	uint32_t sz)
+{
+	/* make R3 contain *len* value (1/2/4) */
+
+	emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, rg[EBPF_REG_3], sz);
+
+	/* make R4 contain (RBP - ldmb.stack_ofs) */
+
+	emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RBP, rg[EBPF_REG_4]);
+	emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, rg[EBPF_REG_4],
+		st->ldmb.stack_ofs);
+
+	/* make R1 contain mbuf ptr */
+
+	emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
+		rg[EBPF_REG_6], rg[EBPF_REG_1]);
+
+	/* call rte_pktmbuf_read */
+	emit_call(st, (uintptr_t)__rte_pktmbuf_read);
+
+	/* check that return value (R0) is not zero */
+	emit_tst_reg(st, EBPF_ALU64, rg[EBPF_REG_0], rg[EBPF_REG_0]);
+	emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
+}
+
+/*
+ * helper function, used by emit_ld_mbuf().
+ * generates final part of code for BPF_ABS/BPF_IND load:
+ * perform data load and endianness conversion.
+ * expects dreg to contain valid data pointer.
+ */
+static void
+emit_ldmb_fin(struct bpf_jit_state *st, uint32_t dreg, uint32_t opsz,
+	uint32_t sz)
+{
+	emit_ld_reg(st, BPF_LDX | BPF_MEM | opsz, dreg, dreg, 0);
+	if (sz != sizeof(uint8_t))
+		emit_be2le(st, dreg, sz * CHAR_BIT);
+}
+
+/*
+ * emit code for BPF_ABS/BPF_IND load.
+ * generates the following construction:
+ * fast_path:
+ *   off = ins->sreg + ins->imm
+ *   if (off + ins->opsz < mbuf->data_len)
+ *      goto slow_path;
+ *   ptr = mbuf->buf_addr + bmf->data_off + off;
+ *   goto fin_part;
+ * slow_path:
+ *   typeof(ins->opsz) buf; //allocate space on the stack
+ *   ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf);
+ *   if (ptr == NULL)
+ *      goto exit_label;
+ * fin_part:
+ *   res = *(typeof(ins->opsz))ptr;
+ *   res = bswap(res);
+ */
+static void
+emit_ld_mbuf(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t imm)
+{
+	uint32_t i, mode, opsz, sz;
+	uint32_t rg[EBPF_REG_7];
+	int32_t ofs[LDMB_OFS_NUM];
+
+	mode = BPF_MODE(op);
+	opsz = BPF_SIZE(op);
+	sz = bpf_size(opsz);
+
+	for (i = 0; i != RTE_DIM(rg); i++)
+		rg[i] = ebpf2x86[i];
+
+	/* fill with fake offsets */
+	for (i = 0; i != RTE_DIM(ofs); i++)
+		ofs[i] = st->sz + INT8_MAX;
+
+	/* dry run first to calculate jump offsets */
+
+	ofs[LDMB_FSP_OFS] = st->sz;
+	emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);
+	ofs[LDMB_SLP_OFS] = st->sz;
+	emit_ldmb_slow_path(st, rg, sz);
+	ofs[LDMB_FIN_OFS] = st->sz;
+	emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);
+
+	RTE_VERIFY(ofs[LDMB_FIN_OFS] - ofs[LDMB_FSP_OFS] <= INT8_MAX);
+
+	/* reset dry-run code and do a proper run */
+
+	st->sz = ofs[LDMB_FSP_OFS];
+	emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);
+	emit_ldmb_slow_path(st, rg, sz);
+	emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);
+}
+
 static void
 emit_prolog(struct bpf_jit_state *st, int32_t stack_size)
 {
@@ -1121,6 +1292,7 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
 	/* reset state fields */
 	st->sz = 0;
 	st->exit.num = 0;
+	st->ldmb.stack_ofs = bpf->stack_sz;
 
 	emit_prolog(st, bpf->stack_sz);
 
@@ -1240,6 +1412,15 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
 			emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm);
 			i++;
 			break;
+		/* load absolute/indirect instructions */
+		case (BPF_LD | BPF_ABS | BPF_B):
+		case (BPF_LD | BPF_ABS | BPF_H):
+		case (BPF_LD | BPF_ABS | BPF_W):
+		case (BPF_LD | BPF_IND | BPF_B):
+		case (BPF_LD | BPF_IND | BPF_H):
+		case (BPF_LD | BPF_IND | BPF_W):
+			emit_ld_mbuf(st, op, sr, ins->imm);
+			break;
 		/* store instructions */
 		case (BPF_STX | BPF_MEM | BPF_B):
 		case (BPF_STX | BPF_MEM | BPF_H):
diff --git a/lib/librte_bpf/bpf_validate.c b/lib/librte_bpf/bpf_validate.c
index fecdda0e1..9214f1503 100644
--- a/lib/librte_bpf/bpf_validate.c
+++ b/lib/librte_bpf/bpf_validate.c
@@ -70,6 +70,7 @@ struct bpf_verifier {
 	uint64_t stack_sz;
 	uint32_t nb_nodes;
 	uint32_t nb_jcc_nodes;
+	uint32_t nb_ldmb_nodes;
 	uint32_t node_colour[MAX_NODE_COLOUR];
 	uint32_t edge_type[MAX_EDGE_TYPE];
 	struct bpf_eval_state *evst;
@@ -2020,6 +2021,14 @@ validate(struct bpf_verifier *bvf)
 			rc |= add_edge(bvf, node, i + 2);
 			i++;
 			break;
+		case (BPF_LD | BPF_ABS | BPF_B):
+		case (BPF_LD | BPF_ABS | BPF_H):
+		case (BPF_LD | BPF_ABS | BPF_W):
+		case (BPF_LD | BPF_IND | BPF_B):
+		case (BPF_LD | BPF_IND | BPF_H):
+		case (BPF_LD | BPF_IND | BPF_W):
+			bvf->nb_ldmb_nodes++;
+			/* fallthrough */
 		default:
 			rc |= add_edge(bvf, node, i + 1);
 			break;
@@ -2320,8 +2329,14 @@ bpf_validate(struct rte_bpf *bpf)
 	free(bvf.in);
 
 	/* copy collected info */
-	if (rc == 0)
+	if (rc == 0) {
 		bpf->stack_sz = bvf.stack_sz;
 
+		/* for LD_ABS/LD_IND, we'll need extra space on the stack */
+		if (bvf.nb_ldmb_nodes != 0)
+			bpf->stack_sz = RTE_ALIGN_CEIL(bpf->stack_sz +
+				sizeof(uint64_t), sizeof(uint64_t));
+	}
+
 	return rc;
 }
-- 
2.17.1


  parent reply	other threads:[~2020-05-18 15:54 UTC|newest]

Thread overview: 17+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-05-18 15:52 [dpdk-dev] [PATCH 0/5] bpf: add support for BPF_ABS/BPF_IND instructions Konstantin Ananyev
2020-05-18 15:52 ` [dpdk-dev] [PATCH 1/5] test/bpf: fix few small issues Konstantin Ananyev
2020-06-24 21:33   ` [dpdk-dev] [dpdk-stable] " Thomas Monjalon
2020-05-18 15:52 ` [dpdk-dev] [PATCH 2/5] bpf: fix add/sub min/max estimations Konstantin Ananyev
2020-05-18 15:52 ` [dpdk-dev] [PATCH 3/5] bpf: add support for packet data load instructions Konstantin Ananyev
2020-05-18 15:52 ` [dpdk-dev] [PATCH 4/5] test/bpf: add new test cases for mbuf " Konstantin Ananyev
2020-05-18 15:52 ` Konstantin Ananyev [this message]
2020-05-24 13:37   ` [dpdk-dev] [PATCH 5/5] bpf: x86 JIT support for packet data loadinstructions Morten Brørup
2020-05-24 23:08     ` Ananyev, Konstantin
2020-05-27 14:16 ` [dpdk-dev] [PATCH v2 0/5] bpf: add support for BPF_ABS/BPF_IND instructions Konstantin Ananyev
2020-05-27 14:16   ` [dpdk-dev] [PATCH v2 1/5] test/bpf: fix few small issues Konstantin Ananyev
2020-05-27 14:16   ` [dpdk-dev] [PATCH v2 2/5] bpf: fix add/sub min/max estimations Konstantin Ananyev
2020-05-27 14:16   ` [dpdk-dev] [PATCH v2 3/5] bpf: add support for packet data load instructions Konstantin Ananyev
2020-05-27 14:16   ` [dpdk-dev] [PATCH v2 4/5] test/bpf: add new test cases for mbuf " Konstantin Ananyev
2020-05-27 14:16   ` [dpdk-dev] [PATCH v2 5/5] bpf: x86 JIT support for packet data " Konstantin Ananyev
2020-05-28  8:50     ` Morten Brørup
2020-06-24 21:43   ` [dpdk-dev] [PATCH v2 0/5] bpf: add support for BPF_ABS/BPF_IND instructions Thomas Monjalon

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20200518155245.11380-6-konstantin.ananyev@intel.com \
    --to=konstantin.ananyev@intel.com \
    --cc=dev@dpdk.org \
    --cc=jerinj@marvell.com \
    --cc=stephen@networkplumber.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).