From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id D71B1A0093; Mon, 18 May 2020 17:54:00 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id A579A1D56E; Mon, 18 May 2020 17:53:24 +0200 (CEST) Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by dpdk.org (Postfix) with ESMTP id 65EE81D423 for ; Mon, 18 May 2020 17:53:22 +0200 (CEST) IronPort-SDR: CbUWYPraOq7dmmbwY5159jEnSODWZFmOm3yU8cacdBxyy++5G5eM+PDfGRxuKCClSHY9GabhOl T2RPUrynPlNA== X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga001.fm.intel.com ([10.253.24.23]) by fmsmga101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 18 May 2020 08:53:22 -0700 IronPort-SDR: QAaSXs44yBgC3NZEF4f24Nrwrq53CQ1dAvABUyJNp+sZeXF1ginsLlUD7AzIGG2injZ//qwfnn dbYS5grdKA8Q== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.73,407,1583222400"; d="scan'208";a="373426294" Received: from sivswdev08.ir.intel.com ([10.237.217.47]) by fmsmga001.fm.intel.com with ESMTP; 18 May 2020 08:53:20 -0700 From: Konstantin Ananyev To: dev@dpdk.org Cc: stephen@networkplumber.org, jerinj@marvell.com, Konstantin Ananyev Date: Mon, 18 May 2020 16:52:45 +0100 Message-Id: <20200518155245.11380-6-konstantin.ananyev@intel.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20200518155245.11380-1-konstantin.ananyev@intel.com> References: <20200518155245.11380-1-konstantin.ananyev@intel.com> Subject: [dpdk-dev] [PATCH 5/5] bpf: x86 JIT support for packet data load instructions X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Make x86 JIT to generate native code for (BPF_ABS | | BPF_LD) and (BPF_IND | | BPF_LD) instructions. Signed-off-by: Konstantin Ananyev --- lib/librte_bpf/bpf_jit_x86.c | 181 ++++++++++++++++++++++++++++++++++ lib/librte_bpf/bpf_validate.c | 17 +++- 2 files changed, 197 insertions(+), 1 deletion(-) diff --git a/lib/librte_bpf/bpf_jit_x86.c b/lib/librte_bpf/bpf_jit_x86.c index f70cd6be5..8a7ad45b3 100644 --- a/lib/librte_bpf/bpf_jit_x86.c +++ b/lib/librte_bpf/bpf_jit_x86.c @@ -87,6 +87,14 @@ enum { REG_TMP1 = R10, }; +/* LD_ABS/LD_IMM offsets */ +enum { + LDMB_FSP_OFS, /* fast-path */ + LDMB_SLP_OFS, /* slow-path */ + LDMB_FIN_OFS, /* final part */ + LDMB_OFS_NUM +}; + /* * callee saved registers list. * keep RBP as the last one. @@ -100,6 +108,9 @@ struct bpf_jit_state { uint32_t num; int32_t off; } exit; + struct { + uint32_t stack_ofs; + } ldmb; uint32_t reguse; int32_t *off; uint8_t *ins; @@ -1024,6 +1035,166 @@ emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX); } +/* + * helper function, used by emit_ld_mbuf(). + * generates code for 'fast_path': + * calculate load offset and check is it inside first packet segment. + */ +static void +emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7], + uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm, + const int32_t ofs[LDMB_OFS_NUM]) +{ + /* make R2 contain *off* value */ + + if (sreg != rg[EBPF_REG_2]) { + emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, + rg[EBPF_REG_2], imm); + if (mode == BPF_IND) + emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X, + sreg, rg[EBPF_REG_2]); + } else + /* BPF_IND with sreg == R2 */ + emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K, + rg[EBPF_REG_2], imm); + + /* R3 = mbuf->data_len */ + emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H, + rg[EBPF_REG_6], rg[EBPF_REG_3], + offsetof(struct rte_mbuf, data_len)); + + /* R3 = R3 - R2 */ + emit_alu_reg(st, EBPF_ALU64 | BPF_SUB | BPF_X, + rg[EBPF_REG_2], rg[EBPF_REG_3]); + + /* JSLE R3, */ + emit_cmp_imm(st, EBPF_ALU64, rg[EBPF_REG_3], sz); + emit_abs_jcc(st, BPF_JMP | EBPF_JSLE | BPF_K, ofs[LDMB_SLP_OFS]); + + /* R3 = mbuf->data_off */ + emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H, + rg[EBPF_REG_6], rg[EBPF_REG_3], + offsetof(struct rte_mbuf, data_off)); + + /* R0 = mbuf->buf_addr */ + emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW, + rg[EBPF_REG_6], rg[EBPF_REG_0], + offsetof(struct rte_mbuf, buf_addr)); + + /* R0 = R0 + R3 */ + emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X, + rg[EBPF_REG_3], rg[EBPF_REG_0]); + + /* R0 = R0 + R2 */ + emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X, + rg[EBPF_REG_2], rg[EBPF_REG_0]); + + /* JMP */ + emit_abs_jmp(st, ofs[LDMB_FIN_OFS]); +} + +/* + * helper function, used by emit_ld_mbuf(). + * generates code for 'slow_path': + * call __rte_pktmbuf_read() and check return value. + */ +static void +emit_ldmb_slow_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7], + uint32_t sz) +{ + /* make R3 contain *len* value (1/2/4) */ + + emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, rg[EBPF_REG_3], sz); + + /* make R4 contain (RBP - ldmb.stack_ofs) */ + + emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RBP, rg[EBPF_REG_4]); + emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, rg[EBPF_REG_4], + st->ldmb.stack_ofs); + + /* make R1 contain mbuf ptr */ + + emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, + rg[EBPF_REG_6], rg[EBPF_REG_1]); + + /* call rte_pktmbuf_read */ + emit_call(st, (uintptr_t)__rte_pktmbuf_read); + + /* check that return value (R0) is not zero */ + emit_tst_reg(st, EBPF_ALU64, rg[EBPF_REG_0], rg[EBPF_REG_0]); + emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off); +} + +/* + * helper function, used by emit_ld_mbuf(). + * generates final part of code for BPF_ABS/BPF_IND load: + * perform data load and endianness conversion. + * expects dreg to contain valid data pointer. + */ +static void +emit_ldmb_fin(struct bpf_jit_state *st, uint32_t dreg, uint32_t opsz, + uint32_t sz) +{ + emit_ld_reg(st, BPF_LDX | BPF_MEM | opsz, dreg, dreg, 0); + if (sz != sizeof(uint8_t)) + emit_be2le(st, dreg, sz * CHAR_BIT); +} + +/* + * emit code for BPF_ABS/BPF_IND load. + * generates the following construction: + * fast_path: + * off = ins->sreg + ins->imm + * if (off + ins->opsz < mbuf->data_len) + * goto slow_path; + * ptr = mbuf->buf_addr + bmf->data_off + off; + * goto fin_part; + * slow_path: + * typeof(ins->opsz) buf; //allocate space on the stack + * ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf); + * if (ptr == NULL) + * goto exit_label; + * fin_part: + * res = *(typeof(ins->opsz))ptr; + * res = bswap(res); + */ +static void +emit_ld_mbuf(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t imm) +{ + uint32_t i, mode, opsz, sz; + uint32_t rg[EBPF_REG_7]; + int32_t ofs[LDMB_OFS_NUM]; + + mode = BPF_MODE(op); + opsz = BPF_SIZE(op); + sz = bpf_size(opsz); + + for (i = 0; i != RTE_DIM(rg); i++) + rg[i] = ebpf2x86[i]; + + /* fill with fake offsets */ + for (i = 0; i != RTE_DIM(ofs); i++) + ofs[i] = st->sz + INT8_MAX; + + /* dry run first to calculate jump offsets */ + + ofs[LDMB_FSP_OFS] = st->sz; + emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs); + ofs[LDMB_SLP_OFS] = st->sz; + emit_ldmb_slow_path(st, rg, sz); + ofs[LDMB_FIN_OFS] = st->sz; + emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz); + + RTE_VERIFY(ofs[LDMB_FIN_OFS] - ofs[LDMB_FSP_OFS] <= INT8_MAX); + + /* reset dry-run code and do a proper run */ + + st->sz = ofs[LDMB_FSP_OFS]; + emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs); + emit_ldmb_slow_path(st, rg, sz); + emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz); +} + static void emit_prolog(struct bpf_jit_state *st, int32_t stack_size) { @@ -1121,6 +1292,7 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf) /* reset state fields */ st->sz = 0; st->exit.num = 0; + st->ldmb.stack_ofs = bpf->stack_sz; emit_prolog(st, bpf->stack_sz); @@ -1240,6 +1412,15 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf) emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm); i++; break; + /* load absolute/indirect instructions */ + case (BPF_LD | BPF_ABS | BPF_B): + case (BPF_LD | BPF_ABS | BPF_H): + case (BPF_LD | BPF_ABS | BPF_W): + case (BPF_LD | BPF_IND | BPF_B): + case (BPF_LD | BPF_IND | BPF_H): + case (BPF_LD | BPF_IND | BPF_W): + emit_ld_mbuf(st, op, sr, ins->imm); + break; /* store instructions */ case (BPF_STX | BPF_MEM | BPF_B): case (BPF_STX | BPF_MEM | BPF_H): diff --git a/lib/librte_bpf/bpf_validate.c b/lib/librte_bpf/bpf_validate.c index fecdda0e1..9214f1503 100644 --- a/lib/librte_bpf/bpf_validate.c +++ b/lib/librte_bpf/bpf_validate.c @@ -70,6 +70,7 @@ struct bpf_verifier { uint64_t stack_sz; uint32_t nb_nodes; uint32_t nb_jcc_nodes; + uint32_t nb_ldmb_nodes; uint32_t node_colour[MAX_NODE_COLOUR]; uint32_t edge_type[MAX_EDGE_TYPE]; struct bpf_eval_state *evst; @@ -2020,6 +2021,14 @@ validate(struct bpf_verifier *bvf) rc |= add_edge(bvf, node, i + 2); i++; break; + case (BPF_LD | BPF_ABS | BPF_B): + case (BPF_LD | BPF_ABS | BPF_H): + case (BPF_LD | BPF_ABS | BPF_W): + case (BPF_LD | BPF_IND | BPF_B): + case (BPF_LD | BPF_IND | BPF_H): + case (BPF_LD | BPF_IND | BPF_W): + bvf->nb_ldmb_nodes++; + /* fallthrough */ default: rc |= add_edge(bvf, node, i + 1); break; @@ -2320,8 +2329,14 @@ bpf_validate(struct rte_bpf *bpf) free(bvf.in); /* copy collected info */ - if (rc == 0) + if (rc == 0) { bpf->stack_sz = bvf.stack_sz; + /* for LD_ABS/LD_IND, we'll need extra space on the stack */ + if (bvf.nb_ldmb_nodes != 0) + bpf->stack_sz = RTE_ALIGN_CEIL(bpf->stack_sz + + sizeof(uint64_t), sizeof(uint64_t)); + } + return rc; } -- 2.17.1