From: <jerinj@marvell.com>
To: <dev@dpdk.org>
Cc: <konstantin.ananyev@intel.com>, <honnappa.nagarahalli@arm.com>,
<thomas@monjalon.net>, <gavin.hu@arm.com>,
Jerin Jacob <jerinj@marvell.com>
Subject: [dpdk-dev] [PATCH 7/8] bpf/arm64: add atomic-exchange-and-add operation
Date: Tue, 3 Sep 2019 16:29:37 +0530 [thread overview]
Message-ID: <20190903105938.33231-8-jerinj@marvell.com> (raw)
In-Reply-To: <20190903105938.33231-1-jerinj@marvell.com>
From: Jerin Jacob <jerinj@marvell.com>
Implement XADD eBPF instruction using STADD arm64 instruction.
If the given platform does not have atomics support,
use LDXR and STXR pair for critical section instead of STADD.
Signed-off-by: Jerin Jacob <jerinj@marvell.com>
---
lib/librte_bpf/bpf_jit_arm64.c | 85 +++++++++++++++++++++++++++++++++-
1 file changed, 84 insertions(+), 1 deletion(-)
diff --git a/lib/librte_bpf/bpf_jit_arm64.c b/lib/librte_bpf/bpf_jit_arm64.c
index c797c9c62..62fa6a505 100644
--- a/lib/librte_bpf/bpf_jit_arm64.c
+++ b/lib/librte_bpf/bpf_jit_arm64.c
@@ -837,6 +837,83 @@ emit_return_zero_if_src_zero(struct a64_jit_ctx *ctx, bool is64, uint8_t src)
emit_b(ctx, jump_to_epilogue);
}
+static void
+emit_stadd(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rn)
+{
+ uint32_t insn;
+
+ insn = 0xb820001f;
+ insn |= (!!is64) << 30;
+ insn |= rs << 16;
+ insn |= rn << 5;
+
+ emit_insn(ctx, insn, check_reg(rs) || check_reg(rn));
+}
+
+static void
+emit_ldxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rt, uint8_t rn)
+{
+ uint32_t insn;
+
+ insn = 0x885f7c00;
+ insn |= (!!is64) << 30;
+ insn |= rn << 5;
+ insn |= rt;
+
+ emit_insn(ctx, insn, check_reg(rt) || check_reg(rn));
+}
+
+static void
+emit_stxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rt,
+ uint8_t rn)
+{
+ uint32_t insn;
+
+ insn = 0x88007c00;
+ insn |= (!!is64) << 30;
+ insn |= rs << 16;
+ insn |= rn << 5;
+ insn |= rt;
+
+ emit_insn(ctx, insn, check_reg(rs) || check_reg(rt) || check_reg(rn));
+}
+
+static int
+has_atomics(void)
+{
+ int rc = 0;
+
+#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
+ rc = 1;
+#endif
+ return rc;
+}
+
+static void
+emit_xadd(struct a64_jit_ctx *ctx, uint8_t op, uint8_t tmp1, uint8_t tmp2,
+ uint8_t tmp3, uint8_t dst, int16_t off, uint8_t src)
+{
+ bool is64 = (BPF_SIZE(op) == EBPF_DW);
+ uint8_t rn;
+
+ if (off) {
+ emit_mov_imm(ctx, 1, tmp1, off);
+ emit_add(ctx, 1, tmp1, dst);
+ rn = tmp1;
+ } else {
+ rn = dst;
+ }
+
+ if (has_atomics()) {
+ emit_stadd(ctx, is64, src, rn);
+ } else {
+ emit_ldxr(ctx, is64, tmp2, rn);
+ emit_add(ctx, is64, tmp2, src);
+ emit_stxr(ctx, is64, tmp3, tmp2, rn);
+ emit_cbnz(ctx, is64, tmp3, -3);
+ }
+}
+
static void
check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
{
@@ -863,7 +940,7 @@ check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
static int
emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
{
- uint8_t op, dst, src, tmp1, tmp2;
+ uint8_t op, dst, src, tmp1, tmp2, tmp3;
const struct ebpf_insn *ins;
uint64_t u64;
int16_t off;
@@ -878,6 +955,7 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
ctx->stack_sz = RTE_ALIGN_MUL_CEIL(bpf->stack_sz, 16);
tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1);
tmp2 = ebpf_to_a64_reg(ctx, TMP_REG_2);
+ tmp3 = ebpf_to_a64_reg(ctx, TMP_REG_3);
emit_prologue(ctx);
@@ -1067,6 +1145,11 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
emit_mov_imm(ctx, 1, tmp2, off);
emit_str(ctx, BPF_SIZE(op), tmp1, dst, tmp2);
break;
+ /* STX XADD: lock *(size *)(dst + off) += src */
+ case (BPF_STX | EBPF_XADD | BPF_W):
+ case (BPF_STX | EBPF_XADD | EBPF_DW):
+ emit_xadd(ctx, op, tmp1, tmp2, tmp3, dst, off, src);
+ break;
/* Return r0 */
case (BPF_JMP | EBPF_EXIT):
emit_epilogue(ctx);
--
2.23.0
next prev parent reply other threads:[~2019-09-03 11:00 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-09-03 10:59 [dpdk-dev] [PATCH 0/8] eBPF arm64 JIT support jerinj
2019-09-03 10:59 ` [dpdk-dev] [PATCH 1/8] bpf/arm64: add build infrastructure jerinj
2019-09-03 10:59 ` [dpdk-dev] [PATCH 2/8] bpf/arm64: add prologue and epilogue jerinj
2019-09-03 10:59 ` [dpdk-dev] [PATCH 3/8] bpf/arm64: add basic arithmetic operations jerinj
2019-09-03 10:59 ` [dpdk-dev] [PATCH 4/8] bpf/arm64: add logical operations jerinj
2019-09-03 10:59 ` [dpdk-dev] [PATCH 5/8] bpf/arm64: add byte swap operations jerinj
2019-09-03 10:59 ` [dpdk-dev] [PATCH 6/8] bpf/arm64: add load and store operations jerinj
2019-09-03 10:59 ` jerinj [this message]
2019-10-18 13:16 ` [dpdk-dev] [PATCH 7/8] bpf/arm64: add atomic-exchange-and-add operation David Marchand
2019-09-03 10:59 ` [dpdk-dev] [PATCH 8/8] bpf/arm64: add branch operation jerinj
2019-09-24 17:03 ` [dpdk-dev] [PATCH 0/8] eBPF arm64 JIT support Ananyev, Konstantin
2019-10-12 12:22 ` Thomas Monjalon
2019-10-03 12:51 ` Thomas Monjalon
2019-10-03 13:07 ` Jerin Jacob
2019-10-03 15:05 ` Ananyev, Konstantin
2019-10-04 4:55 ` Honnappa Nagarahalli
2019-10-04 9:54 ` Steve Capper
2019-10-04 10:53 ` Thomas Monjalon
2019-10-04 14:09 ` Daniel Borkmann
2019-10-04 14:43 ` Jerin Jacob
2019-10-05 0:00 ` Daniel Borkmann
2019-10-05 14:39 ` Jerin Jacob
2019-10-07 11:57 ` Ananyev, Konstantin
2019-10-24 4:22 ` Jerin Jacob
2020-04-06 11:05 ` Ananyev, Konstantin
2019-10-04 15:39 ` Jerin Jacob
2019-10-07 12:33 ` Thomas Monjalon
2019-10-07 13:00 ` Jerin Jacob
2019-10-07 18:04 ` Thomas Monjalon
2019-10-07 19:29 ` Jerin Jacob
2019-10-07 20:15 ` Thomas Monjalon
2019-10-08 6:57 ` Jerin Jacob
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190903105938.33231-8-jerinj@marvell.com \
--to=jerinj@marvell.com \
--cc=dev@dpdk.org \
--cc=gavin.hu@arm.com \
--cc=honnappa.nagarahalli@arm.com \
--cc=konstantin.ananyev@intel.com \
--cc=thomas@monjalon.net \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).