bpf: Add support for certain atomics in bpf_arena to x86 JIT Support atomics in bpf_arena that can be JITed as a single x86 instruction. Instructions that are JITed as loops are not supported at the moment, since they require more complex extable and loop logic. JITs can choose to do smarter things with bpf_jit_supports_insn(). Like arm64 may decide to support all bpf atomics instructions when emit_lse_atomic is available and none in ll_sc mode. bpf_jit_supports_percpu_insn(), bpf_jit_supports_ptr_xchg() and other such callbacks can be replaced with bpf_jit_supports_insn() in the future. Signed-off-by: Alexei Starovoitov <ast@kernel.org> Acked-by: Eduard Zingerman <eddyz87@gmail.com> Link: https://lore.kernel.org/r/20240405231134.17274-1-alexei.starovoitov@gmail.com Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index fc13306..60d5c45 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c
@@ -1324,9 +1324,8 @@ static int emit_atomic_rmw(u8 **pprog, u32 atomic_op, return 0; } -static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size, - u32 dst_reg, u32 src_reg, u32 index_reg, - int off) +static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size, + u32 dst_reg, u32 src_reg, u32 index_reg, int off) { u8 *prog = *pprog; @@ -1339,7 +1338,7 @@ static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size, EMIT1(add_3mod(0x48, dst_reg, src_reg, index_reg)); break; default: - pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n"); + pr_err("bpf_jit: 1 and 2 byte atomics are not supported\n"); return -EFAULT; } @@ -1373,81 +1372,6 @@ static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size, return 0; } -static int emit_atomic_ld_st(u8 **pprog, u32 atomic_op, u32 dst_reg, - u32 src_reg, s16 off, u8 bpf_size) -{ - switch (atomic_op) { - case BPF_LOAD_ACQ: - /* dst_reg = smp_load_acquire(src_reg + off16) */ - emit_ldx(pprog, bpf_size, dst_reg, src_reg, off); - break; - case BPF_STORE_REL: - /* smp_store_release(dst_reg + off16, src_reg) */ - emit_stx(pprog, bpf_size, dst_reg, src_reg, off); - break; - default: - pr_err("bpf_jit: unknown atomic load/store opcode %02x\n", - atomic_op); - return -EFAULT; - } - - return 0; -} - -static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, - u32 dst_reg, u32 src_reg, u32 index_reg, - int off) -{ - switch (atomic_op) { - case BPF_LOAD_ACQ: - /* dst_reg = smp_load_acquire(src_reg + idx_reg + off16) */ - emit_ldx_index(pprog, size, dst_reg, src_reg, index_reg, off); - break; - case BPF_STORE_REL: - /* smp_store_release(dst_reg + idx_reg + off16, src_reg) */ - emit_stx_index(pprog, size, dst_reg, src_reg, index_reg, off); - break; - default: - pr_err("bpf_jit: unknown atomic load/store opcode %02x\n", - atomic_op); - return -EFAULT; - } - - return 0; -} - -/* - * Metadata encoding for exception handling in JITed code. - * - * Format of `fixup` and `data` fields in `struct exception_table_entry`: - * - * Bit layout of `fixup` (32-bit): - * - * +-----------+--------+-----------+---------+----------+ - * | 31 | 30-24 | 23-16 | 15-8 | 7-0 | - * | | | | | | - * | ARENA_ACC | Unused | ARENA_REG | DST_REG | INSN_LEN | - * +-----------+--------+-----------+---------+----------+ - * - * - INSN_LEN (8 bits): Length of faulting insn (max x86 insn = 15 bytes (fits in 8 bits)). - * - DST_REG (8 bits): Offset of dst_reg from reg2pt_regs[] (max offset = 112 (fits in 8 bits)). - * This is set to DONT_CLEAR if the insn is a store. - * - ARENA_REG (8 bits): Offset of the register that is used to calculate the - * address for load/store when accessing the arena region. - * - ARENA_ACCESS (1 bit): This bit is set when the faulting instruction accessed the arena region. - * - * Bit layout of `data` (32-bit): - * - * +--------------+--------+--------------+ - * | 31-16 | 15-8 | 7-0 | - * | | | | - * | ARENA_OFFSET | Unused | EX_TYPE_BPF | - * +--------------+--------+--------------+ - * - * - ARENA_OFFSET (16 bits): Offset used to calculate the address for load/store when - * accessing the arena region. - */ - #define DONT_CLEAR 1 #define FIXUP_INSN_LEN_MASK GENMASK(7, 0) #define FIXUP_REG_MASK GENMASK(15, 8) @@ -2391,25 +2315,11 @@ st: if (is_imm8(insn->off)) return err; break; - case BPF_STX | BPF_PROBE_ATOMIC | BPF_B: - case BPF_STX | BPF_PROBE_ATOMIC | BPF_H: - if (!bpf_atomic_is_load_store(insn)) { - pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n"); - return -EFAULT; - } - fallthrough; case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: start_of_ldx = prog; - - if (bpf_atomic_is_load_store(insn)) - err = emit_atomic_ld_st_index(&prog, insn->imm, - BPF_SIZE(insn->code), dst_reg, - src_reg, X86_REG_R12, insn->off); - else - err = emit_atomic_rmw_index(&prog, insn->imm, BPF_SIZE(insn->code), - dst_reg, src_reg, X86_REG_R12, - insn->off); + err = emit_atomic_index(&prog, insn->imm, BPF_SIZE(insn->code), + dst_reg, src_reg, X86_REG_R12, insn->off); if (err) return err; goto populate_extable;
diff --git a/include/linux/filter.h b/include/linux/filter.h index f5c859b..ac9630f 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h
@@ -1143,9 +1143,6 @@ bool bpf_jit_supports_exceptions(void); bool bpf_jit_supports_ptr_xchg(void); bool bpf_jit_supports_arena(void); bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); -bool bpf_jit_supports_private_stack(void); -bool bpf_jit_supports_timed_may_goto(void); -u64 bpf_arch_uaddress_limit(void); void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); u64 arch_bpf_timed_may_goto(void); u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index d595fe5..4924cbf 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c
@@ -3101,15 +3101,6 @@ bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) return false; } -u64 __weak bpf_arch_uaddress_limit(void) -{ -#if defined(CONFIG_64BIT) && defined(CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE) - return TASK_SIZE; -#else - return 0; -#endif -} - /* Return TRUE if the JIT backend satisfies the following two conditions: * 1) JIT backend supports atomic_xchg() on pointer-sized words. * 2) Under the specific arch, the implementation of xchg() is the same
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ff40e5e..7947700 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c
@@ -7704,71 +7704,9 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn } static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, - bool allow_trust_mismatch); + bool allow_trust_missmatch); -static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, - bool strict_alignment_once, bool is_ldsx, - bool allow_trust_mismatch, const char *ctx) -{ - struct bpf_reg_state *regs = cur_regs(env); - enum bpf_reg_type src_reg_type; - int err; - - /* check src operand */ - err = check_reg_arg(env, insn->src_reg, SRC_OP); - if (err) - return err; - - /* check dst operand */ - err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); - if (err) - return err; - - src_reg_type = regs[insn->src_reg].type; - - /* Check if (src_reg + off) is readable. The state of dst_reg will be - * updated by this call. - */ - err = check_mem_access(env, env->insn_idx, insn->src_reg, insn->off, - BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, - strict_alignment_once, is_ldsx); - err = err ?: save_aux_ptr_type(env, src_reg_type, - allow_trust_mismatch); - err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx); - - return err; -} - -static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, - bool strict_alignment_once) -{ - struct bpf_reg_state *regs = cur_regs(env); - enum bpf_reg_type dst_reg_type; - int err; - - /* check src1 operand */ - err = check_reg_arg(env, insn->src_reg, SRC_OP); - if (err) - return err; - - /* check src2 operand */ - err = check_reg_arg(env, insn->dst_reg, SRC_OP); - if (err) - return err; - - dst_reg_type = regs[insn->dst_reg].type; - - /* Check if (dst_reg + off) is writeable. */ - err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off, - BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, - strict_alignment_once, false); - err = err ?: save_aux_ptr_type(env, dst_reg_type, false); - - return err; -} - -static int check_atomic_rmw(struct bpf_verifier_env *env, - struct bpf_insn *insn) +static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) { int load_reg; int err; @@ -7807,7 +7745,11 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return -EACCES; } - if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) { + if (is_ctx_reg(env, insn->dst_reg) || + is_pkt_reg(env, insn->dst_reg) || + is_flow_key_reg(env, insn->dst_reg) || + is_sk_reg(env, insn->dst_reg) || + (is_arena_reg(env, insn->dst_reg) && !bpf_jit_supports_insn(insn, true))) { verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", insn->dst_reg, reg_type_str(env, reg_state(env, insn->dst_reg)->type)); @@ -21304,33 +21246,12 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) insn->code == (BPF_ST | BPF_MEM | BPF_W) || insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { type = BPF_WRITE; - } else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_B) || - insn->code == (BPF_STX | BPF_ATOMIC | BPF_H) || - insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) || + } else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) || insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) && env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) { insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); env->prog->aux->num_exentries++; continue; - } else if (insn->code == (BPF_JMP | BPF_EXIT) && - epilogue_cnt && - i + delta < subprogs[1].start) { - /* Generate epilogue for the main prog */ - if (epilogue_idx) { - /* jump back to the earlier generated epilogue */ - insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1); - cnt = 1; - } else { - memcpy(insn_buf, epilogue_buf, - epilogue_cnt * sizeof(*epilogue_buf)); - cnt = epilogue_cnt; - /* epilogue_idx cannot be 0. It must have at - * least one ctx ptr saving insn before the - * epilogue. - */ - epilogue_idx = i + delta; - } - goto patch_insn_buf; } else { continue; }