| From 4be98754f14316b6ab86ff08b955b892ab146676 Mon Sep 17 00:00:00 2001 |
| From: Sasha Levin <sashal@kernel.org> |
| Date: Tue, 13 Jul 2021 08:18:31 +0000 |
| Subject: bpf: Introduce BPF nospec instruction for mitigating Spectre v4 |
| |
| From: Daniel Borkmann <daniel@iogearbox.net> |
| |
| [ Upstream commit f5e81d1117501546b7be050c5fbafa6efd2c722c ] |
| |
| In case of JITs, each of the JIT backends compiles the BPF nospec instruction |
| /either/ to a machine instruction which emits a speculation barrier /or/ to |
| /no/ machine instruction in case the underlying architecture is not affected |
| by Speculative Store Bypass or has different mitigations in place already. |
| |
| This covers both x86 and (implicitly) arm64: In case of x86, we use 'lfence' |
| instruction for mitigation. In case of arm64, we rely on the firmware mitigation |
| as controlled via the ssbd kernel parameter. Whenever the mitigation is enabled, |
| it works for all of the kernel code with no need to provide any additional |
| instructions here (hence only comment in arm64 JIT). Other archs can follow |
| as needed. The BPF nospec instruction is specifically targeting Spectre v4 |
| since i) we don't use a serialization barrier for the Spectre v1 case, and |
| ii) mitigation instructions for v1 and v4 might be different on some archs. |
| |
| The BPF nospec is required for a future commit, where the BPF verifier does |
| annotate intermediate BPF programs with speculation barriers. |
| |
| Co-developed-by: Piotr Krysiuk <piotras@gmail.com> |
| Co-developed-by: Benedict Schlueter <benedict.schlueter@rub.de> |
| Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> |
| Signed-off-by: Piotr Krysiuk <piotras@gmail.com> |
| Signed-off-by: Benedict Schlueter <benedict.schlueter@rub.de> |
| Acked-by: Alexei Starovoitov <ast@kernel.org> |
| Signed-off-by: Sasha Levin <sashal@kernel.org> |
| --- |
| arch/arm/net/bpf_jit_32.c | 3 +++ |
| arch/arm64/net/bpf_jit_comp.c | 13 +++++++++++++ |
| arch/mips/net/ebpf_jit.c | 3 +++ |
| arch/powerpc/net/bpf_jit_comp64.c | 6 ++++++ |
| arch/riscv/net/bpf_jit_comp32.c | 4 ++++ |
| arch/riscv/net/bpf_jit_comp64.c | 4 ++++ |
| arch/s390/net/bpf_jit_comp.c | 5 +++++ |
| arch/sparc/net/bpf_jit_comp_64.c | 3 +++ |
| arch/x86/net/bpf_jit_comp.c | 7 +++++++ |
| arch/x86/net/bpf_jit_comp32.c | 6 ++++++ |
| include/linux/filter.h | 15 +++++++++++++++ |
| kernel/bpf/core.c | 19 ++++++++++++++++++- |
| kernel/bpf/disasm.c | 16 +++++++++------- |
| 13 files changed, 96 insertions(+), 8 deletions(-) |
| |
| diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c |
| index 0207b6ea6e8a..ce8b04326352 100644 |
| --- a/arch/arm/net/bpf_jit_32.c |
| +++ b/arch/arm/net/bpf_jit_32.c |
| @@ -1602,6 +1602,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) |
| rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); |
| emit_ldx_r(dst, rn, off, ctx, BPF_SIZE(code)); |
| break; |
| + /* speculation barrier */ |
| + case BPF_ST | BPF_NOSPEC: |
| + break; |
| /* ST: *(size *)(dst + off) = imm */ |
| case BPF_ST | BPF_MEM | BPF_W: |
| case BPF_ST | BPF_MEM | BPF_H: |
| diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c |
| index ef9f1d5e989d..345066b8e9fc 100644 |
| --- a/arch/arm64/net/bpf_jit_comp.c |
| +++ b/arch/arm64/net/bpf_jit_comp.c |
| @@ -829,6 +829,19 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, |
| return ret; |
| break; |
| |
| + /* speculation barrier */ |
| + case BPF_ST | BPF_NOSPEC: |
| + /* |
| + * Nothing required here. |
| + * |
| + * In case of arm64, we rely on the firmware mitigation of |
| + * Speculative Store Bypass as controlled via the ssbd kernel |
| + * parameter. Whenever the mitigation is enabled, it works |
| + * for all of the kernel code with no need to provide any |
| + * additional instructions. |
| + */ |
| + break; |
| + |
| /* ST: *(size *)(dst + off) = imm */ |
| case BPF_ST | BPF_MEM | BPF_W: |
| case BPF_ST | BPF_MEM | BPF_H: |
| diff --git a/arch/mips/net/ebpf_jit.c b/arch/mips/net/ebpf_jit.c |
| index 561154cbcc40..b31b91e57c34 100644 |
| --- a/arch/mips/net/ebpf_jit.c |
| +++ b/arch/mips/net/ebpf_jit.c |
| @@ -1355,6 +1355,9 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, |
| } |
| break; |
| |
| + case BPF_ST | BPF_NOSPEC: /* speculation barrier */ |
| + break; |
| + |
| case BPF_ST | BPF_B | BPF_MEM: |
| case BPF_ST | BPF_H | BPF_MEM: |
| case BPF_ST | BPF_W | BPF_MEM: |
| diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c |
| index 022103c6a201..658ca2bab13c 100644 |
| --- a/arch/powerpc/net/bpf_jit_comp64.c |
| +++ b/arch/powerpc/net/bpf_jit_comp64.c |
| @@ -646,6 +646,12 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, |
| } |
| break; |
| |
| + /* |
| + * BPF_ST NOSPEC (speculation barrier) |
| + */ |
| + case BPF_ST | BPF_NOSPEC: |
| + break; |
| + |
| /* |
| * BPF_ST(X) |
| */ |
| diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c |
| index 579575f9cdae..f300f93ba645 100644 |
| --- a/arch/riscv/net/bpf_jit_comp32.c |
| +++ b/arch/riscv/net/bpf_jit_comp32.c |
| @@ -1251,6 +1251,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, |
| return -1; |
| break; |
| |
| + /* speculation barrier */ |
| + case BPF_ST | BPF_NOSPEC: |
| + break; |
| + |
| case BPF_ST | BPF_MEM | BPF_B: |
| case BPF_ST | BPF_MEM | BPF_H: |
| case BPF_ST | BPF_MEM | BPF_W: |
| diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c |
| index 8a56b5293117..c113ae818b14 100644 |
| --- a/arch/riscv/net/bpf_jit_comp64.c |
| +++ b/arch/riscv/net/bpf_jit_comp64.c |
| @@ -939,6 +939,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, |
| emit_ld(rd, 0, RV_REG_T1, ctx); |
| break; |
| |
| + /* speculation barrier */ |
| + case BPF_ST | BPF_NOSPEC: |
| + break; |
| + |
| /* ST: *(size *)(dst + off) = imm */ |
| case BPF_ST | BPF_MEM | BPF_B: |
| emit_imm(RV_REG_T1, imm, ctx); |
| diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c |
| index fc44dce59536..dee01d3b23a4 100644 |
| --- a/arch/s390/net/bpf_jit_comp.c |
| +++ b/arch/s390/net/bpf_jit_comp.c |
| @@ -1153,6 +1153,11 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, |
| break; |
| } |
| break; |
| + /* |
| + * BPF_NOSPEC (speculation barrier) |
| + */ |
| + case BPF_ST | BPF_NOSPEC: |
| + break; |
| /* |
| * BPF_ST(X) |
| */ |
| diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c |
| index 3364e2a00989..fef734473c0f 100644 |
| --- a/arch/sparc/net/bpf_jit_comp_64.c |
| +++ b/arch/sparc/net/bpf_jit_comp_64.c |
| @@ -1287,6 +1287,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) |
| return 1; |
| break; |
| } |
| + /* speculation barrier */ |
| + case BPF_ST | BPF_NOSPEC: |
| + break; |
| /* ST: *(size *)(dst + off) = imm */ |
| case BPF_ST | BPF_MEM | BPF_W: |
| case BPF_ST | BPF_MEM | BPF_H: |
| diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c |
| index d5fa77256058..0a962cd6bac1 100644 |
| --- a/arch/x86/net/bpf_jit_comp.c |
| +++ b/arch/x86/net/bpf_jit_comp.c |
| @@ -1141,6 +1141,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, |
| } |
| break; |
| |
| + /* speculation barrier */ |
| + case BPF_ST | BPF_NOSPEC: |
| + if (boot_cpu_has(X86_FEATURE_XMM2)) |
| + /* Emit 'lfence' */ |
| + EMIT3(0x0F, 0xAE, 0xE8); |
| + break; |
| + |
| /* ST: *(u8*)(dst_reg + off) = imm */ |
| case BPF_ST | BPF_MEM | BPF_B: |
| if (is_ereg(dst_reg)) |
| diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c |
| index 2cf4d217840d..4bd0f98df700 100644 |
| --- a/arch/x86/net/bpf_jit_comp32.c |
| +++ b/arch/x86/net/bpf_jit_comp32.c |
| @@ -1705,6 +1705,12 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, |
| i++; |
| break; |
| } |
| + /* speculation barrier */ |
| + case BPF_ST | BPF_NOSPEC: |
| + if (boot_cpu_has(X86_FEATURE_XMM2)) |
| + /* Emit 'lfence' */ |
| + EMIT3(0x0F, 0xAE, 0xE8); |
| + break; |
| /* ST: *(u8*)(dst_reg + off) = imm */ |
| case BPF_ST | BPF_MEM | BPF_H: |
| case BPF_ST | BPF_MEM | BPF_B: |
| diff --git a/include/linux/filter.h b/include/linux/filter.h |
| index e2ffa02f9067..822b701c803d 100644 |
| --- a/include/linux/filter.h |
| +++ b/include/linux/filter.h |
| @@ -72,6 +72,11 @@ struct ctl_table_header; |
| /* unused opcode to mark call to interpreter with arguments */ |
| #define BPF_CALL_ARGS 0xe0 |
| |
| +/* unused opcode to mark speculation barrier for mitigating |
| + * Speculative Store Bypass |
| + */ |
| +#define BPF_NOSPEC 0xc0 |
| + |
| /* As per nm, we expose JITed images as text (code) section for |
| * kallsyms. That way, tools like perf can find it to match |
| * addresses. |
| @@ -372,6 +377,16 @@ static inline bool insn_is_zext(const struct bpf_insn *insn) |
| .off = 0, \ |
| .imm = 0 }) |
| |
| +/* Speculation barrier */ |
| + |
| +#define BPF_ST_NOSPEC() \ |
| + ((struct bpf_insn) { \ |
| + .code = BPF_ST | BPF_NOSPEC, \ |
| + .dst_reg = 0, \ |
| + .src_reg = 0, \ |
| + .off = 0, \ |
| + .imm = 0 }) |
| + |
| /* Internal classic blocks for direct assignment */ |
| |
| #define __BPF_STMT(CODE, K) \ |
| diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c |
| index 75c2d184018a..d12efb2550d3 100644 |
| --- a/kernel/bpf/core.c |
| +++ b/kernel/bpf/core.c |
| @@ -32,6 +32,8 @@ |
| #include <linux/perf_event.h> |
| #include <linux/extable.h> |
| #include <linux/log2.h> |
| + |
| +#include <asm/barrier.h> |
| #include <asm/unaligned.h> |
| |
| /* Registers */ |
| @@ -1380,6 +1382,7 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack) |
| /* Non-UAPI available opcodes. */ |
| [BPF_JMP | BPF_CALL_ARGS] = &&JMP_CALL_ARGS, |
| [BPF_JMP | BPF_TAIL_CALL] = &&JMP_TAIL_CALL, |
| + [BPF_ST | BPF_NOSPEC] = &&ST_NOSPEC, |
| [BPF_LDX | BPF_PROBE_MEM | BPF_B] = &&LDX_PROBE_MEM_B, |
| [BPF_LDX | BPF_PROBE_MEM | BPF_H] = &&LDX_PROBE_MEM_H, |
| [BPF_LDX | BPF_PROBE_MEM | BPF_W] = &&LDX_PROBE_MEM_W, |
| @@ -1624,7 +1627,21 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack) |
| COND_JMP(s, JSGE, >=) |
| COND_JMP(s, JSLE, <=) |
| #undef COND_JMP |
| - /* STX and ST and LDX*/ |
| + /* ST, STX and LDX*/ |
| + ST_NOSPEC: |
| + /* Speculation barrier for mitigating Speculative Store Bypass. |
| + * In case of arm64, we rely on the firmware mitigation as |
| + * controlled via the ssbd kernel parameter. Whenever the |
| + * mitigation is enabled, it works for all of the kernel code |
| + * with no need to provide any additional instructions here. |
| + * In case of x86, we use 'lfence' insn for mitigation. We |
| + * reuse preexisting logic from Spectre v1 mitigation that |
| + * happens to produce the required code on x86 for v4 as well. |
| + */ |
| +#ifdef CONFIG_X86 |
| + barrier_nospec(); |
| +#endif |
| + CONT; |
| #define LDST(SIZEOP, SIZE) \ |
| STX_MEM_##SIZEOP: \ |
| *(SIZE *)(unsigned long) (DST + insn->off) = SRC; \ |
| diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c |
| index b44d8c447afd..ff1dd7d45b58 100644 |
| --- a/kernel/bpf/disasm.c |
| +++ b/kernel/bpf/disasm.c |
| @@ -162,15 +162,17 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, |
| else |
| verbose(cbs->private_data, "BUG_%02x\n", insn->code); |
| } else if (class == BPF_ST) { |
| - if (BPF_MODE(insn->code) != BPF_MEM) { |
| + if (BPF_MODE(insn->code) == BPF_MEM) { |
| + verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d\n", |
| + insn->code, |
| + bpf_ldst_string[BPF_SIZE(insn->code) >> 3], |
| + insn->dst_reg, |
| + insn->off, insn->imm); |
| + } else if (BPF_MODE(insn->code) == 0xc0 /* BPF_NOSPEC, no UAPI */) { |
| + verbose(cbs->private_data, "(%02x) nospec\n", insn->code); |
| + } else { |
| verbose(cbs->private_data, "BUG_st_%02x\n", insn->code); |
| - return; |
| } |
| - verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d\n", |
| - insn->code, |
| - bpf_ldst_string[BPF_SIZE(insn->code) >> 3], |
| - insn->dst_reg, |
| - insn->off, insn->imm); |
| } else if (class == BPF_LDX) { |
| if (BPF_MODE(insn->code) != BPF_MEM) { |
| verbose(cbs->private_data, "BUG_ldx_%02x\n", insn->code); |
| -- |
| 2.30.2 |
| |