| From e4bcb0bfdfa00318766d053f7a0607a8cae584d2 Mon Sep 17 00:00:00 2001 |
| From: Daniel Borkmann <daniel@iogearbox.net> |
| Date: Mon, 28 Jan 2019 21:23:22 +0100 |
| Subject: bpf: move tmp variable into ax register in interpreter |
| |
| [ commit 144cd91c4c2bced6eb8a7e25e590f6618a11e854 upstream ] |
| |
| This change moves the on-stack 64 bit tmp variable in ___bpf_prog_run() |
| into the hidden ax register. The latter is currently only used in JITs |
| for constant blinding as a temporary scratch register, meaning the BPF |
| interpreter will never see the use of ax. Therefore it is safe to use |
| it for the cases where tmp has been used earlier. This is needed to later |
| on allow restricted hidden use of ax in both interpreter and JITs. |
| |
| Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> |
| Acked-by: Alexei Starovoitov <ast@kernel.org> |
| Signed-off-by: Alexei Starovoitov <ast@kernel.org> |
| Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> |
| Signed-off-by: Sasha Levin <sashal@kernel.org> |
| --- |
| include/linux/filter.h | 3 ++- |
| kernel/bpf/core.c | 34 +++++++++++++++++----------------- |
| 2 files changed, 19 insertions(+), 18 deletions(-) |
| |
| diff --git a/include/linux/filter.h b/include/linux/filter.h |
| index 25a556589ae8..68b97eda236d 100644 |
| --- a/include/linux/filter.h |
| +++ b/include/linux/filter.h |
| @@ -60,7 +60,8 @@ struct sock_reuseport; |
| * constants. See JIT pre-step in bpf_jit_blind_constants(). |
| */ |
| #define BPF_REG_AX MAX_BPF_REG |
| -#define MAX_BPF_JIT_REG (MAX_BPF_REG + 1) |
| +#define MAX_BPF_EXT_REG (MAX_BPF_REG + 1) |
| +#define MAX_BPF_JIT_REG MAX_BPF_EXT_REG |
| |
| /* unused opcode to mark special call to bpf_tail_call() helper */ |
| #define BPF_TAIL_CALL 0xf0 |
| diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c |
| index b2890c268cb3..ada876048ca4 100644 |
| --- a/kernel/bpf/core.c |
| +++ b/kernel/bpf/core.c |
| @@ -52,6 +52,7 @@ |
| #define DST regs[insn->dst_reg] |
| #define SRC regs[insn->src_reg] |
| #define FP regs[BPF_REG_FP] |
| +#define AX regs[BPF_REG_AX] |
| #define ARG1 regs[BPF_REG_ARG1] |
| #define CTX regs[BPF_REG_CTX] |
| #define IMM insn->imm |
| @@ -1055,7 +1056,6 @@ bool bpf_opcode_in_insntable(u8 code) |
| */ |
| static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack) |
| { |
| - u64 tmp; |
| #define BPF_INSN_2_LBL(x, y) [BPF_##x | BPF_##y] = &&x##_##y |
| #define BPF_INSN_3_LBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = &&x##_##y##_##z |
| static const void *jumptable[256] = { |
| @@ -1129,36 +1129,36 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack) |
| (*(s64 *) &DST) >>= IMM; |
| CONT; |
| ALU64_MOD_X: |
| - div64_u64_rem(DST, SRC, &tmp); |
| - DST = tmp; |
| + div64_u64_rem(DST, SRC, &AX); |
| + DST = AX; |
| CONT; |
| ALU_MOD_X: |
| - tmp = (u32) DST; |
| - DST = do_div(tmp, (u32) SRC); |
| + AX = (u32) DST; |
| + DST = do_div(AX, (u32) SRC); |
| CONT; |
| ALU64_MOD_K: |
| - div64_u64_rem(DST, IMM, &tmp); |
| - DST = tmp; |
| + div64_u64_rem(DST, IMM, &AX); |
| + DST = AX; |
| CONT; |
| ALU_MOD_K: |
| - tmp = (u32) DST; |
| - DST = do_div(tmp, (u32) IMM); |
| + AX = (u32) DST; |
| + DST = do_div(AX, (u32) IMM); |
| CONT; |
| ALU64_DIV_X: |
| DST = div64_u64(DST, SRC); |
| CONT; |
| ALU_DIV_X: |
| - tmp = (u32) DST; |
| - do_div(tmp, (u32) SRC); |
| - DST = (u32) tmp; |
| + AX = (u32) DST; |
| + do_div(AX, (u32) SRC); |
| + DST = (u32) AX; |
| CONT; |
| ALU64_DIV_K: |
| DST = div64_u64(DST, IMM); |
| CONT; |
| ALU_DIV_K: |
| - tmp = (u32) DST; |
| - do_div(tmp, (u32) IMM); |
| - DST = (u32) tmp; |
| + AX = (u32) DST; |
| + do_div(AX, (u32) IMM); |
| + DST = (u32) AX; |
| CONT; |
| ALU_END_TO_BE: |
| switch (IMM) { |
| @@ -1414,7 +1414,7 @@ STACK_FRAME_NON_STANDARD(___bpf_prog_run); /* jump table */ |
| static unsigned int PROG_NAME(stack_size)(const void *ctx, const struct bpf_insn *insn) \ |
| { \ |
| u64 stack[stack_size / sizeof(u64)]; \ |
| - u64 regs[MAX_BPF_REG]; \ |
| + u64 regs[MAX_BPF_EXT_REG]; \ |
| \ |
| FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ |
| ARG1 = (u64) (unsigned long) ctx; \ |
| @@ -1427,7 +1427,7 @@ static u64 PROG_NAME_ARGS(stack_size)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, \ |
| const struct bpf_insn *insn) \ |
| { \ |
| u64 stack[stack_size / sizeof(u64)]; \ |
| - u64 regs[MAX_BPF_REG]; \ |
| + u64 regs[MAX_BPF_EXT_REG]; \ |
| \ |
| FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ |
| BPF_R1 = r1; \ |
| -- |
| 2.19.1 |
| |