| From 304244412443f6513ebac8d4d3f95845011d9ad7 Mon Sep 17 00:00:00 2001 |
| From: Sasha Levin <sashal@kernel.org> |
| Date: Tue, 28 Sep 2021 11:13:10 +0200 |
| Subject: bpf, arm: Fix register clobbering in div/mod implementation |
| |
| From: Johan Almbladh <johan.almbladh@anyfinetworks.com> |
| |
| [ Upstream commit 79e3445b38e0cab94264a3894c0c3d57c930b97e ] |
| |
| On ARM CPUs that lack div/mod instructions, ALU32 BPF_DIV and BPF_MOD are |
| implemented using a call to a helper function. Before, the emitted code |
| for those function calls failed to preserve caller-saved ARM registers. |
| Since some of those registers happen to be mapped to BPF registers, it |
| resulted in eBPF register values being overwritten. |
| |
| This patch emits code to push and pop the remaining caller-saved ARM |
| registers r2-r3 into the stack during the div/mod function call. ARM |
| registers r0-r1 are used as arguments and return value, and those were |
| already saved and restored correctly. |
| |
| Fixes: 39c13c204bb1 ("arm: eBPF JIT compiler") |
| Signed-off-by: Johan Almbladh <johan.almbladh@anyfinetworks.com> |
| Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> |
| Signed-off-by: Sasha Levin <sashal@kernel.org> |
| --- |
| arch/arm/net/bpf_jit_32.c | 19 +++++++++++++++++++ |
| 1 file changed, 19 insertions(+) |
| |
| diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c |
| index 79b12e744537..dade3a3ba666 100644 |
| --- a/arch/arm/net/bpf_jit_32.c |
| +++ b/arch/arm/net/bpf_jit_32.c |
| @@ -39,6 +39,10 @@ |
| * +-----+ |
| * |RSVD | JIT scratchpad |
| * current ARM_SP => +-----+ <= (BPF_FP - STACK_SIZE + SCRATCH_SIZE) |
| + * | ... | caller-saved registers |
| + * +-----+ |
| + * | ... | arguments passed on stack |
| + * ARM_SP during call => +-----| |
| * | | |
| * | ... | Function call stack |
| * | | |
| @@ -66,6 +70,12 @@ |
| * |
| * When popping registers off the stack at the end of a BPF function, we |
| * reference them via the current ARM_FP register. |
| + * |
| + * Some eBPF operations are implemented via a call to a helper function. |
| + * Such calls are "invisible" in the eBPF code, so it is up to the calling |
| + * program to preserve any caller-saved ARM registers during the call. The |
| + * JIT emits code to push and pop those registers onto the stack, immediately |
| + * above the callee stack frame. |
| */ |
| #define CALLEE_MASK (1 << ARM_R4 | 1 << ARM_R5 | 1 << ARM_R6 | \ |
| 1 << ARM_R7 | 1 << ARM_R8 | 1 << ARM_R9 | \ |
| @@ -73,6 +83,8 @@ |
| #define CALLEE_PUSH_MASK (CALLEE_MASK | 1 << ARM_LR) |
| #define CALLEE_POP_MASK (CALLEE_MASK | 1 << ARM_PC) |
| |
| +#define CALLER_MASK (1 << ARM_R0 | 1 << ARM_R1 | 1 << ARM_R2 | 1 << ARM_R3) |
| + |
| enum { |
| /* Stack layout - these are offsets from (top of stack - 4) */ |
| BPF_R2_HI, |
| @@ -467,6 +479,7 @@ static inline int epilogue_offset(const struct jit_ctx *ctx) |
| |
| static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op) |
| { |
| + const int exclude_mask = BIT(ARM_R0) | BIT(ARM_R1); |
| const s8 *tmp = bpf2a32[TMP_REG_1]; |
| |
| #if __LINUX_ARM_ARCH__ == 7 |
| @@ -498,11 +511,17 @@ static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op) |
| emit(ARM_MOV_R(ARM_R0, rm), ctx); |
| } |
| |
| + /* Push caller-saved registers on stack */ |
| + emit(ARM_PUSH(CALLER_MASK & ~exclude_mask), ctx); |
| + |
| /* Call appropriate function */ |
| emit_mov_i(ARM_IP, op == BPF_DIV ? |
| (u32)jit_udiv32 : (u32)jit_mod32, ctx); |
| emit_blx_r(ARM_IP, ctx); |
| |
| + /* Restore caller-saved registers from stack */ |
| + emit(ARM_POP(CALLER_MASK & ~exclude_mask), ctx); |
| + |
| /* Save return value */ |
| if (rd != ARM_R0) |
| emit(ARM_MOV_R(rd, ARM_R0), ctx); |
| -- |
| 2.33.0 |
| |