| From 86be36f6502c52ddb4b85938145324fd07332da1 Mon Sep 17 00:00:00 2001 |
| From: "Naveen N. Rao" <naveen.n.rao@linux.vnet.ibm.com> |
| Date: Fri, 15 Mar 2019 20:21:19 +0530 |
| Subject: powerpc: bpf: Fix generation of load/store DW instructions |
| |
| From: Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> |
| |
| commit 86be36f6502c52ddb4b85938145324fd07332da1 upstream. |
| |
| Yauheni Kaliuta pointed out that PTR_TO_STACK store/load verifier test |
| was failing on powerpc64 BE, and rightfully indicated that the PPC_LD() |
| macro is not masking away the last two bits of the offset per the ISA, |
| resulting in the generation of 'lwa' instruction instead of the intended |
| 'ld' instruction. |
| |
| Segher also pointed out that we can't simply mask away the last two bits |
| as that will result in loading/storing from/to a memory location that |
| was not intended. |
| |
| This patch addresses this by using ldx/stdx if the offset is not |
| word-aligned. We load the offset into a temporary register (TMP_REG_2) |
| and use that as the index register in a subsequent ldx/stdx. We fix |
| PPC_LD() macro to mask off the last two bits, but enhance PPC_BPF_LL() |
| and PPC_BPF_STL() to factor in the offset value and generate the proper |
| instruction sequence. We also convert all existing users of PPC_LD() and |
| PPC_STD() to use these macros. All existing uses of these macros have |
| been audited to ensure that TMP_REG_2 can be clobbered. |
| |
| Fixes: 156d0e290e96 ("powerpc/ebpf/jit: Implement JIT compiler for extended BPF") |
| Cc: stable@vger.kernel.org # v4.9+ |
| |
| Reported-by: Yauheni Kaliuta <yauheni.kaliuta@redhat.com> |
| Signed-off-by: Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> |
| Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| |
| --- |
| arch/powerpc/include/asm/ppc-opcode.h | 2 ++ |
| arch/powerpc/net/bpf_jit.h | 17 +++++------------ |
| arch/powerpc/net/bpf_jit32.h | 4 ++++ |
| arch/powerpc/net/bpf_jit64.h | 20 ++++++++++++++++++++ |
| arch/powerpc/net/bpf_jit_comp64.c | 12 ++++++------ |
| 5 files changed, 37 insertions(+), 18 deletions(-) |
| |
| --- a/arch/powerpc/include/asm/ppc-opcode.h |
| +++ b/arch/powerpc/include/asm/ppc-opcode.h |
| @@ -288,6 +288,7 @@ |
| /* Misc instructions for BPF compiler */ |
| #define PPC_INST_LBZ 0x88000000 |
| #define PPC_INST_LD 0xe8000000 |
| +#define PPC_INST_LDX 0x7c00002a |
| #define PPC_INST_LHZ 0xa0000000 |
| #define PPC_INST_LWZ 0x80000000 |
| #define PPC_INST_LHBRX 0x7c00062c |
| @@ -295,6 +296,7 @@ |
| #define PPC_INST_STB 0x98000000 |
| #define PPC_INST_STH 0xb0000000 |
| #define PPC_INST_STD 0xf8000000 |
| +#define PPC_INST_STDX 0x7c00012a |
| #define PPC_INST_STDU 0xf8000001 |
| #define PPC_INST_STW 0x90000000 |
| #define PPC_INST_STWU 0x94000000 |
| --- a/arch/powerpc/net/bpf_jit.h |
| +++ b/arch/powerpc/net/bpf_jit.h |
| @@ -51,6 +51,8 @@ |
| #define PPC_LIS(r, i) PPC_ADDIS(r, 0, i) |
| #define PPC_STD(r, base, i) EMIT(PPC_INST_STD | ___PPC_RS(r) | \ |
| ___PPC_RA(base) | ((i) & 0xfffc)) |
| +#define PPC_STDX(r, base, b) EMIT(PPC_INST_STDX | ___PPC_RS(r) | \ |
| + ___PPC_RA(base) | ___PPC_RB(b)) |
| #define PPC_STDU(r, base, i) EMIT(PPC_INST_STDU | ___PPC_RS(r) | \ |
| ___PPC_RA(base) | ((i) & 0xfffc)) |
| #define PPC_STW(r, base, i) EMIT(PPC_INST_STW | ___PPC_RS(r) | \ |
| @@ -65,7 +67,9 @@ |
| #define PPC_LBZ(r, base, i) EMIT(PPC_INST_LBZ | ___PPC_RT(r) | \ |
| ___PPC_RA(base) | IMM_L(i)) |
| #define PPC_LD(r, base, i) EMIT(PPC_INST_LD | ___PPC_RT(r) | \ |
| - ___PPC_RA(base) | IMM_L(i)) |
| + ___PPC_RA(base) | ((i) & 0xfffc)) |
| +#define PPC_LDX(r, base, b) EMIT(PPC_INST_LDX | ___PPC_RT(r) | \ |
| + ___PPC_RA(base) | ___PPC_RB(b)) |
| #define PPC_LWZ(r, base, i) EMIT(PPC_INST_LWZ | ___PPC_RT(r) | \ |
| ___PPC_RA(base) | IMM_L(i)) |
| #define PPC_LHZ(r, base, i) EMIT(PPC_INST_LHZ | ___PPC_RT(r) | \ |
| @@ -85,17 +89,6 @@ |
| ___PPC_RA(a) | ___PPC_RB(b)) |
| #define PPC_BPF_STDCX(s, a, b) EMIT(PPC_INST_STDCX | ___PPC_RS(s) | \ |
| ___PPC_RA(a) | ___PPC_RB(b)) |
| - |
| -#ifdef CONFIG_PPC64 |
| -#define PPC_BPF_LL(r, base, i) do { PPC_LD(r, base, i); } while(0) |
| -#define PPC_BPF_STL(r, base, i) do { PPC_STD(r, base, i); } while(0) |
| -#define PPC_BPF_STLU(r, base, i) do { PPC_STDU(r, base, i); } while(0) |
| -#else |
| -#define PPC_BPF_LL(r, base, i) do { PPC_LWZ(r, base, i); } while(0) |
| -#define PPC_BPF_STL(r, base, i) do { PPC_STW(r, base, i); } while(0) |
| -#define PPC_BPF_STLU(r, base, i) do { PPC_STWU(r, base, i); } while(0) |
| -#endif |
| - |
| #define PPC_CMPWI(a, i) EMIT(PPC_INST_CMPWI | ___PPC_RA(a) | IMM_L(i)) |
| #define PPC_CMPDI(a, i) EMIT(PPC_INST_CMPDI | ___PPC_RA(a) | IMM_L(i)) |
| #define PPC_CMPW(a, b) EMIT(PPC_INST_CMPW | ___PPC_RA(a) | \ |
| --- a/arch/powerpc/net/bpf_jit32.h |
| +++ b/arch/powerpc/net/bpf_jit32.h |
| @@ -122,6 +122,10 @@ DECLARE_LOAD_FUNC(sk_load_byte_msh); |
| #define PPC_NTOHS_OFFS(r, base, i) PPC_LHZ_OFFS(r, base, i) |
| #endif |
| |
| +#define PPC_BPF_LL(r, base, i) do { PPC_LWZ(r, base, i); } while(0) |
| +#define PPC_BPF_STL(r, base, i) do { PPC_STW(r, base, i); } while(0) |
| +#define PPC_BPF_STLU(r, base, i) do { PPC_STWU(r, base, i); } while(0) |
| + |
| #define SEEN_DATAREF 0x10000 /* might call external helpers */ |
| #define SEEN_XREG 0x20000 /* X reg is used */ |
| #define SEEN_MEM 0x40000 /* SEEN_MEM+(1<<n) = use mem[n] for temporary |
| --- a/arch/powerpc/net/bpf_jit64.h |
| +++ b/arch/powerpc/net/bpf_jit64.h |
| @@ -86,6 +86,26 @@ DECLARE_LOAD_FUNC(sk_load_byte); |
| (imm >= SKF_LL_OFF ? func##_negative_offset : func) : \ |
| func##_positive_offset) |
| |
| +/* |
| + * WARNING: These can use TMP_REG_2 if the offset is not at word boundary, |
| + * so ensure that it isn't in use already. |
| + */ |
| +#define PPC_BPF_LL(r, base, i) do { \ |
| + if ((i) % 4) { \ |
| + PPC_LI(b2p[TMP_REG_2], (i)); \ |
| + PPC_LDX(r, base, b2p[TMP_REG_2]); \ |
| + } else \ |
| + PPC_LD(r, base, i); \ |
| + } while(0) |
| +#define PPC_BPF_STL(r, base, i) do { \ |
| + if ((i) % 4) { \ |
| + PPC_LI(b2p[TMP_REG_2], (i)); \ |
| + PPC_STDX(r, base, b2p[TMP_REG_2]); \ |
| + } else \ |
| + PPC_STD(r, base, i); \ |
| + } while(0) |
| +#define PPC_BPF_STLU(r, base, i) do { PPC_STDU(r, base, i); } while(0) |
| + |
| #define SEEN_FUNC 0x1000 /* might call external helpers */ |
| #define SEEN_STACK 0x2000 /* uses BPF stack */ |
| #define SEEN_SKB 0x4000 /* uses sk_buff */ |
| --- a/arch/powerpc/net/bpf_jit_comp64.c |
| +++ b/arch/powerpc/net/bpf_jit_comp64.c |
| @@ -261,7 +261,7 @@ static void bpf_jit_emit_tail_call(u32 * |
| * if (tail_call_cnt > MAX_TAIL_CALL_CNT) |
| * goto out; |
| */ |
| - PPC_LD(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); |
| + PPC_BPF_LL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); |
| PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT); |
| PPC_BCC(COND_GT, out); |
| |
| @@ -274,7 +274,7 @@ static void bpf_jit_emit_tail_call(u32 * |
| /* prog = array->ptrs[index]; */ |
| PPC_MULI(b2p[TMP_REG_1], b2p_index, 8); |
| PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array); |
| - PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs)); |
| + PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs)); |
| |
| /* |
| * if (prog == NULL) |
| @@ -284,7 +284,7 @@ static void bpf_jit_emit_tail_call(u32 * |
| PPC_BCC(COND_EQ, out); |
| |
| /* goto *(prog->bpf_func + prologue_size); */ |
| - PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func)); |
| + PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func)); |
| #ifdef PPC64_ELF_ABI_v1 |
| /* skip past the function descriptor */ |
| PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], |
| @@ -616,7 +616,7 @@ bpf_alu32_trunc: |
| * the instructions generated will remain the |
| * same across all passes |
| */ |
| - PPC_STD(dst_reg, 1, bpf_jit_stack_local(ctx)); |
| + PPC_BPF_STL(dst_reg, 1, bpf_jit_stack_local(ctx)); |
| PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx)); |
| PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]); |
| break; |
| @@ -672,7 +672,7 @@ emit_clear: |
| PPC_LI32(b2p[TMP_REG_1], imm); |
| src_reg = b2p[TMP_REG_1]; |
| } |
| - PPC_STD(src_reg, dst_reg, off); |
| + PPC_BPF_STL(src_reg, dst_reg, off); |
| break; |
| |
| /* |
| @@ -719,7 +719,7 @@ emit_clear: |
| break; |
| /* dst = *(u64 *)(ul) (src + off) */ |
| case BPF_LDX | BPF_MEM | BPF_DW: |
| - PPC_LD(dst_reg, src_reg, off); |
| + PPC_BPF_LL(dst_reg, src_reg, off); |
| break; |
| |
| /* |